mirror of https://github.com/scrapy/scrapy.git
fix: LocalCache with limit=0 incorrectly stores items
When limit=0 is passed to LocalCache (e.g. when DNSCACHE_ENABLED=False), the condition 'if self.limit' evaluates to False due to Python's truthiness rules, causing items to be stored despite the cache being disabled. This leads to an unbounded memory leak during long crawls when DNS caching is explicitly disabled. Fix changes the condition to 'if self.limit is not None' and adds an early return when limit=0 to correctly handle the disabled cache case.
This commit is contained in:
parent
88ccc73b73
commit
b82c04704c
|
|
@ -152,7 +152,9 @@ class LocalCache(OrderedDict[_KT, _VT]):
|
|||
self.limit: int | None = limit
|
||||
|
||||
def __setitem__(self, key: _KT, value: _VT) -> None:
|
||||
if self.limit:
|
||||
if self.limit is not None:
|
||||
if self.limit == 0:
|
||||
return
|
||||
while len(self) >= self.limit:
|
||||
self.popitem(last=False)
|
||||
super().__setitem__(key, value)
|
||||
|
|
|
|||
|
|
@ -309,6 +309,17 @@ class TestLocalCache:
|
|||
assert str(x) in cache
|
||||
assert cache[str(x)] == x
|
||||
|
||||
def test_cache_with_zero_limit(self):
|
||||
# limit=0 means cache is disabled, no items should be stored
|
||||
cache = LocalCache(limit=0)
|
||||
cache["a"] = 1
|
||||
cache["b"] = 2
|
||||
cache["c"] = 3
|
||||
assert len(cache) == 0
|
||||
assert "a" not in cache
|
||||
assert "b" not in cache
|
||||
assert "c" not in cache
|
||||
|
||||
|
||||
class TestLocalWeakReferencedCache:
|
||||
def test_cache_with_limit(self):
|
||||
|
|
|
|||
Loading…
Reference in New Issue