From 439a3e59b8e858441f8d97dbc32f398db392330d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 4 Nov 2019 10:35:58 -0300 Subject: [PATCH] Fix scrapy.utils.datatypes.LocalCache limit issue --- scrapy/utils/datatypes.py | 5 +++-- tests/test_utils_datatypes.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index df2b99c28..f7e3240c1 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict): self.limit = limit def __setitem__(self, key, value): - while len(self) >= self.limit: - self.popitem(last=False) + if self.limit: + while len(self) >= self.limit: + self.popitem(last=False) super(LocalCache, self).__setitem__(key, value) diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py index 535095b8d..6ffd7c73c 100644 --- a/tests/test_utils_datatypes.py +++ b/tests/test_utils_datatypes.py @@ -7,7 +7,7 @@ if six.PY2: else: from collections.abc import Mapping, MutableMapping -from scrapy.utils.datatypes import CaselessDict, SequenceExclude +from scrapy.utils.datatypes import CaselessDict, SequenceExclude, LocalCache __doctests__ = ['scrapy.utils.datatypes'] @@ -242,6 +242,31 @@ class SequenceExcludeTest(unittest.TestCase): for v in [-3, "test", 1.1]: self.assertNotIn(v, d) + +class LocalCacheTest(unittest.TestCase): + + def test_cache_with_limit(self): + cache = LocalCache(limit=2) + cache['a'] = 1 + cache['b'] = 2 + cache['c'] = 3 + self.assertEqual(len(cache), 2) + self.assertNotIn('a', cache) + self.assertIn('b', cache) + self.assertIn('c', cache) + self.assertEqual(cache['b'], 2) + self.assertEqual(cache['c'], 3) + + def test_cache_without_limit(self): + max = 10**4 + cache = LocalCache() + for x in range(max): + cache[str(x)] = x + self.assertEqual(len(cache), max) + for x in range(max): + self.assertIn(str(x), cache) + self.assertEqual(cache[str(x)], x) + + if __name__ == "__main__": unittest.main() -