Merge pull request #4123 from elacuesta/utils-local-cache-limit

Fix LocalCache limit issue, add tests

scrapy/utils/datatypes.py

@@ -315,8 +315,9 @@ class LocalCache(collections.OrderedDict):
         self.limit = limit
 
     def __setitem__(self, key, value):
-        while len(self) >= self.limit:
-            self.popitem(last=False)
+        if self.limit:
+            while len(self) >= self.limit:
+                self.popitem(last=False)
         super(LocalCache, self).__setitem__(key, value)
 
 

tests/test_utils_datatypes.py

@@ -7,7 +7,7 @@ if six.PY2:
 else:
     from collections.abc import Mapping, MutableMapping
 
-from scrapy.utils.datatypes import CaselessDict, SequenceExclude
+from scrapy.utils.datatypes import CaselessDict, LocalCache, SequenceExclude
 
 
 __doctests__ = ['scrapy.utils.datatypes']
@@ -242,5 +242,31 @@ class SequenceExcludeTest(unittest.TestCase):
         for v in [-3, "test", 1.1]:
             self.assertNotIn(v, d)
 
+
+class LocalCacheTest(unittest.TestCase):
+
+    def test_cache_with_limit(self):
+        cache = LocalCache(limit=2)
+        cache['a'] = 1
+        cache['b'] = 2
+        cache['c'] = 3
+        self.assertEqual(len(cache), 2)
+        self.assertNotIn('a', cache)
+        self.assertIn('b', cache)
+        self.assertIn('c', cache)
+        self.assertEqual(cache['b'], 2)
+        self.assertEqual(cache['c'], 3)
+
+    def test_cache_without_limit(self):
+        maximum = 10**4
+        cache = LocalCache()
+        for x in range(maximum):
+            cache[str(x)] = x
+        self.assertEqual(len(cache), maximum)
+        for x in range(maximum):
+            self.assertIn(str(x), cache)
+            self.assertEqual(cache[str(x)], x)
+
+
 if __name__ == "__main__":
     unittest.main()