From e7a58fe1573176415a9ca054428c53c1ca29931a Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 10:16:18 +0530 Subject: [PATCH 1/7] Code cleanup scrapy.utils.python.WeakKeyCache #4684 --- scrapy/utils/python.py | 12 ------------ tests/test_utils_python.py | 18 +----------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 9204977cf..7a393925e 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -273,18 +273,6 @@ def equal_attributes(obj1, obj2, attributes): return True -class WeakKeyCache: - - def __init__(self, default_factory): - self.default_factory = default_factory - self._weakdict = weakref.WeakKeyDictionary() - - def __getitem__(self, key): - if key not in self._weakdict: - self._weakdict[key] = self.default_factory(key) - return self._weakdict[key] - - @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index ebce3c079..b23ae2e52 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,7 +9,7 @@ from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - WeakKeyCache, get_func_args, to_bytes, to_unicode, + get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) @@ -155,22 +155,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_weakkeycache(self): - class _Weakme: - pass - - _values = count() - wk = WeakKeyCache(lambda k: next(_values)) - k = _Weakme() - v = wk[k] - self.assertEqual(v, wk[k]) - self.assertNotEqual(v, wk[_Weakme()]) - self.assertEqual(v, wk[k]) - del k - for _ in range(100): - if wk._weakdict: - gc.collect() - self.assertFalse(len(wk._weakdict)) def test_get_func_args(self): def f1(a, b, c): From 403bc7020a5e1ba2b59eced2cc5f4453c7650666 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 29 Jul 2020 18:05:33 +0530 Subject: [PATCH 2/7] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- tests/test_utils_python.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index b23ae2e52..5a53d89e4 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,9 +1,7 @@ import functools -import gc import operator import platform import unittest -from itertools import count from sys import version_info from warnings import catch_warnings @@ -12,7 +10,6 @@ from scrapy.utils.python import ( get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) - __doctests__ = ['scrapy.utils.python'] @@ -155,7 +152,6 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) - def test_get_func_args(self): def f1(a, b, c): pass From 49337bd2ae094d97d364948569f59b8211c8dbbe Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Thu, 30 Jul 2020 12:25:21 +0530 Subject: [PATCH 4/7] Code cleanup scrapy.utils.python.WeakKeyCache #4684 and fixing ci alerts --- scrapy/utils/python.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 7a393925e..c8f921ff3 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -127,6 +127,7 @@ def re_rsearch(pattern, text, chunk_size=1024): In case the pattern wasn't found, None is returned, otherwise it returns a tuple containing the start position of the match, and the ending (regarding the entire text). """ + def _chunk_iter(): offset = len(text) while True: @@ -158,6 +159,7 @@ def memoizemethod_noargs(method): if self not in cache: cache[self] = method(self, *args, **kwargs) return cache[self] + return new_method @@ -273,6 +275,19 @@ def equal_attributes(obj1, obj2, attributes): return True +@deprecated +class WeakKeyCache: + + def __init__(self, default_factory): + self.default_factory = default_factory + self._weakdict = weakref.WeakKeyDictionary() + + def __getitem__(self, key): + if key not in self._weakdict: + self._weakdict[key] = self.default_factory(key) + return self._weakdict[key] + + @deprecated def retry_on_eintr(function, *args, **kw): """Run a function and retry it while getting EINTR errors""" From 9d84289109b2368d5929d8b60ce583529c19fe4c Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:11:59 +0530 Subject: [PATCH 5/7] deprecated weakkeycache by specifying in __init__ --- scrapy/utils/python.py | 4 +++- scrapy/utils/tester.py | 3 +++ tests/test_utils_python.py | 22 +++++++++++++++++++++- 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index c8f921ff3..4756b07b6 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -7,10 +7,12 @@ import inspect import re import sys import weakref +import warnings from functools import partial, wraps from itertools import chain from scrapy.utils.decorators import deprecated +from scrapy.exceptions import ScrapyDeprecationWarning def flatten(x): @@ -275,10 +277,10 @@ def equal_attributes(obj1, obj2, attributes): return True -@deprecated class WeakKeyCache: def __init__(self, default_factory): + warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary() diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py new file mode 100644 index 000000000..691e9bc1a --- /dev/null +++ b/scrapy/utils/tester.py @@ -0,0 +1,3 @@ +from scrapy.utils.decorators import deprecated + + diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 5a53d89e4..ebce3c079 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -1,15 +1,18 @@ import functools +import gc import operator import platform import unittest +from itertools import count from sys import version_info from warnings import catch_warnings from scrapy.utils.python import ( memoizemethod_noargs, binary_is_text, equal_attributes, - get_func_args, to_bytes, to_unicode, + WeakKeyCache, get_func_args, to_bytes, to_unicode, without_none_values, MutableChain) + __doctests__ = ['scrapy.utils.python'] @@ -152,6 +155,23 @@ class UtilsPythonTestCase(unittest.TestCase): a.meta['z'] = 2 self.assertFalse(equal_attributes(a, b, [compare_z, 'x'])) + def test_weakkeycache(self): + class _Weakme: + pass + + _values = count() + wk = WeakKeyCache(lambda k: next(_values)) + k = _Weakme() + v = wk[k] + self.assertEqual(v, wk[k]) + self.assertNotEqual(v, wk[_Weakme()]) + self.assertEqual(v, wk[k]) + del k + for _ in range(100): + if wk._weakdict: + gc.collect() + self.assertFalse(len(wk._weakdict)) + def test_get_func_args(self): def f1(a, b, c): pass From b35d1f2b2c430f4d12cb8f9d408dfa0c0051746d Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 09:14:04 +0530 Subject: [PATCH 6/7] deleted tester.py --- scrapy/utils/tester.py | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 scrapy/utils/tester.py diff --git a/scrapy/utils/tester.py b/scrapy/utils/tester.py deleted file mode 100644 index 691e9bc1a..000000000 --- a/scrapy/utils/tester.py +++ /dev/null @@ -1,3 +0,0 @@ -from scrapy.utils.decorators import deprecated - - From 983b7ddf2e39c480efc6d104054f92f570714ac8 Mon Sep 17 00:00:00 2001 From: Kshitij Sharma Date: Wed, 5 Aug 2020 16:13:52 +0530 Subject: [PATCH 7/7] aesthetic fixes --- scrapy/utils/python.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index 4756b07b6..59f1b8371 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -6,13 +6,13 @@ import gc import inspect import re import sys -import weakref import warnings +import weakref from functools import partial, wraps from itertools import chain -from scrapy.utils.decorators import deprecated from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils.decorators import deprecated def flatten(x): @@ -280,7 +280,7 @@ def equal_attributes(obj1, obj2, attributes): class WeakKeyCache: def __init__(self, default_factory): - warnings.warn("Call to deprecated Class WeakKeyCache", category=ScrapyDeprecationWarning, stacklevel=2) + warnings.warn("The WeakKeyCache class is deprecated", category=ScrapyDeprecationWarning, stacklevel=2) self.default_factory = default_factory self._weakdict = weakref.WeakKeyDictionary()