replaced old memoizemethod decorator with a more efficient one (memoizemethod_noargs)

scrapy/http/response/html.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 import re
 
 from scrapy.http.response.text import TextResponse
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
 
 class HtmlResponse(TextResponse):
 
@@ -26,7 +26,7 @@ class HtmlResponse(TextResponse):
     def body_encoding(self):
         return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding()
 
-    @memoizemethod('cache')
+    @memoizemethod_noargs
     def _body_declared_encoding(self):
         chunk = self.body[:5000]
         match = self.METATAG_RE.search(chunk) or self.METATAG_RE2.search(chunk)

scrapy/http/response/text.py

@@ -10,7 +10,7 @@ import re
 from scrapy.xlib.BeautifulSoup import UnicodeDammit
 
 from scrapy.http.response import Response
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
 
 class TextResponse(Response):
 
@@ -41,17 +41,15 @@ class TextResponse(Response):
     def encoding(self):
         return self._encoding or self.headers_encoding() or self.body_encoding()
 
-    @memoizemethod('cache')
-    def headers_encoding(self, headers=None):
-        if headers is None:
-            headers = self.headers
-        content_type = headers.get('Content-Type')
+    @memoizemethod_noargs
+    def headers_encoding(self):
+        content_type = self.headers.get('Content-Type')
         if content_type:
             encoding = self._ENCODING_RE.search(content_type)
             if encoding:
                 return encoding.group(1)
 
-    @memoizemethod('cache')
+    @memoizemethod_noargs
     def body_as_unicode(self):
         """Return body as unicode"""
         possible_encodings = (self._encoding, self.headers_encoding(), \

scrapy/http/response/xml.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 import re
 
 from scrapy.http.response.text import TextResponse
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
 
 class XmlResponse(TextResponse):
 
@@ -21,7 +21,7 @@ class XmlResponse(TextResponse):
     def body_encoding(self):
         return self._body_declared_encoding() or super(XmlResponse, self).body_encoding()
 
-    @memoizemethod('cache')
+    @memoizemethod_noargs
     def _body_declared_encoding(self):
         chunk = self.body[:5000]
         match = self.XMLDECL_RE.search(chunk)

scrapy/tests/test_utils_python.py

@@ -1,6 +1,7 @@
 import unittest
 
-from scrapy.utils.python import str_to_unicode, unicode_to_str, memoizemethod, isbinarytext
+from scrapy.utils.python import str_to_unicode, unicode_to_str, \
+    memoizemethod_noargs, isbinarytext
 
 class UtilsPythonTestCase(unittest.TestCase):
     def test_str_to_unicode(self):
@@ -29,23 +30,24 @@ class UtilsPythonTestCase(unittest.TestCase):
         # converting a strange object should raise TypeError
         self.assertRaises(TypeError, unicode_to_str, unittest)
 
-    def test_memoizemethod(self):
+    def test_memoizemethod_noargs(self):
         class A(object):
             def __init__(self):
                 self.cache = {}
 
-            @memoizemethod('cache')
-            def heavyfunc(self, arg1=None, arg2=None):
-                return [arg1, arg2]
+            @memoizemethod_noargs
+            def cached(self):
+                return object()
+
+            def noncached(self):
+                return object()
 
         a = A()
-        one = a.heavyfunc()
-        two = a.heavyfunc()
-        three = a.heavyfunc('two')
-        four = a.heavyfunc('two')
+        one = a.cached()
+        two = a.cached()
+        three = a.noncached()
         assert one is two
         assert one is not three
-        assert three is four
 
     def test_isbinarytext(self):
 

scrapy/utils/python.py

@@ -9,6 +9,8 @@ import re
 import os
 import fnmatch
 import inspect
+import weakref
+from functools import wraps
 from shutil import copy2, copystat
 from sgmllib import SGMLParser
 
@@ -119,51 +121,17 @@ def re_rsearch(pattern, text, chunk_size=1024):
             return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
     return None
 
-def memoizemethod(cacheattr):
-    """A memoize decorator for methods, which caches calls to instance methods
-    into an attribute of the same instance (which must be dict). Calls with
-    different arguments will be cached in different buckets.
-
-    This has the advantage that, when the instance is collected (by the garbage
-    collector) the cache is collected as well.
-
-    Descriptors are required to implement this functionality. See why at:
-    http://blog.ianbicking.org/2008/10/24/decorators-and-descriptors/
-
-    Example:
-
-    class A(object):
-            def __init__(self):
-            self.cache = {}
-            @memoizemethod('cache')
-        def calculate(self, arg1, arg2):
-            # expensive code here
-
-    All calls to calculate() method of A instances will be cached in their
-    cache (instance) attribute, which must be a dict.
-
+def memoizemethod_noargs(method):
+    """Decorator to cache the result of a method (without arguments) using a
+    weak reference to its object
     """
-
-    class MemoizeMethod(object):
-
-        def __init__(self, function):
-            self.function = function
-
-        def __get__(self, obj, objtype=None):
-            if obj is None:
-                return self
-            new_func = self.function.__get__(obj, objtype)
-            return self.__class__(new_func)
-
-        def __call__(self, *args, **kwargs):
-            method = self.function
-            cache = getattr(method.im_self, cacheattr)
-            key = (method.im_func, tuple(args), frozenset(kwargs.items()))
-            if key not in cache:
-                cache[key] = method(*args, **kwargs)
-            return cache[key]
-
-    return MemoizeMethod
+    cache = weakref.WeakKeyDictionary()
+    @wraps(method)
+    def new_method(self, *args, **kwargs):
+        if self not in cache:
+            cache[self] = method(self, *args, **kwargs)
+        return cache[self]
+    return new_method
 
 _BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])