diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py
index 61269f86c..f1557e6f7 100644
--- a/scrapy/http/response/html.py
+++ b/scrapy/http/response/html.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
import re
from scrapy.http.response.text import TextResponse
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
class HtmlResponse(TextResponse):
@@ -26,7 +26,7 @@ class HtmlResponse(TextResponse):
def body_encoding(self):
return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding()
- @memoizemethod('cache')
+ @memoizemethod_noargs
def _body_declared_encoding(self):
chunk = self.body[:5000]
match = self.METATAG_RE.search(chunk) or self.METATAG_RE2.search(chunk)
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0fa5b54ef..7f0a32a40 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -10,7 +10,7 @@ import re
from scrapy.xlib.BeautifulSoup import UnicodeDammit
from scrapy.http.response import Response
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
class TextResponse(Response):
@@ -41,17 +41,15 @@ class TextResponse(Response):
def encoding(self):
return self._encoding or self.headers_encoding() or self.body_encoding()
- @memoizemethod('cache')
- def headers_encoding(self, headers=None):
- if headers is None:
- headers = self.headers
- content_type = headers.get('Content-Type')
+ @memoizemethod_noargs
+ def headers_encoding(self):
+ content_type = self.headers.get('Content-Type')
if content_type:
encoding = self._ENCODING_RE.search(content_type)
if encoding:
return encoding.group(1)
- @memoizemethod('cache')
+ @memoizemethod_noargs
def body_as_unicode(self):
"""Return body as unicode"""
possible_encodings = (self._encoding, self.headers_encoding(), \
diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py
index 485514179..95f57cd39 100644
--- a/scrapy/http/response/xml.py
+++ b/scrapy/http/response/xml.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
import re
from scrapy.http.response.text import TextResponse
-from scrapy.utils.python import memoizemethod
+from scrapy.utils.python import memoizemethod_noargs
class XmlResponse(TextResponse):
@@ -21,7 +21,7 @@ class XmlResponse(TextResponse):
def body_encoding(self):
return self._body_declared_encoding() or super(XmlResponse, self).body_encoding()
- @memoizemethod('cache')
+ @memoizemethod_noargs
def _body_declared_encoding(self):
chunk = self.body[:5000]
match = self.XMLDECL_RE.search(chunk)
diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py
index 3f2fc3afd..38796436d 100644
--- a/scrapy/tests/test_utils_python.py
+++ b/scrapy/tests/test_utils_python.py
@@ -1,6 +1,7 @@
import unittest
-from scrapy.utils.python import str_to_unicode, unicode_to_str, memoizemethod, isbinarytext
+from scrapy.utils.python import str_to_unicode, unicode_to_str, \
+ memoizemethod_noargs, isbinarytext
class UtilsPythonTestCase(unittest.TestCase):
def test_str_to_unicode(self):
@@ -29,23 +30,24 @@ class UtilsPythonTestCase(unittest.TestCase):
# converting a strange object should raise TypeError
self.assertRaises(TypeError, unicode_to_str, unittest)
- def test_memoizemethod(self):
+ def test_memoizemethod_noargs(self):
class A(object):
def __init__(self):
self.cache = {}
- @memoizemethod('cache')
- def heavyfunc(self, arg1=None, arg2=None):
- return [arg1, arg2]
+ @memoizemethod_noargs
+ def cached(self):
+ return object()
+
+ def noncached(self):
+ return object()
a = A()
- one = a.heavyfunc()
- two = a.heavyfunc()
- three = a.heavyfunc('two')
- four = a.heavyfunc('two')
+ one = a.cached()
+ two = a.cached()
+ three = a.noncached()
assert one is two
assert one is not three
- assert three is four
def test_isbinarytext(self):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index db45bb843..7d43c4566 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -9,6 +9,8 @@ import re
import os
import fnmatch
import inspect
+import weakref
+from functools import wraps
from shutil import copy2, copystat
from sgmllib import SGMLParser
@@ -119,51 +121,17 @@ def re_rsearch(pattern, text, chunk_size=1024):
return (offset + matches[-1].span()[0], offset + matches[-1].span()[1])
return None
-def memoizemethod(cacheattr):
- """A memoize decorator for methods, which caches calls to instance methods
- into an attribute of the same instance (which must be dict). Calls with
- different arguments will be cached in different buckets.
-
- This has the advantage that, when the instance is collected (by the garbage
- collector) the cache is collected as well.
-
- Descriptors are required to implement this functionality. See why at:
- http://blog.ianbicking.org/2008/10/24/decorators-and-descriptors/
-
- Example:
-
- class A(object):
- def __init__(self):
- self.cache = {}
- @memoizemethod('cache')
- def calculate(self, arg1, arg2):
- # expensive code here
-
- All calls to calculate() method of A instances will be cached in their
- cache (instance) attribute, which must be a dict.
-
+def memoizemethod_noargs(method):
+ """Decorator to cache the result of a method (without arguments) using a
+ weak reference to its object
"""
-
- class MemoizeMethod(object):
-
- def __init__(self, function):
- self.function = function
-
- def __get__(self, obj, objtype=None):
- if obj is None:
- return self
- new_func = self.function.__get__(obj, objtype)
- return self.__class__(new_func)
-
- def __call__(self, *args, **kwargs):
- method = self.function
- cache = getattr(method.im_self, cacheattr)
- key = (method.im_func, tuple(args), frozenset(kwargs.items()))
- if key not in cache:
- cache[key] = method(*args, **kwargs)
- return cache[key]
-
- return MemoizeMethod
+ cache = weakref.WeakKeyDictionary()
+ @wraps(method)
+ def new_method(self, *args, **kwargs):
+ if self not in cache:
+ cache[self] = method(self, *args, **kwargs)
+ return cache[self]
+ return new_method
_BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])