From 79bf4c817ae8f48071407559ab669546843dcd64 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 24 Aug 2009 10:21:04 -0300 Subject: [PATCH] replaced old memoizemethod decorator with a more efficient one (memoizemethod_noargs) --- scrapy/http/response/html.py | 4 +-- scrapy/http/response/text.py | 12 +++---- scrapy/http/response/xml.py | 4 +-- scrapy/tests/test_utils_python.py | 22 ++++++------ scrapy/utils/python.py | 56 +++++++------------------------ 5 files changed, 33 insertions(+), 65 deletions(-) diff --git a/scrapy/http/response/html.py b/scrapy/http/response/html.py index 61269f86c..f1557e6f7 100644 --- a/scrapy/http/response/html.py +++ b/scrapy/http/response/html.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import re from scrapy.http.response.text import TextResponse -from scrapy.utils.python import memoizemethod +from scrapy.utils.python import memoizemethod_noargs class HtmlResponse(TextResponse): @@ -26,7 +26,7 @@ class HtmlResponse(TextResponse): def body_encoding(self): return self._body_declared_encoding() or super(HtmlResponse, self).body_encoding() - @memoizemethod('cache') + @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] match = self.METATAG_RE.search(chunk) or self.METATAG_RE2.search(chunk) diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0fa5b54ef..7f0a32a40 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -10,7 +10,7 @@ import re from scrapy.xlib.BeautifulSoup import UnicodeDammit from scrapy.http.response import Response -from scrapy.utils.python import memoizemethod +from scrapy.utils.python import memoizemethod_noargs class TextResponse(Response): @@ -41,17 +41,15 @@ class TextResponse(Response): def encoding(self): return self._encoding or self.headers_encoding() or self.body_encoding() - @memoizemethod('cache') - def headers_encoding(self, headers=None): - if headers is None: - headers = self.headers - content_type = headers.get('Content-Type') + @memoizemethod_noargs + def headers_encoding(self): + content_type = self.headers.get('Content-Type') if content_type: encoding = self._ENCODING_RE.search(content_type) if encoding: return encoding.group(1) - @memoizemethod('cache') + @memoizemethod_noargs def body_as_unicode(self): """Return body as unicode""" possible_encodings = (self._encoding, self.headers_encoding(), \ diff --git a/scrapy/http/response/xml.py b/scrapy/http/response/xml.py index 485514179..95f57cd39 100644 --- a/scrapy/http/response/xml.py +++ b/scrapy/http/response/xml.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import re from scrapy.http.response.text import TextResponse -from scrapy.utils.python import memoizemethod +from scrapy.utils.python import memoizemethod_noargs class XmlResponse(TextResponse): @@ -21,7 +21,7 @@ class XmlResponse(TextResponse): def body_encoding(self): return self._body_declared_encoding() or super(XmlResponse, self).body_encoding() - @memoizemethod('cache') + @memoizemethod_noargs def _body_declared_encoding(self): chunk = self.body[:5000] match = self.XMLDECL_RE.search(chunk) diff --git a/scrapy/tests/test_utils_python.py b/scrapy/tests/test_utils_python.py index 3f2fc3afd..38796436d 100644 --- a/scrapy/tests/test_utils_python.py +++ b/scrapy/tests/test_utils_python.py @@ -1,6 +1,7 @@ import unittest -from scrapy.utils.python import str_to_unicode, unicode_to_str, memoizemethod, isbinarytext +from scrapy.utils.python import str_to_unicode, unicode_to_str, \ + memoizemethod_noargs, isbinarytext class UtilsPythonTestCase(unittest.TestCase): def test_str_to_unicode(self): @@ -29,23 +30,24 @@ class UtilsPythonTestCase(unittest.TestCase): # converting a strange object should raise TypeError self.assertRaises(TypeError, unicode_to_str, unittest) - def test_memoizemethod(self): + def test_memoizemethod_noargs(self): class A(object): def __init__(self): self.cache = {} - @memoizemethod('cache') - def heavyfunc(self, arg1=None, arg2=None): - return [arg1, arg2] + @memoizemethod_noargs + def cached(self): + return object() + + def noncached(self): + return object() a = A() - one = a.heavyfunc() - two = a.heavyfunc() - three = a.heavyfunc('two') - four = a.heavyfunc('two') + one = a.cached() + two = a.cached() + three = a.noncached() assert one is two assert one is not three - assert three is four def test_isbinarytext(self): diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index db45bb843..7d43c4566 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -9,6 +9,8 @@ import re import os import fnmatch import inspect +import weakref +from functools import wraps from shutil import copy2, copystat from sgmllib import SGMLParser @@ -119,51 +121,17 @@ def re_rsearch(pattern, text, chunk_size=1024): return (offset + matches[-1].span()[0], offset + matches[-1].span()[1]) return None -def memoizemethod(cacheattr): - """A memoize decorator for methods, which caches calls to instance methods - into an attribute of the same instance (which must be dict). Calls with - different arguments will be cached in different buckets. - - This has the advantage that, when the instance is collected (by the garbage - collector) the cache is collected as well. - - Descriptors are required to implement this functionality. See why at: - http://blog.ianbicking.org/2008/10/24/decorators-and-descriptors/ - - Example: - - class A(object): - def __init__(self): - self.cache = {} - @memoizemethod('cache') - def calculate(self, arg1, arg2): - # expensive code here - - All calls to calculate() method of A instances will be cached in their - cache (instance) attribute, which must be a dict. - +def memoizemethod_noargs(method): + """Decorator to cache the result of a method (without arguments) using a + weak reference to its object """ - - class MemoizeMethod(object): - - def __init__(self, function): - self.function = function - - def __get__(self, obj, objtype=None): - if obj is None: - return self - new_func = self.function.__get__(obj, objtype) - return self.__class__(new_func) - - def __call__(self, *args, **kwargs): - method = self.function - cache = getattr(method.im_self, cacheattr) - key = (method.im_func, tuple(args), frozenset(kwargs.items())) - if key not in cache: - cache[key] = method(*args, **kwargs) - return cache[key] - - return MemoizeMethod + cache = weakref.WeakKeyDictionary() + @wraps(method) + def new_method(self, *args, **kwargs): + if self not in cache: + cache[self] = method(self, *args, **kwargs) + return cache[self] + return new_method _BINARYCHARS = set(map(chr, range(32))) - set(["\0", "\t", "\n", "\r"])