ported get_base_url and get_meta_refresh to use WeakKeyDictionary (instead of Request.cache)

scrapy/utils/response.py

@@ -21,26 +21,25 @@ def body_or_str(obj, unicode=True):
         return obj if unicode else obj.encode('utf-8')
 
 BASEURL_RE = re.compile(r'<base\s+href\s*=\s*[\"\']\s*([^\"\'\s]+)\s*[\"\']', re.I)
+_baseurl_cache = weakref.WeakKeyDictionary()
 def get_base_url(response):
     """ Return the base url of the given response used to resolve relative links. """
-    # re is used instead of xpath for eficiency reasons. In a quick
-    # benchmark using timeit we got (for 50 repetitions) 0.0017 seconds
-    # using re and 0.7452 using xpath
-    if 'base_url' not in response.cache:
+    if response not in _baseurl_cache:
         match = BASEURL_RE.search(response.body[0:4096])
-        response.cache['base_url'] = match.group(1) if match else response.url
-    return response.cache['base_url']
+        _baseurl_cache[response] = match.group(1) if match else response.url
+    return _baseurl_cache[response]
 
 META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE)
+_metaref_cache = weakref.WeakKeyDictionary()
 def get_meta_refresh(response):
     """ Return a tuple of two strings containing the interval and url included
     in the http-equiv parameter of the HTML meta element. If no url is included
     (None, None) is returned [instead of (interval, None)]
     """
-    if 'meta_refresh_url' not in response.cache:
+    if response not in _metaref_cache:
         match = META_REFRESH_RE.search(response.body[0:4096])
-        response.cache['meta_refresh_url'] = match.groups() if match else (None, None)
-    return response.cache['meta_refresh_url']
+        _metaref_cache[response] = match.groups() if match else (None, None)
+    return _metaref_cache[response]
 
 _beautifulsoup_cache = weakref.WeakKeyDictionary()
 def get_cached_beautifulsoup(response):