diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 17d204377..842b36784 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -21,26 +21,25 @@ def body_or_str(obj, unicode=True): return obj if unicode else obj.encode('utf-8') BASEURL_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE) +_metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """ Return a tuple of two strings containing the interval and url included in the http-equiv parameter of the HTML meta element. If no url is included (None, None) is returned [instead of (interval, None)] """ - if 'meta_refresh_url' not in response.cache: + if response not in _metaref_cache: match = META_REFRESH_RE.search(response.body[0:4096]) - response.cache['meta_refresh_url'] = match.groups() if match else (None, None) - return response.cache['meta_refresh_url'] + _metaref_cache[response] = match.groups() if match else (None, None) + return _metaref_cache[response] _beautifulsoup_cache = weakref.WeakKeyDictionary() def get_cached_beautifulsoup(response):