From 36df87b4de49702ccd159270b974ed64d4a23260 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Mon, 14 Nov 2011 16:54:13 -0200 Subject: [PATCH] ignore meta-refresh redirects embedded in + """) self.assertEqual(get_meta_refresh(r1), (5.0, 'http://example.org/newpage')) self.assertEqual(get_meta_refresh(r2), (None, None)) + self.assertEqual(get_meta_refresh(r3), (None, None)) if __name__ == "__main__": unittest.main() diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index daa3a62bc..e3098a300 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -35,12 +35,14 @@ def get_base_url(response): return _baseurl_cache[response] _noscript_re = re.compile(u'', re.IGNORECASE | re.DOTALL) +_script_re = re.compile(u'.*?', re.IGNORECASE | re.DOTALL) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): """Parse the http-equiv refrsh parameter from the given response""" if response not in _metaref_cache: text = response.body_as_unicode()[0:4096] text = _noscript_re.sub(u'', text) + text = _script_re.sub(u'', text) _metaref_cache[response] = html.get_meta_refresh(text, response.url, \ response.encoding) return _metaref_cache[response]