diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index 9fea54b70..898feb89f 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -4,11 +4,11 @@ from scrapy import log from scrapy.http import Request, Response from scrapy.core.exceptions import HttpException from scrapy.utils.url import urljoin_rfc as urljoin +from scrapy.utils.response import get_meta_refresh class RedirectLoop(Exception): pass -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);url=([^"\']+)', re.IGNORECASE) # some sites use meta-refresh for redirecting to a session expired page, so we # restrict automatic redirection to a maximum delay (in number of seconds) META_REFRESH_MAXSEC = 100 @@ -53,10 +53,9 @@ class RedirectMiddleware(object): def process_response(self, request, response, spider): if isinstance(response, Response): - m = META_REFRESH_RE.search(response.body[0:4096]) - if m and int(m.group(1)) < META_REFRESH_MAXSEC: - redirected = request.copy() - redirected.url = urljoin(request.url, m.group(2)) + interval, url = get_meta_refresh(response) + if url and int(interval) < META_REFRESH_MAXSEC: + redirected = request.replace(url=urljoin(request.url, url)) log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name) return redirected return response diff --git a/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py new file mode 100644 index 000000000..7fd0e98ff --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_downloadermiddleware_redirect.py @@ -0,0 +1,68 @@ +import unittest + +from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware +from scrapy.core.exceptions import HttpException +from scrapy.spider import spiders +from scrapy.http import Request, Response, Headers + +class RedirectMiddlewareTest(unittest.TestCase): + + def setUp(self): + spiders.spider_modules = ['scrapy.tests.test_spiders'] + spiders.reload() + self.spider = spiders.fromdomain('scrapytest.org') + + def test_process_exception(self): + + mw = RedirectMiddleware() + + url = 'http://www.example.com/301' + url2 = 'http://www.example.com/redirected' + req = Request(url) + hdr = Headers({'Location': [url2]}) + rsp = Response(url, headers=hdr) + exc = HttpException('301', None, rsp) + + req2 = mw.process_exception(req, exc, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + + url = 'http://www.example.com/302' + url2 = 'http://www.example.com/redirected' + req = Request(url, method='POST') + hdr = Headers({'Location': [url2]}) + rsp = Response(url, headers=hdr) + exc = HttpException('302', None, rsp) + + req2 = mw.process_exception(req, exc, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, 'GET') + assert not req2.body + + def test_process_response(self): + + mw = RedirectMiddleware() + + body = """ + + """ + req = Request(url='http://example.org') + rsp = Response(url='http://example.org', body=body) + req2 = mw.process_response(req, rsp, self.spider) + + assert isinstance(req2, Request) + self.assertEqual(req2.url, 'http://example.org/newpage') + + # meta-refresh with high intervals don't trigger redirects + body = """ + + """ + req = Request(url='http://example.org') + rsp = Response(url='http://example.org', body=body) + rsp2 = mw.process_response(req, rsp, self.spider) + + assert rsp is rsp2 + +if __name__ == "__main__": + unittest.main() diff --git a/scrapy/trunk/scrapy/tests/test_utils_response.py b/scrapy/trunk/scrapy/tests/test_utils_response.py index 13779a431..986488670 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_response.py +++ b/scrapy/trunk/scrapy/tests/test_utils_response.py @@ -1,6 +1,6 @@ import unittest from scrapy.http import Response, TextResponse -from scrapy.utils.response import body_or_str, get_base_url +from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh class ResponseUtilsTest(unittest.TestCase): dummy_response = TextResponse(url='http://example.org/', body='dummy_response') @@ -32,6 +32,24 @@ class ResponseUtilsTest(unittest.TestCase): """) self.assertEqual(get_base_url(response), 'http://example.org/something') + def test_get_meta_refresh(self): + body=""" + + Dummy + blahablsdfsal& + """ + response = Response(url='http://example.org', body=body) + self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + + # refresh without url should return (None, None) + body="""""" + response = Response(url='http://example.org', body=body) + self.assertEqual(get_meta_refresh(response), (None, None)) + + body="""""" + response = Response(url='http://example.org', body=body) + self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) if __name__ == "__main__": unittest.main() diff --git a/scrapy/trunk/scrapy/utils/response.py b/scrapy/trunk/scrapy/utils/response.py index 97304e57e..cfdc33c47 100644 --- a/scrapy/trunk/scrapy/utils/response.py +++ b/scrapy/trunk/scrapy/utils/response.py @@ -25,3 +25,14 @@ def get_base_url(response): match = BASEURL_RE.search(response.body[0:4096]) response.cache['base_url'] = match.group(1) if match else response.url return response.cache['base_url'] + +META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.IGNORECASE) +def get_meta_refresh(response): + """ Return a tuple of two strings containing the interval and url included + in the http-equiv parameter of the HTML meta element. If no url is included + (None, None) is returned [instead of (interval, None)] + """ + if 'meta_refresh_url' not in response.cache: + match = META_REFRESH_RE.search(response.body[0:4096]) + response.cache['meta_refresh_url'] = match.groups() if match else (None, None) + return response.cache['meta_refresh_url']