diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index f6ae43733..b8085f40a 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -14,8 +14,6 @@ class RedirectMiddleware(object): self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') def process_response(self, request, response, spider): - domain = spider.domain_name - if response.status in [302, 303] and 'Location' in response.headers: redirected_url = urljoin_rfc(request.url, response.headers['location']) redirected = request.replace(url=redirected_url, method='GET', body='') @@ -29,8 +27,8 @@ class RedirectMiddleware(object): return self._redirect(redirected, request, spider, response.status) interval, url = get_meta_refresh(response) - if url and int(interval) < self.max_metarefresh_delay: - redirected = request.replace(url=urljoin_rfc(request.url, url)) + if url and interval < self.max_metarefresh_delay: + redirected = request.replace(url=url) return self._redirect(redirected, request, spider, 'meta refresh') return response diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 1f7c39773..8a58a68ca 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -42,7 +42,7 @@ class ResponseUtilsTest(unittest.TestCase): blahablsdfsal& """ response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # refresh without url should return (None, None) body = """""" @@ -52,7 +52,7 @@ class ResponseUtilsTest(unittest.TestCase): body = """""" response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage')) # meta refresh in multiple lines body = """ @@ -60,7 +60,38 @@ class ResponseUtilsTest(unittest.TestCase): HTTP-EQUIV="Refresh" CONTENT="1; URL=http://example.org/newpage">""" response = Response(url='http://example.org', body=body) - self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage')) + self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage')) + + # entities in the redirect url + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other')) + + # relative redirects + body = """""" + response = Response(url='http://example.com/page/this.html', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html')) + + # non-standard encodings (utf-16) + body = """""" + body = body.decode('ascii').encode('utf-16') + response = TextResponse(url='http://example.com', body=body, encoding='utf-16') + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect')) + + # non-ascii chars in the url (default encoding - utf8) + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) + + # non-ascii chars in the url (custom encoding - latin1) + body = """""" + response = TextResponse(url='http://example.com', body=body, encoding='latin1') + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3')) + + # wrong encodings (possibly caused by truncated chunks) + body = """""" + response = Response(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT')) def test_response_httprepr(self): r1 = Response("http://www.example.com") diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index 8cffa75e5..c3fa69a6e 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -13,6 +13,8 @@ from tempfile import NamedTemporaryFile from twisted.web import http from twisted.web.http import RESPONSES +from scrapy.utils.markup import remove_entities +from scrapy.utils.url import safe_url_string, urljoin_rfc from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse @@ -34,16 +36,29 @@ def get_base_url(response): _baseurl_cache[response] = match.group(1) if match else response.url return _baseurl_cache[response] -META_REFRESH_RE = re.compile(r']*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE) +META_REFRESH_RE = re.compile(ur']*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P["\'])(?P\d+)\s*;\s*url=(?P.*?)(?P=quote)', re.DOTALL | re.IGNORECASE) _metaref_cache = weakref.WeakKeyDictionary() def get_meta_refresh(response): - """ Return a tuple of two strings containing the interval and url included - in the http-equiv parameter of the HTML meta element. If no url is included - (None, None) is returned [instead of (interval, None)] + """Parse the http-equiv parameter of the HTML meta element from the given + response and return a tuple (interval, url) where interval is an integer + containing the delay in seconds (or zero if not present) and url is a + string with the absolute url to redirect. + + If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - match = META_REFRESH_RE.search(response.body[0:4096]) - _metaref_cache[response] = match.groups() if match else (None, None) + encoding = getattr(response, 'encoding', 'utf-8') + body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \ + errors='ignore')) + match = META_REFRESH_RE.search(body_chunk) + if match: + interval = int(match.group('int')) + url = safe_url_string(match.group('url').strip(' "\'')) + url = urljoin_rfc(response.url, url) + _metaref_cache[response] = (interval, url) + else: + _metaref_cache[response] = (None, None) + #_metaref_cache[response] = match.groups() if match else (None, None) return _metaref_cache[response] _beautifulsoup_cache = weakref.WeakKeyDictionary()