diff --git a/scrapy/tests/test_utils_response.py b/scrapy/tests/test_utils_response.py index 606ed4ee4..3b2b2226f 100644 --- a/scrapy/tests/test_utils_response.py +++ b/scrapy/tests/test_utils_response.py @@ -112,6 +112,16 @@ class ResponseUtilsTest(unittest.TestCase): response = TextResponse(url='http://example.org') self.assertEqual(get_meta_refresh(response), (None, None)) + # html commented meta refresh header must not directed + body = """""" + response = TextResponse(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (None, None)) + + # html comments must not interfere with uncommented meta refresh header + body = """-->""" + response = TextResponse(url='http://example.com', body=body) + self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/')) + def test_response_httprepr(self): r1 = Response("http://www.example.com") self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n') diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py index ca258ef44..95ae3df32 100644 --- a/scrapy/utils/response.py +++ b/scrapy/utils/response.py @@ -12,7 +12,7 @@ import tempfile from twisted.web import http from twisted.web.http import RESPONSES -from scrapy.utils.markup import remove_entities +from scrapy.utils.markup import remove_entities, remove_comments from scrapy.utils.url import safe_url_string, urljoin_rfc from scrapy.xlib.BeautifulSoup import BeautifulSoup from scrapy.http import Response, HtmlResponse @@ -48,7 +48,7 @@ def get_meta_refresh(response): If no meta redirect is found, (None, None) is returned. """ if response not in _metaref_cache: - body_chunk = remove_entities(response.body_as_unicode()[0:4096]) + body_chunk = remove_comments(remove_entities(response.body_as_unicode()[0:4096])) match = META_REFRESH_RE.search(body_chunk) if match: interval = int(match.group('int'))