Automated merge with ssh://hg.scrapy.org/scrapy

scrapy/tests/test_utils_response.py

@@ -112,6 +112,16 @@ class ResponseUtilsTest(unittest.TestCase):
         response = TextResponse(url='http://example.org')
         self.assertEqual(get_meta_refresh(response), (None, None))
 
+        # html commented meta refresh header must not directed
+        body = """<!--<meta http-equiv="refresh" content="3; url=http://example.com/">-->"""
+        response = TextResponse(url='http://example.com', body=body)
+        self.assertEqual(get_meta_refresh(response), (None, None))
+
+        # html comments must not interfere with uncommented meta refresh header
+        body = """<!-- commented --><meta http-equiv="refresh" content="3; url=http://example.com/">-->"""
+        response = TextResponse(url='http://example.com', body=body)
+        self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/'))
+
     def test_response_httprepr(self):
         r1 = Response("http://www.example.com")
         self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')

scrapy/utils/response.py

@@ -12,7 +12,7 @@ import tempfile
 from twisted.web import http
 from twisted.web.http import RESPONSES
 
-from scrapy.utils.markup import remove_entities
+from scrapy.utils.markup import remove_entities, remove_comments
 from scrapy.utils.url import safe_url_string, urljoin_rfc
 from scrapy.xlib.BeautifulSoup import BeautifulSoup
 from scrapy.http import Response, HtmlResponse
@@ -48,7 +48,7 @@ def get_meta_refresh(response):
     If no meta redirect is found, (None, None) is returned.
     """
     if response not in _metaref_cache:
-        body_chunk = remove_entities(response.body_as_unicode()[0:4096])
+        body_chunk = remove_comments(remove_entities(response.body_as_unicode()[0:4096]))
         match = META_REFRESH_RE.search(body_chunk)
         if match:
             interval = int(match.group('int'))