RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixed bug with html meta refresh in multiple lines (thanks Molvo for the patch)

This commit is contained in:
Pablo Hoffman 2009-08-12 16:56:30 -03:00
parent 1dc592882b
commit cf566d6238
2 changed files with 9 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/tests/test_utils_response.py
View File

@ -54,6 +54,14 @@ class ResponseUtilsTest(unittest.TestCase):
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
# meta refresh in multiple lines
body = """<html><head>
<META
HTTP-EQUIV="Refresh"
CONTENT="1; URL=http://example.org/newpage">"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage'))
def test_response_httprepr(self):
r1 = Response("http://www.example.com")
self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')

2
scrapy/utils/response.py
View File

@ -31,7 +31,7 @@ def get_base_url(response):
response.cache['base_url'] = match.group(1) if match else response.url
return response.cache['base_url']
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.IGNORECASE)
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE)
def get_meta_refresh(response):
""" Return a tuple of two strings containing the interval and url included
in the http-equiv parameter of the HTML meta element. If no url is included