RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed bug with float values in meta refresh

This commit is contained in:
Martin Olveyra 2010-07-01 11:46:06 -03:00
parent b23af5ccf6
commit b258fc3305
2 changed files with 11 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
scrapy/tests/test_utils_response.py
View File

@ -122,6 +122,15 @@ class ResponseUtilsTest(unittest.TestCase):
response = TextResponse(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/'))
# float refresh intervals
body = """<meta http-equiv="refresh" content=".1;URL=index.html" />"""
response = TextResponse(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (0.1, 'http://example.com/index.html'))
body = """<meta http-equiv="refresh" content="3.1;URL=index.html" />"""
response = TextResponse(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3.1, 'http://example.com/index.html'))
def test_response_httprepr(self):
r1 = Response("http://www.example.com")
self.assertEqual(response_httprepr(r1), 'HTTP/1.1 200 OK\r\n\r\n')

4
scrapy/utils/response.py
View File

@ -36,7 +36,7 @@ def get_base_url(response):
_baseurl_cache[response] = urljoin_rfc(response.url, match.group(1)) if match else response.url
return _baseurl_cache[response]
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', \
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>(\d*\.)?\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', \
re.DOTALL | re.IGNORECASE)
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
@ -51,7 +51,7 @@ def get_meta_refresh(response):
body_chunk = remove_comments(remove_entities(response.body_as_unicode()[0:4096]))
match = META_REFRESH_RE.search(body_chunk)
if match:
interval = int(match.group('int'))
interval = float(match.group('int'))
url = safe_url_string(match.group('url').strip(' "\''))
url = urljoin_rfc(response.url, url)
_metaref_cache[response] = (interval, url)