RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

make get_meta_refresh() function more robust and changed interface to return (int, str) as (interval, absolute_url)

This commit is contained in:
Pablo Hoffman 2009-10-21 09:37:04 -02:00
parent 720bc166cf
commit 939e302b6e
3 changed files with 57 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -14,8 +14,6 @@ class RedirectMiddleware(object):
self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
def process_response(self, request, response, spider):
domain = spider.domain_name
if response.status in [302, 303] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url, method='GET', body='')
@ -29,8 +27,8 @@ class RedirectMiddleware(object):
return self._redirect(redirected, request, spider, response.status)
interval, url = get_meta_refresh(response)
if url and int(interval) < self.max_metarefresh_delay:
redirected = request.replace(url=urljoin_rfc(request.url, url))
if url and interval < self.max_metarefresh_delay:
redirected = request.replace(url=url)
return self._redirect(redirected, request, spider, 'meta refresh')
return response

37
scrapy/tests/test_utils_response.py
View File

@ -42,7 +42,7 @@ class ResponseUtilsTest(unittest.TestCase):
<body>blahablsdfsal&amp;</body>
</html>"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
# refresh without url should return (None, None)
body = """<meta http-equiv="refresh" content="5" />"""
@ -52,7 +52,7 @@ class ResponseUtilsTest(unittest.TestCase):
body = """<meta http-equiv="refresh" content="5;
url=http://example.org/newpage" /></head>"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
self.assertEqual(get_meta_refresh(response), (5, 'http://example.org/newpage'))
# meta refresh in multiple lines
body = """<html><head>
@ -60,7 +60,38 @@ class ResponseUtilsTest(unittest.TestCase):
HTTP-EQUIV="Refresh"
CONTENT="1; URL=http://example.org/newpage">"""
response = Response(url='http://example.org', body=body)
self.assertEqual(get_meta_refresh(response), ('1', 'http://example.org/newpage'))
self.assertEqual(get_meta_refresh(response), (1, 'http://example.org/newpage'))
# entities in the redirect url
body = """<meta http-equiv="refresh" content="3; url=&#39;http://www.example.com/other&#39;">"""
response = Response(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://www.example.com/other'))
# relative redirects
body = """<meta http-equiv="refresh" content="3; url=other.html">"""
response = Response(url='http://example.com/page/this.html', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/page/other.html'))
# non-standard encodings (utf-16)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/redirect">"""
body = body.decode('ascii').encode('utf-16')
response = TextResponse(url='http://example.com', body=body, encoding='utf-16')
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/redirect'))
# non-ascii chars in the url (default encoding - utf8)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xc2\xa3">"""
response = Response(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
# non-ascii chars in the url (custom encoding - latin1)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/to\xa3">"""
response = TextResponse(url='http://example.com', body=body, encoding='latin1')
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/to%C2%A3'))
# wrong encodings (possibly caused by truncated chunks)
body = """<meta http-equiv="refresh" content="3; url=http://example.com/this\xc2_THAT">"""
response = Response(url='http://example.com', body=body)
self.assertEqual(get_meta_refresh(response), (3, 'http://example.com/thisTHAT'))
def test_response_httprepr(self):
r1 = Response("http://www.example.com")

27
scrapy/utils/response.py
View File

@ -13,6 +13,8 @@ from tempfile import NamedTemporaryFile
from twisted.web import http
from twisted.web.http import RESPONSES
from scrapy.utils.markup import remove_entities
from scrapy.utils.url import safe_url_string, urljoin_rfc
from scrapy.xlib.BeautifulSoup import BeautifulSoup
from scrapy.http import Response, HtmlResponse
@ -34,16 +36,29 @@ def get_base_url(response):
_baseurl_cache[response] = match.group(1) if match else response.url
return _baseurl_cache[response]
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.DOTALL | re.IGNORECASE)
META_REFRESH_RE = re.compile(ur'<meta[^>]*http-equiv[^>]*refresh[^>]*content\s*=\s*(?P<quote>["\'])(?P<int>\d+)\s*;\s*url=(?P<url>.*?)(?P=quote)', re.DOTALL | re.IGNORECASE)
_metaref_cache = weakref.WeakKeyDictionary()
def get_meta_refresh(response):
""" Return a tuple of two strings containing the interval and url included
in the http-equiv parameter of the HTML meta element. If no url is included
(None, None) is returned [instead of (interval, None)]
"""Parse the http-equiv parameter of the HTML meta element from the given
response and return a tuple (interval, url) where interval is an integer
containing the delay in seconds (or zero if not present) and url is a
string with the absolute url to redirect.
If no meta redirect is found, (None, None) is returned.
"""
if response not in _metaref_cache:
match = META_REFRESH_RE.search(response.body[0:4096])
_metaref_cache[response] = match.groups() if match else (None, None)
encoding = getattr(response, 'encoding', 'utf-8')
body_chunk = remove_entities(unicode(response.body[0:4096], encoding, \
errors='ignore'))
match = META_REFRESH_RE.search(body_chunk)
if match:
interval = int(match.group('int'))
url = safe_url_string(match.group('url').strip(' "\''))
url = urljoin_rfc(response.url, url)
_metaref_cache[response] = (interval, url)
else:
_metaref_cache[response] = (None, None)
#_metaref_cache[response] = match.groups() if match else (None, None)
return _metaref_cache[response]
_beautifulsoup_cache = weakref.WeakKeyDictionary()