mirror of https://github.com/scrapy/scrapy.git
- added get_meta_refresh to scrapy.utils.response
- added tests for get_meta_refresh_url - added RedirectMiddleware tests --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40840
This commit is contained in:
Pablo Hoffman
parent
d376b20996
commit
f4224be411
|
|
@ -4,11 +4,11 @@ from scrapy import log
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.utils.url import urljoin_rfc as urljoin
|
||||
from scrapy.utils.response import get_meta_refresh
|
||||
|
||||
class RedirectLoop(Exception):
|
||||
pass
|
||||
|
||||
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);url=([^"\']+)', re.IGNORECASE)
|
||||
# some sites use meta-refresh for redirecting to a session expired page, so we
|
||||
# restrict automatic redirection to a maximum delay (in number of seconds)
|
||||
META_REFRESH_MAXSEC = 100
|
||||
|
|
@ -53,10 +53,9 @@ class RedirectMiddleware(object):
|
|||
|
||||
def process_response(self, request, response, spider):
|
||||
if isinstance(response, Response):
|
||||
m = META_REFRESH_RE.search(response.body[0:4096])
|
||||
if m and int(m.group(1)) < META_REFRESH_MAXSEC:
|
||||
redirected = request.copy()
|
||||
redirected.url = urljoin(request.url, m.group(2))
|
||||
interval, url = get_meta_refresh(response)
|
||||
if url and int(interval) < META_REFRESH_MAXSEC:
|
||||
redirected = request.replace(url=urljoin(request.url, url))
|
||||
log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name)
|
||||
return redirected
|
||||
return response
|
||||
|
|
|
|||
|
|
@ -0,0 +1,68 @@
|
|||
import unittest
|
||||
|
||||
from scrapy.contrib.downloadermiddleware.redirect import RedirectMiddleware
|
||||
from scrapy.core.exceptions import HttpException
|
||||
from scrapy.spider import spiders
|
||||
from scrapy.http import Request, Response, Headers
|
||||
|
||||
class RedirectMiddlewareTest(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
spiders.spider_modules = ['scrapy.tests.test_spiders']
|
||||
spiders.reload()
|
||||
self.spider = spiders.fromdomain('scrapytest.org')
|
||||
|
||||
def test_process_exception(self):
|
||||
|
||||
mw = RedirectMiddleware()
|
||||
|
||||
url = 'http://www.example.com/301'
|
||||
url2 = 'http://www.example.com/redirected'
|
||||
req = Request(url)
|
||||
hdr = Headers({'Location': [url2]})
|
||||
rsp = Response(url, headers=hdr)
|
||||
exc = HttpException('301', None, rsp)
|
||||
|
||||
req2 = mw.process_exception(req, exc, self.spider)
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, url2)
|
||||
|
||||
url = 'http://www.example.com/302'
|
||||
url2 = 'http://www.example.com/redirected'
|
||||
req = Request(url, method='POST')
|
||||
hdr = Headers({'Location': [url2]})
|
||||
rsp = Response(url, headers=hdr)
|
||||
exc = HttpException('302', None, rsp)
|
||||
|
||||
req2 = mw.process_exception(req, exc, self.spider)
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, url2)
|
||||
self.assertEqual(req2.method, 'GET')
|
||||
assert not req2.body
|
||||
|
||||
def test_process_response(self):
|
||||
|
||||
mw = RedirectMiddleware()
|
||||
|
||||
body = """<html>
|
||||
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
|
||||
</html>"""
|
||||
req = Request(url='http://example.org')
|
||||
rsp = Response(url='http://example.org', body=body)
|
||||
req2 = mw.process_response(req, rsp, self.spider)
|
||||
|
||||
assert isinstance(req2, Request)
|
||||
self.assertEqual(req2.url, 'http://example.org/newpage')
|
||||
|
||||
# meta-refresh with high intervals don't trigger redirects
|
||||
body = """<html>
|
||||
<head><meta http-equiv="refresh" content="1000;url=http://example.org/newpage" /></head>
|
||||
</html>"""
|
||||
req = Request(url='http://example.org')
|
||||
rsp = Response(url='http://example.org', body=body)
|
||||
rsp2 = mw.process_response(req, rsp, self.spider)
|
||||
|
||||
assert rsp is rsp2
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
@ -1,6 +1,6 @@
|
|||
import unittest
|
||||
from scrapy.http import Response, TextResponse
|
||||
from scrapy.utils.response import body_or_str, get_base_url
|
||||
from scrapy.utils.response import body_or_str, get_base_url, get_meta_refresh
|
||||
|
||||
class ResponseUtilsTest(unittest.TestCase):
|
||||
dummy_response = TextResponse(url='http://example.org/', body='dummy_response')
|
||||
|
|
@ -32,6 +32,24 @@ class ResponseUtilsTest(unittest.TestCase):
|
|||
</html>""")
|
||||
self.assertEqual(get_base_url(response), 'http://example.org/something')
|
||||
|
||||
def test_get_meta_refresh(self):
|
||||
body="""
|
||||
<html>
|
||||
<head><title>Dummy</title><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>
|
||||
<body>blahablsdfsal&</body>
|
||||
</html>"""
|
||||
response = Response(url='http://example.org', body=body)
|
||||
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
|
||||
|
||||
# refresh without url should return (None, None)
|
||||
body="""<meta http-equiv="refresh" content="5" />"""
|
||||
response = Response(url='http://example.org', body=body)
|
||||
self.assertEqual(get_meta_refresh(response), (None, None))
|
||||
|
||||
body="""<meta http-equiv="refresh" content="5;
|
||||
url=http://example.org/newpage" /></head>"""
|
||||
response = Response(url='http://example.org', body=body)
|
||||
self.assertEqual(get_meta_refresh(response), ('5', 'http://example.org/newpage'))
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
|
|
@ -25,3 +25,14 @@ def get_base_url(response):
|
|||
match = BASEURL_RE.search(response.body[0:4096])
|
||||
response.cache['base_url'] = match.group(1) if match else response.url
|
||||
return response.cache['base_url']
|
||||
|
||||
META_REFRESH_RE = re.compile(r'<meta[^>]*http-equiv[^>]*refresh[^>].*?(\d+);\s*url=([^"\']+)', re.IGNORECASE)
|
||||
def get_meta_refresh(response):
|
||||
""" Return a tuple of two strings containing the interval and url included
|
||||
in the http-equiv parameter of the HTML meta element. If no url is included
|
||||
(None, None) is returned [instead of (interval, None)]
|
||||
"""
|
||||
if 'meta_refresh_url' not in response.cache:
|
||||
match = META_REFRESH_RE.search(response.body[0:4096])
|
||||
response.cache['meta_refresh_url'] = match.groups() if match else (None, None)
|
||||
return response.cache['meta_refresh_url']
|
||||
|
|
|
|||
Loading…
Reference in New Issue