RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

downloadermiddleware/redirect: always do "HEAD" if origin request method is HEAD 

Signed-off-by: Ping Yin <pkufranky@gmail.com>
This commit is contained in:
Ping Yin 2010-05-04 16:11:45 +08:00
parent 031eb1e5ed
commit fcdc4ee7d9
2 changed files with 39 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -15,6 +15,14 @@ class RedirectMiddleware(object):
self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST')
def process_response(self, request, response, spider):
if request.method.upper() == 'HEAD':
if response.status in [301, 302, 303, 307] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
else:
return response
if response.status in [302, 303] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected = self._redirect_request_using_get(request, redirected_url)

41
scrapy/tests/test_downloadermiddleware_redirect.py
View File

@ -18,18 +18,24 @@ class RedirectMiddlewareTest(unittest.TestCase):
assert req2.priority > req.priority
def test_redirect_301(self):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'
req = Request(url)
rsp = Response(url, headers={'Location': url2}, status=301)
def _test(method):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'
req = Request(url, method=method)
rsp = Response(url, headers={'Location': url2}, status=301)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
self.assertEqual(req2.method, method)
# response without Location header but with status code is 3XX should be ignored
del rsp.headers['Location']
assert self.mw.process_response(req, rsp, self.spider) is rsp
# response without Location header but with status code is 3XX should be ignored
del rsp.headers['Location']
assert self.mw.process_response(req, rsp, self.spider) is rsp
_test('GET')
_test('POST')
_test('HEAD')
def test_redirect_302(self):
url = 'http://www.example.com/302'
@ -53,6 +59,21 @@ class RedirectMiddlewareTest(unittest.TestCase):
del rsp.headers['Location']
assert self.mw.process_response(req, rsp, self.spider) is rsp
def test_redirect_302_head(self):
url = 'http://www.example.com/302'
url2 = 'http://www.example.com/redirected2'
req = Request(url, method='HEAD')
rsp = Response(url, headers={'Location': url2}, status=302)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
self.assertEqual(req2.url, url2)
self.assertEqual(req2.method, 'HEAD')
# response without Location header but with status code is 3XX should be ignored
del rsp.headers['Location']
assert self.mw.process_response(req, rsp, self.spider) is rsp
def test_meta_refresh(self):
body = """<html>
<head><meta http-equiv="refresh" content="5;url=http://example.org/newpage" /></head>