From fcdc4ee7d9983615df7d4600ee94eedbb1a5f0ed Mon Sep 17 00:00:00 2001 From: Ping Yin Date: Tue, 4 May 2010 16:11:45 +0800 Subject: [PATCH] downloadermiddleware/redirect: always do "HEAD" if origin request method is HEAD Signed-off-by: Ping Yin --- .../contrib/downloadermiddleware/redirect.py | 8 ++++ .../test_downloadermiddleware_redirect.py | 41 ++++++++++++++----- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 1c6297b49..8bd3887db 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -15,6 +15,14 @@ class RedirectMiddleware(object): self.priority_adjust = settings.getint('REDIRECT_PRIORITY_ADJUST') def process_response(self, request, response, spider): + if request.method.upper() == 'HEAD': + if response.status in [301, 302, 303, 307] and 'Location' in response.headers: + redirected_url = urljoin_rfc(request.url, response.headers['location']) + redirected = request.replace(url=redirected_url) + return self._redirect(redirected, request, spider, response.status) + else: + return response + if response.status in [302, 303] and 'Location' in response.headers: redirected_url = urljoin_rfc(request.url, response.headers['location']) redirected = self._redirect_request_using_get(request, redirected_url) diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 8409fec6d..be705b958 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -18,18 +18,24 @@ class RedirectMiddlewareTest(unittest.TestCase): assert req2.priority > req.priority def test_redirect_301(self): - url = 'http://www.example.com/301' - url2 = 'http://www.example.com/redirected' - req = Request(url) - rsp = Response(url, headers={'Location': url2}, status=301) + def _test(method): + url = 'http://www.example.com/301' + url2 = 'http://www.example.com/redirected' + req = Request(url, method=method) + rsp = Response(url, headers={'Location': url2}, status=301) - req2 = self.mw.process_response(req, rsp, self.spider) - assert isinstance(req2, Request) - self.assertEqual(req2.url, url2) + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, method) - # response without Location header but with status code is 3XX should be ignored - del rsp.headers['Location'] - assert self.mw.process_response(req, rsp, self.spider) is rsp + # response without Location header but with status code is 3XX should be ignored + del rsp.headers['Location'] + assert self.mw.process_response(req, rsp, self.spider) is rsp + + _test('GET') + _test('POST') + _test('HEAD') def test_redirect_302(self): url = 'http://www.example.com/302' @@ -53,6 +59,21 @@ class RedirectMiddlewareTest(unittest.TestCase): del rsp.headers['Location'] assert self.mw.process_response(req, rsp, self.spider) is rsp + def test_redirect_302_head(self): + url = 'http://www.example.com/302' + url2 = 'http://www.example.com/redirected2' + req = Request(url, method='HEAD') + rsp = Response(url, headers={'Location': url2}, status=302) + + req2 = self.mw.process_response(req, rsp, self.spider) + assert isinstance(req2, Request) + self.assertEqual(req2.url, url2) + self.assertEqual(req2.method, 'HEAD') + + # response without Location header but with status code is 3XX should be ignored + del rsp.headers['Location'] + assert self.mw.process_response(req, rsp, self.spider) is rsp + def test_meta_refresh(self): body = """