diff --git a/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/contrib/downloadermiddleware/redirect.py index 13da6db0f..b9249abfa 100644 --- a/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/contrib/downloadermiddleware/redirect.py @@ -21,14 +21,14 @@ class RedirectMiddleware(object): status = exception.status response = exception.response - if status in [302, 303]: + if status in [302, 303] and 'Location' in response.headers: redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url, method='GET', body='') redirected.headers.pop('Content-Type', None) redirected.headers.pop('Content-Length', None) return self._redirect(redirected, request, spider, status) - if status in [301, 307]: + if status in [301, 307] and 'Location' in response.headers: redirected_url = urljoin(request.url, response.headers['location']) redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, status) diff --git a/scrapy/tests/test_downloadermiddleware_redirect.py b/scrapy/tests/test_downloadermiddleware_redirect.py index 5d24ae4e3..c2aa82bec 100644 --- a/scrapy/tests/test_downloadermiddleware_redirect.py +++ b/scrapy/tests/test_downloadermiddleware_redirect.py @@ -30,6 +30,10 @@ class RedirectMiddlewareTest(unittest.TestCase): assert isinstance(req2, Request) self.assertEqual(req2.url, url2) + # response without Location header but with status code is 3XX should be ignored + del rsp.headers['Location'] + assert self.mw.process_exception(req, exc, self.spider) is None + def test_redirect_302(self): url = 'http://www.example.com/302' url2 = 'http://www.example.com/redirected2' @@ -50,6 +54,10 @@ class RedirectMiddlewareTest(unittest.TestCase): assert not req2.body, \ "Redirected body must be empty, not '%s'" % req2.body + # response without Location header but with status code is 3XX should be ignored + del rsp.headers['Location'] + assert self.mw.process_exception(req, exc, self.spider) is None + def test_meta_refresh(self): body = """