From 5dc9a88c347db3497b03949938184ca339f4e9cb Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:10:58 +0200 Subject: [PATCH 1/3] Handle HTTP 308 Permanent Redirect --- scrapy/downloadermiddlewares/redirect.py | 4 ++-- tests/test_downloadermiddleware_redirect.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 26677e527..30cae3fee 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): request.meta.get('handle_httpstatus_all', False)): return response - allowed_status = (301, 302, 303, 307) + allowed_status = (301, 302, 303, 307, 308) if 'Location' not in response.headers or response.status not in allowed_status: return response @@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware): redirected_url = urljoin(request.url, location) - if response.status in (301, 307) or request.method == 'HEAD': + if response.status in (301, 307, 308) or request.method == 'HEAD': redirected = request.replace(url=redirected_url) return self._redirect(redirected, request, spider, response.status) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index e8c92affa..a2da4aa8f 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase): req2 = self.mw.process_response(req, rsp, self.spider) assert req2.priority > req.priority - def test_redirect_301(self): - def _test(method): - url = 'http://www.example.com/301' + def test_redirect_3xx_permanent(self): + def _test(method, status=301): + url = 'http://www.example.com/{}'.format(status) url2 = 'http://www.example.com/redirected' req = Request(url, method=method) - rsp = Response(url, headers={'Location': url2}, status=301) + rsp = Response(url, headers={'Location': url2}, status=status) req2 = self.mw.process_response(req, rsp, self.spider) assert isinstance(req2, Request) @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=308) + _test('POST', status=308) + _test('HEAD', status=308) + def test_dont_redirect(self): url = 'http://www.example.com/301' url2 = 'http://www.example.com/redirected' From 1fdc10684fc427e5446350cada465ec934330a3f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 24 Jul 2017 18:25:11 +0200 Subject: [PATCH 2/3] HTTP Cache: treat 308 as 301 --- tests/test_downloadermiddleware_httpcache.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py index 12b69860a..22946b98c 100644 --- a/tests/test_downloadermiddleware_httpcache.py +++ b/tests/test_downloadermiddleware_httpcache.py @@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest): (True, 203, {'Last-Modified': self.yesterday}), (True, 300, {'Last-Modified': self.yesterday}), (True, 301, {'Last-Modified': self.yesterday}), + (True, 308, {'Last-Modified': self.yesterday}), (True, 401, {'Last-Modified': self.yesterday}), (True, 404, {'Cache-Control': 'public, max-age=600'}), (True, 302, {'Expires': self.tomorrow}), From 15a5c533fa6f448b7e5cd72ef099725c2295ef6f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 26 Jul 2017 19:07:57 +0200 Subject: [PATCH 3/3] Add tests for HTTP 307 permanent redirects --- tests/test_downloadermiddleware_redirect.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py index a2da4aa8f..35e474418 100644 --- a/tests/test_downloadermiddleware_redirect.py +++ b/tests/test_downloadermiddleware_redirect.py @@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase): _test('POST') _test('HEAD') + _test('GET', status=307) + _test('POST', status=307) + _test('HEAD', status=307) + _test('GET', status=308) _test('POST', status=308) _test('HEAD', status=308)