RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Handle HTTP 308 Permanent Redirect

This commit is contained in:
Paul Tremberth 2017-07-24 18:10:58 +02:00
parent 17bbd71433
commit 5dc9a88c34
2 changed files with 10 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/downloadermiddlewares/redirect.py
View File

@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
request.meta.get('handle_httpstatus_all', False)):
return response
allowed_status = (301, 302, 303, 307)
allowed_status = (301, 302, 303, 307, 308)
if 'Location' not in response.headers or response.status not in allowed_status:
return response
@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
redirected_url = urljoin(request.url, location)
if response.status in (301, 307) or request.method == 'HEAD':
if response.status in (301, 307, 308) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)

12
tests/test_downloadermiddleware_redirect.py
View File

@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase):
req2 = self.mw.process_response(req, rsp, self.spider)
assert req2.priority > req.priority
def test_redirect_301(self):
def _test(method):
url = 'http://www.example.com/301'
def test_redirect_3xx_permanent(self):
def _test(method, status=301):
url = 'http://www.example.com/{}'.format(status)
url2 = 'http://www.example.com/redirected'
req = Request(url, method=method)
rsp = Response(url, headers={'Location': url2}, status=301)
rsp = Response(url, headers={'Location': url2}, status=status)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
@ -42,6 +42,10 @@ class RedirectMiddlewareTest(unittest.TestCase):
_test('POST')
_test('HEAD')
_test('GET', status=308)
_test('POST', status=308)
_test('HEAD', status=308)
def test_dont_redirect(self):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'