RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2847 from redapple/redirect-308 

Handle HTTP 308 Permanent Redirect
This commit is contained in:
Daniel Graña 2017-07-26 15:29:23 -03:00 committed by GitHub
parent 17bbd71433 15a5c533fa
commit 881a5e3a57
3 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/downloadermiddlewares/redirect.py
View File

@ -64,7 +64,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
request.meta.get('handle_httpstatus_all', False)):
return response
allowed_status = (301, 302, 303, 307)
allowed_status = (301, 302, 303, 307, 308)
if 'Location' not in response.headers or response.status not in allowed_status:
return response
@ -72,7 +72,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
redirected_url = urljoin(request.url, location)
if response.status in (301, 307) or request.method == 'HEAD':
if response.status in (301, 307, 308) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)

1
tests/test_downloadermiddleware_httpcache.py
View File

@ -322,6 +322,7 @@ class RFC2616PolicyTest(DefaultStorageTest):
(True, 203, {'Last-Modified': self.yesterday}),
(True, 300, {'Last-Modified': self.yesterday}),
(True, 301, {'Last-Modified': self.yesterday}),
(True, 308, {'Last-Modified': self.yesterday}),
(True, 401, {'Last-Modified': self.yesterday}),
(True, 404, {'Cache-Control': 'public, max-age=600'}),
(True, 302, {'Expires': self.tomorrow}),

16
tests/test_downloadermiddleware_redirect.py
View File

@ -22,12 +22,12 @@ class RedirectMiddlewareTest(unittest.TestCase):
req2 = self.mw.process_response(req, rsp, self.spider)
assert req2.priority > req.priority
def test_redirect_301(self):
def _test(method):
url = 'http://www.example.com/301'
def test_redirect_3xx_permanent(self):
def _test(method, status=301):
url = 'http://www.example.com/{}'.format(status)
url2 = 'http://www.example.com/redirected'
req = Request(url, method=method)
rsp = Response(url, headers={'Location': url2}, status=301)
rsp = Response(url, headers={'Location': url2}, status=status)
req2 = self.mw.process_response(req, rsp, self.spider)
assert isinstance(req2, Request)
@ -42,6 +42,14 @@ class RedirectMiddlewareTest(unittest.TestCase):
_test('POST')
_test('HEAD')
_test('GET', status=307)
_test('POST', status=307)
_test('HEAD', status=307)
_test('GET', status=308)
_test('POST', status=308)
_test('HEAD', status=308)
def test_dont_redirect(self):
url = 'http://www.example.com/301'
url2 = 'http://www.example.com/redirected'