RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactoring redirect logic

This commit is contained in:
Elias Dorneles 2015-09-13 11:49:59 -03:00
parent 81950f773d
commit b06d0706e6
1 changed files with 11 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrapy/downloadermiddlewares/redirect.py
View File

@ -61,23 +61,21 @@ class RedirectMiddleware(BaseRedirectMiddleware):
request.meta.get('handle_httpstatus_all', False)):
return response
location = None
if 'Location' in response.headers:
# HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
location = to_native_str(response.headers['location'].decode('latin1'))
allowed_status = (301, 302, 303, 307)
if 'Location' not in response.headers or response.status not in allowed_status:
return response
if location is not None and response.status in [301, 302, 303, 307]:
redirected_url = urljoin(request.url, location)
# HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
location = to_native_str(response.headers['location'].decode('latin1'))
if response.status in [301, 307] or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
redirected_url = urljoin(request.url, location)
if response.status in [302, 303]:
redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status)
if response.status in (301, 307) or request.method == 'HEAD':
redirected = request.replace(url=redirected_url)
return self._redirect(redirected, request, spider, response.status)
return response
redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status)
class MetaRefreshMiddleware(BaseRedirectMiddleware):