RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

factorize redirectmw code that handles 302 and meta-refresh redirections 

--HG--
extra : rebase_source : f78fee093ac076d4a0630982dc4bade16755fa3b
This commit is contained in:
Daniel Grana 2009-10-21 12:46:18 -02:00
parent 5371d900a5
commit 6405efa507
1 changed files with 8 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -16,9 +16,7 @@ class RedirectMiddleware(object):
def process_response(self, request, response, spider):
if response.status in [302, 303] and 'Location' in response.headers:
redirected_url = urljoin_rfc(request.url, response.headers['location'])
redirected = request.replace(url=redirected_url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
redirected = self._redirect_request_using_get(request, redirected_url)
return self._redirect(redirected, request, spider, response.status)
if response.status in [301, 307] and 'Location' in response.headers:
@ -28,9 +26,7 @@ class RedirectMiddleware(object):
interval, url = get_meta_refresh(response)
if url and interval < self.max_metarefresh_delay:
redirected = request.replace(url=url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
redirected = self._redirect_request_using_get(request, url)
return self._redirect(redirected, request, spider, 'meta refresh')
return response
@ -52,4 +48,10 @@ class RedirectMiddleware(object):
domain=spider.domain_name, level=log.DEBUG)
raise IgnoreRequest
def _redirect_request_using_get(self, request, redirect_url):
redirected = request.replace(url=redirect_url, method='GET', body='')
redirected.headers.pop('Content-Type', None)
redirected.headers.pop('Content-Length', None)
return redirected