RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

cleanup redirection middleware 

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40888
This commit is contained in:
Daniel Grana 2009-02-19 19:42:42 +00:00
parent 65e8be46bc
commit ac735a185e
1 changed files with 21 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py
View File

@ -16,46 +16,29 @@ MAX_REDIRECT_LOOP = 10
class RedirectMiddleware(object):
def process_exception(self, request, exception, spider):
if isinstance(exception, HttpException):
status = exception.status
response = exception.response
if not isinstance(exception, HttpException):
return
if status in [302, 303]:
redirected_url = urljoin(request.url, response.headers['location'][0])
if not getattr(spider, "no_redirect", False):
redirected = request.replace(url=redirected_url, method='GET', body=None)
# This is needed to avoid redirection loops with requests that contain dont_filter = True
# Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks
if isinstance(redirected.dont_filter, int):
if not hasattr(redirected, "original_dont_filter"):
redirected.original_dont_filter = redirected.dont_filter
if redirected.dont_filter <= -MAX_REDIRECT_LOOP:
raise RedirectLoop("Exited redirect loop with %s consecutive visits to the same url." % (redirected.original_dont_filter + MAX_REDIRECT_LOOP) )
redirected.dont_filter -= 1
else:
redirected.dont_filter = False
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
log.msg("Ignored redirecting (%d) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
return response
status = exception.status
response = exception.response
if status in [301, 307]:
redirected_url = urljoin(request.url, response.headers['location'][0])
if not getattr(spider, "no_redirect", False):
redirected = request.replace(url=redirected_url)
# This is needed to avoid redirection loops with requests that contain dont_filter = True
# Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks
redirected.dont_filter = False
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
log.msg("Ignored redirecting (%d) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)
return response
if status in set([302, 303]):
redirected_url = urljoin(request.url, response.headers['location'][0])
redirected = request.replace(url=redirected_url, method='GET', body=None)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
if status in [301, 307]:
redirected_url = urljoin(request.url, response.headers['location'][0])
redirected = request.replace(url=redirected_url)
log.msg("Redirecting (%d) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
def process_response(self, request, response, spider):
if isinstance(response, Response):
interval, url = get_meta_refresh(response)
if url and int(interval) < META_REFRESH_MAXSEC:
redirected = request.replace(url=urljoin(request.url, url))
log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
interval, url = get_meta_refresh(response)
if url and int(interval) < META_REFRESH_MAXSEC:
redirected = request.replace(url=urljoin(request.url, url))
log.msg("Redirecting (meta refresh) to %s from %s" % (redirected, request), level=log.DEBUG, domain=spider.domain_name)
return redirected
return response