From 7a195a97949ea88a0d2e8227789b603786181ca2 Mon Sep 17 00:00:00 2001 From: olveyra Date: Fri, 1 Aug 2008 18:13:07 +0000 Subject: [PATCH] allow usage of integer "dont_filter", meaning number of times dont_filter will apply when redirecting. Under this scheme, dont_filter=True is the same as dont_filter=1 and (convenient?) dont filter <= 0 means dont_filter = True forever --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40143 --- scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py index 85052c020..3b0608256 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/redirect.py @@ -25,7 +25,10 @@ class RedirectMiddleware(object): redirected.body = None # This is needed to avoid redirection loops with requests that contain dont_filter = True # Example (9 May 2008): http://www.55max.com/product/001_photography.asp?3233,0,0,0,Michael+Banks - redirected.dont_filter = False + if isinstance(redirected.dont_filter, int): + redirected.dont_filter -= 1 + else: + redirected.dont_filter = False log.msg("Redirecting (%s) to %s from %s" % (status, redirected, request), level=log.DEBUG, domain=spider.domain_name) return redirected log.msg("Ignored redirecting (%s) to %s from %s (disabled by spider)" % (status, redirected_url, request), level=log.DEBUG, domain=spider.domain_name)