diff --git a/scrapy/trunk/scrapy/conf/core_settings.py b/scrapy/trunk/scrapy/conf/core_settings.py index ad140af03..84bb44833 100644 --- a/scrapy/trunk/scrapy/conf/core_settings.py +++ b/scrapy/trunk/scrapy/conf/core_settings.py @@ -26,5 +26,5 @@ PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer' EXTENSIONS = [] # contrib.middleware.retry.RetryMiddleware default settings -RETRY_TIMES = 3 +RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200'] diff --git a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py index d21deb160..e7ed17ea5 100644 --- a/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py @@ -38,26 +38,19 @@ class RetryMiddleware(object): def __init__(self): self.failed_count = {} - self.max_retries = settings.getint('RETRY_TIMES') + self.retry_times = settings.getint('RETRY_TIMES') + self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES')) def process_exception(self, request, exception, spider): - retry = False - - if isinstance(exception, self.EXCEPTIONS_TO_RETRY): - retry = True - elif isinstance(exception, HttpException): - if exception.status in settings.getlist('RETRY_HTTP_CODES'): - retry = True - - if retry: + if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)): fp = request.fingerprint() - count = self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 + self.failed_count[fp] = self.failed_count.get(fp, 0) + 1 - if self.failed_count[fp] < self.max_retries: - log.msg("Retrying %s (failed %d times): %s" % (request, count, exception), level=log.DEBUG, domain=spider.domain_name) + if self.failed_count[fp] <= self.retry_times: + log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG) retryreq = request.copy() retryreq.dont_filter = True return retryreq else: - log.msg("Discarding %s (failed %d times): %s" % (request, count, exception), domain=spider.domain_name, level=log.DEBUG) + log.msg("Discarding %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG) diff --git a/scrapy/trunk/scrapy/core/downloader/middleware.py b/scrapy/trunk/scrapy/core/downloader/middleware.py index 32081e66f..e0c2537a7 100644 --- a/scrapy/trunk/scrapy/core/downloader/middleware.py +++ b/scrapy/trunk/scrapy/core/downloader/middleware.py @@ -70,6 +70,7 @@ class DownloaderMiddlewareManager(object): process_response MUST return a Response object. It could alter the given response, or it could create a brand-new Response. + To drop the response entirely an IgnoreRequest exception must be raised. process_exception(self, request, exception, spider) diff --git a/scrapy/trunk/scrapy/tests/test_middleware_retry.py b/scrapy/trunk/scrapy/tests/test_middleware_retry.py new file mode 100644 index 000000000..b028508c2 --- /dev/null +++ b/scrapy/trunk/scrapy/tests/test_middleware_retry.py @@ -0,0 +1,27 @@ +import unittest + +from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware +from scrapy.core.exceptions import HttpException +from scrapy.spider import spiders +from scrapy.http import Request, Response + +class RetryTest(unittest.TestCase): + def setUp(self): + spiders.spider_modules = ['scrapy.tests.test_spiders'] + spiders.reload() + self.spider = spiders.fromdomain('scrapytest.org') + + def test_process_exception(self): + exception_404 = (Request('http://www.scrapytest.org/404'), HttpException('404', None, Response('scrapytest.org', 'http://www.scrapytest.org/404', body='')), self.spider) + exception_503 = (Request('http://www.scrapytest.org/503'), HttpException('503', None, Response('scrapytest.org', 'http://www.scrapytest.org/503', body='')), self.spider) + + mw = RetryMiddleware() + mw.retry_times = 1 + + self.assertTrue(mw.process_exception(*exception_404) is None) + + self.assertTrue(isinstance(mw.process_exception(*exception_503), Request)) + self.assertTrue(mw.process_exception(*exception_503) is None) + +if __name__ == "__main__": + unittest.main()