* adjusted RETRY_TIMES setting and behaviour

* refactored scrapy.contrib.downloadermiddleware.retry.RetryMiddleware
* created test for it (scrapy.tests.test_middleware_retry.RetryTest)
* updated docstring at scrapy.core.downloader.middleware.DownloaderMiddlewareManager

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40488

scrapy/trunk/scrapy/conf/core_settings.py

@@ -26,5 +26,5 @@ PRIORITIZER = 'scrapy.core.prioritizers.RandomPrioritizer'
 EXTENSIONS = []
 
 # contrib.middleware.retry.RetryMiddleware default settings
-RETRY_TIMES = 3
+RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = ['500', '503', '504', '400', '408', '200']

scrapy/trunk/scrapy/contrib/downloadermiddleware/retry.py

@@ -38,26 +38,19 @@ class RetryMiddleware(object):
 
     def __init__(self):
         self.failed_count = {}
-        self.max_retries = settings.getint('RETRY_TIMES')
+        self.retry_times = settings.getint('RETRY_TIMES')
+        self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES'))
 
     def process_exception(self, request, exception, spider):
-        retry = False
-
-        if isinstance(exception, self.EXCEPTIONS_TO_RETRY):
-            retry = True
-        elif isinstance(exception, HttpException):
-            if exception.status in settings.getlist('RETRY_HTTP_CODES'):
-                retry = True
-
-        if retry:
+        if isinstance(exception, self.EXCEPTIONS_TO_RETRY) or (isinstance(exception, HttpException) and (int(exception.status) in self.retry_http_codes)):
             fp = request.fingerprint()
-            count = self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
+            self.failed_count[fp] = self.failed_count.get(fp, 0) + 1
 
-            if self.failed_count[fp] < self.max_retries:
-                log.msg("Retrying %s (failed %d times): %s" % (request, count, exception), level=log.DEBUG, domain=spider.domain_name)
+            if self.failed_count[fp] <= self.retry_times:
+                log.msg("Retrying %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG)
                 retryreq = request.copy()
                 retryreq.dont_filter = True
                 return retryreq
             else:
-                log.msg("Discarding %s (failed %d times): %s" % (request, count, exception), domain=spider.domain_name, level=log.DEBUG)
+                log.msg("Discarding %s (failed %d times): %s" % (request, self.failed_count[fp], exception), domain=spider.domain_name, level=log.DEBUG)
 

scrapy/trunk/scrapy/core/downloader/middleware.py

@@ -70,6 +70,7 @@ class DownloaderMiddlewareManager(object):
 
         process_response MUST return a Response object. It could alter the given
         response, or it could create a brand-new Response.
+        To drop the response entirely an IgnoreRequest exception must be raised.
 
 
     process_exception(self, request, exception, spider)

scrapy/trunk/scrapy/tests/test_middleware_retry.py

@@ -0,0 +1,27 @@
+import unittest
+
+from scrapy.contrib.downloadermiddleware.retry import RetryMiddleware
+from scrapy.core.exceptions import HttpException
+from scrapy.spider import spiders
+from scrapy.http import Request, Response
+
+class RetryTest(unittest.TestCase):
+    def setUp(self):
+        spiders.spider_modules = ['scrapy.tests.test_spiders']
+        spiders.reload()
+        self.spider = spiders.fromdomain('scrapytest.org')
+
+    def test_process_exception(self):
+        exception_404 = (Request('http://www.scrapytest.org/404'), HttpException('404', None, Response('scrapytest.org', 'http://www.scrapytest.org/404', body='')), self.spider)
+        exception_503 = (Request('http://www.scrapytest.org/503'), HttpException('503', None, Response('scrapytest.org', 'http://www.scrapytest.org/503', body='')), self.spider)
+
+        mw = RetryMiddleware()
+        mw.retry_times = 1
+
+        self.assertTrue(mw.process_exception(*exception_404) is None)
+
+        self.assertTrue(isinstance(mw.process_exception(*exception_503), Request))
+        self.assertTrue(mw.process_exception(*exception_503) is None)
+
+if __name__ == "__main__":
+    unittest.main()