retry middleware: added RETRY_ENABLED setting and documented the other settings more properly, also improved messages when no longer retrying requests

docs/topics/downloader-middleware.rst

@@ -324,8 +324,8 @@ to ``scrapy.contrib.httpcache.DbmCacheStorage``.
 By default, it uses the anydbm_ module, but you can change it with the
 :setting:`HTTPCACHE_DBM_MODULE` setting.
 
-Settings
-~~~~~~~~
+HTTPCache middleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 The :class:`HttpCacheMiddleware` can be configured through the following
 settings:
@@ -504,12 +504,13 @@ Once there are no more failed pages to retry, this middleware sends a signal
 The :class:`RetryMiddleware` can be configured through the following
 settings (see the settings documentation for more info):
 
-* :setting:`RETRY_TIMES` - how many times to retry a failed page
-* :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry
+* :setting:`RETRY_ENABLED`
+* :setting:`RETRY_TIMES`
+* :setting:`RETRY_HTTP_CODES`
 
 About HTTP errors to consider:
 
-You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the
+You may want to remove 400 from :setting:`RETRY_HTTP_CODES`, if you stick to the
 HTTP protocol. It's included by default because it's a common code used
 to indicate server overload, which would be something we want to retry.
 
@@ -518,6 +519,39 @@ to indicate server overload, which would be something we want to retry.
 If :attr:`Request.meta <scrapy.http.Request.meta>` contains the ``dont_retry``
 key, the request will be ignored by this middleware.
 
+RetryMiddleware Settings
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: RETRY_ENABLED
+
+RETRY_ENABLED
+^^^^^^^^^^^^^
+
+.. versionadded:: 0.13
+
+Default: ``True``
+
+Whether the Retry middleware will be enabled.
+
+.. setting:: RETRY_TIMES
+
+RETRY_TIMES
+^^^^^^^^^^^
+
+Default: ``2``
+
+Maximum number of times to retry, in addition to the first download.
+
+.. setting:: RETRY_HTTP_CODES
+
+RETRY_HTTP_CODES
+^^^^^^^^^^^^^^^^
+
+Default: ``[500, 503, 504, 400, 408]``
+
+Which HTTP response codes to retry. Other errors (DNS lookup issues,
+connections lost, etc) are always retried.
+
 .. _topics-dlmw-robots:
 
 RobotsTxtMiddleware

scrapy/contrib/downloadermiddleware/retry.py

@@ -25,6 +25,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError
 from twisted.web.client import PartialDownloadError
 
 from scrapy import log
+from scrapy.exceptions import NotConfigured
 from scrapy.utils.response import response_status_message
 from scrapy.conf import settings
 
@@ -37,8 +38,10 @@ class RetryMiddleware(object):
                            ConnectionLost, PartialDownloadError, IOError)
 
     def __init__(self):
+        if not settings.getbool('RETRY_ENABLED'):
+            raise NotConfigured
         self.max_retry_times = settings.getint('RETRY_TIMES')
-        self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES'))
+        self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))
         self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
 
     def process_response(self, request, response, spider):
@@ -66,6 +69,6 @@ class RetryMiddleware(object):
             retryreq.priority = request.priority + self.priority_adjust
             return retryreq
         else:
-            log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason),
+            log.msg("Gave up retrying %s (failed %d times): %s" % (request, retries, reason),
                     spider=spider, level=log.DEBUG)
 

scrapy/settings/default_settings.py

@@ -211,9 +211,9 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
 REDIRECT_PRIORITY_ADJUST = +2
 
-# contrib.middleware.retry.RetryMiddleware default settings
+RETRY_ENABLED = True
 RETRY_TIMES = 2 # initial response + 2 retries = 3 requests
-RETRY_HTTP_CODES = ['500', '503', '504', '400', '408']
+RETRY_HTTP_CODES = [500, 503, 504, 400, 408]
 RETRY_PRIORITY_ADJUST = -1
 
 ROBOTSTXT_OBEY = False