diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 3e4c3cff7..4ff3ed049 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -324,8 +324,8 @@ to ``scrapy.contrib.httpcache.DbmCacheStorage``. By default, it uses the anydbm_ module, but you can change it with the :setting:`HTTPCACHE_DBM_MODULE` setting. -Settings -~~~~~~~~ +HTTPCache middleware settings +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The :class:`HttpCacheMiddleware` can be configured through the following settings: @@ -504,12 +504,13 @@ Once there are no more failed pages to retry, this middleware sends a signal The :class:`RetryMiddleware` can be configured through the following settings (see the settings documentation for more info): -* :setting:`RETRY_TIMES` - how many times to retry a failed page -* :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry +* :setting:`RETRY_ENABLED` +* :setting:`RETRY_TIMES` +* :setting:`RETRY_HTTP_CODES` About HTTP errors to consider: -You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the +You may want to remove 400 from :setting:`RETRY_HTTP_CODES`, if you stick to the HTTP protocol. It's included by default because it's a common code used to indicate server overload, which would be something we want to retry. @@ -518,6 +519,39 @@ to indicate server overload, which would be something we want to retry. If :attr:`Request.meta ` contains the ``dont_retry`` key, the request will be ignored by this middleware. +RetryMiddleware Settings +~~~~~~~~~~~~~~~~~~~~~~~~ + +.. setting:: RETRY_ENABLED + +RETRY_ENABLED +^^^^^^^^^^^^^ + +.. versionadded:: 0.13 + +Default: ``True`` + +Whether the Retry middleware will be enabled. + +.. setting:: RETRY_TIMES + +RETRY_TIMES +^^^^^^^^^^^ + +Default: ``2`` + +Maximum number of times to retry, in addition to the first download. + +.. setting:: RETRY_HTTP_CODES + +RETRY_HTTP_CODES +^^^^^^^^^^^^^^^^ + +Default: ``[500, 503, 504, 400, 408]`` + +Which HTTP response codes to retry. Other errors (DNS lookup issues, +connections lost, etc) are always retried. + .. _topics-dlmw-robots: RobotsTxtMiddleware diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index f1ab5ab10..ed6da4a37 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -25,6 +25,7 @@ from twisted.internet.defer import TimeoutError as UserTimeoutError from twisted.web.client import PartialDownloadError from scrapy import log +from scrapy.exceptions import NotConfigured from scrapy.utils.response import response_status_message from scrapy.conf import settings @@ -37,8 +38,10 @@ class RetryMiddleware(object): ConnectionLost, PartialDownloadError, IOError) def __init__(self): + if not settings.getbool('RETRY_ENABLED'): + raise NotConfigured self.max_retry_times = settings.getint('RETRY_TIMES') - self.retry_http_codes = map(int, settings.getlist('RETRY_HTTP_CODES')) + self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES')) self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') def process_response(self, request, response, spider): @@ -66,6 +69,6 @@ class RetryMiddleware(object): retryreq.priority = request.priority + self.priority_adjust return retryreq else: - log.msg("Discarding %s (failed %d times): %s" % (request, retries, reason), + log.msg("Gave up retrying %s (failed %d times): %s" % (request, retries, reason), spider=spider, level=log.DEBUG) diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index cc4bc2387..ceeb37b4c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -211,9 +211,9 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100 REDIRECT_MAX_TIMES = 20 # uses Firefox default setting REDIRECT_PRIORITY_ADJUST = +2 -# contrib.middleware.retry.RetryMiddleware default settings +RETRY_ENABLED = True RETRY_TIMES = 2 # initial response + 2 retries = 3 requests -RETRY_HTTP_CODES = ['500', '503', '504', '400', '408'] +RETRY_HTTP_CODES = [500, 503, 504, 400, 408] RETRY_PRIORITY_ADJUST = -1 ROBOTSTXT_OBEY = False