From f1c943543a81f051576d3818df0fa15d2d7f0572 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 9 Sep 2010 21:43:44 -0300 Subject: [PATCH] Added dont_retry request.meta key to make RetryMiddleware ignore requests. Closes #234 --- docs/topics/downloader-middleware.rst | 29 +++++++++++-------- docs/topics/request-response.rst | 5 ++-- scrapy/contrib/downloadermiddleware/retry.py | 2 ++ .../tests/test_downloadermiddleware_retry.py | 8 +++++ 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index 1c82b2c6e..d4139ff54 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -400,22 +400,27 @@ RetryMiddleware A middlware to retry failed requests that are potentially caused by temporary problems such as a connection timeout or HTTP 500 error. - Failed pages are collected on the scraping process and rescheduled at the - end, once the spider has finished crawling all regular (non failed) pages. - Once there are no more failed pages to retry, this middleware sends a signal - (retry_complete), so other extensions could connect to that signal. +Failed pages are collected on the scraping process and rescheduled at the +end, once the spider has finished crawling all regular (non failed) pages. +Once there are no more failed pages to retry, this middleware sends a signal +(retry_complete), so other extensions could connect to that signal. - The :class:`RetryMiddleware` can be configured through the following - settings (see the settings documentation for more info): +The :class:`RetryMiddleware` can be configured through the following +settings (see the settings documentation for more info): - * :setting:`RETRY_TIMES` - how many times to retry a failed page - * :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry +* :setting:`RETRY_TIMES` - how many times to retry a failed page +* :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry - About HTTP errors to consider: +About HTTP errors to consider: - You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the - HTTP protocol. It's included by default because it's a common code used - to indicate server overload, which would be something we want to retry. +You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the +HTTP protocol. It's included by default because it's a common code used +to indicate server overload, which would be something we want to retry. + +.. reqmeta:: dont_retry + +If :attr:`Request.meta ` contains the ``dont_retry`` +key, the request will be ignored by this middleware. .. _topics-dlmw-robots: diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index c7744ef0c..bdf319f67 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -209,8 +209,8 @@ Using Request.meta:: .. _topics-request-meta: -Request.meta values -=================== +Request.meta special keys +========================= The :attr:`Request.meta` attribute can contain any arbitrary data, but there are some special keys recognized by Scrapy and its built-in extensions. @@ -218,6 +218,7 @@ are some special keys recognized by Scrapy and its built-in extensions. Those are: * :reqmeta:`dont_redirect` +* :reqmeta:`dont_retry` .. _topics-request-response-ref-request-subclasses: diff --git a/scrapy/contrib/downloadermiddleware/retry.py b/scrapy/contrib/downloadermiddleware/retry.py index bd8d9b866..3d064364f 100644 --- a/scrapy/contrib/downloadermiddleware/retry.py +++ b/scrapy/contrib/downloadermiddleware/retry.py @@ -44,6 +44,8 @@ class RetryMiddleware(object): self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST') def process_response(self, request, response, spider): + if 'dont_retry' in request.meta: + return response if response.status in self.retry_http_codes: reason = response_status_message(response.status) return self._retry(request, reason, spider) or response diff --git a/scrapy/tests/test_downloadermiddleware_retry.py b/scrapy/tests/test_downloadermiddleware_retry.py index 96e7cd297..edb27da7b 100644 --- a/scrapy/tests/test_downloadermiddleware_retry.py +++ b/scrapy/tests/test_downloadermiddleware_retry.py @@ -27,6 +27,14 @@ class RetryTest(unittest.TestCase): # dont retry 404s assert self.mw.process_response(req, rsp, self.spider) is rsp + def test_dont_retry(self): + req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True}) + rsp = Response('http://www.scrapytest.org/503', body='', status=503) + + # first retry + r = self.mw.process_response(req, rsp, self.spider) + assert r is rsp + def test_503(self): req = Request('http://www.scrapytest.org/503') rsp = Response('http://www.scrapytest.org/503', body='', status=503)