Added dont_retry request.meta key to make RetryMiddleware ignore requests. Closes #234

docs/topics/downloader-middleware.rst

@@ -400,22 +400,27 @@ RetryMiddleware
    A middlware to retry failed requests that are potentially caused by
    temporary problems such as a connection timeout or HTTP 500 error.
 
-   Failed pages are collected on the scraping process and rescheduled at the
-   end, once the spider has finished crawling all regular (non failed) pages.
-   Once there are no more failed pages to retry, this middleware sends a signal
-   (retry_complete), so other extensions could connect to that signal.
+Failed pages are collected on the scraping process and rescheduled at the
+end, once the spider has finished crawling all regular (non failed) pages.
+Once there are no more failed pages to retry, this middleware sends a signal
+(retry_complete), so other extensions could connect to that signal.
 
-   The :class:`RetryMiddleware` can be configured through the following
-   settings (see the settings documentation for more info):
+The :class:`RetryMiddleware` can be configured through the following
+settings (see the settings documentation for more info):
 
-   * :setting:`RETRY_TIMES` - how many times to retry a failed page
-   * :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry
+* :setting:`RETRY_TIMES` - how many times to retry a failed page
+* :setting:`RETRY_HTTP_CODES` - which HTTP response codes to retry
 
-   About HTTP errors to consider:
+About HTTP errors to consider:
 
-   You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the
-   HTTP protocol. It's included by default because it's a common code used
-   to indicate server overload, which would be something we want to retry.
+You may want to remove 400 from RETRY_HTTP_CODES, if you stick to the
+HTTP protocol. It's included by default because it's a common code used
+to indicate server overload, which would be something we want to retry.
+
+.. reqmeta:: dont_retry
+
+If :attr:`Request.meta <scrapy.http.Request.meta>` contains the ``dont_retry``
+key, the request will be ignored by this middleware.
 
 .. _topics-dlmw-robots:
 

docs/topics/request-response.rst

@@ -209,8 +209,8 @@ Using Request.meta::
 
 .. _topics-request-meta:
 
-Request.meta values
-===================
+Request.meta special keys
+=========================
 
 The :attr:`Request.meta` attribute can contain any arbitrary data, but there
 are some special keys recognized by Scrapy and its built-in extensions.
@@ -218,6 +218,7 @@ are some special keys recognized by Scrapy and its built-in extensions.
 Those are:
 
 * :reqmeta:`dont_redirect`
+* :reqmeta:`dont_retry`
 
 .. _topics-request-response-ref-request-subclasses:
 

scrapy/contrib/downloadermiddleware/retry.py

@@ -44,6 +44,8 @@ class RetryMiddleware(object):
         self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')
 
     def process_response(self, request, response, spider):
+        if 'dont_retry' in request.meta:
+            return response
         if response.status in self.retry_http_codes:
             reason = response_status_message(response.status)
             return self._retry(request, reason, spider) or response

scrapy/tests/test_downloadermiddleware_retry.py

@@ -27,6 +27,14 @@ class RetryTest(unittest.TestCase):
         # dont retry 404s
         assert self.mw.process_response(req, rsp, self.spider) is rsp
 
+    def test_dont_retry(self):
+        req = Request('http://www.scrapytest.org/503', meta={'dont_retry': True})
+        rsp = Response('http://www.scrapytest.org/503', body='', status=503)
+
+        # first retry
+        r = self.mw.process_response(req, rsp, self.spider)
+        assert r is rsp
+
     def test_503(self):
         req = Request('http://www.scrapytest.org/503')
         rsp = Response('http://www.scrapytest.org/503', body='', status=503)