Merge pull request #2643 from harshasrinivas/set-retry-times-per-request

[MRG+1] Add feature to set RETRY_TIMES per request (#2642)

docs/topics/downloader-middleware.rst

@@ -852,6 +852,11 @@ Default: ``2``
 
 Maximum number of times to retry, in addition to the first download.
 
+Maximum number of retries can also be specified per-request using
+:reqmeta:`max_retry_times` attribute of :attr:`Request.meta <scrapy.http.Request.meta>`.
+When initialized, the :reqmeta:`max_retry_times` meta key takes higher
+precedence over the :setting:`RETRY_TIMES` setting.
+
 .. setting:: RETRY_HTTP_CODES
 
 RETRY_HTTP_CODES

docs/topics/request-response.rst

@@ -308,6 +308,7 @@ Those are:
 * ``ftp_user`` (See :setting:`FTP_USER` for more info)
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
 * :reqmeta:`referrer_policy`
+* :reqmeta:`max_retry_times`
 
 .. reqmeta:: bindaddress
 
@@ -342,6 +343,15 @@ download_fail_on_dataloss
 Whether or not to fail on broken responses. See:
 :setting:`DOWNLOAD_FAIL_ON_DATALOSS`.
 
+.. reqmeta:: max_retry_times
+
+max_retry_times
+---------------
+
+The meta key is used set retry times per request. When initialized, the
+:reqmeta:`max_retry_times` meta key takes higher precedence over the
+:setting:`RETRY_TIMES` setting.
+
 .. _topics-request-response-ref-request-subclasses:
 
 Request subclasses

scrapy/downloadermiddlewares/retry.py

@@ -63,8 +63,13 @@ class RetryMiddleware(object):
     def _retry(self, request, reason, spider):
         retries = request.meta.get('retry_times', 0) + 1
 
+        retry_times = self.max_retry_times
+
+        if 'max_retry_times' in request.meta:
+            retry_times = request.meta['max_retry_times']
+
         stats = spider.crawler.stats
-        if retries <= self.max_retry_times:
+        if retries <= retry_times:
             logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",
                          {'request': request, 'retries': retries, 'reason': reason},
                          extra={'spider': spider})

tests/test_downloadermiddleware_retry.py

@@ -104,5 +104,83 @@ class RetryTest(unittest.TestCase):
         self.assertEqual(req, None)
 
 
+class MaxRetryTimesTest(unittest.TestCase):
+    def setUp(self):
+        self.crawler = get_crawler(Spider)
+        self.spider = self.crawler._create_spider('foo')
+        self.mw = RetryMiddleware.from_crawler(self.crawler)
+        self.mw.max_retry_times = 2
+        self.invalid_url = 'http://www.scrapytest.org/invalid_url'
+
+    def test_with_settings_zero(self):
+
+        # SETTINGS: RETRY_TIMES = 0
+        self.mw.max_retry_times = 0
+
+        req = Request(self.invalid_url)
+        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_metakey_zero(self):
+
+        # SETTINGS: meta(max_retry_times) = 0
+        meta_max_retry_times = 0
+        
+        req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times)
+
+    def test_without_metakey(self):
+
+        # SETTINGS: RETRY_TIMES is NON-ZERO
+        self.mw.max_retry_times = 5
+
+        req = Request(self.invalid_url)
+        self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_metakey_greater(self):
+        
+        # SETINGS: RETRY_TIMES < meta(max_retry_times)
+        self.mw.max_retry_times = 2
+        meta_max_retry_times = 3
+
+        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req2 = Request(self.invalid_url)
+
+        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
+        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_metakey_lesser(self):
+        
+        # SETINGS: RETRY_TIMES > meta(max_retry_times)
+        self.mw.max_retry_times = 5
+        meta_max_retry_times = 4
+
+        req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times})
+        req2 = Request(self.invalid_url)
+
+        self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times)
+        self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times)
+
+    def test_with_dont_retry(self):
+
+        # SETTINGS: meta(max_retry_times) = 4
+        meta_max_retry_times = 4
+
+        req = Request(self.invalid_url, meta= \
+            {'max_retry_times': meta_max_retry_times, 'dont_retry': True})
+
+        self._test_retry(req, DNSLookupError('foo'), 0)
+
+
+    def _test_retry(self, req, exception, max_retry_times):
+        
+        for i in range(0, max_retry_times):
+            req = self.mw.process_exception(req, exception, self.spider)
+            assert isinstance(req, Request)
+
+        # discard it
+        req = self.mw.process_exception(req, exception, self.spider)
+        self.assertEqual(req, None)
+
+
 if __name__ == "__main__":
     unittest.main()