diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst index c3a454279..0d168017f 100644 --- a/docs/topics/downloader-middleware.rst +++ b/docs/topics/downloader-middleware.rst @@ -852,6 +852,11 @@ Default: ``2`` Maximum number of times to retry, in addition to the first download. +Maximum number of retries can also be specified per-request using +:reqmeta:`max_retry_times` attribute of :attr:`Request.meta `. +When initialized, the :reqmeta:`max_retry_times` meta key takes higher +precedence over the :setting:`RETRY_TIMES` setting. + .. setting:: RETRY_HTTP_CODES RETRY_HTTP_CODES diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 5410654ef..f1552572a 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -308,6 +308,7 @@ Those are: * ``ftp_user`` (See :setting:`FTP_USER` for more info) * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info) * :reqmeta:`referrer_policy` +* :reqmeta:`max_retry_times` .. reqmeta:: bindaddress @@ -342,6 +343,15 @@ download_fail_on_dataloss Whether or not to fail on broken responses. See: :setting:`DOWNLOAD_FAIL_ON_DATALOSS`. +.. reqmeta:: max_retry_times + +max_retry_times +--------------- + +The meta key is used set retry times per request. When initialized, the +:reqmeta:`max_retry_times` meta key takes higher precedence over the +:setting:`RETRY_TIMES` setting. + .. _topics-request-response-ref-request-subclasses: Request subclasses diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 549d74f46..07e979628 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,8 +63,13 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + retry_times = self.max_retry_times + + if 'max_retry_times' in request.meta: + retry_times = request.meta['max_retry_times'] + stats = spider.crawler.stats - if retries <= self.max_retry_times: + if retries <= retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider}) diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py index b833cb448..51b79b6c3 100644 --- a/tests/test_downloadermiddleware_retry.py +++ b/tests/test_downloadermiddleware_retry.py @@ -104,5 +104,83 @@ class RetryTest(unittest.TestCase): self.assertEqual(req, None) +class MaxRetryTimesTest(unittest.TestCase): + def setUp(self): + self.crawler = get_crawler(Spider) + self.spider = self.crawler._create_spider('foo') + self.mw = RetryMiddleware.from_crawler(self.crawler) + self.mw.max_retry_times = 2 + self.invalid_url = 'http://www.scrapytest.org/invalid_url' + + def test_with_settings_zero(self): + + # SETTINGS: RETRY_TIMES = 0 + self.mw.max_retry_times = 0 + + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_zero(self): + + # SETTINGS: meta(max_retry_times) = 0 + meta_max_retry_times = 0 + + req = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + self._test_retry(req, DNSLookupError('foo'), meta_max_retry_times) + + def test_without_metakey(self): + + # SETTINGS: RETRY_TIMES is NON-ZERO + self.mw.max_retry_times = 5 + + req = Request(self.invalid_url) + self._test_retry(req, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_greater(self): + + # SETINGS: RETRY_TIMES < meta(max_retry_times) + self.mw.max_retry_times = 2 + meta_max_retry_times = 3 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_metakey_lesser(self): + + # SETINGS: RETRY_TIMES > meta(max_retry_times) + self.mw.max_retry_times = 5 + meta_max_retry_times = 4 + + req1 = Request(self.invalid_url, meta={'max_retry_times': meta_max_retry_times}) + req2 = Request(self.invalid_url) + + self._test_retry(req1, DNSLookupError('foo'), meta_max_retry_times) + self._test_retry(req2, DNSLookupError('foo'), self.mw.max_retry_times) + + def test_with_dont_retry(self): + + # SETTINGS: meta(max_retry_times) = 4 + meta_max_retry_times = 4 + + req = Request(self.invalid_url, meta= \ + {'max_retry_times': meta_max_retry_times, 'dont_retry': True}) + + self._test_retry(req, DNSLookupError('foo'), 0) + + + def _test_retry(self, req, exception, max_retry_times): + + for i in range(0, max_retry_times): + req = self.mw.process_exception(req, exception, self.spider) + assert isinstance(req, Request) + + # discard it + req = self.mw.process_exception(req, exception, self.spider) + self.assertEqual(req, None) + + if __name__ == "__main__": unittest.main()