Add feature to set RETRY_TIMES per request (#2642)

2017-03-12 05:06:12 +05:30 · 2017-03-12 05:06:12 +05:30 · 810658bcc5
parent cfb56400b2
commit 810658bcc5
1 changed files with 3 additions and 0 deletions
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@ -63,6 +63,9 @@ class RetryMiddleware(object):
    def _retry(self, request, reason, spider):
        retries = request.meta.get('retry_times', 0) + 1

+        if 'max_retry_times' in request.meta:
+            self.max_retry_times = request.meta['max_retry_times']
+
        stats = spider.crawler.stats
        if retries <= self.max_retry_times:
            logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",