From 810658bcc5b1897d57c0882a0f7ab4a0d264a778 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Sun, 12 Mar 2017 05:06:12 +0530 Subject: [PATCH] Add feature to set RETRY_TIMES per request (#2642) --- scrapy/downloadermiddlewares/retry.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 549d74f46..a5342995f 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,6 +63,9 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + if 'max_retry_times' in request.meta: + self.max_retry_times = request.meta['max_retry_times'] + stats = spider.crawler.stats if retries <= self.max_retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s",