From 0d57b5cd43343a335fcf2e923b29e76d09dd0b51 Mon Sep 17 00:00:00 2001 From: harshasrinivas Date: Mon, 13 Mar 2017 02:10:23 +0530 Subject: [PATCH] Prevent max_retry_times override --- scrapy/downloadermiddlewares/retry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index a5342995f..07e979628 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -63,11 +63,13 @@ class RetryMiddleware(object): def _retry(self, request, reason, spider): retries = request.meta.get('retry_times', 0) + 1 + retry_times = self.max_retry_times + if 'max_retry_times' in request.meta: - self.max_retry_times = request.meta['max_retry_times'] + retry_times = request.meta['max_retry_times'] stats = spider.crawler.stats - if retries <= self.max_retry_times: + if retries <= retry_times: logger.debug("Retrying %(request)s (failed %(retries)d times): %(reason)s", {'request': request, 'retries': retries, 'reason': reason}, extra={'spider': spider})