From 175a4b5957c82c814ef386f0e3276225ad86f3f4 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Thu, 1 Dec 2011 18:44:26 -0200 Subject: [PATCH] allow spider to set autothrottle max concurrency --- scrapy/contrib/throttle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index a1afd2c5f..69edea1ca 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -76,6 +76,9 @@ class AutoThrottle(object): def spider_opened(self, spider): spider.download_delay = self.START_DELAY + if hasattr(spider, "max_concurrent_requests"): + self.MAX_CONCURRENCY = spider.max_concurrent_requests + # override in order to avoid to initialize slot with concurrency > 1 spider.max_concurrent_requests = 1 self.last_latencies = [self.START_DELAY] self.last_lat = self.START_DELAY, 0.0