RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

allow spider to set autothrottle max concurrency

This commit is contained in:
Martin Olveyra 2011-12-01 18:44:26 -02:00 committed by Pablo Hoffman
parent 0f649f0e30
commit 175a4b5957
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
scrapy/contrib/throttle.py
View File

@ -76,6 +76,9 @@ class AutoThrottle(object):
def spider_opened(self, spider):
spider.download_delay = self.START_DELAY
if hasattr(spider, "max_concurrent_requests"):
self.MAX_CONCURRENCY = spider.max_concurrent_requests
# override in order to avoid to initialize slot with concurrency > 1
spider.max_concurrent_requests = 1
self.last_latencies = [self.START_DELAY]
self.last_lat = self.START_DELAY, 0.0