RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #80 from kalessin/master 

autothrottle code improvements (download delay + style)
This commit is contained in:
Pablo Hoffman 2012-01-16 12:24:04 -08:00
parent fc52d8d5cf 1c6a5a9374
commit a0f41f100c
1 changed files with 10 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
scrapy/contrib/throttle.py
View File

@ -68,21 +68,24 @@ class AutoThrottle(object):
self.START_DELAY = settings.getfloat("AUTOTHROTTLE_START_DELAY", 5.0)
self.CONCURRENCY_CHECK_PERIOD = settings.getint("AUTOTHROTTLE_CONCURRENCY_CHECK_PERIOD", 10)
self.MAX_CONCURRENCY = settings.getint("AUTOTHROTTLE_MAX_CONCURRENCY", 8)
self.DEBUG = settings.getint("AUTOTHROTTLE_DEBUG", False)
self.DEBUG = settings.getbool("AUTOTHROTTLE_DEBUG")
self.MIN_DOWNLOAD_DELAY = settings.getint("AUTOTHROTTLE_MIN_DOWNLOAD_DELAY")
self.last_latencies = [self.START_DELAY]
self.last_lat = self.START_DELAY, 0.0
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def spider_opened(self, spider):
if hasattr(spider, "download_delay"):
self.MIN_DOWNLOAD_DELAY = spider.download_delay
spider.download_delay = self.START_DELAY
if hasattr(spider, "max_concurrent_requests"):
self.MAX_CONCURRENCY = spider.max_concurrent_requests
# override in order to avoid to initialize slot with concurrency > 1
spider.max_concurrent_requests = 1
self.last_latencies = [self.START_DELAY]
self.last_lat = self.START_DELAY, 0.0
def response_received(self, response, spider):
slot = self._get_slot(response.request)
latency = response.meta.get('download_latency')
@ -124,6 +127,9 @@ class AutoThrottle(object):
# if latency is bigger than old delay, then use latency instead of mean. Works better with problematic sites
new_delay = (slot.delay + latency) / 2.0 if latency < slot.delay else latency
if new_delay < self.MIN_DOWNLOAD_DELAY:
new_delay = self.MIN_DOWNLOAD_DELAY
# dont adjust delay if response status != 200 and new delay is smaller than old one,
# as error pages (and redirections) are usually small and so tend to reduce latency, thus provoking a positive feedback
# by reducing delay instead of increase.