diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index fc7cdf953..a66d86f25 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -1,4 +1,5 @@ from scrapy.xlib.pydispatch import dispatcher +from scrapy.utils.python import setattr_default from scrapy import signals class AutoThrottle(object): @@ -70,16 +71,23 @@ class AutoThrottle(object): self.last_lat = {} def spider_opened(self, spider): + setattr_default(spider, 'dont_throttle', False) + if spider.dont_throttle: + return spider.download_delay = self.START_DELAY spider.max_concurrent_requests = 1 self.last_latencies[spider] = [self.START_DELAY] self.last_lat[spider] = self.START_DELAY, 0.0 def spider_closed(self, spider): + if spider.dont_throttle: + return del self.last_latencies[spider] del self.last_lat[spider] def response_received(self, response, spider): + if spider.dont_throttle: + return latency = response.meta.get('download_latency') if not latency: return