RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

added Spider.dont_throttle attribute to disable AutoThrottle extension per spider

This commit is contained in:
Pablo Hoffman 2011-05-22 18:26:38 -03:00
parent 88dbe2ae87
commit 110cd05296
1 changed files with 8 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/throttle.py
View File

@ -1,4 +1,5 @@
from scrapy.xlib.pydispatch import dispatcher
from scrapy.utils.python import setattr_default
from scrapy import signals
class AutoThrottle(object):
@ -70,16 +71,23 @@ class AutoThrottle(object):
self.last_lat = {}
def spider_opened(self, spider):
setattr_default(spider, 'dont_throttle', False)
if spider.dont_throttle:
return
spider.download_delay = self.START_DELAY
spider.max_concurrent_requests = 1
self.last_latencies[spider] = [self.START_DELAY]
self.last_lat[spider] = self.START_DELAY, 0.0
def spider_closed(self, spider):
if spider.dont_throttle:
return
del self.last_latencies[spider]
del self.last_lat[spider]
def response_received(self, response, spider):
if spider.dont_throttle:
return
latency = response.meta.get('download_latency')
if not latency:
return