RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

identify autothrottle debug mode stats with slot key, in order to allow to 

track concurrency/delay issues with spiders which crawls more than one
site.
This commit is contained in:
Martin Olveyra 2012-02-17 16:53:48 -02:00
parent 7b8942a648
commit b094cd4c57
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/contrib/throttle.py
View File

@ -87,7 +87,7 @@ class AutoThrottle(object):
spider.max_concurrent_requests = 1
def response_received(self, response, spider):
slot = self._get_slot(response.request)
key, slot = self._get_slot(response.request)
latency = response.meta.get('download_latency')
if not latency or not slot:
@ -97,8 +97,8 @@ class AutoThrottle(object):
self._check_concurrency(slot, latency)
if self.DEBUG:
spider.log("conc:%2d | delay:%5d ms | latency:%5d ms | size:%6d bytes" % \
(slot.concurrency, slot.delay*1000, \
spider.log("slot: %s | conc:%2d | delay:%5d ms | latency:%5d ms | size:%6d bytes" % \
(key, slot.concurrency, slot.delay*1000, \
latency*1000, len(response.body)))
def _get_slot(self, request):
@ -106,7 +106,7 @@ class AutoThrottle(object):
key = urlparse_cached(request).hostname or ''
if downloader.ip_concurrency:
key = dnscache.get(key, key)
return downloader.slots.get(key)
return key, downloader.slots.get(key)
def _check_concurrency(self, slot, latency):
latencies = self.last_latencies