From b094cd4c57cbc25a6dd43005ebd8eeee109a75b6 Mon Sep 17 00:00:00 2001 From: Martin Olveyra Date: Fri, 17 Feb 2012 16:53:48 -0200 Subject: [PATCH] identify autothrottle debug mode stats with slot key, in order to allow to track concurrency/delay issues with spiders which crawls more than one site. --- scrapy/contrib/throttle.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/contrib/throttle.py b/scrapy/contrib/throttle.py index 1b119b7e6..423f3726c 100644 --- a/scrapy/contrib/throttle.py +++ b/scrapy/contrib/throttle.py @@ -87,7 +87,7 @@ class AutoThrottle(object): spider.max_concurrent_requests = 1 def response_received(self, response, spider): - slot = self._get_slot(response.request) + key, slot = self._get_slot(response.request) latency = response.meta.get('download_latency') if not latency or not slot: @@ -97,8 +97,8 @@ class AutoThrottle(object): self._check_concurrency(slot, latency) if self.DEBUG: - spider.log("conc:%2d | delay:%5d ms | latency:%5d ms | size:%6d bytes" % \ - (slot.concurrency, slot.delay*1000, \ + spider.log("slot: %s | conc:%2d | delay:%5d ms | latency:%5d ms | size:%6d bytes" % \ + (key, slot.concurrency, slot.delay*1000, \ latency*1000, len(response.body))) def _get_slot(self, request): @@ -106,7 +106,7 @@ class AutoThrottle(object): key = urlparse_cached(request).hostname or '' if downloader.ip_concurrency: key = dnscache.get(key, key) - return downloader.slots.get(key) + return key, downloader.slots.get(key) def _check_concurrency(self, slot, latency): latencies = self.last_latencies