diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecbf33039..965471851 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -1,6 +1,8 @@ +from __future__ import absolute_import import random import warnings from time import time +from datetime import datetime from collections import deque from twisted.internet import reactor, defer, task @@ -8,7 +10,6 @@ from twisted.internet import reactor, defer, task from scrapy.utils.defer import mustbe_deferred from scrapy.utils.httpobj import urlparse_cached from scrapy.resolver import dnscache -from scrapy.exceptions import ScrapyDeprecationWarning from scrapy import signals from .middleware import DownloaderMiddlewareManager from .handlers import DownloadHandlers @@ -17,10 +18,11 @@ from .handlers import DownloadHandlers class Slot(object): """Downloader slot""" - def __init__(self, concurrency, delay, settings): + def __init__(self, concurrency, delay, randomize_delay): self.concurrency = concurrency self.delay = delay - self.randomize_delay = settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') + self.randomize_delay = randomize_delay + self.active = set() self.queue = deque() self.transferring = set() @@ -39,6 +41,21 @@ class Slot(object): if self.latercall and self.latercall.active(): self.latercall.cancel() + def __repr__(self): + cls_name = self.__class__.__name__ + return "%s(concurrency=%r, delay=%0.2f, randomize_delay=%r)" % ( + cls_name, self.concurrency, self.delay, self.randomize_delay) + + def __str__(self): + return ( + "" % ( + self.concurrency, self.delay, self.randomize_delay, + len(self.active), len(self.queue), len(self.transferring), + datetime.fromtimestamp(self.lastseen).isoformat() + ) + ) + def _get_concurrency_delay(concurrency, spider, settings): delay = settings.getfloat('DOWNLOAD_DELAY') @@ -66,6 +83,7 @@ class Downloader(object): self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') + self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) @@ -87,7 +105,7 @@ class Downloader(object): if key not in self.slots: conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.settings) + self.slots[key] = Slot(conc, delay, self.randomize_delay) return key, self.slots[key]