From 8da9836db3d6071eff6d0267ed47a7af81b830d3 Mon Sep 17 00:00:00 2001 From: Mikhail Korobov Date: Wed, 24 Jun 2015 03:20:46 +0500 Subject: [PATCH] don't access settings from downloader.Slot --- scrapy/core/downloader/__init__.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index ecbf33039..682c95a15 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -17,10 +17,11 @@ from .handlers import DownloadHandlers class Slot(object): """Downloader slot""" - def __init__(self, concurrency, delay, settings): + def __init__(self, concurrency, delay, randomize_delay): self.concurrency = concurrency self.delay = delay - self.randomize_delay = settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') + self.randomize_delay = randomize_delay + self.active = set() self.queue = deque() self.transferring = set() @@ -66,6 +67,7 @@ class Downloader(object): self.total_concurrency = self.settings.getint('CONCURRENT_REQUESTS') self.domain_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self.ip_concurrency = self.settings.getint('CONCURRENT_REQUESTS_PER_IP') + self.randomize_delay = self.settings.getbool('RANDOMIZE_DOWNLOAD_DELAY') self.middleware = DownloaderMiddlewareManager.from_crawler(crawler) self._slot_gc_loop = task.LoopingCall(self._slot_gc) self._slot_gc_loop.start(60) @@ -87,7 +89,7 @@ class Downloader(object): if key not in self.slots: conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency conc, delay = _get_concurrency_delay(conc, spider, self.settings) - self.slots[key] = Slot(conc, delay, self.settings) + self.slots[key] = Slot(conc, delay, self.randomize_delay) return key, self.slots[key]