From cb9c937f50aaeaf67aba3f54cecdf6bb854aabf7 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Tue, 26 Jul 2011 18:49:01 -0300 Subject: [PATCH] minor code rearrangement for consistency --- scrapy/core/downloader/__init__.py | 1 - scrapy/core/engine.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index c8d94dd71..adb7f5d9b 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -57,7 +57,6 @@ class Downloader(object): self.slots = {} self.handlers = DownloadHandlers() self.middleware = DownloaderMiddlewareManager.from_settings(settings) - self.concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') def fetch(self, request, spider): slot = self.slots[spider] diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b2c8baf07..7759ccd89 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -59,6 +59,7 @@ class ExecutionEngine(object): self.scheduler = load_object(settings['SCHEDULER'])() self.downloader = Downloader() self.scraper = Scraper(self, self.settings) + self._concurrent_spiders = settings.getint('CONCURRENT_SPIDERS') self._spider_closed_callback = spider_closed_callback @defer.inlineCallbacks @@ -164,7 +165,7 @@ class ExecutionEngine(object): def has_capacity(self): """Does the engine have capacity to handle more spiders""" - return len(self.downloader.slots) < self.downloader.concurrent_spiders + return len(self.slots) < self._concurrent_spiders def crawl(self, request, spider): assert spider in self.open_spiders, \