diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index c544956a4..20a22bb90 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -123,7 +123,7 @@ class ExecutionEngine(object): return not self.running \ or slot.closing \ or self.downloader.needs_backout() \ - or self.scraper.slots[spider].needs_backout() + or self.scraper.slot.needs_backout() def _next_request_from_scheduler(self, spider): slot = self.slots[spider] @@ -151,8 +151,7 @@ class ExecutionEngine(object): return d def spider_is_idle(self, spider): - scraper_idle = spider in self.scraper.slots \ - and self.scraper.slots[spider].is_idle() + scraper_idle = self.scraper.slot.is_idle() pending = self.slots[spider].scheduler.has_pending_requests() downloading = bool(self.downloader.active) idle = scraper_idle and not (pending or downloading) diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 187c1e9b3..bf2d9fdc1 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -60,7 +60,7 @@ class Slot(object): class Scraper(object): def __init__(self, crawler): - self.slots = {} + self.slot = None self.spidermw = SpiderMiddlewareManager.from_crawler(crawler) itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR']) self.itemproc = itemproc_cls.from_crawler(crawler) @@ -72,14 +72,12 @@ class Scraper(object): @defer.inlineCallbacks def open_spider(self, spider): """Open the given spider for scraping and allocate resources for it""" - assert spider not in self.slots, "Spider already opened: %s" % spider - self.slots[spider] = Slot() + self.slot = Slot() yield self.itemproc.open_spider(spider) def close_spider(self, spider): """Close a spider being scraped and release its resources""" - assert spider in self.slots, "Spider not opened: %s" % spider - slot = self.slots[spider] + slot = self.slot slot.closing = defer.Deferred() slot.closing.addCallback(self.itemproc.close_spider) self._check_if_closing(spider, slot) @@ -87,15 +85,14 @@ class Scraper(object): def is_idle(self): """Return True if there isn't any more spiders to process""" - return not self.slots + return not self.slot def _check_if_closing(self, spider, slot): if slot.closing and slot.is_idle(): - del self.slots[spider] slot.closing.callback(spider) def enqueue_scrape(self, response, request, spider): - slot = self.slots[spider] + slot = self.slot dfd = slot.add_response_request(response, request) def finish_scraping(_): slot.finish_response(response, request) @@ -169,7 +166,7 @@ class Scraper(object): spider=spider) self.crawler.engine.crawl(request=output, spider=spider) elif isinstance(output, BaseItem): - self.slots[spider].itemproc_size += 1 + self.slot.itemproc_size += 1 dfd = self.itemproc.process_item(output, spider) dfd.addBoth(self._itemproc_finished, output, response, spider) return dfd @@ -196,7 +193,7 @@ class Scraper(object): def _itemproc_finished(self, output, item, response, spider): """ItemProcessor finished for the given ``item`` and returned ``output`` """ - self.slots[spider].itemproc_size -= 1 + self.slot.itemproc_size -= 1 if isinstance(output, Failure): ex = output.value if isinstance(ex, DropItem): diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py index 26fe361ef..9371aa1b3 100644 --- a/scrapy/utils/engine.py +++ b/scrapy/utils/engine.py @@ -11,7 +11,7 @@ def get_engine_status(engine): "len(engine.downloader.slots)", "len(engine.downloader.active)", "engine.scraper.is_idle()", - "len(engine.scraper.slots)", + "1", ] spider_tests = [ "engine.spider_is_idle(spider)", @@ -19,11 +19,11 @@ def get_engine_status(engine): "len(engine.slots[spider].inprogress)", "len(engine.slots[spider].scheduler.dqs or [])", "len(engine.slots[spider].scheduler.mqs)", - "len(engine.scraper.slots[spider].queue)", - "len(engine.scraper.slots[spider].active)", - "engine.scraper.slots[spider].active_size", - "engine.scraper.slots[spider].itemproc_size", - "engine.scraper.slots[spider].needs_backout()", + "len(engine.scraper.slot.queue)", + "len(engine.scraper.slot.active)", + "engine.scraper.slot.active_size", + "engine.scraper.slot.itemproc_size", + "engine.scraper.slot.needs_backout()", ] status = {'global': [], 'spiders': {}}