Slots removed, now Scraper can handle just one spider

scrapy/core/engine.py

@@ -123,7 +123,7 @@ class ExecutionEngine(object):
         return not self.running \
             or slot.closing \
             or self.downloader.needs_backout() \
-            or self.scraper.slots[spider].needs_backout()
+            or self.scraper.slot.needs_backout()
 
     def _next_request_from_scheduler(self, spider):
         slot = self.slots[spider]
@@ -151,8 +151,7 @@ class ExecutionEngine(object):
         return d
 
     def spider_is_idle(self, spider):
-        scraper_idle = spider in self.scraper.slots \
-            and self.scraper.slots[spider].is_idle()
+        scraper_idle = self.scraper.slot.is_idle()
         pending = self.slots[spider].scheduler.has_pending_requests()
         downloading = bool(self.downloader.active)
         idle = scraper_idle and not (pending or downloading)

scrapy/core/scraper.py

@@ -60,7 +60,7 @@ class Slot(object):
 class Scraper(object):
 
     def __init__(self, crawler):
-        self.slots = {}
+        self.slot = None
         self.spidermw = SpiderMiddlewareManager.from_crawler(crawler)
         itemproc_cls = load_object(crawler.settings['ITEM_PROCESSOR'])
         self.itemproc = itemproc_cls.from_crawler(crawler)
@@ -72,14 +72,12 @@ class Scraper(object):
     @defer.inlineCallbacks
     def open_spider(self, spider):
         """Open the given spider for scraping and allocate resources for it"""
-        assert spider not in self.slots, "Spider already opened: %s" % spider
-        self.slots[spider] = Slot()
+        self.slot = Slot()
         yield self.itemproc.open_spider(spider)
 
     def close_spider(self, spider):
         """Close a spider being scraped and release its resources"""
-        assert spider in self.slots, "Spider not opened: %s" % spider
-        slot = self.slots[spider]
+        slot = self.slot
         slot.closing = defer.Deferred()
         slot.closing.addCallback(self.itemproc.close_spider)
         self._check_if_closing(spider, slot)
@@ -87,15 +85,14 @@ class Scraper(object):
 
     def is_idle(self):
         """Return True if there isn't any more spiders to process"""
-        return not self.slots
+        return not self.slot
 
     def _check_if_closing(self, spider, slot):
         if slot.closing and slot.is_idle():
-            del self.slots[spider]
             slot.closing.callback(spider)
 
     def enqueue_scrape(self, response, request, spider):
-        slot = self.slots[spider]
+        slot = self.slot
         dfd = slot.add_response_request(response, request)
         def finish_scraping(_):
             slot.finish_response(response, request)
@@ -169,7 +166,7 @@ class Scraper(object):
                 spider=spider)
             self.crawler.engine.crawl(request=output, spider=spider)
         elif isinstance(output, BaseItem):
-            self.slots[spider].itemproc_size += 1
+            self.slot.itemproc_size += 1
             dfd = self.itemproc.process_item(output, spider)
             dfd.addBoth(self._itemproc_finished, output, response, spider)
             return dfd
@@ -196,7 +193,7 @@ class Scraper(object):
     def _itemproc_finished(self, output, item, response, spider):
         """ItemProcessor finished for the given ``item`` and returned ``output``
         """
-        self.slots[spider].itemproc_size -= 1
+        self.slot.itemproc_size -= 1
         if isinstance(output, Failure):
             ex = output.value
             if isinstance(ex, DropItem):

scrapy/utils/engine.py

@@ -11,7 +11,7 @@ def get_engine_status(engine):
         "len(engine.downloader.slots)",
         "len(engine.downloader.active)",
         "engine.scraper.is_idle()",
-        "len(engine.scraper.slots)",
+        "1",
     ]
     spider_tests = [
         "engine.spider_is_idle(spider)",
@@ -19,11 +19,11 @@ def get_engine_status(engine):
         "len(engine.slots[spider].inprogress)",
         "len(engine.slots[spider].scheduler.dqs or [])",
         "len(engine.slots[spider].scheduler.mqs)",
-        "len(engine.scraper.slots[spider].queue)",
-        "len(engine.scraper.slots[spider].active)",
-        "engine.scraper.slots[spider].active_size",
-        "engine.scraper.slots[spider].itemproc_size",
-        "engine.scraper.slots[spider].needs_backout()",
+        "len(engine.scraper.slot.queue)",
+        "len(engine.scraper.slot.active)",
+        "engine.scraper.slot.active_size",
+        "engine.scraper.slot.itemproc_size",
+        "engine.scraper.slot.needs_backout()",
     ]
 
     status = {'global': [], 'spiders': {}}