Simplified engine by removing the configure() and kill() methods. Also simplified the Spider Manager by removing the close_spider() method

scrapy/core/engine.py

@@ -9,10 +9,8 @@ from time import time
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy import log
+from scrapy import log, signals
 from scrapy.stats import stats
-from scrapy.conf import settings
-from scrapy import signals
 from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import IgnoreRequest, DontCloseSpider
@@ -23,25 +21,17 @@ from scrapy.utils.defer import mustbe_deferred
 
 class ExecutionEngine(object):
 
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.configured = False
+    def __init__(self, settings, spider_closed_callback):
+        self.settings = settings
         self.closing = {} # dict (spider -> reason) of spiders being closed
         self.closing_dfds = {} # dict (spider -> deferred) of spiders being closed
         self.running = False
-        self.killed = False
         self.paused = False
         self._next_request_calls = {}
         self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED'])
-
-    def configure(self, spider_closed_callback):
-        """
-        Configure execution engine with the given scheduling policy and downloader.
-        """
         self.scheduler = load_object(settings['SCHEDULER'])()
         self.downloader = Downloader()
-        self.scraper = Scraper(self, self.crawler.settings)
-        self.configured = True
+        self.scraper = Scraper(self, self.settings)
         self._spider_closed_callback = spider_closed_callback
 
     @defer.inlineCallbacks
@@ -59,13 +49,6 @@ class ExecutionEngine(object):
         dfd = self._close_all_spiders()
         return dfd.addBoth(lambda _: self._finish_stopping_engine())
 
-    def kill(self):
-        """Forces shutdown without waiting for pending transfers to finish.
-        stop() must have been called first
-        """
-        assert not self.running, "Call engine.stop() before engine.kill()"
-        self.killed = True
-
     def pause(self):
         """Pause the execution engine"""
         self.paused = True
@@ -256,8 +239,6 @@ class ExecutionEngine(object):
         dfd.addErrback(log.err, "Unhandled error in scraper.close_spider()", \
             spider=spider)
         dfd.addBoth(lambda _: self._finish_closing_spider(spider))
-        if self.killed:
-            return self._finish_closing_spider(spider)
         return dfd
 
     def _close_all_spiders(self):
@@ -277,9 +258,6 @@ class ExecutionEngine(object):
         dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
         dfd.addErrback(log.err, "Unhandled error in stats.close_spider()",
             spider=spider)
-        dfd.addBoth(lambda _: self.crawler.spiders.close_spider(spider))
-        dfd.addErrback(log.err, "Unhandled error in spiders.close_spider()",
-            spider=spider)
         dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
         dfd.addBoth(lambda _: self.closing_dfds.pop(spider).callback(spider))
         dfd.addBoth(lambda _: self._spider_closed_callback(spider))

scrapy/crawler.py

@@ -30,13 +30,12 @@ class Crawler(object):
         if self.configured:
             return
         self.configured = True
-        self.engine = ExecutionEngine(self)
         self.extensions = ExtensionManager.from_settings(self.settings)
         spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
         self.spiders = spman_cls.from_settings(self.settings)
         queue_cls = load_object(self.settings['QUEUE_CLASS'])
         self.queue = queue_cls(self.spiders)
-        self.engine.configure(self._spider_closed)
+        self.engine = ExecutionEngine(self.settings, self._spider_closed)
 
     @defer.inlineCallbacks
     def _start_next_spider(self):

scrapy/spidermanager.py

@@ -62,6 +62,3 @@ class SpiderManager(object):
     def list(self):
         """Returns list of spiders available."""
         return self._spiders.keys()
-
-    def close_spider(self, spider):
-        pass