From d1e260a8d40a3a639526cbfaaa861a525649c0db Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Thu, 26 Aug 2010 22:20:04 -0300
Subject: [PATCH] Simplified engine by removing the configure() and kill()
 methods. Also simplified the Spider Manager by removing the close_spider()
 method

---
 scrapy/core/engine.py   | 30 ++++--------------------------
 scrapy/crawler.py       |  3 +--
 scrapy/spidermanager.py |  3 ---
 3 files changed, 5 insertions(+), 31 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 6516fed89..48f5dc0e9 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -9,10 +9,8 @@ from time import time
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy import log
+from scrapy import log, signals
 from scrapy.stats import stats
-from scrapy.conf import settings
-from scrapy import signals
 from scrapy.core.downloader import Downloader
 from scrapy.core.scraper import Scraper
 from scrapy.exceptions import IgnoreRequest, DontCloseSpider
@@ -23,25 +21,17 @@ from scrapy.utils.defer import mustbe_deferred
 
 class ExecutionEngine(object):
 
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.configured = False
+    def __init__(self, settings, spider_closed_callback):
+        self.settings = settings
         self.closing = {} # dict (spider -> reason) of spiders being closed
         self.closing_dfds = {} # dict (spider -> deferred) of spiders being closed
         self.running = False
-        self.killed = False
         self.paused = False
         self._next_request_calls = {}
         self._crawled_logline = load_object(settings['LOG_FORMATTER_CRAWLED'])
-
-    def configure(self, spider_closed_callback):
-        """
-        Configure execution engine with the given scheduling policy and downloader.
-        """
         self.scheduler = load_object(settings['SCHEDULER'])()
         self.downloader = Downloader()
-        self.scraper = Scraper(self, self.crawler.settings)
-        self.configured = True
+        self.scraper = Scraper(self, self.settings)
         self._spider_closed_callback = spider_closed_callback
 
     @defer.inlineCallbacks
@@ -59,13 +49,6 @@ class ExecutionEngine(object):
         dfd = self._close_all_spiders()
         return dfd.addBoth(lambda _: self._finish_stopping_engine())
 
-    def kill(self):
-        """Forces shutdown without waiting for pending transfers to finish.
-        stop() must have been called first
-        """
-        assert not self.running, "Call engine.stop() before engine.kill()"
-        self.killed = True
-
     def pause(self):
         """Pause the execution engine"""
         self.paused = True
@@ -256,8 +239,6 @@ class ExecutionEngine(object):
         dfd.addErrback(log.err, "Unhandled error in scraper.close_spider()", \
             spider=spider)
         dfd.addBoth(lambda _: self._finish_closing_spider(spider))
-        if self.killed:
-            return self._finish_closing_spider(spider)
         return dfd
 
     def _close_all_spiders(self):
@@ -277,9 +258,6 @@ class ExecutionEngine(object):
         dfd.addBoth(lambda _: stats.close_spider(spider, reason=reason))
         dfd.addErrback(log.err, "Unhandled error in stats.close_spider()",
             spider=spider)
-        dfd.addBoth(lambda _: self.crawler.spiders.close_spider(spider))
-        dfd.addErrback(log.err, "Unhandled error in spiders.close_spider()",
-            spider=spider)
         dfd.addBoth(lambda _: log.msg("Spider closed (%s)" % reason, spider=spider))
         dfd.addBoth(lambda _: self.closing_dfds.pop(spider).callback(spider))
         dfd.addBoth(lambda _: self._spider_closed_callback(spider))
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e6793d369..ccc4f6d33 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -30,13 +30,12 @@ class Crawler(object):
         if self.configured:
             return
         self.configured = True
-        self.engine = ExecutionEngine(self)
         self.extensions = ExtensionManager.from_settings(self.settings)
         spman_cls = load_object(self.settings['SPIDER_MANAGER_CLASS'])
         self.spiders = spman_cls.from_settings(self.settings)
         queue_cls = load_object(self.settings['QUEUE_CLASS'])
         self.queue = queue_cls(self.spiders)
-        self.engine.configure(self._spider_closed)
+        self.engine = ExecutionEngine(self.settings, self._spider_closed)
 
     @defer.inlineCallbacks
     def _start_next_spider(self):
diff --git a/scrapy/spidermanager.py b/scrapy/spidermanager.py
index 0b0a3c673..dd33a2643 100644
--- a/scrapy/spidermanager.py
+++ b/scrapy/spidermanager.py
@@ -62,6 +62,3 @@ class SpiderManager(object):
     def list(self):
         """Returns list of spiders available."""
         return self._spiders.keys()
-
-    def close_spider(self, spider):
-        pass