From 8d24175c8159cf5bd209f53bdcfc133b78c5a70f Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Wed, 25 Aug 2010 05:24:02 -0300
Subject: [PATCH] Added CrawlerProcess class, isolating all (Twisted)
 reactor-controlling code into this class and leaving the Crawler class free
 of any reactor control.

This allows embedding the Scrapy crawler in other Twisted applications, or any
Twisted asynchronous code.

The "scrapy" command will use the new CrawlerProcess class, which resembles the
behaviour of the old (all-in-one) Crawler class.

Closes #214.

Also moved log.start() call from Crawler class to scrapy.cmdline module.
---
 scrapy/cmdline.py |  3 ++-
 scrapy/crawler.py | 46 ++++++++++++++++++++++++++++++----------------
 scrapy/project.py |  4 ++--
 3 files changed, 34 insertions(+), 19 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 8d437a1c5..a2b0edc6b 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -114,7 +114,8 @@ def execute(argv=None):
     settings.defaults.update(cmd.default_settings)
     del args[0]  # remove command name from args
     from scrapy.project import crawler
-    crawler.configure(control_reactor=True)
+    log.start()
+    crawler.configure()
     ret = _run_command(cmd, args, opts)
     if ret is False:
         parser.print_help()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index d89cf601f..16edff92a 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -2,29 +2,23 @@ import signal
 
 from twisted.internet import reactor, defer
 
+from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core.engine import ExecutionEngine
 from scrapy.core.queue import ExecutionQueue
 from scrapy.extension import ExtensionManager
-from scrapy import log
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+from scrapy import log, signals
 
 
 class Crawler(object):
 
     def __init__(self, settings, spiders):
         self.configured = False
-        self.control_reactor = True
         self.settings = settings
         self.spiders = spiders
         self.engine = ExecutionEngine(self)
 
-    def configure(self, control_reactor=True, queue=None):
-        self.control_reactor = control_reactor
-        if control_reactor:
-            install_shutdown_handlers(self._signal_shutdown)
-
-        if not log.started:
-            log.start()
+    def configure(self, queue=None):
         self.extensions = ExtensionManager.from_settings(self.settings)
         if not self.spiders.loaded:
             self.spiders.load()
@@ -63,9 +57,6 @@ class Crawler(object):
     def start(self):
         yield defer.maybeDeferred(self.engine.start)
         self._nextcall = reactor.callLater(0, self._start_next_spider)
-        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
-        if self.control_reactor:
-            reactor.run(installSignalHandlers=False)
 
     @defer.inlineCallbacks
     def stop(self):
@@ -73,22 +64,45 @@ class Crawler(object):
             self._nextcall.cancel()
         if self.engine.running:
             yield defer.maybeDeferred(self.engine.stop)
+
+
+class CrawlerProcess(Crawler):
+    """A class to run a single Scrapy crawler in a process. It provides
+    automatic control of the Twisted reactor and installs some convenient
+    signals for shutting down the crawl.
+    """
+
+    def __init__(self, *a, **kw):
+        super(CrawlerProcess, self).__init__(*a, **kw)
+        dispatcher.connect(self.stop, signals.engine_stopped)
+        install_shutdown_handlers(self._signal_shutdown)
+
+    def start(self):
+        super(CrawlerProcess, self).start()
+        reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
+        reactor.run(installSignalHandlers=False) # blocking call
+
+    def stop(self):
+        d = super(CrawlerProcess, self).stop()
+        d.addBoth(self._stop_reactor)
+        return d
+
+    def _stop_reactor(self, _=None):
         try:
             reactor.stop()
         except RuntimeError: # raised if already stopped or in shutdown stage
             pass
 
     def _signal_shutdown(self, signum, _):
+        install_shutdown_handlers(self._signal_kill)
         signame = signal_names[signum]
         log.msg("Received %s, shutting down gracefully. Send again to force " \
             "unclean shutdown" % signame, level=log.INFO)
         reactor.callFromThread(self.stop)
-        install_shutdown_handlers(self._signal_kill)
 
     def _signal_kill(self, signum, _):
+        install_shutdown_handlers(signal.SIG_IGN)
         signame = signal_names[signum]
         log.msg('Received %s twice, forcing unclean shutdown' % signame, \
             level=log.INFO)
-        log.log_level = log.SILENT # disable logging of confusing tracebacks
-        reactor.callFromThread(self.engine.kill)
-        install_shutdown_handlers(signal.SIG_IGN)
+        reactor.callFromThread(self._stop_reactor)
diff --git a/scrapy/project.py b/scrapy/project.py
index 038ad66e2..b11a049ac 100644
--- a/scrapy/project.py
+++ b/scrapy/project.py
@@ -1,6 +1,6 @@
-from scrapy.crawler import Crawler
+from scrapy.crawler import CrawlerProcess
 from scrapy.utils.misc import load_object
 from scrapy.conf import settings
 
 _spiders = load_object(settings['SPIDER_MANAGER_CLASS'])()
-crawler = Crawler(settings, _spiders)
+crawler = CrawlerProcess(settings, _spiders)