Several core changes:

Execution Manager:

* added control_reactor argument to delegate external twisted
  reactor control (for example by twistd)
* now it loads spiders (if not already loaded)
* now it stars the log (if not already started)
* removed *args from configure() method
* removed **opts from runonce and start methods

Execution engine:

* added control_reactor argument to to delegate external twisted
  reactor control (for example by twistd)
* changed some functions and method names for clarity
* improve handling of exceptions in st() method
* regrouped close_domain, closed_domain, and _close_domain method
  for legibilty

Scheduler:

* replaced pending_domains_count (dict) by pending_domains (set)
* simplified some doc

scrapy/command/commands/crawl.py

@@ -57,4 +57,4 @@ class Command(ScrapyCommand):
                 log.msg('Cannot create any requests with the provided arguments', log.ERROR)
                 return
 
-        scrapymanager.runonce(*args, **opts.__dict__)
+        scrapymanager.runonce(*args)

scrapy/command/commands/start.py

@@ -13,4 +13,4 @@ class Command(ScrapyCommand):
         ScrapyCommand.add_options(self, parser)
 
     def run(self, args, opts):
-        scrapymanager.start(*args, **opts.__dict__)
+        scrapymanager.start(*args)

scrapy/core/manager.py

@@ -1,5 +1,7 @@
 import signal
 
+from twisted.internet import reactor
+
 from scrapy.extension import extensions
 from scrapy import log
 from scrapy.http import Request
@@ -20,11 +22,21 @@ class ExecutionManager(object):
     def __init__(self):
         self.interrupted = False
         self.configured = False
+        self.control_reactor = True
 
-    def configure(self, *args, **opts):
-        self._install_signals()
+    def configure(self, control_reactor=True):
+        self.control_reactor = control_reactor
+        if control_reactor:
+            self._install_signals()
+        else:
+            reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
 
-        extensions.load()
+        if not log.started:
+            log.start()
+        if not extensions.loaded:
+            extensions.load()
+        if not spiders.loaded:
+            spiders.load()
         log.msg("Enabled extensions: %s" % ", ".join(extensions.enabled.iterkeys()))
 
         scheduler = load_object(settings['SCHEDULER'])()
@@ -35,35 +47,37 @@ class ExecutionManager(object):
         """Schedule the given args for crawling. args is a list of urls or domains"""
 
         requests = self._parse_args(args)
-        # schedule initial requets to be scraped at engine start
+        # schedule initial requests to be scraped at engine start
         for domain in requests or ():
             spider = spiders.fromdomain(domain) 
             priority = self.domainprio.get_priority(domain)
             for request in requests[domain]:
                 scrapyengine.crawl(request, spider, domain_priority=priority)
 
-    def runonce(self, *args, **opts):
+    def runonce(self, *args):
         """Run the engine until it finishes scraping all domains and then exit"""
-        self.configure(*args, **opts)
+        self.configure()
         self.crawl(*args)
         scrapyengine.start()
 
-    def start(self, **opts):
+    def start(self, control_reactor=True):
         """Start the scrapy server, without scheduling any domains"""
-        self.configure(**opts)
+        self.configure(control_reactor)
         scrapyengine.keep_alive = True
-        scrapyengine.start()# blocking call
-        self.stop()
+        scrapyengine.start(control_reactor=control_reactor)
+        if control_reactor:
+            self.stop()
 
     def stop(self):
         """Stop the scrapy server, shutting down the execution engine"""
         self.interrupted = True
         scrapyengine.stop()
         log.log_level = -999 # disable logging
-        signal.signal(signal.SIGTERM, signal.SIG_IGN)
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
-        if hasattr(signal, "SIGBREAK"):
-            signal.signal(signal.SIGBREAK, signal.SIG_IGN)
+        if self.control_reactor:
+            signal.signal(signal.SIGTERM, signal.SIG_IGN)
+            signal.signal(signal.SIGINT, signal.SIG_IGN)
+            if hasattr(signal, "SIGBREAK"):
+                signal.signal(signal.SIGBREAK, signal.SIG_IGN)
 
     def reload_spiders(self):
         """Reload all enabled spiders except for the ones that are currently