From 02e228ad760a9a35d697aac7c70e38b3353eb242 Mon Sep 17 00:00:00 2001
From: Pablo Hoffman <pablo@pablohoffman.com>
Date: Tue, 15 Sep 2009 09:27:30 -0300
Subject: [PATCH] added support for returning deferreds in spider manager
 close_domain() method, and making sure engine_stopped signal is always sent
 (even when no spiders have run)

---
 scrapy/core/engine.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index a64a6b3c4..004a65b98 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -60,9 +60,10 @@ class ExecutionEngine(object):
         if not self.running:
             return
         self.running = False
-        for spider in self.open_spiders:
-            reactor.addSystemEventTrigger('before', 'shutdown', \
-                self.close_spider, spider, reason='shutdown')
+        def before_shutdown():
+            dfd = self._close_all_spiders()
+            return dfd.addBoth(lambda _: self._finish_stopping_engine())
+        reactor.addSystemEventTrigger('before', 'shutdown', before_shutdown)
         if self._mainloop_task.running:
             self._mainloop_task.stop()
         try:
@@ -284,10 +285,15 @@ class ExecutionEngine(object):
             return self._finish_closing_spider_if_idle(spider)
         return defer.succeed(None)
 
+    def _close_all_spiders(self):
+        dfds = [self.close_spider(s, reason='shutdown') for s in self.open_spiders]
+        dlist = defer.DeferredList(dfds)
+        return dlist
+
     def _finish_closing_spider_if_idle(self, spider):
         """Call _finish_closing_spider if domain is idle"""
         if self.spider_is_idle(spider) or self.killed:
-            self._finish_closing_spider(spider)
+            return self._finish_closing_spider(spider)
         else:
             dfd = defer.Deferred()
             dfd.addCallback(self._finish_closing_spider_if_idle)
@@ -304,10 +310,12 @@ class ExecutionEngine(object):
         send_catch_log(signal=signals.domain_closed, sender=self.__class__, \
             domain=domain, spider=spider, reason=reason)
         stats.close_domain(domain, reason=reason)
-        spiders.close_spider(spider)
-        log.msg("Domain closed (%s)" % reason, domain=domain) 
-        self._mainloop()
-        if not self.open_spiders:
-            send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
+        dfd = defer.maybeDeferred(spiders.close_spider, spider)
+        dfd.addBoth(log.msg, "Domain closed (%s)" % reason, domain=domain)
+        reactor.callLater(0, self._mainloop)
+        return dfd
+
+    def _finish_stopping_engine(self):
+        send_catch_log(signal=signals.engine_stopped, sender=self.__class__)
 
 scrapyengine = ExecutionEngine()