Log full traceback of signal handler errors in send_catch_log() - closes #194. Also made engine use send_catch_log for spider_idle signal

scrapy/core/engine.py

@@ -8,7 +8,6 @@ from time import time
 
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
-from scrapy.xlib.pydispatch import dispatcher
 
 from scrapy import log
 from scrapy.stats import stats
@@ -228,15 +227,13 @@ class ExecutionEngine(object):
         next loop and this function is guaranteed to be called (at least) once
         again for this spider.
         """
-        try:
-            dispatcher.send(signal=signals.spider_idle, sender=self.__class__, \
-                spider=spider)
-        except DontCloseSpider:
+        res = send_catch_log(signal=signals.spider_idle, sender=self.__class__, \
+            spider=spider, dont_log=DontCloseSpider)
+        if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
+                for _, x in res):
             reactor.callLater(5, self.next_request, spider)
             return
-        except Exception, e:
-            log.msg("Exception caught on 'spider_idle' signal dispatch: %r" % e, \
-                level=log.ERROR)
+
         if self.spider_is_idle(spider):
             self.close_spider(spider, reason='finished')
 

scrapy/utils/signal.py

@@ -1,16 +1,29 @@
 """Helper functinos for working with signals"""
 
-from scrapy.xlib.pydispatch import dispatcher
-from scrapy.xlib.pydispatch.robust import sendRobust
+from twisted.python.failure import Failure
+
+from scrapy.xlib.pydispatch.dispatcher import Any, Anonymous, liveReceivers, getAllReceivers
+from scrapy.xlib.pydispatch.robustapply import robustApply
+
 from scrapy import log
 
-def send_catch_log(*args, **kwargs):
-    """Same as dispatcher.robust.sendRobust but logs any exceptions raised by
-    the signal handlers
+def send_catch_log(signal=Any, sender=Anonymous, *arguments, **named):
+    """Like pydispatcher.robust.sendRobust but it also logs errors and returns
+    Failures instead of exceptions.
     """
-    results = sendRobust(*args, **kwargs)
-    for receiver, response in results:
-        if isinstance(response, Exception):
-            log.msg("Exception caught on signal dispatch: receiver=%r, " \
-                " exception=%r" % (receiver, response), level=log.ERROR)
-    return results
+    dont_log = named.pop('dont_log', None)
+    spider = named.get('spider', None)
+    responses = []
+    for receiver in liveReceivers(getAllReceivers(sender, signal)):
+        try:
+            response = robustApply(receiver, signal=signal, sender=sender,
+                *arguments, **named)
+        except dont_log:
+            result = Failure()
+        except Exception:
+            result = Failure()
+            log.err(result, "Signal handler error", spider=spider)
+        else:
+            result = response
+        responses.append((receiver, result))
+    return responses