Improve spider errors logging which were previously logged as confusing "Unhandled errors" - closes #196

scrapy/core/scraper.py

@@ -4,7 +4,7 @@ extracts information from them"""
 from twisted.python.failure import Failure
 from twisted.internet import defer
 
-from scrapy.utils.defer import defer_result, defer_succeed, parallel
+from scrapy.utils.defer import defer_result, defer_succeed, parallel, iter_errback
 from scrapy.utils.spider import iterate_spider_output
 from scrapy.utils.misc import load_object
 from scrapy.utils.signal import send_catch_log
@@ -143,7 +143,7 @@ class Scraper(object):
 
     def handle_spider_error(self, _failure, request, spider, propagated_failure=None):
         referer = request.headers.get('Referer', None)
-        msg = "Spider exception caught while processing <%s> (referer: <%s>)" % \
+        msg = "Spider error processing <%s> (referer: <%s>)" % \
             (request.url, referer)
         log.err(_failure, msg, spider=spider)
         stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \
@@ -152,7 +152,8 @@ class Scraper(object):
     def handle_spider_output(self, result, request, response, spider):
         if not result:
             return defer_succeed(None)
-        dfd = parallel(iter(result), self.concurrent_items,
+        it = iter_errback(result, self.handle_spider_error, request, spider)
+        dfd = parallel(it, self.concurrent_items,
             self._process_spidermw_output, request, response, spider)
         return dfd
 

scrapy/tests/test_utils_defer.py

@@ -3,7 +3,7 @@ from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
 from scrapy.utils.defer import mustbe_deferred, process_chain, \
-    process_chain_both, process_parallel
+    process_chain_both, process_parallel, iter_errback
 
 
 class MustbeDeferredTest(unittest.TestCase):
@@ -76,3 +76,29 @@ class DeferUtilsTest(unittest.TestCase):
         self.failUnlessFailure(d, TypeError)
         self.flushLoggedErrors()
         return d
+
+
+class IterErrbackTest(unittest.TestCase):
+
+    def test_iter_errback_good(self):
+        def itergood():
+            for x in xrange(10):
+                yield x
+
+        errors = []
+        out = list(iter_errback(itergood(), errors.append))
+        self.failUnlessEqual(out, range(10))
+        self.failIf(errors)
+
+    def test_iter_errback_bad(self):
+        def iterbad():
+            for x in xrange(10):
+                if x == 5:
+                    a = 1/0
+                yield x
+
+        errors = []
+        out = list(iter_errback(iterbad(), errors.append))
+        self.failUnlessEqual(out, [0, 1, 2, 3, 4])
+        self.failUnlessEqual(len(errors), 1)
+        self.failUnless(isinstance(errors[0].value, ZeroDivisionError))

scrapy/utils/defer.py

@@ -85,3 +85,16 @@ def process_parallel(callbacks, input, *a, **kw):
     d = defer.gatherResults(dfds)
     d.addErrback(lambda _: _.value.subFailure)
     return d
+
+def iter_errback(iterable, errback, *a, **kw):
+    """Wraps an iterable calling an errback if an error is caught while
+    iterating it.
+    """
+    it = iter(iterable)
+    while 1:
+        try:
+            yield it.next()
+        except StopIteration:
+            break
+        except:
+            errback(failure.Failure(), *a, **kw)