diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index d3875826b..6343f02f3 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -186,6 +186,24 @@ spider_idle :param spider: the spider which has gone idle :type spider: :class:`~scrapy.spider.BaseSpider` object +spider_error +------------ + +.. signal:: spider_error +.. function:: spider_error(failure, response, spider) + + Sent when a spider callback generates an error (ie. raises an exception). + + :param failure: the exception raised as a Twisted `Failure`_ object + :type failure: `Failure`_ object + + :param response: the response being processed when the exception was raised + :type response: :class:`~scrapy.http.Response` object + + :param spider: the spider which raised the exception + :type spider: :class:`~scrapy.spider.BaseSpider` object + + request_received ---------------- @@ -241,3 +259,4 @@ response_downloaded :param spider: the spider for which the response is intended :type spider: :class:`~scrapy.spider.BaseSpider` object +.. _Failure: http://twistedmatrix.com/documents/current/api/twisted.python.failure.Failure.html diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2104a728e..1550c344f 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -121,7 +121,7 @@ class Scraper(object): assert isinstance(response, (Response, Failure)) dfd = self._scrape2(response, request, spider) # returns spiders processed output - dfd.addErrback(self.handle_spider_error, request, spider) + dfd.addErrback(self.handle_spider_error, request, response, spider) dfd.addCallback(self.handle_spider_output, request, response, spider) return dfd @@ -142,18 +142,20 @@ class Scraper(object): dfd.addCallbacks(request.callback or spider.parse, request.errback) return dfd.addCallback(iterate_spider_output) - def handle_spider_error(self, _failure, request, spider, propagated_failure=None): + def handle_spider_error(self, _failure, request, response, spider, propagated_failure=None): referer = request.headers.get('Referer', None) msg = "Spider error processing <%s> (referer: <%s>)" % \ (request.url, referer) log.err(_failure, msg, spider=spider) + send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ + spider=spider) stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ spider=spider) def handle_spider_output(self, result, request, response, spider): if not result: return defer_succeed(None) - it = iter_errback(result, self.handle_spider_error, request, spider) + it = iter_errback(result, self.handle_spider_error, request, response, spider) dfd = parallel(it, self.concurrent_items, self._process_spidermw_output, request, response, spider) return dfd diff --git a/scrapy/signals.py b/scrapy/signals.py index e907e5fcb..c08a11382 100644 --- a/scrapy/signals.py +++ b/scrapy/signals.py @@ -10,6 +10,7 @@ engine_stopped = object() spider_opened = object() spider_idle = object() spider_closed = object() +spider_error = object() request_received = object() response_received = object() response_downloaded = object()