diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index e5120ec0d..3409a0e7c 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -57,6 +57,7 @@ class Slot(object): def needs_backout(self): return self.active_size > self.max_active_size + class Scraper(object): def __init__(self, crawler): @@ -100,8 +101,8 @@ class Scraper(object): self._scrape_next(spider, slot) return _ dfd.addBoth(finish_scraping) - dfd.addErrback(log.err, 'Scraper bug processing %s' % request, \ - spider=spider) + dfd.addErrback( + log.err, 'Scraper bug processing %s' % request, spider=spider) self._scrape_next(spider, slot) return dfd @@ -124,13 +125,13 @@ class Scraper(object): """Handle the different cases of request's result been a Response or a Failure""" if not isinstance(request_result, Failure): - return self.spidermw.scrape_response(self.call_spider, \ - request_result, request, spider) + return self.spidermw.scrape_response( + self.call_spider, request_result, request, spider) else: # FIXME: don't ignore errors in spider middleware dfd = self.call_spider(request_result, request, spider) - return dfd.addErrback(self._log_download_errors, \ - request_result, request, spider) + return dfd.addErrback( + self._log_download_errors, request_result, request, spider) def call_spider(self, result, request, spider): result.request = request @@ -143,11 +144,21 @@ class Scraper(object): if isinstance(exc, CloseSpider): self.crawler.engine.close_spider(spider, exc.reason or 'cancelled') return - log.err(_failure, "Spider error processing %s" % request, spider=spider) - self.signals.send_catch_log(signal=signals.spider_error, failure=_failure, response=response, \ - spider=spider) - self.crawler.stats.inc_value("spider_exceptions/%s" % _failure.value.__class__.__name__, \ - spider=spider) + referer = request.headers.get('Referer') + log.err( + _failure, + "Spider error processing %s (referer: %s)" % (request, referer), + spider=spider + ) + self.signals.send_catch_log( + signal=signals.spider_error, + failure=_failure, response=response, + spider=spider + ) + self.crawler.stats.inc_value( + "spider_exceptions/%s" % _failure.value.__class__.__name__, + spider=spider + ) def handle_spider_output(self, result, request, response, spider): if not result: @@ -180,8 +191,8 @@ class Scraper(object): """Log and silence errors that come from the engine (typically download errors that got propagated thru here) """ - if isinstance(download_failure, Failure) \ - and not download_failure.check(IgnoreRequest): + if (isinstance(download_failure, Failure) and + not download_failure.check(IgnoreRequest)): if download_failure.frames: log.err(download_failure, 'Error downloading %s' % request, spider=spider) @@ -204,13 +215,15 @@ class Scraper(object): if isinstance(ex, DropItem): logkws = self.logformatter.dropped(item, ex, response, spider) log.msg(spider=spider, **logkws) - return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \ - item=item, response=response, spider=spider, exception=output.value) + return self.signals.send_catch_log_deferred( + signal=signals.item_dropped, item=item, response=response, + spider=spider, exception=output.value) else: log.err(output, 'Error processing %s' % item, spider=spider) else: logkws = self.logformatter.scraped(output, response, spider) log.msg(spider=spider, **logkws) - return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \ - item=output, response=response, spider=spider) + return self.signals.send_catch_log_deferred( + signal=signals.item_scraped, item=output, response=response, + spider=spider)