diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 2942dfa58..6f69d668e 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -374,32 +374,24 @@ class Scraper: errors that got propagated thru here). spider_failure: the value passed into the errback of self.call_spider() + (likely raised in the request errback) + download_failure: the value passed into _scrape2() from ExecutionEngine._handle_downloader_output() as "result" + (likely raised in the download handler or a downloader middleware) """ if not download_failure.check(IgnoreRequest): assert self.crawler.spider - if download_failure.frames: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - exc_info=failure_to_exc_info(download_failure), - ) - else: - errmsg = download_failure.getErrorMessage() - if errmsg: - logkws = self.logformatter.download_error( - download_failure, request, self.crawler.spider, errmsg - ) - logger.log( - *logformatter_adapter(logkws), - extra={"spider": self.crawler.spider}, - ) - + logkws = self.logformatter.download_error( + download_failure, request, self.crawler.spider + ) + logger.log( + *logformatter_adapter(logkws), + extra={"spider": self.crawler.spider}, + exc_info=failure_to_exc_info(download_failure), + ) if spider_failure is not download_failure: + # a request errback raised a different exception, it needs to be handled later return spider_failure return None diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f49deac1f..a8174d537 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import logging import unittest @@ -723,3 +725,78 @@ class TestCrawlSpider(TestCase): assert crawler.spider.meta[ "failure" ].value.response.headers == crawler.spider.meta.get("headers_received") + + @defer.inlineCallbacks + def test_spider_errback(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_silence(self): + failures = [] + + def eb(failure: Failure) -> None: + failures.append(failure) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert len(failures) == 1 + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "Spider error processing" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error(self): + failures = [] + + def eb(failure: Failure) -> Failure: + failures.append(failure) + return failure + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert len(failures) == 1 + assert "Error downloading" in str(log) + assert "Spider error processing" not in str(log) + + @defer.inlineCallbacks + def test_spider_errback_exception_downloader_error(self): + def eb(failure: Failure) -> None: + raise ValueError("foo") + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "Error downloading" in str(log) + assert "Spider error processing" in str(log)