diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9378f2651..975344103 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -188,6 +188,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -204,7 +205,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -219,6 +220,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -309,7 +312,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -327,7 +330,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" if isinstance(result, AsyncIterator): @@ -352,7 +355,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def _process_spidermw_output(self, output: Any, response: Response) -> None: + async def _process_spidermw_output( + self, output: Any, response: Response | Failure + ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -368,7 +373,9 @@ class Scraper: ) @deferred_f_from_coro_f - async def start_itemproc(self, item: Any, *, response: Response | None) -> None: + async def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> None: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response):