diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 9fd68bce5..1f0d57c63 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -189,6 +189,7 @@ class Scraper: ) assert self.crawler.spider + output: Iterable[Any] | AsyncIterator[Any] if isinstance(result, Response): try: # call the spider middlewares and the request callback with the response @@ -203,7 +204,7 @@ class Scraper: try: # call the request errback with the downloader error - await self.call_spider_async(result, request) + output = await self.call_spider_async(result, request) except Exception as spider_exc: # the errback didn't silence the exception if not result.check(IgnoreRequest): @@ -218,6 +219,8 @@ class Scraper: if spider_exc is not result.value: # the errback raised a different exception, handle it self.handle_spider_error(Failure(), request, result) + else: + await self.handle_spider_output_async(output, request, result) def call_spider( self, result: Response | Failure, request: Request, spider: Spider | None = None @@ -308,7 +311,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, spider: Spider | None = None, ) -> Deferred[None]: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" @@ -326,7 +329,7 @@ class Scraper: self, result: Iterable[_T] | AsyncIterator[_T], request: Request, - response: Response, + response: Response | Failure, ) -> None: """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.""" it: Iterable[_T] | AsyncIterator[_T] @@ -361,7 +364,7 @@ class Scraper: ) def _process_spidermw_output( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> Deferred[None]: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -371,7 +374,7 @@ class Scraper: return deferred_from_coro(self._process_spidermw_output_async(output, response)) async def _process_spidermw_output_async( - self, output: Any, response: Response + self, output: Any, response: Response | Failure ) -> None: """Process each Request/Item (given in the output parameter) returned from the given spider. @@ -385,7 +388,9 @@ class Scraper: if output is not None: await self.start_itemproc_async(output, response=response) - def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]: + def start_itemproc( + self, item: Any, *, response: Response | Failure | None + ) -> Deferred[None]: """Send *item* to the item pipelines for processing. *response* is the source of the item data. If the item does not come @@ -394,7 +399,7 @@ class Scraper: return deferred_from_coro(self.start_itemproc_async(item, response=response)) async def start_itemproc_async( - self, item: Any, *, response: Response | None + self, item: Any, *, response: Response | Failure | None ) -> None: """Send *item* to the item pipelines for processing. diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py index 4f08918ae..e81a9ec93 100644 --- a/scrapy/logformatter.py +++ b/scrapy/logformatter.py @@ -116,7 +116,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item is dropped while it is passing through the item pipeline.""" @@ -137,7 +137,7 @@ class LogFormatter: self, item: Any, exception: BaseException, - response: Response | None, + response: Response | Failure | None, spider: Spider, ) -> LogFormatterResult: """Logs a message when an item causes an error while it is passing diff --git a/tests/test_crawl.py b/tests/test_crawl.py index b90706027..8289b2243 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -5,6 +5,7 @@ import logging import unittest from ipaddress import IPv4Address from socket import gethostbyname +from typing import Any from urllib.parse import urlparse import pytest @@ -419,6 +420,8 @@ with multiples lines class TestCrawlSpider(TestCase): + mockserver: MockServer + @classmethod def setUpClass(cls): cls.mockserver = MockServer() @@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase): ) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/status?n=400"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_spider_errback_downloader_error(self): failures = [] @@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase): assert "Spider error processing" not in str(log) @defer.inlineCallbacks - def test_spider_errback_exception_downloader_error(self): + def test_spider_errback_downloader_error_exception(self): def eb(failure: Failure) -> None: raise ValueError("foo") @@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase): assert "Error downloading" in str(log) assert "Spider error processing" in str(log) + @defer.inlineCallbacks + def test_spider_errback_downloader_error_item(self): + def eb(failure: Failure) -> Any: + return {"foo": "bar"} + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "'item_scraped_count': 1" in str(log) + + @defer.inlineCallbacks + def test_spider_errback_downloader_error_request(self): + def eb(failure: Failure) -> Request: + return Request(self.mockserver.url("/")) + + crawler = get_crawler(SingleRequestSpider) + with LogCapture() as log: + yield crawler.crawl( + seed=self.mockserver.url("/drop?abort=1"), errback_func=eb + ) + assert "HTTP status code is not handled or not allowed" not in str(log) + assert "Spider error processing" not in str(log) + assert "Crawled (200)" in str(log) + @defer.inlineCallbacks def test_raise_closespider(self): def cb(response):