Fix a regression in errback result handling. (#6863)

scrapy/core/scraper.py

@@ -189,6 +189,7 @@ class Scraper:
             )
 
         assert self.crawler.spider
+        output: Iterable[Any] | AsyncIterator[Any]
         if isinstance(result, Response):
             try:
                 # call the spider middlewares and the request callback with the response
@@ -203,7 +204,7 @@ class Scraper:
 
         try:
             # call the request errback with the downloader error
-            await self.call_spider_async(result, request)
+            output = await self.call_spider_async(result, request)
         except Exception as spider_exc:
             # the errback didn't silence the exception
             if not result.check(IgnoreRequest):
@@ -218,6 +219,8 @@ class Scraper:
             if spider_exc is not result.value:
                 # the errback raised a different exception, handle it
                 self.handle_spider_error(Failure(), request, result)
+        else:
+            await self.handle_spider_output_async(output, request, result)
 
     def call_spider(
         self, result: Response | Failure, request: Request, spider: Spider | None = None
@@ -308,7 +311,7 @@ class Scraper:
         self,
         result: Iterable[_T] | AsyncIterator[_T],
         request: Request,
-        response: Response,
+        response: Response | Failure,
         spider: Spider | None = None,
     ) -> Deferred[None]:
         """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
@@ -326,7 +329,7 @@ class Scraper:
         self,
         result: Iterable[_T] | AsyncIterator[_T],
         request: Request,
-        response: Response,
+        response: Response | Failure,
     ) -> None:
         """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
         it: Iterable[_T] | AsyncIterator[_T]
@@ -361,7 +364,7 @@ class Scraper:
         )
 
     def _process_spidermw_output(
-        self, output: Any, response: Response
+        self, output: Any, response: Response | Failure
     ) -> Deferred[None]:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider.
@@ -371,7 +374,7 @@ class Scraper:
         return deferred_from_coro(self._process_spidermw_output_async(output, response))
 
     async def _process_spidermw_output_async(
-        self, output: Any, response: Response
+        self, output: Any, response: Response | Failure
     ) -> None:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider.
@@ -385,7 +388,9 @@ class Scraper:
         if output is not None:
             await self.start_itemproc_async(output, response=response)
 
-    def start_itemproc(self, item: Any, *, response: Response | None) -> Deferred[None]:
+    def start_itemproc(
+        self, item: Any, *, response: Response | Failure | None
+    ) -> Deferred[None]:
         """Send *item* to the item pipelines for processing.
 
         *response* is the source of the item data. If the item does not come
@@ -394,7 +399,7 @@ class Scraper:
         return deferred_from_coro(self.start_itemproc_async(item, response=response))
 
     async def start_itemproc_async(
-        self, item: Any, *, response: Response | None
+        self, item: Any, *, response: Response | Failure | None
     ) -> None:
         """Send *item* to the item pipelines for processing.
 

scrapy/logformatter.py

@@ -116,7 +116,7 @@ class LogFormatter:
         self,
         item: Any,
         exception: BaseException,
-        response: Response | None,
+        response: Response | Failure | None,
         spider: Spider,
     ) -> LogFormatterResult:
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
@@ -137,7 +137,7 @@ class LogFormatter:
         self,
         item: Any,
         exception: BaseException,
-        response: Response | None,
+        response: Response | Failure | None,
         spider: Spider,
     ) -> LogFormatterResult:
         """Logs a message when an item causes an error while it is passing

tests/test_crawl.py

@@ -5,6 +5,7 @@ import logging
 import unittest
 from ipaddress import IPv4Address
 from socket import gethostbyname
+from typing import Any
 from urllib.parse import urlparse
 
 import pytest
@@ -419,6 +420,8 @@ with multiples lines
 
 
 class TestCrawlSpider(TestCase):
+    mockserver: MockServer
+
     @classmethod
     def setUpClass(cls):
         cls.mockserver = MockServer()
@@ -756,6 +759,34 @@ class TestCrawlSpider(TestCase):
             )
         assert "Spider error processing" in str(log)
 
+    @defer.inlineCallbacks
+    def test_spider_errback_item(self):
+        def eb(failure: Failure) -> Any:
+            return {"foo": "bar"}
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "'item_scraped_count': 1" in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_request(self):
+        def eb(failure: Failure) -> Request:
+            return Request(self.mockserver.url("/"))
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "Crawled (200)" in str(log)
+
     @defer.inlineCallbacks
     def test_spider_errback_downloader_error(self):
         failures = []
@@ -774,7 +805,7 @@ class TestCrawlSpider(TestCase):
         assert "Spider error processing" not in str(log)
 
     @defer.inlineCallbacks
-    def test_spider_errback_exception_downloader_error(self):
+    def test_spider_errback_downloader_error_exception(self):
         def eb(failure: Failure) -> None:
             raise ValueError("foo")
 
@@ -786,6 +817,34 @@ class TestCrawlSpider(TestCase):
         assert "Error downloading" in str(log)
         assert "Spider error processing" in str(log)
 
+    @defer.inlineCallbacks
+    def test_spider_errback_downloader_error_item(self):
+        def eb(failure: Failure) -> Any:
+            return {"foo": "bar"}
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "'item_scraped_count': 1" in str(log)
+
+    @defer.inlineCallbacks
+    def test_spider_errback_downloader_error_request(self):
+        def eb(failure: Failure) -> Request:
+            return Request(self.mockserver.url("/"))
+
+        crawler = get_crawler(SingleRequestSpider)
+        with LogCapture() as log:
+            yield crawler.crawl(
+                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+            )
+        assert "HTTP status code is not handled or not allowed" not in str(log)
+        assert "Spider error processing" not in str(log)
+        assert "Crawled (200)" in str(log)
+
     @defer.inlineCallbacks
     def test_raise_closespider(self):
         def cb(response):