diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 1c9ee0c77..fad8ac56e 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -122,14 +122,20 @@ one or more of these methods: This method is an :term:`asynchronous generator` called with the results from the spider after the spider has processed the response. + .. versionchanged:: VERSION + It is also called with the output of a request :ref:`errback + `, with *response* set to ``None``, when the errback runs + because a download failed. + *result* is lazy: a generator callback runs as *result* is iterated, so code that runs before that iteration runs before the callback body. .. seealso:: :ref:`universal-spider-middleware`. :param response: the response which generated this output from the - spider - :type response: :class:`~scrapy.http.Response` object + spider, or ``None`` if the output comes from an errback called + because a download failed + :type response: :class:`~scrapy.http.Response` object or ``None`` :param result: the results from the spider :type result: an :term:`asynchronous iterable` of @@ -145,9 +151,14 @@ one or more of these methods: .. method:: process_spider_exception(response, exception) - This method is called when a spider callback or a - :meth:`process_spider_output` method (from a previous spider - middleware) raises an exception. + This method is called when a spider callback, a request :ref:`errback + ` or a :meth:`process_spider_output` method (from a previous + spider middleware) raises an exception. + + .. versionchanged:: VERSION + It is also called for exceptions raised while iterating the output + of an errback that runs because a download failed, with *response* + set to ``None``. :meth:`process_spider_exception` should return either ``None`` or an iterable of :class:`~scrapy.Request` or :ref:`item ` @@ -163,8 +174,8 @@ one or more of these methods: :meth:`process_spider_exception` will be called. :param response: the response being processed when the exception was - raised - :type response: :class:`~scrapy.http.Response` object + raised, or ``None`` if there was no response + :type response: :class:`~scrapy.http.Response` object or ``None`` :param exception: the exception raised :type exception: :exc:`Exception` object diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 351375d7b..691746e2b 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -265,8 +265,11 @@ class Scraper: return try: - # call the request errback with the downloader error - output = await self.call_spider_async(result, request) + # call the request errback with the downloader error and the spider + # middlewares with its output + output = await self.spidermw._scrape_failure_async( + self.call_spider_async, result, request + ) except Exception as spider_exc: # the errback didn't silence the exception assert self.crawler.spider diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index fbc6f2530..086e444a3 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -99,7 +99,7 @@ class SpiderMiddlewareManager(MiddlewareManager): async def _evaluate_iterable( self, - response: Response, + response: Response | None, iterable: AsyncIterator[_T], exception_processor_index: int, recover_to: MutableAsyncChain[_T], @@ -115,7 +115,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_exception( self, - response: Response, + response: Response | None, exception: Exception, start_index: int = 0, ) -> MutableAsyncChain[_T]: @@ -151,7 +151,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def _process_spider_output( self, - response: Response, + response: Response | None, result: AsyncIterator[_T], start_index: int = 0, ) -> MutableAsyncChain[_T]: @@ -172,7 +172,7 @@ class SpiderMiddlewareManager(MiddlewareManager): return MutableAsyncChain(result, recovered) async def _process_callback_output( - self, response: Response, result: AsyncIterator[_T] + self, response: Response | None, result: AsyncIterator[_T] ) -> MutableAsyncChain[_T]: recovered: MutableAsyncChain[_T] = MutableAsyncChain() result = self._evaluate_iterable(response, result, 0, recovered) @@ -227,6 +227,20 @@ class SpiderMiddlewareManager(MiddlewareManager): await _defer_sleep_async() return self._process_spider_exception(response, ex) + async def _scrape_failure_async( + self, + scrape_func: ScrapeFunc[_T], + failure: Failure, + request: Request, + ) -> MutableAsyncChain[_T]: + # There is no response to run the process_spider_input chain on, so the + # errback output enters the process_spider_output chain with None as the + # response. Exceptions from the errback itself are left to the caller, + # which tells apart the download error from a different one. + it: Iterable[_T] | AsyncIterator[_T] = await scrape_func(failure, request) + ait = it if isinstance(it, AsyncIterator) else as_async_generator(it) + return await self._process_callback_output(None, ait) + async def process_start( self, spider: Spider | None = None ) -> AsyncIterator[Any] | None: diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py index 3b14d2b0a..651df036c 100644 --- a/scrapy/spidermiddlewares/base.py +++ b/scrapy/spidermiddlewares/base.py @@ -86,8 +86,9 @@ class BaseSpiderMiddleware: :type request: :class:`~scrapy.Request` object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object or ``None`` for - start requests + :type response: :class:`~scrapy.http.Response` object, or ``None`` for + start requests and for the output of a request errback called + because a download failed :return: the processed request or ``None`` """ @@ -104,8 +105,9 @@ class BaseSpiderMiddleware: :type item: item object :param response: the response being processed - :type response: :class:`~scrapy.http.Response` object or ``None`` for - start items + :type response: :class:`~scrapy.http.Response` object, or ``None`` for + start items and for the output of a request errback called because + a download failed :return: the processed item or ``None`` """ diff --git a/scrapy/spidermiddlewares/start.py b/scrapy/spidermiddlewares/start.py index 5d76b60d2..678815a07 100644 --- a/scrapy/spidermiddlewares/start.py +++ b/scrapy/spidermiddlewares/start.py @@ -1,12 +1,13 @@ from __future__ import annotations -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any + +from scrapy import Request from .base import BaseSpiderMiddleware if TYPE_CHECKING: - from scrapy.http import Request - from scrapy.http.response import Response + from collections.abc import AsyncIterator class StartSpiderMiddleware(BaseSpiderMiddleware): @@ -23,9 +24,8 @@ class StartSpiderMiddleware(BaseSpiderMiddleware): `. """ - def get_processed_request( - self, request: Request, response: Response | None - ) -> Request | None: - if response is None: - request.meta.setdefault("is_start_request", True) - return request + async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]: + async for o in start: + if isinstance(o, Request): + o.meta.setdefault("is_start_request", True) + yield o diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dacc90b27..160d6f6aa 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -243,6 +243,51 @@ class GeneratorOutputChainSpider(Spider): yield {"processed": ["parse-second-item"]} +# ================================================================================ +# (5) errback output after a download error +class LogOutputMiddleware(_BaseSpiderMiddleware): + async def process_spider_output(self, response, result): + async for o in result: + self.crawler.spider.logger.info( + f"Middleware: output {o} with response {response}" + ) + yield o + + def process_spider_exception(self, response, exception): + self.crawler.spider.logger.info( + f"Middleware: {exception.__class__.__name__} exception caught" + f" with response {response}" + ) + return [] + + +class DownloadErrorSpider(Spider): + name = "DownloadErrorSpider" + custom_settings = { + "SPIDER_MIDDLEWARES": { + LogOutputMiddleware: 10, + }, + } + + async def start(self): + yield Request(self.mockserver.url("/drop?abort=1"), errback=self.errback) + + def errback(self, failure): + yield {"from": "errback"} + yield Request(self.mockserver.url("/status?n=200"), callback=self.parse) + + def parse(self, response): + self.logger.info(f"is_start_request: {response.meta.get('is_start_request')}") + + +class DownloadErrorFailSpider(DownloadErrorSpider): + name = "DownloadErrorFailSpider" + + def errback(self, failure): + yield {"from": "errback"} + raise LookupError + + # ================================================================================ class TestSpiderMiddleware: mockserver: MockServer @@ -426,3 +471,30 @@ class TestSpiderMiddleware: assert str(item_from_callback) in log4 assert str(item_recovered) in log4 assert "parse-second-item" not in log4 + + @coroutine_test + async def test_download_error_errback_output( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """ + (5.1) The output of an errback called because a download failed goes + through the process_spider_output chain, with None as the response. + """ + log5 = await self.crawl_log(DownloadErrorSpider, caplog) + assert "Middleware: output {'from': 'errback'} with response None" in log5 + assert "'item_scraped_count': 1" in log5 + assert "Crawled (200)" in log5 + assert "is_start_request: None" in log5 + + @coroutine_test + async def test_download_error_errback_exception( + self, caplog: pytest.LogCaptureFixture + ) -> None: + """ + (5.2) An exception from such an errback goes through the + process_spider_exception chain. + """ + log5 = await self.crawl_log(DownloadErrorFailSpider, caplog) + assert "Middleware: output {'from': 'errback'} with response None" in log5 + assert "Middleware: LookupError exception caught with response None" in log5 + assert "'item_scraped_count': 1" in log5