diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 5d1ec246c..177d8822a 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,6 +8,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterator, Callable, Coroutine, Iterable +from contextlib import suppress from functools import wraps from inspect import isasyncgenfunction, iscoroutine from itertools import islice @@ -108,6 +109,8 @@ class SpiderMiddlewareManager(MiddlewareManager): async for r in iterable: yield r except Exception as ex: + if getattr(ex, "_spidermw_unhandled", False): + raise exception_result: MutableAsyncChain[_T] = self._process_spider_exception( response, ex, exception_processor_index ) @@ -147,6 +150,11 @@ class SpiderMiddlewareManager(MiddlewareManager): f"or an iterable, got {type(result)}" ) raise _InvalidOutput(msg) + # Every remaining middleware declined to handle the exception, so the + # outer process_spider_output layers must let it through instead of + # offering it to those middlewares again. + with suppress(AttributeError): + exception._spidermw_unhandled = True # type: ignore[attr-defined] raise exception def _process_spider_output( diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index dacc90b27..9aa124693 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -243,6 +243,39 @@ class GeneratorOutputChainSpider(Spider): yield {"processed": ["parse-second-item"]} +# ================================================================================ +# (5) an exception from a spider callback (generator) that no process_spider_exception +# method handles, with middlewares that define a process_spider_output method +class FirstUnhandledMiddleware(_GeneratorDoNothingMiddleware): + pass + + +class SecondUnhandledMiddleware(_GeneratorDoNothingMiddleware): + pass + + +class ThirdUnhandledMiddleware(_GeneratorDoNothingMiddleware): + pass + + +class UnhandledExceptionSpider(Spider): + name = "UnhandledExceptionSpider" + custom_settings = { + "SPIDER_MIDDLEWARES": { + FirstUnhandledMiddleware: 30, + SecondUnhandledMiddleware: 20, + ThirdUnhandledMiddleware: 10, + }, + } + + async def start(self): + yield Request(self.mockserver.url("/status?n=200")) + + def parse(self, response): + yield {"processed": ["parse-first-item"]} + raise ImportError + + # ================================================================================ class TestSpiderMiddleware: mockserver: MockServer @@ -426,3 +459,20 @@ class TestSpiderMiddleware: assert str(item_from_callback) in log4 assert str(item_recovered) in log4 assert "parse-second-item" not in log4 + + @coroutine_test + async def test_unhandled_exception(self, caplog: pytest.LogCaptureFixture) -> None: + """ + (5) An exception that no process_spider_exception method handles should be + offered to each of them only once, and then reach the spider error log. + """ + log5 = await self.crawl_log(UnhandledExceptionSpider, caplog) + for middleware in ( + FirstUnhandledMiddleware, + SecondUnhandledMiddleware, + ThirdUnhandledMiddleware, + ): + method = f"{middleware.__name__}.process_spider_exception" + assert log5.count(f"{method}: ImportError caught") == 1 + assert "Spider error processing" in log5 + assert "'item_scraped_count': 1" in log5