From c285f4cb18c310f131e2ce8912a950d533837180 Mon Sep 17 00:00:00 2001 From: Adrian Date: Mon, 10 Aug 2026 10:06:54 +0200 Subject: [PATCH] Support CloseSpider during spider startup (#7905) --- docs/topics/item-pipeline.rst | 7 +++++ docs/topics/signals.rst | 7 +++++ scrapy/core/engine.py | 33 +++++++++++++++------- scrapy/crawler.py | 18 ++++++++---- scrapy/exceptions.py | 7 ++--- tests/test_engine.py | 52 ++++++++++++++++++++++++++++++++++- 6 files changed, 104 insertions(+), 20 deletions(-) diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst index 35891ce8e..a6aac78ac 100644 --- a/docs/topics/item-pipeline.rst +++ b/docs/topics/item-pipeline.rst @@ -47,6 +47,13 @@ Additionally, they may also implement the following methods: This method is called when the spider is opened. + .. versionchanged:: VERSION + Added support for :exc:`~scrapy.exceptions.CloseSpider`. + + It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before + it starts crawling, e.g. if a resource that the pipeline needs is + unavailable. + .. method:: close_spider(self) This method is called when the spider is closed, before the diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst index acbf7ddef..0a85c3c05 100644 --- a/docs/topics/signals.rst +++ b/docs/topics/signals.rst @@ -290,6 +290,13 @@ spider_opened reserve per-spider resources, but can be used for any task that needs to be performed when a spider is opened. + .. versionchanged:: VERSION + Added support for :exc:`~scrapy.exceptions.CloseSpider`. + + You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the + spider before it starts crawling, e.g. if a resource that the spider needs + is unavailable. + This signal supports :ref:`asynchronous handlers `. :param spider: the spider which has been opened diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b63d136a4..104daf399 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -248,7 +248,7 @@ class ExecutionEngine: ) return deferred_from_coro(self.close_async()) - async def close_async(self) -> None: + async def close_async(self, *, reason: str = "shutdown") -> None: """ Gracefully close the execution engine. If it has already been started, stop it. In all cases, close the spider and the downloader. @@ -256,9 +256,7 @@ class ExecutionEngine: if self.running: await self.stop_async() # will also close spider and downloader elif self.spider is not None: - await self.close_spider_async( - reason="shutdown" - ) # will also close downloader + await self.close_spider_async(reason=reason) # will also close downloader elif hasattr(self, "downloader"): self.downloader.close() @@ -557,10 +555,20 @@ class ExecutionEngine: nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) self._slot = _Slot(close_if_idle, nextcall, scheduler) - self._start = await self.scraper.spidermw.process_start() - if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)): - await maybe_deferred_to_future(d) - await self.scraper.open_spider_async() + # A component that fails to start can ask for the spider to be closed. + # The rest of the startup runs anyway, so that components that are + # started also get stopped, and the request is honored once the spider + # is open. + close_spider_exc: CloseSpider | None = None + try: + self._start = await self.scraper.spidermw.process_start() + if hasattr(scheduler, "open") and ( + d := scheduler.open(self.crawler.spider) + ): + await maybe_deferred_to_future(d) + await self.scraper.open_spider_async() + except CloseSpider as exc: + close_spider_exc = exc stats = self.crawler.stats if argument_is_required(stats.open_spider, "spider"): warnings.warn( @@ -572,9 +580,14 @@ class ExecutionEngine: stats.open_spider(spider=self.crawler.spider) else: stats.open_spider() - await self.signals.send_catch_log_async( - signals.spider_opened, spider=self.crawler.spider + results = await self.signals.send_catch_log_async( + signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider ) + for _, result in results: + if isinstance(result, CloseSpider): + close_spider_exc = close_spider_exc or result + if close_spider_exc is not None: + raise close_spider_exc def _spider_idle(self) -> None: """ diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 44c4ffdcf..f0cfba6b9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks from scrapy import Spider from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine -from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning from scrapy.extension import ExtensionManager from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager @@ -270,8 +270,12 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - yield deferred_from_coro(self.engine.open_spider_async()) - yield deferred_from_coro(self.engine.start_async()) + try: + yield deferred_from_coro(self.engine.open_spider_async()) + except CloseSpider as exc: + yield deferred_from_coro(self.engine.close_async(reason=exc.reason)) + else: + yield deferred_from_coro(self.engine.start_async()) except Exception: self.crawling = False if self._engine is not None: @@ -300,8 +304,12 @@ class Crawler: self._apply_settings() self._update_root_log_handler() self.engine = self._create_engine() - await self.engine.open_spider_async() - await self.engine.start_async() + try: + await self.engine.open_spider_async() + except CloseSpider as exc: + await self.engine.close_async(reason=exc.reason) + else: + await self.engine.start_async() except Exception: self.crawling = False if self._engine is not None: diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py index fa5bc1c1e..cd2560df1 100644 --- a/scrapy/exceptions.py +++ b/scrapy/exceptions.py @@ -56,12 +56,11 @@ class DontCloseSpider(Exception): class CloseSpider(Exception): - """Raised from a :ref:`spider callback ` or from - :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped. + """Raised from a :ref:`spider callback `, or while the + spider is starting, to request the spider to be closed/stopped. .. versionchanged:: VERSION - Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead - of being reported as a start error. + Added support for raising it while the spider is starting. *reason* is a string with the reason for closing. diff --git a/tests/test_engine.py b/tests/test_engine.py index 8a3bceccb..53eb4a1f6 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -29,7 +29,9 @@ from tests.utils.engine import ( ) if TYPE_CHECKING: - from collections.abc import AsyncIterator + from collections.abc import AsyncIterator, Generator + + from twisted.internet.defer import Deferred from tests.mockserver.http import MockServer @@ -230,3 +232,51 @@ async def test_request_scheduled_signal(): f"{scheduler.enqueued!r} != [{keep_request!r}]" ) crawler.signals.disconnect(signal_handler, signals.request_scheduled) + + +class ClosingPipeline: + def open_spider(self): + raise CloseSpider("pipeline_reason") + + +class TestCloseSpiderOnStartup: + @coroutine_test + async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None: + closed: list[str] = [] + + def spider_closed(reason: str) -> None: + closed.append(reason) + + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + crawler.signals.connect(spider_closed, signals.spider_closed) + with caplog.at_level(logging.INFO): + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason" + assert closed == ["pipeline_reason"] + assert "Traceback" not in caplog.text + + @coroutine_test + async def test_spider_opened(self) -> None: + def spider_opened(spider: Spider) -> None: + raise CloseSpider("signal_reason") + + crawler = get_crawler(DefaultSpider) + crawler.signals.connect(spider_opened, signals.spider_opened) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "signal_reason" + + @coroutine_test + async def test_startup_wins_over_spider_opened(self) -> None: + def spider_opened(spider: Spider) -> None: + raise CloseSpider("signal_reason") + + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + crawler.signals.connect(spider_opened, signals.spider_opened) + await crawler.crawl_async() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason" + + @inline_callbacks_test + def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]: + crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}}) + yield crawler.crawl() + assert crawler.stats.get_value("finish_reason") == "pipeline_reason"