Support CloseSpider during spider startup (#7905)

docs/topics/item-pipeline.rst

@@ -47,6 +47,13 @@ Additionally, they may also implement the following methods:
 
    This method is called when the spider is opened.
 
+   .. versionchanged:: VERSION
+      Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+   It may raise :exc:`~scrapy.exceptions.CloseSpider` to close the spider before
+   it starts crawling, e.g. if a resource that the pipeline needs is
+   unavailable.
+
 .. method:: close_spider(self)
 
    This method is called when the spider is closed, before the

docs/topics/signals.rst

@@ -290,6 +290,13 @@ spider_opened
     reserve per-spider resources, but can be used for any task that needs to be
     performed when a spider is opened.
 
+    .. versionchanged:: VERSION
+       Added support for :exc:`~scrapy.exceptions.CloseSpider`.
+
+    You may raise a :exc:`~scrapy.exceptions.CloseSpider` exception to close the
+    spider before it starts crawling, e.g. if a resource that the spider needs
+    is unavailable.
+
     This signal supports :ref:`asynchronous handlers <signal-deferred>`.
 
     :param spider: the spider which has been opened

scrapy/core/engine.py

@@ -248,7 +248,7 @@ class ExecutionEngine:
         )
         return deferred_from_coro(self.close_async())
 
-    async def close_async(self) -> None:
+    async def close_async(self, *, reason: str = "shutdown") -> None:
         """
         Gracefully close the execution engine.
         If it has already been started, stop it. In all cases, close the spider and the downloader.
@@ -256,9 +256,7 @@ class ExecutionEngine:
         if self.running:
             await self.stop_async()  # will also close spider and downloader
         elif self.spider is not None:
-            await self.close_spider_async(
-                reason="shutdown"
-            )  # will also close downloader
+            await self.close_spider_async(reason=reason)  # will also close downloader
         elif hasattr(self, "downloader"):
             self.downloader.close()
 
@@ -557,10 +555,20 @@ class ExecutionEngine:
         nextcall = CallLaterOnce(self._start_scheduled_requests)
         scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
         self._slot = _Slot(close_if_idle, nextcall, scheduler)
-        self._start = await self.scraper.spidermw.process_start()
-        if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
-            await maybe_deferred_to_future(d)
-        await self.scraper.open_spider_async()
+        # A component that fails to start can ask for the spider to be closed.
+        # The rest of the startup runs anyway, so that components that are
+        # started also get stopped, and the request is honored once the spider
+        # is open.
+        close_spider_exc: CloseSpider | None = None
+        try:
+            self._start = await self.scraper.spidermw.process_start()
+            if hasattr(scheduler, "open") and (
+                d := scheduler.open(self.crawler.spider)
+            ):
+                await maybe_deferred_to_future(d)
+            await self.scraper.open_spider_async()
+        except CloseSpider as exc:
+            close_spider_exc = exc
         stats = self.crawler.stats
         if argument_is_required(stats.open_spider, "spider"):
             warnings.warn(
@@ -572,9 +580,14 @@ class ExecutionEngine:
             stats.open_spider(spider=self.crawler.spider)
         else:
             stats.open_spider()
-        await self.signals.send_catch_log_async(
-            signals.spider_opened, spider=self.crawler.spider
+        results = await self.signals.send_catch_log_async(
+            signals.spider_opened, spider=self.crawler.spider, dont_log=CloseSpider
         )
+        for _, result in results:
+            if isinstance(result, CloseSpider):
+                close_spider_exc = close_spider_exc or result
+        if close_spider_exc is not None:
+            raise close_spider_exc
 
     def _spider_idle(self) -> None:
         """

scrapy/crawler.py

@@ -15,7 +15,7 @@ from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
 from scrapy import Spider
 from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
-from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
@@ -270,8 +270,12 @@ class Crawler:
             self._apply_settings()
             self._update_root_log_handler()
             self.engine = self._create_engine()
-            yield deferred_from_coro(self.engine.open_spider_async())
-            yield deferred_from_coro(self.engine.start_async())
+            try:
+                yield deferred_from_coro(self.engine.open_spider_async())
+            except CloseSpider as exc:
+                yield deferred_from_coro(self.engine.close_async(reason=exc.reason))
+            else:
+                yield deferred_from_coro(self.engine.start_async())
         except Exception:
             self.crawling = False
             if self._engine is not None:
@@ -300,8 +304,12 @@ class Crawler:
             self._apply_settings()
             self._update_root_log_handler()
             self.engine = self._create_engine()
-            await self.engine.open_spider_async()
-            await self.engine.start_async()
+            try:
+                await self.engine.open_spider_async()
+            except CloseSpider as exc:
+                await self.engine.close_async(reason=exc.reason)
+            else:
+                await self.engine.start_async()
         except Exception:
             self.crawling = False
             if self._engine is not None:

scrapy/exceptions.py

@@ -56,12 +56,11 @@ class DontCloseSpider(Exception):
 
 
 class CloseSpider(Exception):
-    """Raised from a :ref:`spider callback <topics-spiders>` or from
-    :meth:`~scrapy.Spider.start` to request the spider to be closed/stopped.
+    """Raised from a :ref:`spider callback <topics-spiders>`, or while the
+    spider is starting, to request the spider to be closed/stopped.
 
     .. versionchanged:: VERSION
-       Raising it from :meth:`~scrapy.Spider.start` closes the spider, instead
-       of being reported as a start error.
+       Added support for raising it while the spider is starting.
 
     *reason* is a string with the reason for closing.
 

tests/test_engine.py

@@ -29,7 +29,9 @@ from tests.utils.engine import (
 )
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator
+    from collections.abc import AsyncIterator, Generator
+
+    from twisted.internet.defer import Deferred
 
     from tests.mockserver.http import MockServer
 
@@ -230,3 +232,51 @@ async def test_request_scheduled_signal():
         f"{scheduler.enqueued!r} != [{keep_request!r}]"
     )
     crawler.signals.disconnect(signal_handler, signals.request_scheduled)
+
+
+class ClosingPipeline:
+    def open_spider(self):
+        raise CloseSpider("pipeline_reason")
+
+
+class TestCloseSpiderOnStartup:
+    @coroutine_test
+    async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None:
+        closed: list[str] = []
+
+        def spider_closed(reason: str) -> None:
+            closed.append(reason)
+
+        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+        crawler.signals.connect(spider_closed, signals.spider_closed)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async()
+        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
+        assert closed == ["pipeline_reason"]
+        assert "Traceback" not in caplog.text
+
+    @coroutine_test
+    async def test_spider_opened(self) -> None:
+        def spider_opened(spider: Spider) -> None:
+            raise CloseSpider("signal_reason")
+
+        crawler = get_crawler(DefaultSpider)
+        crawler.signals.connect(spider_opened, signals.spider_opened)
+        await crawler.crawl_async()
+        assert crawler.stats.get_value("finish_reason") == "signal_reason"
+
+    @coroutine_test
+    async def test_startup_wins_over_spider_opened(self) -> None:
+        def spider_opened(spider: Spider) -> None:
+            raise CloseSpider("signal_reason")
+
+        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+        crawler.signals.connect(spider_opened, signals.spider_opened)
+        await crawler.crawl_async()
+        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
+
+    @inline_callbacks_test
+    def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]:
+        crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
+        yield crawler.crawl()
+        assert crawler.stats.get_value("finish_reason") == "pipeline_reason"