Fix open_spider exception handling to set exit code 1 (#7255)

scrapy/crawler.py

@@ -439,12 +439,16 @@ class CrawlerRunner(CrawlerRunnerBase):
         self.crawlers.add(crawler)
         d = crawler.crawl(*args, **kwargs)
         self._active.add(d)
+        failed = False
         try:
             yield d
+        except Exception:
+            failed = True
+            raise
         finally:
             self.crawlers.discard(crawler)
             self._active.discard(d)
-            self.bootstrap_failed |= not getattr(crawler, "spider", None)
+            self.bootstrap_failed |= not getattr(crawler, "spider", None) or failed
 
     def stop(self) -> Deferred[Any]:
         """
@@ -538,7 +542,15 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
         loop = asyncio.get_event_loop()
         self.crawlers.add(crawler)
-        task = loop.create_task(crawler.crawl_async(*args, **kwargs))
+
+        async def _crawl_and_track() -> None:
+            try:
+                await crawler.crawl_async(*args, **kwargs)
+            except Exception:
+                self.bootstrap_failed = True
+                raise  # re-raise so asyncio still logs it to stderr naturally
+
+        task = loop.create_task(_crawl_and_track())
         self._active.add(task)
 
         def _done(_: asyncio.Task[None]) -> None:

tests/test_cmdline_crawl_with_pipeline/__init__.py

@@ -2,8 +2,6 @@ import sys
 from pathlib import Path
 from subprocess import PIPE, Popen
 
-from tests import TWISTED_KEEPS_TRACEBACKS
-
 
 class TestCmdlineCrawlPipeline:
     def _execute(self, spname):
@@ -18,10 +16,6 @@ class TestCmdlineCrawlPipeline:
         assert returncode == 0
 
     def test_exception_at_open_spider_in_pipeline(self):
-        returncode, stderr = self._execute("exception")
-        # An unhandled exception in a pipeline should not stop the crawl
-        assert returncode == 0
-        if TWISTED_KEEPS_TRACEBACKS:
-            assert b'RuntimeError("exception")' in stderr
-        else:
-            assert b"RuntimeError: exception" in stderr
+        returncode, _ = self._execute("exception")
+        # An exception in pipeline's open_spider should result in a non-zero exit code
+        assert returncode == 1