Merge 47e1bfbfa5 into e28e56aa61

scrapy/commands/check.py

@@ -119,4 +119,6 @@ class Command(ScrapyCommand):
 
                 result.printErrors()
                 result.printSummary(start_time, stop)
-                self.exitcode = int(not result.wasSuccessful())
+                self.exitcode = int(
+                    not result.wasSuccessful() or self.crawler_process.bootstrap_failed
+                )

scrapy/commands/fetch.py

@@ -97,3 +97,5 @@ class Command(ScrapyCommand):
 
         self.crawler_process.crawl(spidercls)
         self.crawler_process.start()
+        if self.crawler_process.bootstrap_failed:
+            self.exitcode = 1

scrapy/commands/parse.py

@@ -269,6 +269,8 @@ class Command(BaseRunSpiderCommand):
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
         self.pcrawler = next(iter(self.crawler_process.crawlers))
         self.crawler_process.start()
+        if self.crawler_process.bootstrap_failed:
+            self.exitcode = 1
 
         if not self.first_response:
             logger.error("No response downloaded for: %(url)s", {"url": url})

tests/test_command_check.py

@@ -61,6 +61,12 @@ class CheckSpider(scrapy.Spider):
         assert "OK" in err
         assert ret == 0
 
+    def test_bootstrap_failure(self, proj_path: Path) -> None:
+        self._write_contract(proj_path, "", "pass")
+        self._break_bootstrap(proj_path / self.project_name)
+        ret, _, _ = proc("check", cwd=proj_path)
+        assert ret == 1
+
     def test_check_returns_requests_contract(self, proj_path: Path) -> None:
         contracts = """
         @returns requests 1
@@ -218,7 +224,9 @@ class CheckSpider(scrapy.Spider):
         )
         cm_cls_mock.return_value = cm_mock = Mock()
         spider_loader_mock = Mock()
-        cmd.crawler_process = Mock(spider_loader=spider_loader_mock)
+        cmd.crawler_process = Mock(
+            spider_loader=spider_loader_mock, bootstrap_failed=False
+        )
         spider_name = "FakeSpider"
         spider_cls_mock = Mock()
         spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[

tests/test_command_fetch.py

@@ -68,3 +68,10 @@ class MySpider(scrapy.Spider):
             "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path
         )
         assert "myspider-user-agent" in out, err
+
+    def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:
+        self._break_bootstrap(proj_path / self.project_name)
+        returncode, _, _ = proc(
+            "fetch", "--spider", "myspider", mockserver.url("/text"), cwd=proj_path
+        )
+        assert returncode == 1

tests/test_command_parse.py

@@ -211,6 +211,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
 """
             )
 
+    def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None:
+        self._break_bootstrap(proj_path / self.project_name)
+        returncode, _, _ = proc(
+            "parse",
+            "--spider",
+            self.spider_name,
+            "-c",
+            "parse",
+            mockserver.url("/html"),
+            cwd=proj_path,
+        )
+        assert returncode == 1
+
     def test_spider_arguments(self, proj_path: Path, mockserver: MockServer) -> None:
         _, _, stderr = proc(
             "parse",

tests/utils/bases/commands.py

@@ -38,3 +38,20 @@ class TestProjectBase:
         """Add text to the end of the project settings.py."""
         with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
             f.write(text)
+
+    @classmethod
+    def _break_bootstrap(cls, proj_mod_path: Path) -> None:
+        """Enable a spider middleware that raises an exception when built, so
+        that crawling any spider of the project fails before it starts."""
+        cls._append_settings(
+            proj_mod_path,
+            """
+
+class BreakingMiddleware:
+    def __init__(self):
+        raise RuntimeError("Breaking middleware")
+
+
+SPIDER_MIDDLEWARES = {f"{__name__}.BreakingMiddleware": 100}
+""",
+        )