diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 2113c19d2..4d9e02a73 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -119,4 +119,6 @@ class Command(ScrapyCommand): result.printErrors() result.printSummary(start_time, stop) - self.exitcode = int(not result.wasSuccessful()) + self.exitcode = int( + not result.wasSuccessful() or self.crawler_process.bootstrap_failed + ) diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py index 0b8311efb..809f6918d 100644 --- a/scrapy/commands/fetch.py +++ b/scrapy/commands/fetch.py @@ -97,3 +97,5 @@ class Command(ScrapyCommand): self.crawler_process.crawl(spidercls) self.crawler_process.start() + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 51caed57f..7b669c140 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -269,6 +269,8 @@ class Command(BaseRunSpiderCommand): self.crawler_process.crawl(self.spidercls, **opts.spargs) self.pcrawler = next(iter(self.crawler_process.crawlers)) self.crawler_process.start() + if self.crawler_process.bootstrap_failed: + self.exitcode = 1 if not self.first_response: logger.error("No response downloaded for: %(url)s", {"url": url}) diff --git a/tests/test_command_check.py b/tests/test_command_check.py index 240f44584..36af92c37 100644 --- a/tests/test_command_check.py +++ b/tests/test_command_check.py @@ -61,6 +61,12 @@ class CheckSpider(scrapy.Spider): assert "OK" in err assert ret == 0 + def test_bootstrap_failure(self, proj_path: Path) -> None: + self._write_contract(proj_path, "", "pass") + self._break_bootstrap(proj_path / self.project_name) + ret, _, _ = proc("check", cwd=proj_path) + assert ret == 1 + def test_check_returns_requests_contract(self, proj_path: Path) -> None: contracts = """ @returns requests 1 @@ -218,7 +224,9 @@ class CheckSpider(scrapy.Spider): ) cm_cls_mock.return_value = cm_mock = Mock() spider_loader_mock = Mock() - cmd.crawler_process = Mock(spider_loader=spider_loader_mock) + cmd.crawler_process = Mock( + spider_loader=spider_loader_mock, bootstrap_failed=False + ) spider_name = "FakeSpider" spider_cls_mock = Mock() spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[ diff --git a/tests/test_command_fetch.py b/tests/test_command_fetch.py index c6a3afc91..e25540f50 100644 --- a/tests/test_command_fetch.py +++ b/tests/test_command_fetch.py @@ -68,3 +68,10 @@ class MySpider(scrapy.Spider): "fetch", "--spider", "myspider", mockserver.url("/echo"), cwd=proj_path ) assert "myspider-user-agent" in out, err + + def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None: + self._break_bootstrap(proj_path / self.project_name) + returncode, _, _ = proc( + "fetch", "--spider", "myspider", mockserver.url("/text"), cwd=proj_path + ) + assert returncode == 1 diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py index e434055b7..f06816b8c 100644 --- a/tests/test_command_parse.py +++ b/tests/test_command_parse.py @@ -211,6 +211,19 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}} """ ) + def test_bootstrap_failure(self, proj_path: Path, mockserver: MockServer) -> None: + self._break_bootstrap(proj_path / self.project_name) + returncode, _, _ = proc( + "parse", + "--spider", + self.spider_name, + "-c", + "parse", + mockserver.url("/html"), + cwd=proj_path, + ) + assert returncode == 1 + def test_spider_arguments(self, proj_path: Path, mockserver: MockServer) -> None: _, _, stderr = proc( "parse", diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.py index 55ef686fa..3a01488a7 100644 --- a/tests/utils/bases/commands.py +++ b/tests/utils/bases/commands.py @@ -38,3 +38,20 @@ class TestProjectBase: """Add text to the end of the project settings.py.""" with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f: f.write(text) + + @classmethod + def _break_bootstrap(cls, proj_mod_path: Path) -> None: + """Enable a spider middleware that raises an exception when built, so + that crawling any spider of the project fails before it starts.""" + cls._append_settings( + proj_mod_path, + """ + +class BreakingMiddleware: + def __init__(self): + raise RuntimeError("Breaking middleware") + + +SPIDER_MIDDLEWARES = {f"{__name__}.BreakingMiddleware": 100} +""", + )