Merge 986b83736c into b3670369b8

scrapy/crawler.py

@@ -30,6 +30,7 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
+from scrapy.utils.python import global_object_name
 from scrapy.utils.reactor import (
     _asyncio_reactor_path,
     install_reactor,
@@ -128,7 +129,7 @@ class Crawler:
                 )
             if reactor_class:
                 # We need to check that the correct reactor is installed.
-                verify_installed_reactor(reactor_class)
+                self._verify_installed_reactor(reactor_class)
                 if is_asyncio_reactor_installed() and event_loop:
                     verify_installed_asyncio_event_loop(event_loop)
 
@@ -151,6 +152,29 @@ class Crawler:
             "Overridden settings:\n%(settings)s", {"settings": pprint.pformat(d)}
         )
 
+    def _verify_installed_reactor(self, reactor_class: str) -> None:
+        try:
+            verify_installed_reactor(reactor_class)
+        except RuntimeError:
+            if (
+                self._init_reactor
+                or self.settings.getbool("FORCE_CRAWLER_PROCESS")
+                or not is_asyncio_reactor_installed()
+            ):
+                raise
+            from twisted.internet import reactor
+
+            raise RuntimeError(
+                f"The installed reactor ({global_object_name(type(reactor))}) "
+                f"does not match the one requested by spider "
+                f"{self.spidercls.name!r} ({reactor_class}). When running "
+                f"'scrapy crawl' (and similar commands), the TWISTED_REACTOR "
+                f"value from per-spider settings is only applied if "
+                f"FORCE_CRAWLER_PROCESS is True. Set FORCE_CRAWLER_PROCESS=True "
+                f"(in the project settings or with '-s FORCE_CRAWLER_PROCESS=True' "
+                f"on the command line) to use the spider's reactor."
+            ) from None
+
     def _apply_reactorless_default_settings(self) -> None:
         """Change some setting defaults when not using a Twisted reactor.
 

tests/test_command_crawl.py

@@ -143,3 +143,49 @@ class MySpider(scrapy.Spider):
         assert "[myspider] DEBUG: It works!" in log
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
+
+    def test_spider_reactor_without_force_crawler_process(
+        self, proj_path: Path
+    ) -> None:
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+    custom_settings = {
+        'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor',
+    }
+
+    async def start(self):
+        self.logger.debug('It works!')
+        return
+        yield
+"""
+        log = self.get_log(spider_code, proj_path)
+        assert "[myspider] DEBUG: It works!" not in log
+        assert "does not match the one requested by spider 'myspider'" in log
+        assert "FORCE_CRAWLER_PROCESS=True" in log
+        assert "Spider closed (finished)" not in log
+
+    def test_spider_reactor_with_force_crawler_process(self, proj_path: Path) -> None:
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+    custom_settings = {
+        'TWISTED_REACTOR': 'twisted.internet.selectreactor.SelectReactor',
+    }
+
+    async def start(self):
+        self.logger.debug('It works!')
+        return
+        yield
+"""
+        log = self.get_log(
+            spider_code, proj_path, args=("-s", "FORCE_CRAWLER_PROCESS=True")
+        )
+        assert "[myspider] DEBUG: It works!" in log
+        assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
+        assert "does not match the one requested by spider 'myspider'" not in log
+        assert "Spider closed (finished)" in log

tests/test_commands.py

@@ -333,9 +333,10 @@ class MySpider(scrapy.Spider):
             assert self.ASYNC_MSG in err
             assert (
                 "The installed reactor (twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
-                " does not match the requested one"
+                f" does not match the one requested by spider {spider!r}"
                 " (twisted.internet.selectreactor.SelectReactor)"
             ) in err
+            assert "FORCE_CRAWLER_PROCESS=True" in err
 
     def test_project_asyncio_spider_settings_select_forced(
         self, proj_path: Path

tests/test_crawler_subprocess.py

@@ -346,10 +346,12 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         log = self.run_script("twisted_reactor_custom_settings_select.py")
         assert "Spider closed (finished)" not in log
         assert (
-            "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) "
-            "does not match the requested one "
+            "The installed reactor "
+            "(twisted.internet.asyncioreactor.AsyncioSelectorReactor) does not "
+            "match the one requested by spider 'asyncio_reactor' "
             "(twisted.internet.selectreactor.SelectReactor)"
         ) in log
+        assert "FORCE_CRAWLER_PROCESS=True" in log
 
     @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self) -> None: