Move installing the reactor from CrawlerProcess to Crawler.

scrapy/crawler.py

@@ -25,6 +25,7 @@ from scrapy.utils.log import (
     configure_logging,
     get_scrapy_root_handler,
     install_scrapy_root_handler,
+    log_reactor_info,
     log_scrapy_info,
     LogCounterHandler,
 )
@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
 
 class Crawler:
 
-    def __init__(self, spidercls, settings=None):
+    def __init__(self, spidercls, settings=None, init_reactor: bool = False):
         if isinstance(spidercls, Spider):
             raise ValueError('The spidercls argument must be a class, not an object')
 
@@ -69,6 +70,19 @@ class Crawler:
 
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
+
+        if init_reactor:
+            # this needs to be done after the spider settings are merged,
+            # but before something imports twisted.internet.reactor
+            if self.settings.get("TWISTED_REACTOR"):
+                install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
+            else:
+                from twisted.internet import default
+                default.install()
+            log_reactor_info()
+        if self.settings.get("TWISTED_REACTOR"):
+            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
+
         self.extensions = ExtensionManager.from_crawler(self)
 
         self.settings.freeze()
@@ -153,7 +167,6 @@ class CrawlerRunner:
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
-        self._handle_twisted_reactor()
 
     @property
     def spiders(self):
@@ -247,10 +260,6 @@ class CrawlerRunner:
         while self._active:
             yield defer.DeferredList(self._active)
 
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
-
 
 class CrawlerProcess(CrawlerRunner):
     """
@@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner):
                     {'signame': signame})
         reactor.callFromThread(self._stop_reactor)
 
+    def _create_crawler(self, spidercls):
+        if isinstance(spidercls, str):
+            spidercls = self.spider_loader.load(spidercls)
+        return Crawler(spidercls, self.settings, init_reactor=True)
+
     def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
@@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner):
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
-
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
-        super()._handle_twisted_reactor()

scrapy/utils/log.py

@@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None:
         if name != "Scrapy"
     ]
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
-    log_reactor_info()
 
 
 def log_reactor_info() -> None:

tests/test_crawler.py

@@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertEqual(runner.bootstrap_failed, True)
 
+    @defer.inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
@@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                CrawlerRunner(settings={
+                runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
+                yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     # https://twistedmatrix.com/trac/ticket/9766
@@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                     runner = CrawlerProcess(settings={
                         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                     })
+                    yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):