From eb62906c3e4c1e1f8e3e6c7965a04d5e65c61907 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 1 Dec 2021 17:40:41 +0500
Subject: [PATCH 01/13] Extract utils.log.log_reactor_info().

---
 scrapy/utils/log.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 0441c0358..9887ecc40 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -143,7 +143,7 @@ def _get_handler(settings):
     return handler
 
 
-def log_scrapy_info(settings):
+def log_scrapy_info(settings: Settings) -> None:
     logger.info("Scrapy %(version)s started (bot: %(bot)s)",
                 {'version': scrapy.__version__, 'bot': settings['BOT_NAME']})
     versions = [
@@ -152,6 +152,10 @@ def log_scrapy_info(settings):
         if name != "Scrapy"
     ]
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
+    log_reactor_info()
+
+
+def log_reactor_info() -> None:
     from twisted.internet import reactor
     logger.debug("Using reactor: %s.%s", reactor.__module__, reactor.__class__.__name__)
     from twisted.internet import asyncioreactor

From 6483dfdbe17cd66c409435b95a05850a3c94b5ee Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 1 Dec 2021 19:53:39 +0500
Subject: [PATCH 02/13] Move install_shutdown_handlers() from __init__() to
 start().

---
 scrapy/crawler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 578016536..357f14dc0 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -278,7 +278,6 @@ class CrawlerProcess(CrawlerRunner):
 
     def __init__(self, settings=None, install_root_handler=True):
         super().__init__(settings)
-        install_shutdown_handlers(self._signal_shutdown)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
 
@@ -318,6 +317,7 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
+        install_shutdown_handlers(self._signal_shutdown)
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
         resolver.install_on_reactor()

From 46ef9cf771789f1db513bbf2f65243d3320ce695 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 22 Dec 2021 21:24:59 +0500
Subject: [PATCH 03/13] Don't install non-working shutdown handlers in `scrapy
 shell`.

---
 scrapy/commands/shell.py | 2 +-
 scrapy/crawler.py        | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index d1944df3d..de81986d8 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -75,6 +75,6 @@ class Command(ScrapyCommand):
 
     def _start_crawler_thread(self):
         t = Thread(target=self.crawler_process.start,
-                   kwargs={'stop_after_crawl': False})
+                   kwargs={'stop_after_crawl': False, 'install_signal_handlers': False})
         t.daemon = True
         t.start()
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 357f14dc0..e54ad9750 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -297,7 +297,7 @@ class CrawlerProcess(CrawlerRunner):
                     {'signame': signame})
         reactor.callFromThread(self._stop_reactor)
 
-    def start(self, stop_after_crawl=True):
+    def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
         size to :setting:`REACTOR_THREADPOOL_MAXSIZE`, and installs a DNS cache
@@ -308,6 +308,9 @@ class CrawlerProcess(CrawlerRunner):
 
         :param bool stop_after_crawl: stop or not the reactor when all
             crawlers have finished
+
+        :param bool install_signal_handlers: whether to install the shutdown
+            handlers (default: True)
         """
         from twisted.internet import reactor
         if stop_after_crawl:
@@ -317,7 +320,8 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        install_shutdown_handlers(self._signal_shutdown)
+        if install_signal_handlers:
+            install_shutdown_handlers(self._signal_shutdown)
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
         resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
         resolver.install_on_reactor()

From 60c8838554a79e70c22a7c6a57baedfcaf521444 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:07:18 +0500
Subject: [PATCH 04/13] Move installing the reactor from CrawlerProcess to
 Crawler.

---
 scrapy/crawler.py     | 31 ++++++++++++++++++++-----------
 scrapy/utils/log.py   |  1 -
 tests/test_crawler.py |  5 ++++-
 3 files changed, 24 insertions(+), 13 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index e54ad9750..95cfb1bd1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -25,6 +25,7 @@ from scrapy.utils.log import (
     configure_logging,
     get_scrapy_root_handler,
     install_scrapy_root_handler,
+    log_reactor_info,
     log_scrapy_info,
     LogCounterHandler,
 )
@@ -38,7 +39,7 @@ logger = logging.getLogger(__name__)
 
 class Crawler:
 
-    def __init__(self, spidercls, settings=None):
+    def __init__(self, spidercls, settings=None, init_reactor: bool = False):
         if isinstance(spidercls, Spider):
             raise ValueError('The spidercls argument must be a class, not an object')
 
@@ -69,6 +70,19 @@ class Crawler:
 
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
+
+        if init_reactor:
+            # this needs to be done after the spider settings are merged,
+            # but before something imports twisted.internet.reactor
+            if self.settings.get("TWISTED_REACTOR"):
+                install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
+            else:
+                from twisted.internet import default
+                default.install()
+            log_reactor_info()
+        if self.settings.get("TWISTED_REACTOR"):
+            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
+
         self.extensions = ExtensionManager.from_crawler(self)
 
         self.settings.freeze()
@@ -153,7 +167,6 @@ class CrawlerRunner:
         self._crawlers = set()
         self._active = set()
         self.bootstrap_failed = False
-        self._handle_twisted_reactor()
 
     @property
     def spiders(self):
@@ -247,10 +260,6 @@ class CrawlerRunner:
         while self._active:
             yield defer.DeferredList(self._active)
 
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
-
 
 class CrawlerProcess(CrawlerRunner):
     """
@@ -297,6 +306,11 @@ class CrawlerProcess(CrawlerRunner):
                     {'signame': signame})
         reactor.callFromThread(self._stop_reactor)
 
+    def _create_crawler(self, spidercls):
+        if isinstance(spidercls, str):
+            spidercls = self.spider_loader.load(spidercls)
+        return Crawler(spidercls, self.settings, init_reactor=True)
+
     def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
         This method starts a :mod:`~twisted.internet.reactor`, adjusts its pool
@@ -341,8 +355,3 @@ class CrawlerProcess(CrawlerRunner):
             reactor.stop()
         except RuntimeError:  # raised if already stopped or in shutdown stage
             pass
-
-    def _handle_twisted_reactor(self):
-        if self.settings.get("TWISTED_REACTOR"):
-            install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
-        super()._handle_twisted_reactor()
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 9887ecc40..78e302d19 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -152,7 +152,6 @@ def log_scrapy_info(settings: Settings) -> None:
         if name != "Scrapy"
     ]
     logger.info("Versions: %(versions)s", {'versions': ", ".join(versions)})
-    log_reactor_info()
 
 
 def log_reactor_info() -> None:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index be067155e..118cb631b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -271,6 +271,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
         self.assertEqual(runner.bootstrap_failed, True)
 
+    @defer.inlineCallbacks
     def test_crawler_runner_asyncio_enabled_true(self):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
@@ -279,9 +280,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
-                CrawlerRunner(settings={
+                runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                 })
+                yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     # https://twistedmatrix.com/trac/ticket/9766
@@ -301,6 +303,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                     runner = CrawlerProcess(settings={
                         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
                     })
+                    yield runner.crawl(NoRequestsSpider)
 
     @defer.inlineCallbacks
     def test_crawler_process_asyncio_enabled_false(self):

From 041699b54cfa6cde9f886a98ff300e3276e2eaad Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:14:47 +0500
Subject: [PATCH 05/13] Remove tests that want to modify the test process
 reactor.

---
 tests/test_crawler.py | 27 ---------------------------
 1 file changed, 27 deletions(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 118cb631b..f445c181e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -285,33 +285,6 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
                 })
                 yield runner.crawl(NoRequestsSpider)
 
-    @defer.inlineCallbacks
-    # https://twistedmatrix.com/trac/ticket/9766
-    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
-            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
-    def test_crawler_process_asyncio_enabled_true(self):
-        with LogCapture(level=logging.DEBUG) as log:
-            if self.reactor_pytest == 'asyncio':
-                runner = CrawlerProcess(settings={
-                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                })
-                yield runner.crawl(NoRequestsSpider)
-                self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
-            else:
-                msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
-                with self.assertRaisesRegex(Exception, msg):
-                    runner = CrawlerProcess(settings={
-                        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                    })
-                    yield runner.crawl(NoRequestsSpider)
-
-    @defer.inlineCallbacks
-    def test_crawler_process_asyncio_enabled_false(self):
-        runner = CrawlerProcess(settings={"TWISTED_REACTOR": None})
-        with LogCapture(level=logging.DEBUG) as log:
-            yield runner.crawl(NoRequestsSpider)
-            self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
-
 
 class ScriptRunnerMixin:
     def run_script(self, script_name, *script_args):

From ebcafdf4a9e0692bf301546b6d60465b3b2c4b06 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:35:26 +0500
Subject: [PATCH 06/13] Add tests for TWISTED_REACTOR in custom_settings.

---
 .../twisted_reactor_custom_settings.py        | 14 +++++++++++
 ...wisted_reactor_custom_settings_conflict.py | 22 +++++++++++++++++
 .../twisted_reactor_custom_settings_same.py   | 21 ++++++++++++++++
 tests/test_crawler.py                         | 24 +++++++++++++++++++
 4 files changed, 81 insertions(+)
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
 create mode 100644 tests/CrawlerProcess/twisted_reactor_custom_settings_same.py

diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings.py b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
new file mode 100644
index 000000000..56304bd23
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings.py
@@ -0,0 +1,14 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
new file mode 100644
index 000000000..9a6c01d72
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -0,0 +1,22 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class PollReactorSpider(scrapy.Spider):
+    name = 'poll_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+    }
+
+
+class AsyncioReactorSpider(scrapy.Spider):
+    name = 'asyncio_reactor'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(PollReactorSpider)
+process.crawl(AsyncioReactorSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
new file mode 100644
index 000000000..1f5a44010
--- /dev/null
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -0,0 +1,21 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class AsyncioReactorSpider1(scrapy.Spider):
+    name = 'asyncio_reactor1'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+class AsyncioReactorSpider2(scrapy.Spider):
+    name = 'asyncio_reactor2'
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+
+process = CrawlerProcess()
+process.crawl(AsyncioReactorSpider1)
+process.crawl(AsyncioReactorSpider2)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index f445c181e..6d6763aec 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -361,6 +361,30 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings(self):
+        log = self.run_script("twisted_reactor_custom_settings.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings_same(self):
+        log = self.run_script("twisted_reactor_custom_settings_same.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    # https://twistedmatrix.com/trac/ticket/9766
+    @skipIf(platform.system() == 'Windows' and sys.version_info >= (3, 8),
+            "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
+    def test_reactor_asyncio_custom_settings_conflict(self):
+        log = self.run_script("twisted_reactor_custom_settings_conflict.py")
+        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
+        self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)
+
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
     @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')

From 002513438204eea5062b5a1d75fb4f261880da4f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 16:45:17 +0500
Subject: [PATCH 07/13] Completely skip WindowsRunSpiderCommandTest outside
 Windows.

---
 tests/test_commands.py | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index 75098a77a..efe9b0531 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -765,6 +765,7 @@ class MySpider(scrapy.Spider):
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
 
+@skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
 
     spider_filename = 'myspider.pyw'
@@ -777,35 +778,27 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
         self.assertIn("start_requests", log)
         self.assertIn("badspider.pyw", log)
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_run_good_spider(self):
         super().test_run_good_spider()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider(self):
         super().test_runspider()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_dnscache_disabled(self):
         super().test_runspider_dnscache_disabled()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_log_level(self):
         super().test_runspider_log_level()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_log_short_names(self):
         super().test_runspider_log_short_names()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_runspider_no_spider_found(self):
         super().test_runspider_no_spider_found()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_output(self):
         super().test_output()
 
-    @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
     def test_overwrite_output(self):
         super().test_overwrite_output()
 

From 9c4bfb48362f736fce81b71a6ca1fa0b3600231d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 17:17:36 +0500
Subject: [PATCH 08/13] Remove an unused import.

---
 tests/test_crawler.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6d6763aec..d68c50026 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -7,7 +7,6 @@ import warnings
 from unittest import skipIf
 
 from pytest import raises, mark
-from testfixtures import LogCapture
 from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.python.versions import Version

From d4565318c7061c2ccd17fa5d5eabcacef8c34826 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Dec 2021 17:40:31 +0500
Subject: [PATCH 09/13] Fix a reactor test on Windows.

---
 .../twisted_reactor_custom_settings_conflict.py           | 8 ++++----
 tests/test_crawler.py                                     | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
index 9a6c01d72..3f219098c 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_conflict.py
@@ -2,10 +2,10 @@ import scrapy
 from scrapy.crawler import CrawlerProcess
 
 
-class PollReactorSpider(scrapy.Spider):
-    name = 'poll_reactor'
+class SelectReactorSpider(scrapy.Spider):
+    name = 'select_reactor'
     custom_settings = {
-        "TWISTED_REACTOR": "twisted.internet.pollreactor.PollReactor",
+        "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
     }
 
 
@@ -17,6 +17,6 @@ class AsyncioReactorSpider(scrapy.Spider):
 
 
 process = CrawlerProcess()
-process.crawl(PollReactorSpider)
+process.crawl(SelectReactorSpider)
 process.crawl(AsyncioReactorSpider)
 process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index d68c50026..e7d5c8132 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -381,8 +381,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "the asyncio reactor is broken on Windows when running Python ≥ 3.8")
     def test_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
-        self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
-        self.assertIn("(twisted.internet.pollreactor.PollReactor) does not match the requested one", log)
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
+        self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')

From 940cc0776ff86f726e79c2ab2018f4b83a833936 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 24 Dec 2021 17:12:50 +0500
Subject: [PATCH 10/13] Add docs about TWISTED_REACTOR and other per-process
 settings.

---
 docs/topics/practices.rst | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 732eba587..bd0dd8ce0 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -102,6 +102,17 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
+.. note::
+    .. versionchanged:: VERSION
+
+    The Twisted reactor is now installed when
+    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
+    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
+    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
+    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
+    they are silently ignored when set there and you need to set these settings
+    in some other way.
+
 .. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
@@ -193,6 +204,25 @@ Same example but running the spiders sequentially by chaining the deferreds:
     crawl()
     reactor.run() # the script will block here until the last crawl call is finished
 
+Different spiders can set different values for the same setting, but when they
+run in the same process it may be impossible, by design or because of some
+limitations, to use these different values. What happens in practice is
+different for different settings:
+
+* :setting:`SPIDER_LOADER_CLASS` and the ones used by its value
+  (:setting:`SPIDER_MODULES`, :setting:`SPIDER_LOADER_WARN_ONLY` for the
+  default one) cannot be read from the per-spider settings. These are applied
+  when the :class:`~scrapy.crawler.CrawlerRunner` or
+  :class:`~scrapy.crawler.CrawlerProcess` object is created.
+* For :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` the first
+  available value is used, and if a spider requests a different reactor an
+  exception will be raised. These are applied when the reactor is installed.
+* For :setting:`REACTOR_THREADPOOL_MAXSIZE`, :setting:`DNS_RESOLVER` and the
+  ones used by the resolver (:setting:`DNSCACHE_ENABLED`,
+  :setting:`DNSCACHE_SIZE`, :setting:`DNS_TIMEOUT` for ones included in Scrapy)
+  the first available value is used. These are applied when the reactor is
+  started.
+
 .. seealso:: :ref:`run-from-script`.
 
 .. _distributed-crawls:

From a986792def6df1b2bbdf1bc996308d3afd8528c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 24 Dec 2021 19:43:14 +0500
Subject: [PATCH 11/13] Add more docs for TWISTED_REACTOR.

---
 docs/topics/settings.rst | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 210c1def7..cff6d80cb 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1638,10 +1638,18 @@ which raises :exc:`Exception`, becomes::
 
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will not attempt to install any specific reactor, and the
-default reactor defined by Twisted for the current platform will be used. This
-is to maintain backward compatibility and avoid possible problems caused by
-using a non-default reactor.
+means that Scrapy will install the default reactor defined by Twisted for the
+current platform will be used. This is to maintain backward compatibility and
+avoid possible problems caused by using a non-default reactor.
+
+.. note::
+    .. versionchanged:: VERSION
+
+    Previously this setting had no effect in a spider
+    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
+    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
+    they must not have different values for this setting, because they will use
+    a single reactor instance.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 

From 64261d9e389737621caa85f320cf81ef2aef1faa Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 15:45:59 +0500
Subject: [PATCH 12/13] Slight refactoring.

---
 scrapy/crawler.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 95cfb1bd1..a638254f1 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -71,17 +71,18 @@ class Crawler:
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
 
+        reactor_class = self.settings.get("TWISTED_REACTOR")
         if init_reactor:
             # this needs to be done after the spider settings are merged,
             # but before something imports twisted.internet.reactor
-            if self.settings.get("TWISTED_REACTOR"):
-                install_reactor(self.settings["TWISTED_REACTOR"], self.settings["ASYNCIO_EVENT_LOOP"])
+            if reactor_class:
+                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
             else:
                 from twisted.internet import default
                 default.install()
             log_reactor_info()
-        if self.settings.get("TWISTED_REACTOR"):
-            verify_installed_reactor(self.settings["TWISTED_REACTOR"])
+        if reactor_class:
+            verify_installed_reactor(reactor_class)
 
         self.extensions = ExtensionManager.from_crawler(self)
 

From 6eaceec735d551f5b777bc641ff8d85dbb3ba98c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 Dec 2021 20:14:24 +0500
Subject: [PATCH 13/13] Implement docs suggestions.

---
 docs/news.rst             | 22 ++++++++++++++++++++++
 docs/topics/practices.rst | 11 -----------
 docs/topics/settings.rst  | 13 ++-----------
 3 files changed, 24 insertions(+), 22 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 509366c17..2afe318f6 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1,3 +1,25 @@
+.. note::
+    .. versionchanged:: VERSION
+
+    The Twisted reactor is now installed when
+    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
+    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
+    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
+    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
+    they are silently ignored when set there and you need to set these settings
+    in some other way.
+
+
+.. note::
+    .. versionchanged:: VERSION
+
+    Previously this setting had no effect in a spider
+    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
+    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
+    they must not have different values for this setting, because they will use
+    a single reactor instance.
+
+
 .. _news:
 
 Release notes
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index bd0dd8ce0..1a9d56143 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -102,17 +102,6 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run() # the script will block here until the crawling is finished
 
-.. note::
-    .. versionchanged:: VERSION
-
-    The Twisted reactor is now installed when
-    :meth:`~scrapy.crawler.CrawlerProcess.crawl` is first called, not when a
-    :class:`scrapy.crawler.CrawlerProcess` object is created. Because of this,
-    :setting:`TWISTED_REACTOR` and :setting:`ASYNCIO_EVENT_LOOP` are now
-    honored in :attr:`~scrapy.Spider.custom_settings`. In older Scrapy versions
-    they are silently ignored when set there and you need to set these settings
-    in some other way.
-
 .. seealso:: :doc:`twisted:core/howto/reactor-basics`
 
 .. _run-multiple-spiders:
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index cff6d80cb..f6c95c502 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1639,17 +1639,8 @@ which raises :exc:`Exception`, becomes::
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
 means that Scrapy will install the default reactor defined by Twisted for the
-current platform will be used. This is to maintain backward compatibility and
-avoid possible problems caused by using a non-default reactor.
-
-.. note::
-    .. versionchanged:: VERSION
-
-    Previously this setting had no effect in a spider
-    :attr:`~scrapy.Spider.custom_settings` attribute. Now it will be used, but
-    if you :ref:`run several spiders in one process <run-multiple-spiders>`,
-    they must not have different values for this setting, because they will use
-    a single reactor instance.
+current platform. This is to maintain backward compatibility and avoid possible
+problems caused by using a non-default reactor.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.