Merge 04a150129f into e28e56aa61

docs/topics/settings.rst

@@ -330,6 +330,12 @@ Reactor settings
 **Reactor settings** are settings tied to the :doc:`Twisted reactor
 <twisted:core/howto/reactor-basics>`.
 
+.. versionchanged:: VERSION
+   :setting:`TWISTED_DNS_RESOLVER`, the settings of the resolver and
+   :setting:`REACTOR_THREADPOOL_MAXSIZE` are now read from the first spider,
+   instead of being read from the project settings and ignored in
+   :ref:`per-spider settings <spider-settings>`.
+
 Because only 1 reactor can be used per process, these settings cannot use a
 different value per spider when :ref:`running multiple spiders in the same
 process <run-multiple-spiders>`.
@@ -359,11 +365,10 @@ These settings are applied when starting the reactor:
 
 -   :setting:`REACTOR_THREADPOOL_MAXSIZE`
 
-They are read from the settings of the
-:class:`~scrapy.crawler.CrawlerProcess` or
-:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a
-spider or an :ref:`add-on <topics-addons>` has no effect. They are ignored
-altogether when using :class:`~scrapy.crawler.CrawlerRunner` or
+They can also be :ref:`set from a spider <spider-settings>`, but only the
+values from the first spider that runs are used; if a later spider defines a
+different value, a warning is issued. They are ignored altogether when using
+:class:`~scrapy.crawler.CrawlerRunner` or
 :class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor.
 
 There is an additional restriction for :setting:`TWISTED_REACTOR` and

scrapy/crawler.py

@@ -57,6 +57,17 @@ logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
 
+# Reactor settings that are applied when starting the reactor, i.e. once per
+# process, and so can only take the value of one of the crawlers.
+_REACTOR_SETTINGS = (
+    "DNSCACHE_ENABLED",
+    "DNSCACHE_SIZE",
+    "DNS_RESOLVER",
+    "DNS_TIMEOUT",
+    "REACTOR_THREADPOOL_MAXSIZE",
+    "TWISTED_DNS_RESOLVER",
+)
+
 
 class _LateAttribute(Generic[_T]):
     """Descriptor for a :class:`Crawler` attribute that only gets a value once
@@ -714,6 +725,7 @@ class CrawlerProcessBase(CrawlerRunnerBase):
         install_root_handler: bool = True,
     ):
         super().__init__(settings)
+        self._reactor_crawler: Crawler | None = None
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
 
@@ -723,6 +735,30 @@ class CrawlerProcessBase(CrawlerRunnerBase):
     ) -> None:
         raise NotImplementedError
 
+    def create_crawler(
+        self, crawler_or_spidercls: type[Spider] | str | Crawler
+    ) -> Crawler:
+        crawler = super().create_crawler(crawler_or_spidercls)
+        if self._reactor_crawler is None:
+            self._reactor_crawler = crawler
+        else:
+            ignored = [
+                setting
+                for setting in _REACTOR_SETTINGS
+                if self._reactor_crawler.settings[setting] != crawler.settings[setting]
+            ]
+            if ignored:
+                warnings.warn(
+                    f"Spider {crawler.spidercls.__name__} defines a different "
+                    f"value than spider "
+                    f"{self._reactor_crawler.spidercls.__name__} for the "
+                    f"following reactor settings: {', '.join(ignored)}. Only "
+                    f"the value of the first spider is used, since the "
+                    f"reactor is shared by every spider in a process.",
+                    stacklevel=2,
+                )
+        return crawler
+
     def _signal_shutdown(self, signum: int, _: Any) -> None:
         from twisted.internet import reactor
 
@@ -755,7 +791,13 @@ class CrawlerProcessBase(CrawlerRunnerBase):
     def _setup_reactor(self, install_signal_handlers: bool) -> None:
         from twisted.internet import reactor
 
-        dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0
+        # Reactor settings are read from the first crawler, so that they can be
+        # defined from a spider, and fall back to the process settings when no
+        # crawler has been created yet.
+        crawler: Crawler | CrawlerProcessBase = self._reactor_crawler or self
+        settings = crawler.settings
+
+        dns_priority = settings.getpriority("DNS_RESOLVER") or 0
         default_priority = SETTINGS_PRIORITIES["default"]
 
         if dns_priority > default_priority:
@@ -766,24 +808,20 @@ class CrawlerProcessBase(CrawlerRunnerBase):
                 stacklevel=2,
             )
 
-            twisted_dns_priority = (
-                self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0
-            )
+            twisted_dns_priority = settings.getpriority("TWISTED_DNS_RESOLVER") or 0
             if twisted_dns_priority > dns_priority:
-                resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+                resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]
             else:
-                resolver_cls_path = self.settings["DNS_RESOLVER"]
+                resolver_cls_path = settings["DNS_RESOLVER"]
         else:
-            resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+            resolver_cls_path = settings["TWISTED_DNS_RESOLVER"]
 
         resolver_class = load_object(resolver_cls_path)
 
-        # We pass self, which is CrawlerProcess, instead of Crawler here,
-        # which works because the default resolvers only use crawler.settings.
-        resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[call-overload]
+        resolver = build_from_crawler(resolver_class, crawler, reactor=reactor)  # type: ignore[call-overload]
         resolver.install_on_reactor()
         tp = reactor.getThreadPool()
-        tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
+        tp.adjustPoolsize(maxthreads=settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
         reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd)
         if install_signal_handlers:
             reactor.addSystemEventTrigger(

tests/AsyncCrawlerProcess/reactor_settings.py

@@ -0,0 +1,35 @@
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.resolver import dnscache
+
+
+class Spider1(scrapy.Spider):
+    name = "spider1"
+    custom_settings = {
+        "DNSCACHE_ENABLED": False,
+        "DNS_TIMEOUT": 11,
+        "REACTOR_THREADPOOL_MAXSIZE": 42,
+    }
+
+    async def start(self):
+        from twisted.internet import reactor
+
+        self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")
+        self.logger.info(f"DNS cache limit: {dnscache.limit}")
+        self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")
+        return
+        yield
+
+
+class Spider2(Spider1):
+    name = "spider2"
+    custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}
+
+
+process = AsyncCrawlerProcess()
+process.crawl(Spider1)
+if len(sys.argv) > 1 and sys.argv[1] == "conflict":
+    process.crawl(Spider2)
+process.start()

tests/CrawlerProcess/reactor_settings.py

@@ -0,0 +1,35 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.resolver import dnscache
+
+
+class Spider1(scrapy.Spider):
+    name = "spider1"
+    custom_settings = {
+        "DNSCACHE_ENABLED": False,
+        "DNS_TIMEOUT": 11,
+        "REACTOR_THREADPOOL_MAXSIZE": 42,
+    }
+
+    async def start(self):
+        from twisted.internet import reactor
+
+        self.logger.info(f"DNS timeout: {reactor.resolver.timeout}")
+        self.logger.info(f"DNS cache limit: {dnscache.limit}")
+        self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}")
+        return
+        yield
+
+
+class Spider2(Spider1):
+    name = "spider2"
+    custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22}
+
+
+process = CrawlerProcess()
+process.crawl(Spider1)
+if len(sys.argv) > 1 and sys.argv[1] == "conflict":
+    process.crawl(Spider2)
+process.start()

tests/test_crawler_subprocess.py

@@ -132,6 +132,23 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Spider closed (finished)" in log
         assert "The DNS_RESOLVER setting is deprecated" in log
 
+    def test_reactor_settings(self) -> None:
+        log = self.run_script("reactor_settings.py")
+        assert "Spider closed (finished)" in log
+        assert "DNS timeout: 11.0" in log
+        assert "DNS cache limit: 0" in log
+        assert "Thread pool size: 42" in log
+        assert "reactor settings" not in log
+
+    def test_reactor_settings_conflict(self) -> None:
+        log = self.run_script("reactor_settings.py", "conflict")
+        assert "Spider closed (finished)" in log
+        assert "DNS timeout: 11.0" in log
+        assert (
+            "Spider Spider2 defines a different value than spider Spider1 for "
+            "the following reactor settings: DNS_TIMEOUT" in log
+        )
+
     def test_twisted_reactor_asyncio(self) -> None:
         log = self.run_script("twisted_reactor_asyncio.py")
         assert "Spider closed (finished)" in log