diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e58815fad..60a6c0b56 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -330,6 +330,12 @@ Reactor settings **Reactor settings** are settings tied to the :doc:`Twisted reactor `. +.. versionchanged:: VERSION + :setting:`TWISTED_DNS_RESOLVER`, the settings of the resolver and + :setting:`REACTOR_THREADPOOL_MAXSIZE` are now read from the first spider, + instead of being read from the project settings and ignored in + :ref:`per-spider settings `. + Because only 1 reactor can be used per process, these settings cannot use a different value per spider when :ref:`running multiple spiders in the same process `. @@ -359,11 +365,10 @@ These settings are applied when starting the reactor: - :setting:`REACTOR_THREADPOOL_MAXSIZE` -They are read from the settings of the -:class:`~scrapy.crawler.CrawlerProcess` or -:class:`~scrapy.crawler.AsyncCrawlerProcess` object, so setting them from a -spider or an :ref:`add-on ` has no effect. They are ignored -altogether when using :class:`~scrapy.crawler.CrawlerRunner` or +They can also be :ref:`set from a spider `, but only the +values from the first spider that runs are used; if a later spider defines a +different value, a warning is issued. They are ignored altogether when using +:class:`~scrapy.crawler.CrawlerRunner` or :class:`~scrapy.crawler.AsyncCrawlerRunner`, which do not start the reactor. There is an additional restriction for :setting:`TWISTED_REACTOR` and diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 444a5fb67..e52f398a9 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -57,6 +57,17 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") +# Reactor settings that are applied when starting the reactor, i.e. once per +# process, and so can only take the value of one of the crawlers. +_REACTOR_SETTINGS = ( + "DNSCACHE_ENABLED", + "DNSCACHE_SIZE", + "DNS_RESOLVER", + "DNS_TIMEOUT", + "REACTOR_THREADPOOL_MAXSIZE", + "TWISTED_DNS_RESOLVER", +) + class _LateAttribute(Generic[_T]): """Descriptor for a :class:`Crawler` attribute that only gets a value once @@ -714,6 +725,7 @@ class CrawlerProcessBase(CrawlerRunnerBase): install_root_handler: bool = True, ): super().__init__(settings) + self._reactor_crawler: Crawler | None = None configure_logging(self.settings, install_root_handler) log_scrapy_info(self.settings) @@ -723,6 +735,30 @@ class CrawlerProcessBase(CrawlerRunnerBase): ) -> None: raise NotImplementedError + def create_crawler( + self, crawler_or_spidercls: type[Spider] | str | Crawler + ) -> Crawler: + crawler = super().create_crawler(crawler_or_spidercls) + if self._reactor_crawler is None: + self._reactor_crawler = crawler + else: + ignored = [ + setting + for setting in _REACTOR_SETTINGS + if self._reactor_crawler.settings[setting] != crawler.settings[setting] + ] + if ignored: + warnings.warn( + f"Spider {crawler.spidercls.__name__} defines a different " + f"value than spider " + f"{self._reactor_crawler.spidercls.__name__} for the " + f"following reactor settings: {', '.join(ignored)}. Only " + f"the value of the first spider is used, since the " + f"reactor is shared by every spider in a process.", + stacklevel=2, + ) + return crawler + def _signal_shutdown(self, signum: int, _: Any) -> None: from twisted.internet import reactor @@ -755,7 +791,13 @@ class CrawlerProcessBase(CrawlerRunnerBase): def _setup_reactor(self, install_signal_handlers: bool) -> None: from twisted.internet import reactor - dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0 + # Reactor settings are read from the first crawler, so that they can be + # defined from a spider, and fall back to the process settings when no + # crawler has been created yet. + crawler: Crawler | CrawlerProcessBase = self._reactor_crawler or self + settings = crawler.settings + + dns_priority = settings.getpriority("DNS_RESOLVER") or 0 default_priority = SETTINGS_PRIORITIES["default"] if dns_priority > default_priority: @@ -766,24 +808,20 @@ class CrawlerProcessBase(CrawlerRunnerBase): stacklevel=2, ) - twisted_dns_priority = ( - self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0 - ) + twisted_dns_priority = settings.getpriority("TWISTED_DNS_RESOLVER") or 0 if twisted_dns_priority > dns_priority: - resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + resolver_cls_path = settings["TWISTED_DNS_RESOLVER"] else: - resolver_cls_path = self.settings["DNS_RESOLVER"] + resolver_cls_path = settings["DNS_RESOLVER"] else: - resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + resolver_cls_path = settings["TWISTED_DNS_RESOLVER"] resolver_class = load_object(resolver_cls_path) - # We pass self, which is CrawlerProcess, instead of Crawler here, - # which works because the default resolvers only use crawler.settings. - resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload] + resolver = build_from_crawler(resolver_class, crawler, reactor=reactor) # type: ignore[call-overload] resolver.install_on_reactor() tp = reactor.getThreadPool() - tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE")) + tp.adjustPoolsize(maxthreads=settings.getint("REACTOR_THREADPOOL_MAXSIZE")) reactor.addSystemEventTrigger("before", "shutdown", self._stop_dfd) if install_signal_handlers: reactor.addSystemEventTrigger( diff --git a/tests/AsyncCrawlerProcess/reactor_settings.py b/tests/AsyncCrawlerProcess/reactor_settings.py new file mode 100644 index 000000000..3b64c22a2 --- /dev/null +++ b/tests/AsyncCrawlerProcess/reactor_settings.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import AsyncCrawlerProcess +from scrapy.resolver import dnscache + + +class Spider1(scrapy.Spider): + name = "spider1" + custom_settings = { + "DNSCACHE_ENABLED": False, + "DNS_TIMEOUT": 11, + "REACTOR_THREADPOOL_MAXSIZE": 42, + } + + async def start(self): + from twisted.internet import reactor + + self.logger.info(f"DNS timeout: {reactor.resolver.timeout}") + self.logger.info(f"DNS cache limit: {dnscache.limit}") + self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}") + return + yield + + +class Spider2(Spider1): + name = "spider2" + custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22} + + +process = AsyncCrawlerProcess() +process.crawl(Spider1) +if len(sys.argv) > 1 and sys.argv[1] == "conflict": + process.crawl(Spider2) +process.start() diff --git a/tests/CrawlerProcess/reactor_settings.py b/tests/CrawlerProcess/reactor_settings.py new file mode 100644 index 000000000..8a4848114 --- /dev/null +++ b/tests/CrawlerProcess/reactor_settings.py @@ -0,0 +1,35 @@ +import sys + +import scrapy +from scrapy.crawler import CrawlerProcess +from scrapy.resolver import dnscache + + +class Spider1(scrapy.Spider): + name = "spider1" + custom_settings = { + "DNSCACHE_ENABLED": False, + "DNS_TIMEOUT": 11, + "REACTOR_THREADPOOL_MAXSIZE": 42, + } + + async def start(self): + from twisted.internet import reactor + + self.logger.info(f"DNS timeout: {reactor.resolver.timeout}") + self.logger.info(f"DNS cache limit: {dnscache.limit}") + self.logger.info(f"Thread pool size: {reactor.getThreadPool().max}") + return + yield + + +class Spider2(Spider1): + name = "spider2" + custom_settings = {**Spider1.custom_settings, "DNS_TIMEOUT": 22} + + +process = CrawlerProcess() +process.crawl(Spider1) +if len(sys.argv) > 1 and sys.argv[1] == "conflict": + process.crawl(Spider2) +process.start() diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py index defee1041..e989ee823 100644 --- a/tests/test_crawler_subprocess.py +++ b/tests/test_crawler_subprocess.py @@ -132,6 +132,23 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin): assert "Spider closed (finished)" in log assert "The DNS_RESOLVER setting is deprecated" in log + def test_reactor_settings(self) -> None: + log = self.run_script("reactor_settings.py") + assert "Spider closed (finished)" in log + assert "DNS timeout: 11.0" in log + assert "DNS cache limit: 0" in log + assert "Thread pool size: 42" in log + assert "reactor settings" not in log + + def test_reactor_settings_conflict(self) -> None: + log = self.run_script("reactor_settings.py", "conflict") + assert "Spider closed (finished)" in log + assert "DNS timeout: 11.0" in log + assert ( + "Spider Spider2 defines a different value than spider Spider1 for " + "the following reactor settings: DNS_TIMEOUT" in log + ) + def test_twisted_reactor_asyncio(self) -> None: log = self.run_script("twisted_reactor_asyncio.py") assert "Spider closed (finished)" in log