Rename DNS_RESOLVER to TWISTED_DNS_RESOLVER (#7361)

docs/faq.rst

@@ -360,7 +360,7 @@ method for this purpose. For example:
 Does Scrapy support IPv6 addresses?
 -----------------------------------
 
-Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
 Note that by doing so, you lose the ability to set a specific timeout for DNS requests
 (the value of the :setting:`DNS_TIMEOUT` setting is ignored).
 

docs/news.rst

@@ -4731,7 +4731,7 @@ Highlights:
 * :ref:`FTP support <media-pipeline-ftp>` for media pipelines
 * New :attr:`Response.certificate <scrapy.http.Response.certificate>`
   attribute
-* IPv6 support through :setting:`DNS_RESOLVER`
+* IPv6 support through ``DNS_RESOLVER``
 
 Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -4839,7 +4839,7 @@ New features
     :class:`twisted.internet.ssl.Certificate` object for HTTPS responses
     (:issue:`2726`, :issue:`4054`)
 
-*   A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support
+*   A new ``DNS_RESOLVER`` setting allows enabling IPv6 support
     (:issue:`1031`, :issue:`4227`)
 
 *   A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring

docs/topics/components.rst

@@ -11,7 +11,7 @@ That includes the classes that you may assign to the following settings:
 
 -   :setting:`ADDONS`
 
--   :setting:`DNS_RESOLVER`
+-   :setting:`TWISTED_DNS_RESOLVER`
 
 -   :setting:`DOWNLOAD_HANDLERS`
 

docs/topics/settings.rst

@@ -332,7 +332,7 @@ These settings are:
 -   :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using
     :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below)
 
--   :setting:`DNS_RESOLVER` and settings used by the corresponding
+-   :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding
     component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE`
     and :setting:`DNS_TIMEOUT` for the default one.
 
@@ -671,10 +671,10 @@ Default: ``10000``
 
 DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
 
-.. setting:: DNS_RESOLVER
+.. setting:: TWISTED_DNS_RESOLVER
 
-DNS_RESOLVER
-------------
+TWISTED_DNS_RESOLVER
+--------------------
 
 Default: ``'scrapy.resolver.CachingThreadedResolver'``
 

scrapy/crawler.py

@@ -16,7 +16,7 @@ from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
-from scrapy.settings import Settings, overridden_settings
+from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
 from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
 from scrapy.utils.defer import deferred_from_coro
@@ -660,7 +660,29 @@ class CrawlerProcessBase(CrawlerRunnerBase):
     def _setup_reactor(self, install_signal_handlers: bool) -> None:
         from twisted.internet import reactor
 
-        resolver_class = load_object(self.settings["DNS_RESOLVER"])
+        dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0
+        default_priority = SETTINGS_PRIORITIES["default"]
+
+        if dns_priority > default_priority:
+            warnings.warn(
+                "The DNS_RESOLVER setting is deprecated, please use "
+                "TWISTED_DNS_RESOLVER instead.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+
+            twisted_dns_priority = (
+                self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0
+            )
+            if twisted_dns_priority > dns_priority:
+                resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+            else:
+                resolver_cls_path = self.settings["DNS_RESOLVER"]
+        else:
+            resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"]
+
+        resolver_class = load_object(resolver_cls_path)
+
         # We pass self, which is CrawlerProcess, instead of Crawler here,
         # which works because the default resolvers only use crawler.settings.
         resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[call-overload]

scrapy/settings/__init__.py

@@ -162,6 +162,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
                 stacklevel=2,
             )
 
+        if name == "DNS_RESOLVER":
+            warnings.warn(
+                "The DNS_RESOLVER setting is deprecated, please use "
+                "TWISTED_DNS_RESOLVER instead.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+
         return self[name] if self[name] is not None else default
 
     def getbool(self, name: _SettingsKey, default: bool = False) -> bool:

scrapy/settings/default_settings.py

@@ -48,7 +48,6 @@ __all__ = [
     "DEPTH_STATS_VERBOSE",
     "DNSCACHE_ENABLED",
     "DNSCACHE_SIZE",
-    "DNS_RESOLVER",
     "DNS_TIMEOUT",
     "DOWNLOADER",
     "DOWNLOADER_CLIENTCONTEXTFACTORY",
@@ -185,6 +184,7 @@ __all__ = [
     "TELNETCONSOLE_PORT",
     "TELNETCONSOLE_USERNAME",
     "TEMPLATES_DIR",
+    "TWISTED_DNS_RESOLVER",
     "TWISTED_REACTOR",
     "TWISTED_REACTOR_ENABLED",
     "URLLENGTH_LIMIT",
@@ -526,6 +526,8 @@ TELNETCONSOLE_PASSWORD = None
 
 TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve())
 
+TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver"
+
 TWISTED_REACTOR_ENABLED = True
 TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 

tests/AsyncCrawlerProcess/caching_hostname_resolver.py

@@ -28,7 +28,7 @@ if __name__ == "__main__":
     process = AsyncCrawlerProcess(
         settings={
             "RETRY_ENABLED": False,
-            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+            "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
         }
     )
     process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])

tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py

@@ -15,7 +15,7 @@ if __name__ == "__main__":
     process = AsyncCrawlerProcess(
         settings={
             "RETRY_ENABLED": False,
-            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+            "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
         }
     )
     process.crawl(CachingHostnameResolverSpider)

tests/CrawlerProcess/caching_hostname_resolver.py

@@ -28,7 +28,7 @@ if __name__ == "__main__":
     process = CrawlerProcess(
         settings={
             "RETRY_ENABLED": False,
-            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+            "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
         }
     )
     process.crawl(CachingHostnameResolverSpider, url=sys.argv[1])

tests/CrawlerProcess/caching_hostname_resolver_ipv6.py

@@ -15,7 +15,7 @@ if __name__ == "__main__":
     process = CrawlerProcess(
         settings={
             "RETRY_ENABLED": False,
-            "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+            "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
         }
     )
     process.crawl(CachingHostnameResolverSpider)