diff --git a/docs/faq.rst b/docs/faq.rst index fdf0582de..e117d2a5f 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -360,7 +360,7 @@ method for this purpose. For example: Does Scrapy support IPv6 addresses? ----------------------------------- -Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. +Yes, by setting :setting:`TWISTED_DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``. Note that by doing so, you lose the ability to set a specific timeout for DNS requests (the value of the :setting:`DNS_TIMEOUT` setting is ignored). diff --git a/docs/news.rst b/docs/news.rst index 356220c3f..e445b0498 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -4731,7 +4731,7 @@ Highlights: * :ref:`FTP support ` for media pipelines * New :attr:`Response.certificate ` attribute -* IPv6 support through :setting:`DNS_RESOLVER` +* IPv6 support through ``DNS_RESOLVER`` Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -4839,7 +4839,7 @@ New features :class:`twisted.internet.ssl.Certificate` object for HTTPS responses (:issue:`2726`, :issue:`4054`) -* A new :setting:`DNS_RESOLVER` setting allows enabling IPv6 support +* A new ``DNS_RESOLVER`` setting allows enabling IPv6 support (:issue:`1031`, :issue:`4227`) * A new :setting:`SCRAPER_SLOT_MAX_ACTIVE_SIZE` setting allows configuring diff --git a/docs/topics/components.rst b/docs/topics/components.rst index d8a604ea7..c0df86922 100644 --- a/docs/topics/components.rst +++ b/docs/topics/components.rst @@ -11,7 +11,7 @@ That includes the classes that you may assign to the following settings: - :setting:`ADDONS` -- :setting:`DNS_RESOLVER` +- :setting:`TWISTED_DNS_RESOLVER` - :setting:`DOWNLOAD_HANDLERS` diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index f0456b4d5..97c8f41de 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -332,7 +332,7 @@ These settings are: - :setting:`ASYNCIO_EVENT_LOOP` (not possible to set per-spider when using :class:`~scrapy.crawler.AsyncCrawlerProcess`, see below) -- :setting:`DNS_RESOLVER` and settings used by the corresponding +- :setting:`TWISTED_DNS_RESOLVER` and settings used by the corresponding component, e.g. :setting:`DNSCACHE_ENABLED`, :setting:`DNSCACHE_SIZE` and :setting:`DNS_TIMEOUT` for the default one. @@ -671,10 +671,10 @@ Default: ``10000`` DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`. -.. setting:: DNS_RESOLVER +.. setting:: TWISTED_DNS_RESOLVER -DNS_RESOLVER ------------- +TWISTED_DNS_RESOLVER +-------------------- Default: ``'scrapy.resolver.CachingThreadedResolver'`` diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 500139566..0a19e9985 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -16,7 +16,7 @@ from scrapy.addons import AddonManager from scrapy.core.engine import ExecutionEngine from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extension import ExtensionManager -from scrapy.settings import Settings, overridden_settings +from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader from scrapy.utils.defer import deferred_from_coro @@ -660,7 +660,29 @@ class CrawlerProcessBase(CrawlerRunnerBase): def _setup_reactor(self, install_signal_handlers: bool) -> None: from twisted.internet import reactor - resolver_class = load_object(self.settings["DNS_RESOLVER"]) + dns_priority = self.settings.getpriority("DNS_RESOLVER") or 0 + default_priority = SETTINGS_PRIORITIES["default"] + + if dns_priority > default_priority: + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + category=ScrapyDeprecationWarning, + stacklevel=2, + ) + + twisted_dns_priority = ( + self.settings.getpriority("TWISTED_DNS_RESOLVER") or 0 + ) + if twisted_dns_priority > dns_priority: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["DNS_RESOLVER"] + else: + resolver_cls_path = self.settings["TWISTED_DNS_RESOLVER"] + + resolver_class = load_object(resolver_cls_path) + # We pass self, which is CrawlerProcess, instead of Crawler here, # which works because the default resolvers only use crawler.settings. resolver = build_from_crawler(resolver_class, self, reactor=reactor) # type: ignore[call-overload] diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py index d5c4b2dd3..456e90e77 100644 --- a/scrapy/settings/__init__.py +++ b/scrapy/settings/__init__.py @@ -162,6 +162,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]): stacklevel=2, ) + if name == "DNS_RESOLVER": + warnings.warn( + "The DNS_RESOLVER setting is deprecated, please use " + "TWISTED_DNS_RESOLVER instead.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return self[name] if self[name] is not None else default def getbool(self, name: _SettingsKey, default: bool = False) -> bool: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 462240133..b80e48601 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -48,7 +48,6 @@ __all__ = [ "DEPTH_STATS_VERBOSE", "DNSCACHE_ENABLED", "DNSCACHE_SIZE", - "DNS_RESOLVER", "DNS_TIMEOUT", "DOWNLOADER", "DOWNLOADER_CLIENTCONTEXTFACTORY", @@ -185,6 +184,7 @@ __all__ = [ "TELNETCONSOLE_PORT", "TELNETCONSOLE_USERNAME", "TEMPLATES_DIR", + "TWISTED_DNS_RESOLVER", "TWISTED_REACTOR", "TWISTED_REACTOR_ENABLED", "URLLENGTH_LIMIT", @@ -526,6 +526,8 @@ TELNETCONSOLE_PASSWORD = None TEMPLATES_DIR = str((Path(__file__).parent / ".." / "templates").resolve()) +TWISTED_DNS_RESOLVER = "scrapy.resolver.CachingThreadedResolver" + TWISTED_REACTOR_ENABLED = True TWISTED_REACTOR = "twisted.internet.asyncioreactor.AsyncioSelectorReactor" diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py index 5f75d5e17..a6520ca22 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py @@ -28,7 +28,7 @@ if __name__ == "__main__": process = AsyncCrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py index 07e2d3684..55d2ef711 100644 --- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py @@ -15,7 +15,7 @@ if __name__ == "__main__": process = AsyncCrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider) diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py index 53d427061..7f687e7d9 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver.py +++ b/tests/CrawlerProcess/caching_hostname_resolver.py @@ -28,7 +28,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider, url=sys.argv[1]) diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py index 5cca94bed..da9c16cb8 100644 --- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py +++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py @@ -15,7 +15,7 @@ if __name__ == "__main__": process = CrawlerProcess( settings={ "RETRY_ENABLED": False, - "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", + "TWISTED_DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver", } ) process.crawl(CachingHostnameResolverSpider)