Merge pull request #4227 from elacuesta/name-resolver

Name resolver with IPv6 support

docs/faq.rst

@@ -353,6 +353,13 @@ method for this purpose. For example::
                     for _ in range(item['multiply_by']):
                         yield deepcopy(item)
 
+Does Scrapy support IPv6 addresses?
+-----------------------------------
+
+Yes, by setting :setting:`DNS_RESOLVER` to ``scrapy.resolver.CachingHostnameResolver``.
+Note that by doing so, you lose the ability to set a specific timeout for DNS requests
+(the value of the :setting:`DNS_TIMEOUT` setting is ignored).
+
 
 .. _user agents: https://en.wikipedia.org/wiki/User_agent
 .. _LIFO: https://en.wikipedia.org/wiki/Stack_(abstract_data_type)

docs/topics/settings.rst

@@ -397,6 +397,19 @@ Default: ``10000``
 
 DNS in-memory cache size.
 
+.. setting:: DNS_RESOLVER
+
+DNS_RESOLVER
+------------
+
+Default: ``'scrapy.resolver.CachingThreadedResolver'``
+
+The class to be used to resolve DNS names. The default ``scrapy.resolver.CachingThreadedResolver``
+supports specifying a timeout for DNS requests via the :setting:`DNS_TIMEOUT` setting,
+but works only with IPv4 addresses. Scrapy provides an alternative resolver,
+``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
+take the :setting:`DNS_TIMEOUT` setting into account.
+
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT

pytest.ini

@@ -159,6 +159,7 @@ flake8-ignore =
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
+    scrapy/resolver.py E501
     scrapy/responsetypes.py E128 E501 E305
     scrapy/robotstxt.py E501
     scrapy/shell.py E501

scrapy/crawler.py

@@ -4,34 +4,35 @@ import signal
 import warnings
 
 from twisted.internet import defer
-from zope.interface.verify import verifyClass, DoesNotImplement
+from zope.interface.verify import DoesNotImplement, verifyClass
 
-from scrapy import Spider
+from scrapy import signals, Spider
 from scrapy.core.engine import ExecutionEngine
-from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderLoader
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
+from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.misc import load_object
 from scrapy.utils.log import (
-    LogCounterHandler, configure_logging, log_scrapy_info,
-    get_scrapy_root_handler, install_scrapy_root_handler)
-from scrapy import signals
+    configure_logging,
+    get_scrapy_root_handler,
+    install_scrapy_root_handler,
+    log_scrapy_info,
+    LogCounterHandler,
+)
+from scrapy.utils.misc import create_instance, load_object
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
 
 logger = logging.getLogger(__name__)
 
 
-class Crawler(object):
+class Crawler:
 
     def __init__(self, spidercls, settings=None):
         if isinstance(spidercls, Spider):
-            raise ValueError(
-                'The spidercls argument must be a class, not an object')
+            raise ValueError('The spidercls argument must be a class, not an object')
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -110,7 +111,7 @@ class Crawler(object):
             yield defer.maybeDeferred(self.engine.stop)
 
 
-class CrawlerRunner(object):
+class CrawlerRunner:
     """
     This is a convenient helper class that keeps track of, manages and runs
     crawlers inside an already setup :mod:`~twisted.internet.reactor`.
@@ -303,24 +304,14 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        reactor.installResolver(self._get_dns_resolver())
+        resolver_class = load_object(self.settings["DNS_RESOLVER"])
+        resolver = create_instance(resolver_class, self.settings, self, reactor=reactor)
+        resolver.install_on_reactor()
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _get_dns_resolver(self):
-        from twisted.internet import reactor
-        if self.settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = self.settings.getint('DNSCACHE_SIZE')
-        else:
-            cache_size = 0
-        return CachingThreadedResolver(
-            reactor=reactor,
-            cache_size=cache_size,
-            timeout=self.settings.getfloat('DNS_TIMEOUT')
-        )
-
     def _graceful_stop_reactor(self):
         d = self.stop()
         d.addBoth(self._stop_reactor)

scrapy/resolver.py

@@ -1,19 +1,37 @@
 from twisted.internet import defer
 from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
+from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
 
-# TODO: cache misses
 
+# TODO: cache misses
 dnscache = LocalCache(10000)
 
 
+@implementer(IResolverSimple)
 class CachingThreadedResolver(ThreadedResolver):
+    """
+    Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
+    """
+
     def __init__(self, reactor, cache_size, timeout):
         super(CachingThreadedResolver, self).__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
+    @classmethod
+    def from_crawler(cls, crawler, reactor):
+        if crawler.settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT'))
+
+    def install_on_reactor(self,):
+        self.reactor.installResolver(self)
+
     def getHostByName(self, name, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
@@ -30,3 +48,58 @@ class CachingThreadedResolver(ThreadedResolver):
     def _cache_result(self, result, name):
         dnscache[name] = result
         return result
+
+
+@implementer(IHostnameResolver)
+class CachingHostnameResolver:
+    """
+    Experimental caching resolver. Resolves IPv4 and IPv6 addresses,
+    does not support setting a timeout value for DNS requests.
+    """
+
+    def __init__(self, reactor, cache_size):
+        self.reactor = reactor
+        self.original_resolver = reactor.nameResolver
+        dnscache.limit = cache_size
+
+    @classmethod
+    def from_crawler(cls, crawler, reactor):
+        if crawler.settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = crawler.settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        return cls(reactor, cache_size)
+
+    def install_on_reactor(self):
+        self.reactor.installNameResolver(self)
+
+    def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+                        addressTypes=None, transportSemantics='TCP'):
+
+        @provider(IResolutionReceiver)
+        class CachingResolutionReceiver(resolutionReceiver):
+
+            def resolutionBegan(self, resolution):
+                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                self.resolution = resolution
+                self.resolved = False
+
+            def addressResolved(self, address):
+                super(CachingResolutionReceiver, self).addressResolved(address)
+                self.resolved = True
+
+            def resolutionComplete(self):
+                super(CachingResolutionReceiver, self).resolutionComplete()
+                if self.resolved:
+                    dnscache[hostName] = self.resolution
+
+        try:
+            return dnscache[hostName]
+        except KeyError:
+            return self.original_resolver.resolveHostName(
+                CachingResolutionReceiver(),
+                hostName,
+                portNumber,
+                addressTypes,
+                transportSemantics
+            )

scrapy/settings/default_settings.py

@@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
 DNSCACHE_SIZE = 10000
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
 DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0

tests/CrawlerProcess/alternative_name_resolver.py

@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+    name = "ipv6_spider"
+    start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={
+    "RETRY_ENABLED": False,
+    "DNS_RESOLVER": "scrapy.resolver.CachingHostnameResolver",
+})
+process.crawl(IPv6Spider)
+process.start()

tests/CrawlerProcess/default_name_resolver.py

@@ -0,0 +1,12 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class IPv6Spider(scrapy.Spider):
+    name = "ipv6_spider"
+    start_urls = ["http://[::1]"]
+
+
+process = CrawlerProcess(settings={"RETRY_ENABLED": False})
+process.crawl(IPv6Spider)
+process.start()

tests/test_crawler.py

@@ -305,3 +305,21 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         log = self.run_script('asyncio_enabled_reactor.py')
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("DEBUG: Asyncio reactor is installed", log)
+
+    def test_ipv6_default_name_resolver(self):
+        log = self.run_script('default_name_resolver.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertIn("twisted.internet.error.DNSLookupError: DNS lookup failed: no results for hostname lookup: ::1.", log)
+        self.assertIn("'downloader/exception_type_count/twisted.internet.error.DNSLookupError': 1,", log)
+
+    def test_ipv6_alternative_name_resolver(self):
+        log = self.run_script('alternative_name_resolver.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertTrue(any([
+            "twisted.internet.error.ConnectionRefusedError" in log,
+            "twisted.internet.error.ConnectError" in log,
+        ]))
+        self.assertTrue(any([
+            "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
+            "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
+        ]))