Ability to choose name resolver

scrapy/crawler.py

@@ -305,23 +305,13 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        reactor.installNameResolver(self._get_dns_resolver())
+        resolver_class = load_object(self.settings["DNS_RESOLVER"])
+        resolver_class.install(reactor, self.settings)
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False)  # blocking call
 
-    def _get_dns_resolver(self):
-        from twisted.internet import reactor
-        if self.settings.getbool('DNSCACHE_ENABLED'):
-            cache_size = self.settings.getint('DNSCACHE_SIZE')
-        else:
-            cache_size = 0
-        return CachingHostnameResolver(
-            resolver=reactor.nameResolver,
-            cache_size=cache_size,
-        )
-
     def _graceful_stop_reactor(self):
         d = self.stop()
         d.addBoth(self._stop_reactor)

scrapy/resolver.py

@@ -1,4 +1,6 @@
-from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple
 from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
@@ -8,8 +10,58 @@ from scrapy.utils.datatypes import LocalCache
 dnscache = LocalCache(10000)
 
 
+@implementer(IResolverSimple)
+class CachingThreadedResolver(ThreadedResolver):
+    """
+    Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests
+    """
+
+    @classmethod
+    def install(cls, reactor, settings):
+        if settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT'))
+        reactor.installResolver(resolver)
+
+    def __init__(self, reactor, cache_size, timeout):
+        super(CachingThreadedResolver, self).__init__(reactor)
+        dnscache.limit = cache_size
+        self.timeout = timeout
+
+    def getHostByName(self, name, timeout=None):
+        if name in dnscache:
+            return defer.succeed(dnscache[name])
+        # in Twisted<=16.6, getHostByName() is always called with
+        # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
+        # so the input argument above is simply overridden
+        # to enforce Scrapy's DNS_TIMEOUT setting's value
+        timeout = (self.timeout,)
+        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
+        return d
+
+    def _cache_result(self, result, name):
+        dnscache[name] = result
+        return result
+
+
 @implementer(IHostnameResolver)
 class CachingHostnameResolver:
+    """
+    Experimental caching resolver, supporting IPv4 and IPv6
+    """
+
+    @classmethod
+    def install(cls, reactor, settings):
+        if settings.getbool('DNSCACHE_ENABLED'):
+            cache_size = settings.getint('DNSCACHE_SIZE')
+        else:
+            cache_size = 0
+        resolver = cls(reactor.nameResolver, cache_size)
+        reactor.installNameResolver(resolver)
 
     def __init__(self, resolver, cache_size):
         self.resolver = resolver

scrapy/settings/default_settings.py

@@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
 DNSCACHE_SIZE = 10000
+DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver'
 DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0