Custom name resolver implementing twisted.internet.interfaces.IHostnameResolver

pytest.ini

@@ -158,6 +158,7 @@ flake8-ignore =
     scrapy/mail.py E402 E128 E501 E502
     scrapy/middleware.py E128 E501
     scrapy/pqueues.py E501
+    scrapy/resolver.py E501
     scrapy/responsetypes.py E128 E501 E305
     scrapy/robotstxt.py E501
     scrapy/shell.py E501

scrapy/crawler.py

@@ -4,34 +4,36 @@ import signal
 import warnings
 
 from twisted.internet import defer
-from zope.interface.verify import verifyClass, DoesNotImplement
+from zope.interface.verify import DoesNotImplement, verifyClass
 
-from scrapy import Spider
+from scrapy import signals, Spider
 from scrapy.core.engine import ExecutionEngine
-from scrapy.resolver import CachingThreadedResolver
-from scrapy.interfaces import ISpiderLoader
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
+from scrapy.interfaces import ISpiderLoader
+from scrapy.resolver import CachingHostnameResolver
 from scrapy.settings import overridden_settings, Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncio import install_asyncio_reactor, is_asyncio_reactor_installed
-from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.misc import load_object
 from scrapy.utils.log import (
-    LogCounterHandler, configure_logging, log_scrapy_info,
-    get_scrapy_root_handler, install_scrapy_root_handler)
-from scrapy import signals
+    configure_logging,
+    get_scrapy_root_handler,
+    install_scrapy_root_handler,
+    log_scrapy_info,
+    LogCounterHandler,
+)
+from scrapy.utils.misc import load_object
+from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 
 
 logger = logging.getLogger(__name__)
 
 
-class Crawler(object):
+class Crawler:
 
     def __init__(self, spidercls, settings=None):
         if isinstance(spidercls, Spider):
-            raise ValueError(
-                'The spidercls argument must be a class, not an object')
+            raise ValueError('The spidercls argument must be a class, not an object')
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
@@ -110,7 +112,7 @@ class Crawler(object):
             yield defer.maybeDeferred(self.engine.stop)
 
 
-class CrawlerRunner(object):
+class CrawlerRunner:
     """
     This is a convenient helper class that keeps track of, manages and runs
     crawlers inside an already setup :mod:`~twisted.internet.reactor`.
@@ -303,7 +305,7 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(self._stop_reactor)
 
-        reactor.installResolver(self._get_dns_resolver())
+        reactor.installNameResolver(self._get_dns_resolver())
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
@@ -315,10 +317,10 @@ class CrawlerProcess(CrawlerRunner):
             cache_size = self.settings.getint('DNSCACHE_SIZE')
         else:
             cache_size = 0
-        return CachingThreadedResolver(
-            reactor=reactor,
+        return CachingHostnameResolver(
+            resolver=reactor.nameResolver,
             cache_size=cache_size,
-            timeout=self.settings.getfloat('DNS_TIMEOUT')
+            timeout=self.settings.getfloat('DNS_TIMEOUT'),
         )
 
     def _graceful_stop_reactor(self):

scrapy/resolver.py

@@ -1,32 +1,39 @@
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver
+from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
 
-# TODO: cache misses
 
+# TODO: cache misses
 dnscache = LocalCache(10000)
 
 
-class CachingThreadedResolver(ThreadedResolver):
-    def __init__(self, reactor, cache_size, timeout):
-        super(CachingThreadedResolver, self).__init__(reactor)
-        dnscache.limit = cache_size
+@implementer(IHostnameResolver)
+class CachingHostnameResolver(object):
+
+    def __init__(self, resolver, cache_size, timeout):
+        self.resolver = resolver
         self.timeout = timeout
+        dnscache.limit = cache_size
 
-    def getHostByName(self, name, timeout=None):
-        if name in dnscache:
-            return defer.succeed(dnscache[name])
-        # in Twisted<=16.6, getHostByName() is always called with
-        # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
-        # so the input argument above is simply overridden
-        # to enforce Scrapy's DNS_TIMEOUT setting's value
-        timeout = (self.timeout,)
-        d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        if dnscache.limit:
-            d.addCallback(self._cache_result, name)
-        return d
+    def resolveHostName(self, resolutionReceiver, hostName, portNumber=0,
+                        addressTypes=None, transportSemantics='TCP'):
 
-    def _cache_result(self, result, name):
-        dnscache[name] = result
-        return result
+        @provider(IResolutionReceiver)
+        class CachingResolutionReceiver(resolutionReceiver):
+            def resolutionBegan(self, resolution):
+                super(CachingResolutionReceiver, self).resolutionBegan(resolution)
+                self.resolution = resolution
+
+            def resolutionComplete(self):
+                super(CachingResolutionReceiver, self).resolutionComplete()
+                dnscache[hostName] = self.resolution
+
+        try:
+            result = dnscache[hostName]
+        except KeyError:
+            result = self.resolver.resolveHostName(
+                CachingResolutionReceiver(), hostName, portNumber, addressTypes, transportSemantics
+            )
+        finally:
+            return result