diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 61851acc3..c5351c08f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -305,23 +305,13 @@ class CrawlerProcess(CrawlerRunner): return d.addBoth(self._stop_reactor) - reactor.installNameResolver(self._get_dns_resolver()) + resolver_class = load_object(self.settings["DNS_RESOLVER"]) + resolver_class.install(reactor, self.settings) tp = reactor.getThreadPool() tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE')) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call - def _get_dns_resolver(self): - from twisted.internet import reactor - if self.settings.getbool('DNSCACHE_ENABLED'): - cache_size = self.settings.getint('DNSCACHE_SIZE') - else: - cache_size = 0 - return CachingHostnameResolver( - resolver=reactor.nameResolver, - cache_size=cache_size, - ) - def _graceful_stop_reactor(self): d = self.stop() d.addBoth(self._stop_reactor) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index ddbae61a9..2bef9f1b8 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -1,4 +1,6 @@ -from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver +from twisted.internet.interfaces import IHostnameResolver, IResolutionReceiver, IResolverSimple from zope.interface.declarations import implementer, provider from scrapy.utils.datatypes import LocalCache @@ -8,8 +10,58 @@ from scrapy.utils.datatypes import LocalCache dnscache = LocalCache(10000) +@implementer(IResolverSimple) +class CachingThreadedResolver(ThreadedResolver): + """ + Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests + """ + + @classmethod + def install(cls, reactor, settings): + if settings.getbool('DNSCACHE_ENABLED'): + cache_size = settings.getint('DNSCACHE_SIZE') + else: + cache_size = 0 + resolver = cls(reactor, cache_size, settings.getfloat('DNS_TIMEOUT')) + reactor.installResolver(resolver) + + def __init__(self, reactor, cache_size, timeout): + super(CachingThreadedResolver, self).__init__(reactor) + dnscache.limit = cache_size + self.timeout = timeout + + def getHostByName(self, name, timeout=None): + if name in dnscache: + return defer.succeed(dnscache[name]) + # in Twisted<=16.6, getHostByName() is always called with + # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple), + # so the input argument above is simply overridden + # to enforce Scrapy's DNS_TIMEOUT setting's value + timeout = (self.timeout,) + d = super(CachingThreadedResolver, self).getHostByName(name, timeout) + if dnscache.limit: + d.addCallback(self._cache_result, name) + return d + + def _cache_result(self, result, name): + dnscache[name] = result + return result + + @implementer(IHostnameResolver) class CachingHostnameResolver: + """ + Experimental caching resolver, supporting IPv4 and IPv6 + """ + + @classmethod + def install(cls, reactor, settings): + if settings.getbool('DNSCACHE_ENABLED'): + cache_size = settings.getint('DNSCACHE_SIZE') + else: + cache_size = 0 + resolver = cls(reactor.nameResolver, cache_size) + reactor.installNameResolver(resolver) def __init__(self, resolver, cache_size): self.resolver = resolver diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d03fd37b0..46ed3be96 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -60,6 +60,7 @@ DEPTH_PRIORITY = 0 DNSCACHE_ENABLED = True DNSCACHE_SIZE = 10000 +DNS_RESOLVER = 'scrapy.resolver.CachingThreadedResolver' DNS_TIMEOUT = 60 DOWNLOAD_DELAY = 0