removed old blocking caching DNS resolver and replaced by a non-blocking one installed as the default reactor resolver

--HG--
rename : scrapy/core/downloader/dnscache.py => scrapy/core/downloader/resolver.py

scrapy/core/downloader/dnscache.py

@@ -1,52 +0,0 @@
-"""
-Dns cache module. 
-This module implements a dns cache to improve the performance of the 
-crawler, reducing the dns lookups.
-"""
-import socket
-from scrapy.core import signals
-from scrapy.xlib.pydispatch import dispatcher
-
-class DNSCache(object):
-    """
-    DNSCache.
-
-    Impelements a DNS Cache to improve the performance of the 
-    DNS lookup that the crawler request when it's running.
-    Use:
-     Create a single instance:
-       >>> import dnscache
-       >>> cachedns = dnscache.DNSCache()
-     
-     To get (and set a new host if not exists) we only call 
-       >> ip = cachedns.get('python.org')
-       >>> print ip
-       '82.94.164.162'
-       >>> 
-    """
-    def __init__(self):
-        dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
-        self._cache = {}
-    
-    def get(self, host):
-        """
-        Returns the ip associated with the host and save it in
-        the cache.
-        """
-        try:
-            ips = self._cache[host]
-        except KeyError:
-            # The socket.gethostbyname_ex throw an
-            # exception when it can't get the host 
-            # ip. So we save the hostname anyway.
-            # we use socket.getaddrinfo that support IP4/IP6
-            try:
-                ips = list(set([x[4][0] for x in socket.getaddrinfo(host,None)])) 
-            except socket.gaierror:
-                ips = [host]
-            self._cache[host] = ips
-        return ips[0]
-
-    def domain_closed(self, domain, spider, reason):
-        if domain in self._cache:
-            del self._cache[domain]

scrapy/core/downloader/handlers.py

@@ -18,7 +18,6 @@ from scrapy.core.exceptions import NotSupported
 from scrapy.utils.defer import defer_succeed
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.signal import send_catch_log
-from scrapy.core.downloader.dnscache import DNSCache
 from scrapy.core.downloader.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
 from scrapy.conf import settings
@@ -27,9 +26,6 @@ from scrapy.conf import settings
 default_timeout = settings.getint('DOWNLOAD_TIMEOUT')
 ssl_supported = 'ssl' in optional_features
 
-# Cache for dns lookups.
-dnscache = DNSCache()
-
 def download_any(request, spider):
     scheme = urlparse_cached(request).scheme
     if scheme == 'http':
@@ -69,19 +65,17 @@ def download_http(request, spider):
     """Return a deferred for the HTTP download"""
     factory = create_factory(request, spider)
     url = urlparse_cached(request)
-    ip = dnscache.get(url.hostname)
     port = url.port
-    reactor.connectTCP(ip, port or 80, factory)
+    reactor.connectTCP(url.hostname, port or 80, factory)
     return factory.deferred
 
 def download_https(request, spider):
     """Return a deferred for the HTTPS download"""
     factory = create_factory(request, spider)
     url = urlparse_cached(request)
-    ip = dnscache.get(url.hostname)
     port = url.port
     contextFactory = ssl.ClientContextFactory()
-    reactor.connectSSL(ip, port or 443, factory, contextFactory)
+    reactor.connectSSL(url.hostname, port or 443, factory, contextFactory)
     return factory.deferred
 
 def download_file(request, spider) :

scrapy/core/downloader/manager.py

@@ -8,11 +8,12 @@ from twisted.internet import reactor, defer
 
 from scrapy.core.exceptions import IgnoreRequest
 from scrapy.spider import spiders
-from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
-from scrapy.core.downloader.handlers import download_any
 from scrapy.conf import settings
 from scrapy.utils.defer import mustbe_deferred
 from scrapy import log
+from .middleware import DownloaderMiddlewareManager
+from .handlers import download_any
+from .resolver import CachingThreadedResolver
 
 
 class SiteInfo(object):
@@ -54,6 +55,8 @@ class Downloader(object):
         self.sites = {}
         self.middleware = DownloaderMiddlewareManager()
         self.concurrent_domains = settings.getint('CONCURRENT_DOMAINS')
+        cached_resolver = CachingThreadedResolver(reactor)
+        reactor.installResolver(cached_resolver)
 
     def fetch(self, request, spider):
         """ Main method to use to request a download

scrapy/core/downloader/resolver.py

@@ -0,0 +1,29 @@
+"""DNS resolver with cache to use with Twisted reactors"""
+
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+
+from scrapy.core import signals
+from scrapy.xlib.pydispatch import dispatcher
+
+class CachingThreadedResolver(ThreadedResolver):
+
+    def __init__(self, reactor):
+        ThreadedResolver.__init__(self, reactor)
+        self._cache = {}
+        dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
+
+    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
+        if name in self._cache:
+            return defer.succeed(self._cache[name])
+        dfd = ThreadedResolver.getHostByName(self, name, timeout)
+        dfd.addCallback(self._cache_result, name)
+        return dfd
+
+    def _cache_result(self, result, name):
+        self._cache[name] = result
+        return result
+
+    def domain_closed(self, spider):
+        for domain in [spider.domain_name] + spider.extra_domain_names:
+            self._cache.pop(domain, None)