Added dns cache support for the crawler, improving the performance of the page download because this reduce the dns lookups.

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40725

scrapy/trunk/scrapy/core/downloader/dnscache.py

@@ -0,0 +1,52 @@
+"""
+Dns cache module. 
+This module implements a dns cache to improve the performance of the 
+crawler, reducing the dns lookups.
+"""
+import socket
+from scrapy.core import signals
+from pydispatch import dispatcher
+
+class DNSCache(object):
+    """
+    DNSCache.
+
+    Impelements a DNS Cache to improve the performance of the 
+    DNS lookup that the crawler request when it's running.
+    Use:
+     Create a single instance:
+       >>> import dnscache
+       >>> cachedns = dnscache.DNSCache()
+     
+     To get (and set a new host if not exists) we only call 
+       >> ip = cachedns.get('python.org')
+       >>> print ip
+       '82.94.164.162'
+       >>> 
+    """
+    def __init__(self):
+        dispatcher.connect(self.domain_closed, signal=signals.domain_closed)
+        self._cache = {}
+    
+    def get(self, host):
+        """
+        Returns the ip associated with the host and save it in
+        the cache.
+        """
+        try:
+            ips = self._cache[host]
+        except KeyError:
+            # The socket.gethostbyname_ex throw an
+            # exception when it can't get the host 
+            # ip. So we save the hostname anyway.
+            # we use socket.getaddrinfo that support IP4/IP6
+            try:
+                ips = list(set([x[4][0] for x in socket.getaddrinfo(host,None)])) 
+            except socket.gaierror:
+                ips = [host]
+            self._cache[host] = ips
+        return ips[0]
+
+    def domain_closed(self, domain, spider, status):
+        if domain in self._cache:
+            del self._cache[domain]

scrapy/trunk/scrapy/core/downloader/handlers.py

@@ -16,6 +16,11 @@ from scrapy.core.exceptions import UsageError, HttpException
 from scrapy.utils.defer import defer_succeed
 from scrapy.conf import settings
 
+from scrapy.core.downloader.dnscache import DNSCache
+
+# Cache for dns lookups.
+dnscache = DNSCache()
+
 def download_any(request, spider):
     u = urlparse.urlparse(request.url)
     if u.scheme == 'file':
@@ -62,12 +67,14 @@ def download_http(request, spider):
     factory.deferred.addCallbacks(_on_success, _on_error)
 
     u = urlparse.urlparse(request.url)
+    ip = dnscache.get(u.hostname)
+    port = u.port
     if u.scheme == 'https' :
         from twisted.internet import ssl
         contextFactory = ssl.ClientContextFactory()
-        reactor.connectSSL(u.hostname, u.port or 443, factory, contextFactory)
+        reactor.connectSSL(ip, port or 443, factory, contextFactory)
     else:
-        reactor.connectTCP(u.hostname, u.port or 80, factory)
+        reactor.connectTCP(ip, port or 80, factory)
     return factory.deferred
 
 def download_file(request, spider) :