diff --git a/scrapy/trunk/scrapy/core/downloader/dnscache.py b/scrapy/trunk/scrapy/core/downloader/dnscache.py new file mode 100644 index 000000000..2bc8acb75 --- /dev/null +++ b/scrapy/trunk/scrapy/core/downloader/dnscache.py @@ -0,0 +1,52 @@ +""" +Dns cache module. +This module implements a dns cache to improve the performance of the +crawler, reducing the dns lookups. +""" +import socket +from scrapy.core import signals +from pydispatch import dispatcher + +class DNSCache(object): + """ + DNSCache. + + Impelements a DNS Cache to improve the performance of the + DNS lookup that the crawler request when it's running. + Use: + Create a single instance: + >>> import dnscache + >>> cachedns = dnscache.DNSCache() + + To get (and set a new host if not exists) we only call + >> ip = cachedns.get('python.org') + >>> print ip + '82.94.164.162' + >>> + """ + def __init__(self): + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + self._cache = {} + + def get(self, host): + """ + Returns the ip associated with the host and save it in + the cache. + """ + try: + ips = self._cache[host] + except KeyError: + # The socket.gethostbyname_ex throw an + # exception when it can't get the host + # ip. So we save the hostname anyway. + # we use socket.getaddrinfo that support IP4/IP6 + try: + ips = list(set([x[4][0] for x in socket.getaddrinfo(host,None)])) + except socket.gaierror: + ips = [host] + self._cache[host] = ips + return ips[0] + + def domain_closed(self, domain, spider, status): + if domain in self._cache: + del self._cache[domain] diff --git a/scrapy/trunk/scrapy/core/downloader/handlers.py b/scrapy/trunk/scrapy/core/downloader/handlers.py index e63e501c4..3ef4f48ed 100644 --- a/scrapy/trunk/scrapy/core/downloader/handlers.py +++ b/scrapy/trunk/scrapy/core/downloader/handlers.py @@ -16,6 +16,11 @@ from scrapy.core.exceptions import UsageError, HttpException from scrapy.utils.defer import defer_succeed from scrapy.conf import settings +from scrapy.core.downloader.dnscache import DNSCache + +# Cache for dns lookups. +dnscache = DNSCache() + def download_any(request, spider): u = urlparse.urlparse(request.url) if u.scheme == 'file': @@ -62,12 +67,14 @@ def download_http(request, spider): factory.deferred.addCallbacks(_on_success, _on_error) u = urlparse.urlparse(request.url) + ip = dnscache.get(u.hostname) + port = u.port if u.scheme == 'https' : from twisted.internet import ssl contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(u.hostname, u.port or 443, factory, contextFactory) + reactor.connectSSL(ip, port or 443, factory, contextFactory) else: - reactor.connectTCP(u.hostname, u.port or 80, factory) + reactor.connectTCP(ip, port or 80, factory) return factory.deferred def download_file(request, spider) :