From a767b7044e8a2b1311ec624098be764fd586b627 Mon Sep 17 00:00:00 2001 From: Pablo Hoffman Date: Thu, 20 Aug 2009 17:11:12 -0300 Subject: [PATCH] removed old blocking caching DNS resolver and replaced by a non-blocking one installed as the default reactor resolver --HG-- rename : scrapy/core/downloader/dnscache.py => scrapy/core/downloader/resolver.py --- scrapy/core/downloader/dnscache.py | 52 ------------------------------ scrapy/core/downloader/handlers.py | 10 ++---- scrapy/core/downloader/manager.py | 7 ++-- scrapy/core/downloader/resolver.py | 29 +++++++++++++++++ 4 files changed, 36 insertions(+), 62 deletions(-) delete mode 100644 scrapy/core/downloader/dnscache.py create mode 100644 scrapy/core/downloader/resolver.py diff --git a/scrapy/core/downloader/dnscache.py b/scrapy/core/downloader/dnscache.py deleted file mode 100644 index 5ad6be7e6..000000000 --- a/scrapy/core/downloader/dnscache.py +++ /dev/null @@ -1,52 +0,0 @@ -""" -Dns cache module. -This module implements a dns cache to improve the performance of the -crawler, reducing the dns lookups. -""" -import socket -from scrapy.core import signals -from scrapy.xlib.pydispatch import dispatcher - -class DNSCache(object): - """ - DNSCache. - - Impelements a DNS Cache to improve the performance of the - DNS lookup that the crawler request when it's running. - Use: - Create a single instance: - >>> import dnscache - >>> cachedns = dnscache.DNSCache() - - To get (and set a new host if not exists) we only call - >> ip = cachedns.get('python.org') - >>> print ip - '82.94.164.162' - >>> - """ - def __init__(self): - dispatcher.connect(self.domain_closed, signal=signals.domain_closed) - self._cache = {} - - def get(self, host): - """ - Returns the ip associated with the host and save it in - the cache. - """ - try: - ips = self._cache[host] - except KeyError: - # The socket.gethostbyname_ex throw an - # exception when it can't get the host - # ip. So we save the hostname anyway. - # we use socket.getaddrinfo that support IP4/IP6 - try: - ips = list(set([x[4][0] for x in socket.getaddrinfo(host,None)])) - except socket.gaierror: - ips = [host] - self._cache[host] = ips - return ips[0] - - def domain_closed(self, domain, spider, reason): - if domain in self._cache: - del self._cache[domain] diff --git a/scrapy/core/downloader/handlers.py b/scrapy/core/downloader/handlers.py index 21cf1ed50..df5bf1903 100644 --- a/scrapy/core/downloader/handlers.py +++ b/scrapy/core/downloader/handlers.py @@ -18,7 +18,6 @@ from scrapy.core.exceptions import NotSupported from scrapy.utils.defer import defer_succeed from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.signal import send_catch_log -from scrapy.core.downloader.dnscache import DNSCache from scrapy.core.downloader.responsetypes import responsetypes from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory from scrapy.conf import settings @@ -27,9 +26,6 @@ from scrapy.conf import settings default_timeout = settings.getint('DOWNLOAD_TIMEOUT') ssl_supported = 'ssl' in optional_features -# Cache for dns lookups. -dnscache = DNSCache() - def download_any(request, spider): scheme = urlparse_cached(request).scheme if scheme == 'http': @@ -69,19 +65,17 @@ def download_http(request, spider): """Return a deferred for the HTTP download""" factory = create_factory(request, spider) url = urlparse_cached(request) - ip = dnscache.get(url.hostname) port = url.port - reactor.connectTCP(ip, port or 80, factory) + reactor.connectTCP(url.hostname, port or 80, factory) return factory.deferred def download_https(request, spider): """Return a deferred for the HTTPS download""" factory = create_factory(request, spider) url = urlparse_cached(request) - ip = dnscache.get(url.hostname) port = url.port contextFactory = ssl.ClientContextFactory() - reactor.connectSSL(ip, port or 443, factory, contextFactory) + reactor.connectSSL(url.hostname, port or 443, factory, contextFactory) return factory.deferred def download_file(request, spider) : diff --git a/scrapy/core/downloader/manager.py b/scrapy/core/downloader/manager.py index 33eb20189..ef513418c 100644 --- a/scrapy/core/downloader/manager.py +++ b/scrapy/core/downloader/manager.py @@ -8,11 +8,12 @@ from twisted.internet import reactor, defer from scrapy.core.exceptions import IgnoreRequest from scrapy.spider import spiders -from scrapy.core.downloader.middleware import DownloaderMiddlewareManager -from scrapy.core.downloader.handlers import download_any from scrapy.conf import settings from scrapy.utils.defer import mustbe_deferred from scrapy import log +from .middleware import DownloaderMiddlewareManager +from .handlers import download_any +from .resolver import CachingThreadedResolver class SiteInfo(object): @@ -54,6 +55,8 @@ class Downloader(object): self.sites = {} self.middleware = DownloaderMiddlewareManager() self.concurrent_domains = settings.getint('CONCURRENT_DOMAINS') + cached_resolver = CachingThreadedResolver(reactor) + reactor.installResolver(cached_resolver) def fetch(self, request, spider): """ Main method to use to request a download diff --git a/scrapy/core/downloader/resolver.py b/scrapy/core/downloader/resolver.py new file mode 100644 index 000000000..596f5b3e4 --- /dev/null +++ b/scrapy/core/downloader/resolver.py @@ -0,0 +1,29 @@ +"""DNS resolver with cache to use with Twisted reactors""" + +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver + +from scrapy.core import signals +from scrapy.xlib.pydispatch import dispatcher + +class CachingThreadedResolver(ThreadedResolver): + + def __init__(self, reactor): + ThreadedResolver.__init__(self, reactor) + self._cache = {} + dispatcher.connect(self.domain_closed, signal=signals.domain_closed) + + def getHostByName(self, name, timeout = (1, 3, 11, 45)): + if name in self._cache: + return defer.succeed(self._cache[name]) + dfd = ThreadedResolver.getHostByName(self, name, timeout) + dfd.addCallback(self._cache_result, name) + return dfd + + def _cache_result(self, result, name): + self._cache[name] = result + return result + + def domain_closed(self, spider): + for domain in [spider.domain_name] + spider.extra_domain_names: + self._cache.pop(domain, None)