diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst index 25dd4c11d..d35923435 100644 --- a/docs/intro/overview.rst +++ b/docs/intro/overview.rst @@ -248,6 +248,8 @@ scraping easy and efficient, such as: * Support for crawling based on URLs discovered through `Sitemaps`_ +* A caching DNS resolver + What's next? ============ diff --git a/scrapy/contrib/resolver.py b/scrapy/contrib/resolver.py deleted file mode 100644 index 8bf04c25f..000000000 --- a/scrapy/contrib/resolver.py +++ /dev/null @@ -1,46 +0,0 @@ -from collections import defaultdict - -from twisted.internet import reactor, defer -from twisted.internet.base import ThreadedResolver - -from scrapy.xlib.pydispatch import dispatcher -from scrapy.utils.httpobj import urlparse_cached -from scrapy import signals - - -class CachingResolver(object): - """Scrapy extension to use a caching resolver, instead of default one""" - - def __init__(self): - self.spider_hostnames = defaultdict(set) - self.resolver = _CachingThreadedResolver(reactor) - reactor.installResolver(self.resolver) - dispatcher.connect(self.request_received, signals.request_received) - dispatcher.connect(self.spider_closed, signal=signals.spider_closed) - - def request_received(self, request, spider): - url_hostname = urlparse_cached(request).hostname - self.spider_hostnames[spider].add(url_hostname) - - def spider_closed(self, spider): - for hostname in self.spider_hostnames[spider]: - self.resolver._cache.pop(hostname, None) - self.spider_hostnames.pop(spider, None) - - -class _CachingThreadedResolver(ThreadedResolver): - - def __init__(self, *args, **kwargs): - ThreadedResolver.__init__(self, *args, **kwargs) - self._cache = {} - - def getHostByName(self, name, timeout = (1, 3, 11, 45)): - if name in self._cache: - return defer.succeed(self._cache[name]) - dfd = ThreadedResolver.getHostByName(self, name, timeout) - dfd.addCallback(self._cache_result, name) - return dfd - - def _cache_result(self, result, name): - self._cache[name] = result - return result diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4c81fa417..ceeae57d6 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,6 +4,7 @@ from twisted.internet import reactor, defer from scrapy.xlib.pydispatch import dispatcher from scrapy.core.engine import ExecutionEngine +from scrapy.resolver import CachingThreadedResolver from scrapy.extension import ExtensionManager from scrapy.utils.ossignal import install_shutdown_handlers, signal_names from scrapy.utils.misc import load_object @@ -69,6 +70,7 @@ class CrawlerProcess(Crawler): def start(self): super(CrawlerProcess, self).start() + reactor.installResolver(CachingThreadedResolver(reactor)) reactor.addSystemEventTrigger('before', 'shutdown', self.stop) reactor.run(installSignalHandlers=False) # blocking call diff --git a/scrapy/resolver.py b/scrapy/resolver.py new file mode 100644 index 000000000..af8f3d69b --- /dev/null +++ b/scrapy/resolver.py @@ -0,0 +1,27 @@ +import socket + +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver + +from scrapy.utils.datatypes import LocalCache + + +dnscache = LocalCache(10000) # XXX: make size a setting? + +def gethostbyname(hostname): + if hostname not in dnscache: + dnscache[hostname] = socket.gethostbyname(hostname) + return dnscache[hostname] + +class CachingThreadedResolver(ThreadedResolver): + + def getHostByName(self, name, timeout = (1, 3, 11, 45)): + if name in dnscache: + return defer.succeed(dnscache[name]) + d = ThreadedResolver.getHostByName(self, name, timeout) + d.addCallback(self._cache_result, name) + return d + + def _cache_result(self, result, name): + dnscache[name] = result + return result diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py index 621934657..73945ea3e 100644 --- a/scrapy/utils/datatypes.py +++ b/scrapy/utils/datatypes.py @@ -9,6 +9,9 @@ import copy from collections import deque, defaultdict from itertools import chain +from scrapy.utils.py27 import OrderedDict + + class MultiValueDictKeyError(KeyError): pass @@ -342,3 +345,19 @@ class PriorityStack(PriorityQueue): else: self.positems[priority].append(item) + +class LocalCache(OrderedDict): + """Dictionary with a finite number of keys. + + Older items expires first. + + """ + + def __init__(self, limit=None): + super(LocalCache, self).__init__() + self.limit = limit + + def __setitem__(self, key, value): + while len(self) >= self.limit: + self.popitem(last=False) + super(LocalCache, self).__setitem__(key, value) diff --git a/scrapy/utils/py27.py b/scrapy/utils/py27.py new file mode 100644 index 000000000..ca523d851 --- /dev/null +++ b/scrapy/utils/py27.py @@ -0,0 +1,10 @@ +""" +Similar to scrapy.utils.py26, but for Python 2.7 +""" + +__all__ = ['OrderedDict'] + +try: + from collections import OrderedDict +except ImportError: + from scrapy.xlib.ordereddict import OrderedDict diff --git a/scrapy/xlib/ordereddict.py b/scrapy/xlib/ordereddict.py new file mode 100644 index 000000000..5b0303f5a --- /dev/null +++ b/scrapy/xlib/ordereddict.py @@ -0,0 +1,127 @@ +# Copyright (c) 2009 Raymond Hettinger +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +from UserDict import DictMixin + +class OrderedDict(dict, DictMixin): + + def __init__(self, *args, **kwds): + if len(args) > 1: + raise TypeError('expected at most 1 arguments, got %d' % len(args)) + try: + self.__end + except AttributeError: + self.clear() + self.update(*args, **kwds) + + def clear(self): + self.__end = end = [] + end += [None, end, end] # sentinel node for doubly linked list + self.__map = {} # key --> [key, prev, next] + dict.clear(self) + + def __setitem__(self, key, value): + if key not in self: + end = self.__end + curr = end[1] + curr[2] = end[1] = self.__map[key] = [key, curr, end] + dict.__setitem__(self, key, value) + + def __delitem__(self, key): + dict.__delitem__(self, key) + key, prev, next = self.__map.pop(key) + prev[2] = next + next[1] = prev + + def __iter__(self): + end = self.__end + curr = end[2] + while curr is not end: + yield curr[0] + curr = curr[2] + + def __reversed__(self): + end = self.__end + curr = end[1] + while curr is not end: + yield curr[0] + curr = curr[1] + + def popitem(self, last=True): + if not self: + raise KeyError('dictionary is empty') + if last: + key = reversed(self).next() + else: + key = iter(self).next() + value = self.pop(key) + return key, value + + def __reduce__(self): + items = [[k, self[k]] for k in self] + tmp = self.__map, self.__end + del self.__map, self.__end + inst_dict = vars(self).copy() + self.__map, self.__end = tmp + if inst_dict: + return (self.__class__, (items,), inst_dict) + return self.__class__, (items,) + + def keys(self): + return list(self) + + setdefault = DictMixin.setdefault + update = DictMixin.update + pop = DictMixin.pop + values = DictMixin.values + items = DictMixin.items + iterkeys = DictMixin.iterkeys + itervalues = DictMixin.itervalues + iteritems = DictMixin.iteritems + + def __repr__(self): + if not self: + return '%s()' % (self.__class__.__name__,) + return '%s(%r)' % (self.__class__.__name__, self.items()) + + def copy(self): + return self.__class__(self) + + @classmethod + def fromkeys(cls, iterable, value=None): + d = cls() + for key in iterable: + d[key] = value + return d + + def __eq__(self, other): + if isinstance(other, OrderedDict): + if len(self) != len(other): + return False + for p, q in zip(self.items(), other.items()): + if p != q: + return False + return True + return dict.__eq__(self, other) + + def __ne__(self, other): + return not self == other