Added cached DNS resolver based on old caching resolver extension from scrapy.contrib.resolver. This new one is *not* an extension, it comes builtin and always enabled.

docs/intro/overview.rst

@@ -248,6 +248,8 @@ scraping easy and efficient, such as:
 
 * Support for crawling based on URLs discovered through `Sitemaps`_
 
+* A caching DNS resolver
+
 What's next?
 ============
 

scrapy/contrib/resolver.py

@@ -1,46 +0,0 @@
-from collections import defaultdict
-
-from twisted.internet import reactor, defer
-from twisted.internet.base import ThreadedResolver
-
-from scrapy.xlib.pydispatch import dispatcher
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy import signals
-
-
-class CachingResolver(object):
-    """Scrapy extension to use a caching resolver, instead of default one"""
-
-    def __init__(self):
-        self.spider_hostnames = defaultdict(set)
-        self.resolver = _CachingThreadedResolver(reactor)
-        reactor.installResolver(self.resolver)
-        dispatcher.connect(self.request_received, signals.request_received)
-        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
-
-    def request_received(self, request, spider):
-        url_hostname = urlparse_cached(request).hostname
-        self.spider_hostnames[spider].add(url_hostname)
-
-    def spider_closed(self, spider):
-        for hostname in self.spider_hostnames[spider]:
-            self.resolver._cache.pop(hostname, None)
-        self.spider_hostnames.pop(spider, None)
-
-
-class _CachingThreadedResolver(ThreadedResolver):
-
-    def __init__(self, *args, **kwargs):
-        ThreadedResolver.__init__(self, *args, **kwargs)
-        self._cache = {}
-
-    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
-        if name in self._cache:
-            return defer.succeed(self._cache[name])
-        dfd = ThreadedResolver.getHostByName(self, name, timeout)
-        dfd.addCallback(self._cache_result, name)
-        return dfd
-
-    def _cache_result(self, result, name):
-        self._cache[name] = result
-        return result

scrapy/crawler.py

@@ -4,6 +4,7 @@ from twisted.internet import reactor, defer
 
 from scrapy.xlib.pydispatch import dispatcher
 from scrapy.core.engine import ExecutionEngine
+from scrapy.resolver import CachingThreadedResolver
 from scrapy.extension import ExtensionManager
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
 from scrapy.utils.misc import load_object
@@ -69,6 +70,7 @@ class CrawlerProcess(Crawler):
 
     def start(self):
         super(CrawlerProcess, self).start()
+        reactor.installResolver(CachingThreadedResolver(reactor))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
         reactor.run(installSignalHandlers=False) # blocking call
 

scrapy/resolver.py

@@ -0,0 +1,27 @@
+import socket
+
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+
+from scrapy.utils.datatypes import LocalCache
+
+
+dnscache = LocalCache(10000) # XXX: make size a setting?
+
+def gethostbyname(hostname):
+    if hostname not in dnscache:
+        dnscache[hostname] = socket.gethostbyname(hostname)
+    return dnscache[hostname]
+
+class CachingThreadedResolver(ThreadedResolver):
+
+    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
+        if name in dnscache:
+            return defer.succeed(dnscache[name])
+        d = ThreadedResolver.getHostByName(self, name, timeout)
+        d.addCallback(self._cache_result, name)
+        return d
+
+    def _cache_result(self, result, name):
+        dnscache[name] = result
+        return result

scrapy/utils/datatypes.py

@@ -9,6 +9,9 @@ import copy
 from collections import deque, defaultdict
 from itertools import chain
 
+from scrapy.utils.py27 import OrderedDict
+
+
 class MultiValueDictKeyError(KeyError):
     pass
 
@@ -342,3 +345,19 @@ class PriorityStack(PriorityQueue):
         else:
             self.positems[priority].append(item)
 
+
+class LocalCache(OrderedDict):
+    """Dictionary with a finite number of keys.
+
+    Older items expires first.
+
+    """
+
+    def __init__(self, limit=None):
+        super(LocalCache, self).__init__()
+        self.limit = limit
+
+    def __setitem__(self, key, value):
+        while len(self) >= self.limit:
+            self.popitem(last=False)
+        super(LocalCache, self).__setitem__(key, value)

scrapy/utils/py27.py

@@ -0,0 +1,10 @@
+"""
+Similar to scrapy.utils.py26, but for Python 2.7
+"""
+
+__all__ = ['OrderedDict']
+
+try:
+    from collections import OrderedDict
+except ImportError:
+    from scrapy.xlib.ordereddict import OrderedDict

scrapy/xlib/ordereddict.py

@@ -0,0 +1,127 @@
+# Copyright (c) 2009 Raymond Hettinger
+#
+# Permission is hereby granted, free of charge, to any person
+# obtaining a copy of this software and associated documentation files
+# (the "Software"), to deal in the Software without restriction,
+# including without limitation the rights to use, copy, modify, merge,
+# publish, distribute, sublicense, and/or sell copies of the Software,
+# and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+#     The above copyright notice and this permission notice shall be
+#     included in all copies or substantial portions of the Software.
+#
+#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+#     OTHER DEALINGS IN THE SOFTWARE.
+
+from UserDict import DictMixin
+
+class OrderedDict(dict, DictMixin):
+
+    def __init__(self, *args, **kwds):
+        if len(args) > 1:
+            raise TypeError('expected at most 1 arguments, got %d' % len(args))
+        try:
+            self.__end
+        except AttributeError:
+            self.clear()
+        self.update(*args, **kwds)
+
+    def clear(self):
+        self.__end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.__map = {}                 # key --> [key, prev, next]
+        dict.clear(self)
+
+    def __setitem__(self, key, value):
+        if key not in self:
+            end = self.__end
+            curr = end[1]
+            curr[2] = end[1] = self.__map[key] = [key, curr, end]
+        dict.__setitem__(self, key, value)
+
+    def __delitem__(self, key):
+        dict.__delitem__(self, key)
+        key, prev, next = self.__map.pop(key)
+        prev[2] = next
+        next[1] = prev
+
+    def __iter__(self):
+        end = self.__end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.__end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def popitem(self, last=True):
+        if not self:
+            raise KeyError('dictionary is empty')
+        if last:
+            key = reversed(self).next()
+        else:
+            key = iter(self).next()
+        value = self.pop(key)
+        return key, value
+
+    def __reduce__(self):
+        items = [[k, self[k]] for k in self]
+        tmp = self.__map, self.__end
+        del self.__map, self.__end
+        inst_dict = vars(self).copy()
+        self.__map, self.__end = tmp
+        if inst_dict:
+            return (self.__class__, (items,), inst_dict)
+        return self.__class__, (items,)
+
+    def keys(self):
+        return list(self)
+
+    setdefault = DictMixin.setdefault
+    update = DictMixin.update
+    pop = DictMixin.pop
+    values = DictMixin.values
+    items = DictMixin.items
+    iterkeys = DictMixin.iterkeys
+    itervalues = DictMixin.itervalues
+    iteritems = DictMixin.iteritems
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, self.items())
+
+    def copy(self):
+        return self.__class__(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        d = cls()
+        for key in iterable:
+            d[key] = value
+        return d
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedDict):
+            if len(self) != len(other):
+                return False
+            for p, q in  zip(self.items(), other.items()):
+                if p != q:
+                    return False
+            return True
+        return dict.__eq__(self, other)
+
+    def __ne__(self, other):
+        return not self == other