From 85aa3c7596c6e9c66daaa5503faadd03a16e1d59 Mon Sep 17 00:00:00 2001
From: Alexander Sibiryakov <sixty-one@yandex.ru>
Date: Thu, 2 Apr 2015 18:30:59 +0200
Subject: [PATCH] Dns cache size and timeout options

---
 docs/topics/settings.rst            | 18 ++++++++++++++++++
 scrapy/crawler.py                   |  6 +++---
 scrapy/resolver.py                  |  9 +++++++--
 scrapy/settings/default_settings.py |  2 ++
 4 files changed, 30 insertions(+), 5 deletions(-)

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c39601b14..1986b2f3e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -290,6 +290,24 @@ Default: ``True``
 
 Whether to enable DNS in-memory cache.
 
+.. setting:: DNSCACHE_SIZE
+
+DNSCACHE_SIZE
+----------------
+
+Default: ``10000``
+
+DNS in-memory cache size.
+
+.. setting:: DNS_TIMEOUT
+
+DNS_TIMEOUT
+----------------
+
+Default: ``60``
+
+Timeout for processing of DNS queries in seconds. Float is supported.
+
 .. setting:: DOWNLOADER
 
 DOWNLOADER
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 52d4069ac..efb181fa4 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -149,9 +149,9 @@ class CrawlerProcess(CrawlerRunner):
                 return
             d.addBoth(lambda _: self._stop_reactor())
 
-        if self.settings.getbool('DNSCACHE_ENABLED'):
-            reactor.installResolver(CachingThreadedResolver(reactor))
-
+        cache_size = self.settings.getint('DNSCACHE_SIZE') if self.settings.getbool('DNSCACHE_ENABLED') else 0
+        reactor.installResolver(CachingThreadedResolver(reactor, cache_size,
+                                                            self.settings.getfloat('DNS_TIMEOUT')))
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint('REACTOR_THREADPOOL_MAXSIZE'))
         reactor.addSystemEventTrigger('before', 'shutdown', self.stop)
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index e1a7aedce..3954fd977 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,15 +4,20 @@ from twisted.internet.base import ThreadedResolver
 from scrapy.utils.datatypes import LocalCache
 
 # TODO: cache misses
-# TODO: make cache size a setting
 
 dnscache = LocalCache(10000)
 
 class CachingThreadedResolver(ThreadedResolver):
+    def __init__(self, reactor, cache_size, timeout):
+        super(CachingThreadedResolver, self).__init__(reactor)
+        dnscache.limit = cache_size
+        self.timeout = timeout
 
-    def getHostByName(self, name, timeout = (1, 3, 11, 45)):
+    def getHostByName(self, name, timeout=None):
         if name in dnscache:
             return defer.succeed(dnscache[name])
+        if not timeout:
+            timeout = self.timeout
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
         d.addCallback(self._cache_result, name)
         return d
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 5b8dc4eaa..22005eb5d 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -52,6 +52,8 @@ DEPTH_STATS = True
 DEPTH_PRIORITY = 0
 
 DNSCACHE_ENABLED = True
+DNSCACHE_SIZE = 10000
+DNS_TIMEOUT = 60
 
 DOWNLOAD_DELAY = 0