Merge pull request #2816 from redapple/dns-cache-disabled

Fix DNS resolver when DNSCACHE_ENABLED=False

scrapy/resolver.py

@@ -22,7 +22,8 @@ class CachingThreadedResolver(ThreadedResolver):
         # to enforce Scrapy's DNS_TIMEOUT setting's value
         timeout = (self.timeout,)
         d = super(CachingThreadedResolver, self).getHostByName(name, timeout)
-        d.addCallback(self._cache_result, name)
+        if dnscache.limit:
+            d.addCallback(self._cache_result, name)
         return d
 
     def _cache_result(self, result, name):

tests/test_commands.py

@@ -226,6 +226,27 @@ class MySpider(scrapy.Spider):
         self.assertNotIn("DEBUG: It Works!", log)
         self.assertIn("INFO: Spider opened", log)
 
+    def test_runspider_dnscache_disabled(self):
+        # see https://github.com/scrapy/scrapy/issues/2811
+        # The spider below should not be able to connect to localhost:12345,
+        # which is intended,
+        # but this should not be because of DNS lookup error
+        # assumption: localhost will resolve in all cases (true?)
+        log = self.get_log("""
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+    start_urls = ['http://localhost:12345']
+
+    def parse(self, response):
+        return {'test': 'value'}
+""",
+                           args=('-s', 'DNSCACHE_ENABLED=False'))
+        print(log)
+        self.assertNotIn("DNSLookupError", log)
+        self.assertIn("INFO: Spider opened", log)
+
     def test_runspider_log_short_names(self):
         log1 = self.get_log(self.debug_log_spider,
                             args=('-s', 'LOG_SHORT_NAMES=1'))