Test DNS resolution using CrawlerProcess

tests/CrawlerProcess/ip_address.py

@@ -0,0 +1,51 @@
+from urllib.parse import urlparse
+
+from twisted.internet import defer
+from twisted.internet.base import ThreadedResolver
+from twisted.internet.interfaces import IResolverSimple
+from zope.interface.declarations import implementer
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerProcess
+
+from tests.mockserver import MockServer
+
+
+@implementer(IResolverSimple)
+class MockThreadedResolver(ThreadedResolver):
+    """
+    Resolves all names to localhost
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler, reactor):
+        return cls(reactor)
+
+    def install_on_reactor(self,):
+        self.reactor.installResolver(self)
+
+    def getHostByName(self, name, timeout=None):
+        return defer.succeed("127.0.0.1")
+
+
+class LocalhostSpider(Spider):
+    name = "localhost_spider"
+
+    def start_requests(self):
+        yield Request(self.url)
+
+    def parse(self, response):
+        netloc = urlparse(response.url).netloc
+        self.logger.info("Host: %s" % netloc.split(":")[0])
+        self.logger.info("Type: %s" % type(response.ip_address))
+        self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mockserver:
+    settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
+    process = CrawlerProcess(settings)
+
+    port = urlparse(mockserver.http_address).port
+    url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
+    process.crawl(LocalhostSpider, url=url)
+    process.start()

tests/test_crawl.py

@@ -1,6 +1,7 @@
 import json
 import logging
 from ipaddress import IPv4Address
+from socket import gethostbyname
 from urllib.parse import urlparse
 
 from testfixtures import LogCapture
@@ -313,15 +314,6 @@ with multiples lines
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
-        from socket import gethostbyname
-
-        crawler = self.runner.create_crawler(SingleRequestSpider)
-        url = 'https://example.org'
-        yield crawler.crawl(seed=url)
-        ip_address = crawler.spider.meta['responses'][0].ip_address
-        self.assertIsInstance(ip_address, IPv4Address)
-        self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc))
-
         crawler = self.runner.create_crawler(SingleRequestSpider)
         url = self.mockserver.url('/status?n=200')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)

tests/test_crawler.py

@@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
 
     def test_spider_custom_settings_log_level(self):
         log_file = self.mktemp()
+
         class MySpider(scrapy.Spider):
             name = 'spider'
             custom_settings = {
@@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log,
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
+
+    def test_response_ip_address(self):
+        log = self.run_script("ip_address.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Host: not.a.real.domain", log)
+        self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
+        self.assertIn("IP address: 127.0.0.1", log)