CrawlerRunner test for Response.ip_address

tests/CrawlerProcess/ip_address.py

@@ -1,51 +0,0 @@
-from urllib.parse import urlparse
-
-from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
-from twisted.internet.interfaces import IResolverSimple
-from zope.interface.declarations import implementer
-
-from scrapy import Spider, Request
-from scrapy.crawler import CrawlerProcess
-
-from tests.mockserver import MockServer
-
-
-@implementer(IResolverSimple)
-class MockThreadedResolver(ThreadedResolver):
-    """
-    Resolves all names to localhost
-    """
-
-    @classmethod
-    def from_crawler(cls, crawler, reactor):
-        return cls(reactor)
-
-    def install_on_reactor(self):
-        self.reactor.installResolver(self)
-
-    def getHostByName(self, name, timeout=None):
-        return defer.succeed("127.0.0.1")
-
-
-class LocalhostSpider(Spider):
-    name = "localhost_spider"
-
-    def start_requests(self):
-        yield Request(self.url)
-
-    def parse(self, response):
-        netloc = urlparse(response.url).netloc
-        self.logger.info("Host: %s" % netloc.split(":")[0])
-        self.logger.info("Type: %s" % type(response.ip_address))
-        self.logger.info("IP address: %s" % response.ip_address)
-
-
-with MockServer() as mockserver:
-    settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"}
-    process = CrawlerProcess(settings)
-
-    port = urlparse(mockserver.http_address).port
-    url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port)
-    process.crawl(LocalhostSpider, url=url)
-    process.start()

tests/CrawlerRunner/ip_address.py

@@ -0,0 +1,37 @@
+from urllib.parse import urlparse
+
+from twisted.internet import reactor
+from twisted.names.client import createResolver
+
+from scrapy import Spider, Request
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+from tests.mockserver import MockServer, MockDNSServer
+
+
+class LocalhostSpider(Spider):
+    name = "localhost_spider"
+
+    def start_requests(self):
+        yield Request(self.url)
+
+    def parse(self, response):
+        netloc = urlparse(response.url).netloc
+        self.logger.info("Host: %s" % netloc.split(":")[0])
+        self.logger.info("Type: %s" % type(response.ip_address))
+        self.logger.info("IP address: %s" % response.ip_address)
+
+
+with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
+    port = urlparse(mock_http_server.http_address).port
+    url = "http://not.a.real.domain:{port}/echo".format(port=port)
+
+    servers = [(mock_dns_server.host, mock_dns_server.port)]
+    reactor.installResolver(createResolver(servers=servers))
+
+    configure_logging()
+    runner = CrawlerRunner()
+    d = runner.crawl(LocalhostSpider, url=url)
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()

tests/mockserver.py

@@ -246,8 +246,11 @@ class MockDNSResolver:
 class MockDNSServer():
 
     def __enter__(self):
-        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'],
+        self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
                           stdout=PIPE, env=get_testenv())
+        host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
+        self.host = host
+        self.port = int(port)
         return self
 
     def __exit__(self, exc_type, exc_value, traceback):
@@ -292,11 +295,11 @@ if __name__ == "__main__":
         clients = [MockDNSResolver()]
         factory = DNSServerFactory(clients=clients)
         protocol = dns.DNSDatagramProtocol(controller=factory)
-        reactor.listenUDP(10053, protocol)
-        reactor.listenTCP(10053, factory)
+        listener = reactor.listenUDP(0, protocol)
 
         def print_listening():
-            print("DNS server running on port 10053")
+            host = listener.getHost()
+            print("%s:%s" % (host.host, host.port))
 
     reactor.callWhenRunning(print_listening)
     reactor.run()

tests/test_crawler.py

@@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
             self.assertNotIn("Asyncio reactor is installed", str(log))
 
 
-class CrawlerProcessSubprocess(unittest.TestCase):
-    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
-
+class ScriptRunnerMixin:
     def run_script(self, script_name):
         script_path = os.path.join(self.script_dir, script_name)
         args = (sys.executable, script_path)
@@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
         stdout, stderr = p.communicate()
         return stderr.decode('utf-8')
 
+
+class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
     def test_simple(self):
         log = self.run_script('simple.py')
         self.assertIn('Spider closed (finished)', log)
@@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase):
             "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log,
         ]))
 
+
+class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
+    script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
+
     def test_response_ip_address(self):
         log = self.run_script("ip_address.py")
-        self.assertIn("Spider closed (finished)", log)
-        self.assertIn("Host: not.a.real.domain", log)
-        self.assertIn("Type: <class 'ipaddress.IPv4Address'>", log)
-        self.assertIn("IP address: 127.0.0.1", log)
+        self.assertIn("INFO: Spider closed (finished)", log)
+        self.assertIn("INFO: Host: not.a.real.domain", log)
+        self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
+        self.assertIn("INFO: IP address: 127.0.0.1", log)