From 80925ab845b7f55be97d9bb91015ceee90efc333 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 5 Aug 2019 11:39:07 -0300 Subject: [PATCH 01/16] Get server IP address for HTTP/1.1 responses --- docs/topics/request-response.rst | 12 +++++++++- scrapy/core/downloader/__init__.py | 2 +- scrapy/core/downloader/handlers/http11.py | 18 ++++++++++----- scrapy/http/response/__init__.py | 5 +++-- tests/test_crawl.py | 27 +++++++++++++++++++++++ 5 files changed, 54 insertions(+), 10 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 8997a7f19..a4cc1a7d7 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -34,7 +34,7 @@ Request objects :type url: string :param callback: the function that will be called with the response of this - request (once its downloaded) as its first parameter. For more information + request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's :meth:`~scrapy.spiders.Spider.parse` method will be used. @@ -611,6 +611,12 @@ Response objects This represents the :class:`Request` that generated this response. :type request: :class:`Request` object + :param ip_address: The IP address of the server from which the Response originated. + :type ip_address: :class:`ipaddress.IPv4Address` object + + .. FIXME: Add ipaddress.IPv6Address once it's supported + + .. attribute:: Response.url A string containing the URL of the response. @@ -679,6 +685,10 @@ Response objects they're shown on the string representation of the Response (`__str__` method) which is used by the engine for logging. + .. attribute:: Response.ip_address + + The IP address of the server from which the Response originated. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 157dc3418..11c9dd908 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -172,7 +172,7 @@ class Downloader(object): return response dfd.addCallback(_downloaded) - # 3. After response arrives, remove the request from transferring + # 3. After response arrives, remove the request from transferring # state to free up the transferring slot so it can be used by the # following requests (perhaps those which came from the downloader # middleware itself) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 5a5f6cf0a..b690f439f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -4,6 +4,7 @@ import logging import re import warnings from io import BytesIO +from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -382,7 +383,7 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None + return txresponse, b'', None, None maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -418,11 +419,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags = result + txresponse, body, flags, ip_address = result status = int(txresponse.code) headers = Headers(txresponse.headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, flags=flags) + return respcls(url=url, status=status, headers=headers, body=body, flags=flags, ip_address=ip_address) @implementer(IBodyProducer) @@ -456,6 +457,11 @@ class _ResponseReader(protocol.Protocol): self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 + self._ip_address = None + + def connectionMade(self): + if self._ip_address is None: + self._ip_address = ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. @@ -488,16 +494,16 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None)) + self._finished.callback((self._txresponse, body, None, self._ip_address)) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'])) + self._finished.callback((self._txresponse, body, ['partial'], self._ip_address)) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'])) + self._finished.callback((self._txresponse, body, ['dataloss'], self._ip_address)) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index f92d0901c..ca5ecc02c 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,13 +17,14 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, ip_address=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) self._set_url(url) self.request = request self.flags = [] if flags is None else list(flags) + self.ip_address = ip_address @property def meta(self): @@ -76,7 +77,7 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags']: + for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'ip_address']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/tests/test_crawl.py b/tests/test_crawl.py index f433fcea6..6281160ae 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,5 +1,7 @@ import json import logging +from ipaddress import IPv4Address +from urllib.parse import urlparse from testfixtures import LogCapture from twisted.internet import defer @@ -308,3 +310,28 @@ with multiples lines self.assertIn("[callback] status 201", str(log)) self.assertIn("[errback] status 404", str(log)) self.assertIn("[errback] status 500", str(log)) + + @defer.inlineCallbacks + def test_dns_server_ip_address(self): + from socket import gethostbyname + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = 'https://example.org' + yield crawler.crawl(seed=url) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/status?n=200') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsNone(ip_address) + + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/echo?body=test') + expected_netloc, _ = urlparse(url).netloc.split(':') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) From e8da7e296691d2b4eb63e2a442bb600e03e5766f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 17:53:39 -0300 Subject: [PATCH 02/16] Test DNS resolution using CrawlerProcess --- tests/CrawlerProcess/ip_address.py | 51 ++++++++++++++++++++++++++++++ tests/test_crawl.py | 10 +----- tests/test_crawler.py | 8 +++++ 3 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 tests/CrawlerProcess/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py new file mode 100644 index 000000000..6b069cc90 --- /dev/null +++ b/tests/CrawlerProcess/ip_address.py @@ -0,0 +1,51 @@ +from urllib.parse import urlparse + +from twisted.internet import defer +from twisted.internet.base import ThreadedResolver +from twisted.internet.interfaces import IResolverSimple +from zope.interface.declarations import implementer + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerProcess + +from tests.mockserver import MockServer + + +@implementer(IResolverSimple) +class MockThreadedResolver(ThreadedResolver): + """ + Resolves all names to localhost + """ + + @classmethod + def from_crawler(cls, crawler, reactor): + return cls(reactor) + + def install_on_reactor(self,): + self.reactor.installResolver(self) + + def getHostByName(self, name, timeout=None): + return defer.succeed("127.0.0.1") + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mockserver: + settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} + process = CrawlerProcess(settings) + + port = urlparse(mockserver.http_address).port + url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) + process.crawl(LocalhostSpider, url=url) + process.start() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 6281160ae..9896058dc 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,7 @@ import json import logging from ipaddress import IPv4Address +from socket import gethostbyname from urllib.parse import urlparse from testfixtures import LogCapture @@ -313,15 +314,6 @@ with multiples lines @defer.inlineCallbacks def test_dns_server_ip_address(self): - from socket import gethostbyname - - crawler = self.runner.create_crawler(SingleRequestSpider) - url = 'https://example.org' - yield crawler.crawl(seed=url) - ip_address = crawler.spider.meta['responses'][0].ip_address - self.assertIsInstance(ip_address, IPv4Address) - self.assertEqual(str(ip_address), gethostbyname(urlparse(url).netloc)) - crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 0ce0674de..dfc1cf448 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -107,6 +107,7 @@ class CrawlerLoggingTestCase(unittest.TestCase): def test_spider_custom_settings_log_level(self): log_file = self.mktemp() + class MySpider(scrapy.Spider): name = 'spider' custom_settings = { @@ -323,3 +324,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectionRefusedError': 1," in log, "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + + def test_response_ip_address(self): + log = self.run_script("ip_address.py") + self.assertIn("Spider closed (finished)", log) + self.assertIn("Host: not.a.real.domain", log) + self.assertIn("Type: ", log) + self.assertIn("IP address: 127.0.0.1", log) From 8529dff41d3d2f6c81ee58c60b16dd9f2b8f72b4 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 26 Jan 2020 18:00:56 -0300 Subject: [PATCH 03/16] Update docs regarding Response.ip_address and IPv6 --- docs/topics/request-response.rst | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index a4cc1a7d7..17eb63064 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -612,10 +612,7 @@ Response objects :type request: :class:`Request` object :param ip_address: The IP address of the server from which the Response originated. - :type ip_address: :class:`ipaddress.IPv4Address` object - - .. FIXME: Add ipaddress.IPv6Address once it's supported - + :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` .. attribute:: Response.url From a2ae380efcaa5a3419a4f6a35541ae0fb71a2e7f Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 13:23:52 -0300 Subject: [PATCH 04/16] Remove unnecessary commas --- scrapy/resolver.py | 2 +- tests/CrawlerProcess/ip_address.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index 554a3a14d..f69894b1e 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py index 6b069cc90..949e97172 100644 --- a/tests/CrawlerProcess/ip_address.py +++ b/tests/CrawlerProcess/ip_address.py @@ -21,7 +21,7 @@ class MockThreadedResolver(ThreadedResolver): def from_crawler(cls, crawler, reactor): return cls(reactor) - def install_on_reactor(self,): + def install_on_reactor(self): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From bb8f7dc609382153df79774ad9d8f6d33d064279 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:14 -0300 Subject: [PATCH 05/16] Mock DNS server --- tests/mockserver.py | 90 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..585741f1b 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,4 @@ +import argparse import json import os import random @@ -6,18 +7,19 @@ from subprocess import Popen, PIPE from urllib.parse import urlencode from OpenSSL import SSL -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from twisted.internet import defer, reactor, ssl +from twisted.internet.task import deferLater +from twisted.names import dns, error +from twisted.names.server import DNSServerFactory +from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource -from twisted.web.server import GzipEncoderFactory -from twisted.web.resource import EncodingResourceWrapper from twisted.web.util import redirectTo -from twisted.internet import reactor, ssl -from twisted.internet.task import deferLater from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 +from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -198,12 +200,10 @@ class Root(Resource): return b'Scrapy mock HTTP server\n' -class MockServer(): +class MockServer: def __enter__(self): - from scrapy.utils.test import get_testenv - - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'], stdout=PIPE, env=get_testenv()) http_address = self.proc.stdout.readline().strip().decode('ascii') https_address = self.proc.stdout.readline().strip().decode('ascii') @@ -224,6 +224,37 @@ class MockServer(): return host + path +class MockDNSResolver: + """ + Implements twisted.internet.interfaces.IResolver partially + """ + + def _resolve(self, name): + record = dns.Record_A(address=b"127.0.0.1") + answer = dns.RRHeader(name=name, payload=record) + return [answer], [], [] + + def query(self, query, timeout=None): + if query.type == dns.A: + return defer.succeed(self._resolve(query.name.name)) + return defer.fail(error.DomainError()) + + def lookupAllRecords(self, name, timeout=None): + return defer.succeed(self._resolve(name)) + + +class MockDNSServer(): + + def __enter__(self): + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + stdout=PIPE, env=get_testenv()) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.proc.kill() + self.proc.communicate() + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( os.path.join(os.path.dirname(__file__), keyfile), @@ -238,19 +269,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": - root = Root() - factory = Site(root) - httpPort = reactor.listenTCP(0, factory) - contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http") + args = parser.parse_args() - def print_listening(): - httpHost = httpPort.getHost() - httpsHost = httpsPort.getHost() - httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) - httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) - print(httpAddress) - print(httpsAddress) + if args.type == "http": + root = Root() + factory = Site(root) + httpPort = reactor.listenTCP(0, factory) + contextFactory = ssl_context_factory() + httpsPort = reactor.listenSSL(0, factory, contextFactory) + + def print_listening(): + httpHost = httpPort.getHost() + httpsHost = httpsPort.getHost() + httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) + httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + + elif args.type == "dns": + clients = [MockDNSResolver()] + factory = DNSServerFactory(clients=clients) + protocol = dns.DNSDatagramProtocol(controller=factory) + reactor.listenUDP(10053, protocol) + reactor.listenTCP(10053, factory) + + def print_listening(): + print("DNS server running on port 10053") reactor.callWhenRunning(print_listening) reactor.run() From 4851efdfb0885a40a44a2834c6c69d0104326801 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 14:50:54 -0300 Subject: [PATCH 06/16] Flake8 adjustments --- tests/mockserver.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 585741f1b..67139534e 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -257,9 +257,9 @@ class MockDNSServer(): def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), - ) + os.path.join(os.path.dirname(__file__), keyfile), + os.path.join(os.path.dirname(__file__), certfile), + ) if cipher_string: ctx = factory.getContext() # disabling TLS1.2+ because it unconditionally enables some strong ciphers From e0ef8ad2d6f958de6ce04cd7756e142efeb1a6a2 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 15:52:15 -0300 Subject: [PATCH 07/16] CrawlerRunner test for Response.ip_address --- tests/CrawlerProcess/ip_address.py | 51 ------------------------------ tests/CrawlerRunner/ip_address.py | 37 ++++++++++++++++++++++ tests/mockserver.py | 11 ++++--- tests/test_crawler.py | 20 ++++++++---- 4 files changed, 57 insertions(+), 62 deletions(-) delete mode 100644 tests/CrawlerProcess/ip_address.py create mode 100644 tests/CrawlerRunner/ip_address.py diff --git a/tests/CrawlerProcess/ip_address.py b/tests/CrawlerProcess/ip_address.py deleted file mode 100644 index 949e97172..000000000 --- a/tests/CrawlerProcess/ip_address.py +++ /dev/null @@ -1,51 +0,0 @@ -from urllib.parse import urlparse - -from twisted.internet import defer -from twisted.internet.base import ThreadedResolver -from twisted.internet.interfaces import IResolverSimple -from zope.interface.declarations import implementer - -from scrapy import Spider, Request -from scrapy.crawler import CrawlerProcess - -from tests.mockserver import MockServer - - -@implementer(IResolverSimple) -class MockThreadedResolver(ThreadedResolver): - """ - Resolves all names to localhost - """ - - @classmethod - def from_crawler(cls, crawler, reactor): - return cls(reactor) - - def install_on_reactor(self): - self.reactor.installResolver(self) - - def getHostByName(self, name, timeout=None): - return defer.succeed("127.0.0.1") - - -class LocalhostSpider(Spider): - name = "localhost_spider" - - def start_requests(self): - yield Request(self.url) - - def parse(self, response): - netloc = urlparse(response.url).netloc - self.logger.info("Host: %s" % netloc.split(":")[0]) - self.logger.info("Type: %s" % type(response.ip_address)) - self.logger.info("IP address: %s" % response.ip_address) - - -with MockServer() as mockserver: - settings = {"DNS_RESOLVER": __name__ + ".MockThreadedResolver"} - process = CrawlerProcess(settings) - - port = urlparse(mockserver.http_address).port - url = "http://not.a.real.domain:{port}/echo?body=test".format(port=port) - process.crawl(LocalhostSpider, url=url) - process.start() diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py new file mode 100644 index 000000000..5a71536d8 --- /dev/null +++ b/tests/CrawlerRunner/ip_address.py @@ -0,0 +1,37 @@ +from urllib.parse import urlparse + +from twisted.internet import reactor +from twisted.names.client import createResolver + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + +from tests.mockserver import MockServer, MockDNSServer + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) + + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) + + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index 67139534e..08a81418c 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -246,8 +246,11 @@ class MockDNSResolver: class MockDNSServer(): def __enter__(self): - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', 'dns'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], stdout=PIPE, env=get_testenv()) + host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") + self.host = host + self.port = int(port) return self def __exit__(self, exc_type, exc_value, traceback): @@ -292,11 +295,11 @@ if __name__ == "__main__": clients = [MockDNSResolver()] factory = DNSServerFactory(clients=clients) protocol = dns.DNSDatagramProtocol(controller=factory) - reactor.listenUDP(10053, protocol) - reactor.listenTCP(10053, factory) + listener = reactor.listenUDP(0, protocol) def print_listening(): - print("DNS server running on port 10053") + host = listener.getHost() + print("%s:%s" % (host.host, host.port)) reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/test_crawler.py b/tests/test_crawler.py index dfc1cf448..5d381c368 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -281,9 +281,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertNotIn("Asyncio reactor is installed", str(log)) -class CrawlerProcessSubprocess(unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') - +class ScriptRunnerMixin: def run_script(self, script_name): script_path = os.path.join(self.script_dir, script_name) args = (sys.executable, script_path) @@ -292,6 +290,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): stdout, stderr = p.communicate() return stderr.decode('utf-8') + +class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) @@ -325,9 +327,13 @@ class CrawlerProcessSubprocess(unittest.TestCase): "'downloader/exception_type_count/twisted.internet.error.ConnectError': 1," in log, ])) + +class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + def test_response_ip_address(self): log = self.run_script("ip_address.py") - self.assertIn("Spider closed (finished)", log) - self.assertIn("Host: not.a.real.domain", log) - self.assertIn("Type: ", log) - self.assertIn("IP address: 127.0.0.1", log) + self.assertIn("INFO: Spider closed (finished)", log) + self.assertIn("INFO: Host: not.a.real.domain", log) + self.assertIn("INFO: Type: ", log) + self.assertIn("INFO: IP address: 127.0.0.1", log) From 13670f0397ba8dcec3dceb1852bad5751406d19d Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 3 Feb 2020 16:16:43 -0300 Subject: [PATCH 08/16] Ignore tests/CrawlerRunner directory --- conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conftest.py b/conftest.py index c0de09909..55294feca 100644 --- a/conftest.py +++ b/conftest.py @@ -11,7 +11,8 @@ collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess - *_py_files("tests/CrawlerProcess") + *_py_files("tests/CrawlerProcess"), + *_py_files("tests/CrawlerRunner"), ] for line in open('tests/ignores.txt'): From ad70497416527c3d882a64f7803e73155f3fa1da Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 4 Feb 2020 13:30:13 -0300 Subject: [PATCH 09/16] Remove unnecessary parentheses in class definition --- tests/mockserver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 08a81418c..30d9bc0e8 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -243,7 +243,7 @@ class MockDNSResolver: return defer.succeed(self._resolve(name)) -class MockDNSServer(): +class MockDNSServer: def __enter__(self): self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], From 13ba9bc629cb0a77ebaca36a10a0a4984d7cce68 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 10 Feb 2020 12:29:39 -0300 Subject: [PATCH 10/16] Note about Response.ip_address --- docs/topics/request-response.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 17eb63064..89e570028 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,6 +685,8 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. .. method:: Response.copy() From 037ae5b22e6d6600dc537ee5073652ce74e5f47b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Mon, 10 Feb 2020 19:54:47 +0100 Subject: [PATCH 11/16] =?UTF-8?q?Explicitly=20indicate=20None=20as=20ip=5F?= =?UTF-8?q?address=E2=80=99s=20default=20value?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/topics/request-response.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 89e570028..8f2504a33 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -685,8 +685,10 @@ Response objects .. attribute:: Response.ip_address The IP address of the server from which the Response originated. + This attribute is currently only populated by the HTTP 1.1 download - handler, i.e. for ``http(s)`` responses. + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`ip_address` is always ``None``. .. method:: Response.copy() From f85bf77da3c8943f0791dcae893e8294c4d118d7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:31:13 -0300 Subject: [PATCH 12/16] Restore unrelated change --- scrapy/resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/resolver.py b/scrapy/resolver.py index f69894b1e..554a3a14d 100644 --- a/scrapy/resolver.py +++ b/scrapy/resolver.py @@ -29,7 +29,7 @@ class CachingThreadedResolver(ThreadedResolver): cache_size = 0 return cls(reactor, cache_size, crawler.settings.getfloat('DNS_TIMEOUT')) - def install_on_reactor(self): + def install_on_reactor(self,): self.reactor.installResolver(self) def getHostByName(self, name, timeout=None): From 889b4718520220d1a81e702ff754ec210a7d3c79 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 23 Feb 2020 18:40:43 -0300 Subject: [PATCH 13/16] Import changes --- scrapy/core/downloader/handlers/http11.py | 4 ++-- tests/test_crawl.py | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e72275021..190ae1d3b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,11 +1,11 @@ """Download handlers for http and https schemes""" +import ipaddress import logging import re import warnings from contextlib import suppress from io import BytesIO -from ipaddress import ip_address from time import time from urllib.parse import urldefrag @@ -468,7 +468,7 @@ class _ResponseReader(protocol.Protocol): self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) if self._ip_address is None: - self._ip_address = ip_address(self.transport._producer.getPeer().host) + self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host) def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 3a9b00ab3..3c110e7a6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -441,13 +441,15 @@ with multiples lines self.assertEqual(cert.getIssuer().commonName, b"localhost") @defer.inlineCallbacks - def test_dns_server_ip_address(self): + def test_dns_server_ip_address_none(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/status?n=200') yield crawler.crawl(seed=url, mockserver=self.mockserver) ip_address = crawler.spider.meta['responses'][0].ip_address self.assertIsNone(ip_address) + @defer.inlineCallbacks + def test_dns_server_ip_address(self): crawler = self.runner.create_crawler(SingleRequestSpider) url = self.mockserver.url('/echo?body=test') expected_netloc, _ = urlparse(url).netloc.split(':') From 91a78eef3ee9de033e66db55c49321b2cc43740e Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sun, 8 Mar 2020 22:32:17 -0300 Subject: [PATCH 14/16] Pass callback results as dicts instead of tuples --- scrapy/core/downloader/handlers/http11.py | 56 ++++++++++++++++------- 1 file changed, 40 insertions(+), 16 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 190ae1d3b..e904cbc05 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -384,7 +384,13 @@ class ScrapyAgent(object): def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None, None + return { + "txresponse": txresponse, + "body": b"", + "flags": None, + "certificate": None, + "ip_address": None, + } maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -420,12 +426,18 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags, certificate, ip_address = result - status = int(txresponse.code) - headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, - flags=flags, certificate=certificate, ip_address=ip_address) + status = int(result["txresponse"].code) + headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + return respcls( + url=url, + status=status, + headers=headers, + body=result["body"], + flags=result["flags"], + certificate=result["certificate"], + ip_address=result["ip_address"], + ) @implementer(IBodyProducer) @@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback( - (self._txresponse, body, None, self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": None, + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(PotentialDataLoss): - self._finished.callback( - (self._txresponse, body, ['partial'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["partial"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback( - (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address) - ) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["dataloss"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return elif not self._fail_on_dataloss_warned: From 1785095707dec53647c835c0b0861b220e8495af Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 11 Mar 2020 20:41:59 -0300 Subject: [PATCH 15/16] Remove single-use variable --- scrapy/core/downloader/handlers/http11.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index e904cbc05..a5b03a62b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -426,12 +426,11 @@ class ScrapyAgent(object): return d def _cb_bodydone(self, result, request, url): - status = int(result["txresponse"].code) headers = Headers(result["txresponse"].headers.getAllRawHeaders()) respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) return respcls( url=url, - status=status, + status=int(result["txresponse"].code), headers=headers, body=result["body"], flags=result["flags"], From c9229922772a4d7f92a26786d6ea441609043a09 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Thu, 16 Apr 2020 11:37:37 -0300 Subject: [PATCH 16/16] Tests: Move code inside __main__ block --- tests/CrawlerRunner/ip_address.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py index 5a71536d8..826374cd4 100644 --- a/tests/CrawlerRunner/ip_address.py +++ b/tests/CrawlerRunner/ip_address.py @@ -23,15 +23,16 @@ class LocalhostSpider(Spider): self.logger.info("IP address: %s" % response.ip_address) -with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: - port = urlparse(mock_http_server.http_address).port - url = "http://not.a.real.domain:{port}/echo".format(port=port) +if __name__ == "__main__": + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) - servers = [(mock_dns_server.host, mock_dns_server.port)] - reactor.installResolver(createResolver(servers=servers)) + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) - configure_logging() - runner = CrawlerRunner() - d = runner.crawl(LocalhostSpider, url=url) - d.addBoth(lambda _: reactor.stop()) - reactor.run() + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run()