diff --git a/conftest.py b/conftest.py index be5fbabf4..b39d644a5 100644 --- a/conftest.py +++ b/conftest.py @@ -12,6 +12,8 @@ collect_ignore = [ "scrapy/utils/testsite.py", # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess *_py_files("tests/CrawlerProcess"), + # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess + *_py_files("tests/CrawlerRunner"), # Py36-only parts of respective tests *_py_files("tests/py36"), ] diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst index 573efc05f..5eb4915cd 100644 --- a/docs/topics/request-response.rst +++ b/docs/topics/request-response.rst @@ -36,7 +36,7 @@ Request objects :type url: string :param callback: the function that will be called with the response of this - request (once its downloaded) as its first parameter. For more information + request (once it's downloaded) as its first parameter. For more information see :ref:`topics-request-response-ref-request-callback-arguments` below. If a Request doesn't specify a callback, the spider's :meth:`~scrapy.spiders.Spider.parse` method will be used. @@ -616,6 +616,9 @@ Response objects :param certificate: an object representing the server's SSL certificate. :type certificate: twisted.internet.ssl.Certificate + :param ip_address: The IP address of the server from which the Response originated. + :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address` + .. attribute:: Response.url A string containing the URL of the response. @@ -705,6 +708,14 @@ Response objects Only populated for ``https`` responses, ``None`` otherwise. + .. attribute:: Response.ip_address + + The IP address of the server from which the Response originated. + + This attribute is currently only populated by the HTTP 1.1 download + handler, i.e. for ``http(s)`` responses. For other handlers, + :attr:`ip_address` is always ``None``. + .. method:: Response.copy() Returns a new Response which is a copy of this Response. diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py index 36aca4dae..dc5cf1ab8 100644 --- a/scrapy/core/downloader/__init__.py +++ b/scrapy/core/downloader/__init__.py @@ -173,7 +173,7 @@ class Downloader: return response dfd.addCallback(_downloaded) - # 3. After response arrives, remove the request from transferring + # 3. After response arrives, remove the request from transferring # state to free up the transferring slot so it can be used by the # following requests (perhaps those which came from the downloader # middleware itself) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 09f828419..9be8ffdfb 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,5 +1,6 @@ """Download handlers for http and https schemes""" +import ipaddress import logging import re import warnings @@ -373,7 +374,13 @@ class ScrapyAgent: def _cb_bodyready(self, txresponse, request): # deliverBody hangs for responses without body if txresponse.length == 0: - return txresponse, b'', None, None + return { + "txresponse": txresponse, + "body": b"", + "flags": None, + "certificate": None, + "ip_address": None, + } maxsize = request.meta.get('download_maxsize', self._maxsize) warnsize = request.meta.get('download_warnsize', self._warnsize) @@ -409,12 +416,17 @@ class ScrapyAgent: return d def _cb_bodydone(self, result, request, url): - txresponse, body, flags, certificate = result - status = int(txresponse.code) - headers = Headers(txresponse.headers.getAllRawHeaders()) - respcls = responsetypes.from_args(headers=headers, url=url, body=body) - return respcls(url=url, status=status, headers=headers, body=body, - flags=flags, certificate=certificate) + headers = Headers(result["txresponse"].headers.getAllRawHeaders()) + respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"]) + return respcls( + url=url, + status=int(result["txresponse"].code), + headers=headers, + body=result["body"], + flags=result["flags"], + certificate=result["certificate"], + ip_address=result["ip_address"], + ) @implementer(IBodyProducer) @@ -449,12 +461,16 @@ class _ResponseReader(protocol.Protocol): self._reached_warnsize = False self._bytes_received = 0 self._certificate = None + self._ip_address = None def connectionMade(self): if self._certificate is None: with suppress(AttributeError): self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate()) + if self._ip_address is None: + self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host) + def dataReceived(self, bodyBytes): # This maybe called several times after cancel was called with buffered data. if self._finished.called: @@ -486,16 +502,34 @@ class _ResponseReader(protocol.Protocol): body = self._bodybuf.getvalue() if reason.check(ResponseDone): - self._finished.callback((self._txresponse, body, None, self._certificate)) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": None, + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(PotentialDataLoss): - self._finished.callback((self._txresponse, body, ['partial'], self._certificate)) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["partial"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons): if not self._fail_on_dataloss: - self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate)) + self._finished.callback({ + "txresponse": self._txresponse, + "body": body, + "flags": ["dataloss"], + "certificate": self._certificate, + "ip_address": self._ip_address, + }) return elif not self._fail_on_dataloss_warned: diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 682cec161..c2c37dd1d 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -17,7 +17,8 @@ from scrapy.utils.trackref import object_ref class Response(object_ref): - def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None): + def __init__(self, url, status=200, headers=None, body=b'', flags=None, + request=None, certificate=None, ip_address=None): self.headers = Headers(headers or {}) self.status = int(status) self._set_body(body) @@ -25,6 +26,7 @@ class Response(object_ref): self.request = request self.flags = [] if flags is None else list(flags) self.certificate = certificate + self.ip_address = ip_address @property def cb_kwargs(self): @@ -87,7 +89,8 @@ class Response(object_ref): """Create a new Response with the same attributes except for those given new values. """ - for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']: + for x in ['url', 'status', 'headers', 'body', + 'request', 'flags', 'certificate', 'ip_address']: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop('cls', self.__class__) return cls(*args, **kwargs) diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py new file mode 100644 index 000000000..826374cd4 --- /dev/null +++ b/tests/CrawlerRunner/ip_address.py @@ -0,0 +1,38 @@ +from urllib.parse import urlparse + +from twisted.internet import reactor +from twisted.names.client import createResolver + +from scrapy import Spider, Request +from scrapy.crawler import CrawlerRunner +from scrapy.utils.log import configure_logging + +from tests.mockserver import MockServer, MockDNSServer + + +class LocalhostSpider(Spider): + name = "localhost_spider" + + def start_requests(self): + yield Request(self.url) + + def parse(self, response): + netloc = urlparse(response.url).netloc + self.logger.info("Host: %s" % netloc.split(":")[0]) + self.logger.info("Type: %s" % type(response.ip_address)) + self.logger.info("IP address: %s" % response.ip_address) + + +if __name__ == "__main__": + with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server: + port = urlparse(mock_http_server.http_address).port + url = "http://not.a.real.domain:{port}/echo".format(port=port) + + servers = [(mock_dns_server.host, mock_dns_server.port)] + reactor.installResolver(createResolver(servers=servers)) + + configure_logging() + runner = CrawlerRunner() + d = runner.crawl(LocalhostSpider, url=url) + d.addBoth(lambda _: reactor.stop()) + reactor.run() diff --git a/tests/mockserver.py b/tests/mockserver.py index a45277db9..30d9bc0e8 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -1,3 +1,4 @@ +import argparse import json import os import random @@ -6,18 +7,19 @@ from subprocess import Popen, PIPE from urllib.parse import urlencode from OpenSSL import SSL -from twisted.web.server import Site, NOT_DONE_YET -from twisted.web.resource import Resource +from twisted.internet import defer, reactor, ssl +from twisted.internet.task import deferLater +from twisted.names import dns, error +from twisted.names.server import DNSServerFactory +from twisted.web.resource import EncodingResourceWrapper, Resource +from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site from twisted.web.static import File from twisted.web.test.test_webclient import PayloadResource -from twisted.web.server import GzipEncoderFactory -from twisted.web.resource import EncodingResourceWrapper from twisted.web.util import redirectTo -from twisted.internet import reactor, ssl -from twisted.internet.task import deferLater from scrapy.utils.python import to_bytes, to_unicode from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 +from scrapy.utils.test import get_testenv def getarg(request, name, default=None, type=None): @@ -198,12 +200,10 @@ class Root(Resource): return b'Scrapy mock HTTP server\n' -class MockServer(): +class MockServer: def __enter__(self): - from scrapy.utils.test import get_testenv - - self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'], + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'], stdout=PIPE, env=get_testenv()) http_address = self.proc.stdout.readline().strip().decode('ascii') https_address = self.proc.stdout.readline().strip().decode('ascii') @@ -224,11 +224,45 @@ class MockServer(): return host + path +class MockDNSResolver: + """ + Implements twisted.internet.interfaces.IResolver partially + """ + + def _resolve(self, name): + record = dns.Record_A(address=b"127.0.0.1") + answer = dns.RRHeader(name=name, payload=record) + return [answer], [], [] + + def query(self, query, timeout=None): + if query.type == dns.A: + return defer.succeed(self._resolve(query.name.name)) + return defer.fail(error.DomainError()) + + def lookupAllRecords(self, name, timeout=None): + return defer.succeed(self._resolve(name)) + + +class MockDNSServer: + + def __enter__(self): + self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'], + stdout=PIPE, env=get_testenv()) + host, port = self.proc.stdout.readline().strip().decode('ascii').split(":") + self.host = host + self.port = int(port) + return self + + def __exit__(self, exc_type, exc_value, traceback): + self.proc.kill() + self.proc.communicate() + + def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None): factory = ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), keyfile), - os.path.join(os.path.dirname(__file__), certfile), - ) + os.path.join(os.path.dirname(__file__), keyfile), + os.path.join(os.path.dirname(__file__), certfile), + ) if cipher_string: ctx = factory.getContext() # disabling TLS1.2+ because it unconditionally enables some strong ciphers @@ -238,19 +272,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if __name__ == "__main__": - root = Root() - factory = Site(root) - httpPort = reactor.listenTCP(0, factory) - contextFactory = ssl_context_factory() - httpsPort = reactor.listenSSL(0, factory, contextFactory) + parser = argparse.ArgumentParser() + parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http") + args = parser.parse_args() - def print_listening(): - httpHost = httpPort.getHost() - httpsHost = httpsPort.getHost() - httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port) - httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port) - print(httpAddress) - print(httpsAddress) + if args.type == "http": + root = Root() + factory = Site(root) + httpPort = reactor.listenTCP(0, factory) + contextFactory = ssl_context_factory() + httpsPort = reactor.listenSSL(0, factory, contextFactory) + + def print_listening(): + httpHost = httpPort.getHost() + httpsHost = httpsPort.getHost() + httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port) + httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port) + print(httpAddress) + print(httpsAddress) + + elif args.type == "dns": + clients = [MockDNSResolver()] + factory = DNSServerFactory(clients=clients) + protocol = dns.DNSDatagramProtocol(controller=factory) + listener = reactor.listenUDP(0, protocol) + + def print_listening(): + host = listener.getHost() + print("%s:%s" % (host.host, host.port)) reactor.callWhenRunning(print_listening) reactor.run() diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c02e6a70b..4215ca56c 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -1,6 +1,9 @@ import json import logging import sys +from ipaddress import IPv4Address +from socket import gethostbyname +from urllib.parse import urlparse from pytest import mark from testfixtures import LogCapture @@ -436,3 +439,21 @@ with multiples lines self.assertIsInstance(cert, Certificate) self.assertEqual(cert.getSubject().commonName, b"localhost") self.assertEqual(cert.getIssuer().commonName, b"localhost") + + @defer.inlineCallbacks + def test_dns_server_ip_address_none(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/status?n=200') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsNone(ip_address) + + @defer.inlineCallbacks + def test_dns_server_ip_address(self): + crawler = self.runner.create_crawler(SingleRequestSpider) + url = self.mockserver.url('/echo?body=test') + expected_netloc, _ = urlparse(url).netloc.split(':') + yield crawler.crawl(seed=url, mockserver=self.mockserver) + ip_address = crawler.spider.meta['responses'][0].ip_address + self.assertIsInstance(ip_address, IPv4Address) + self.assertEqual(str(ip_address), gethostbyname(expected_netloc)) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 169e763f0..b4144ea1d 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -274,9 +274,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase): self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log)) -class CrawlerProcessSubprocess(unittest.TestCase): - script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') - +class ScriptRunnerMixin: def run_script(self, script_name): script_path = os.path.join(self.script_dir, script_name) args = (sys.executable, script_path) @@ -285,6 +283,10 @@ class CrawlerProcessSubprocess(unittest.TestCase): stdout, stderr = p.communicate() return stderr.decode('utf-8') + +class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess') + def test_simple(self): log = self.run_script('simple.py') self.assertIn('Spider closed (finished)', log) @@ -332,3 +334,14 @@ class CrawlerProcessSubprocess(unittest.TestCase): log = self.run_script("twisted_reactor_asyncio.py") self.assertIn("Spider closed (finished)", log) self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log) + + +class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase): + script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner') + + def test_response_ip_address(self): + log = self.run_script("ip_address.py") + self.assertIn("INFO: Spider closed (finished)", log) + self.assertIn("INFO: Host: not.a.real.domain", log) + self.assertIn("INFO: Type: ", log) + self.assertIn("INFO: IP address: 127.0.0.1", log)