mirror of https://github.com/scrapy/scrapy.git
Merge pull request #3940 from elacuesta/response_ip_address
Get server IP address for HTTP/1.1 Responses
This commit is contained in:
Mikhail Korobov
GitHub
commit
c33fd47a80
|
|
@ -12,6 +12,8 @@ collect_ignore = [
|
|||
"scrapy/utils/testsite.py",
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
|
||||
*_py_files("tests/CrawlerProcess"),
|
||||
# contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess
|
||||
*_py_files("tests/CrawlerRunner"),
|
||||
# Py36-only parts of respective tests
|
||||
*_py_files("tests/py36"),
|
||||
]
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ Request objects
|
|||
:type url: string
|
||||
|
||||
:param callback: the function that will be called with the response of this
|
||||
request (once its downloaded) as its first parameter. For more information
|
||||
request (once it's downloaded) as its first parameter. For more information
|
||||
see :ref:`topics-request-response-ref-request-callback-arguments` below.
|
||||
If a Request doesn't specify a callback, the spider's
|
||||
:meth:`~scrapy.spiders.Spider.parse` method will be used.
|
||||
|
|
@ -616,6 +616,9 @@ Response objects
|
|||
:param certificate: an object representing the server's SSL certificate.
|
||||
:type certificate: twisted.internet.ssl.Certificate
|
||||
|
||||
:param ip_address: The IP address of the server from which the Response originated.
|
||||
:type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
|
||||
|
||||
.. attribute:: Response.url
|
||||
|
||||
A string containing the URL of the response.
|
||||
|
|
@ -705,6 +708,14 @@ Response objects
|
|||
|
||||
Only populated for ``https`` responses, ``None`` otherwise.
|
||||
|
||||
.. attribute:: Response.ip_address
|
||||
|
||||
The IP address of the server from which the Response originated.
|
||||
|
||||
This attribute is currently only populated by the HTTP 1.1 download
|
||||
handler, i.e. for ``http(s)`` responses. For other handlers,
|
||||
:attr:`ip_address` is always ``None``.
|
||||
|
||||
.. method:: Response.copy()
|
||||
|
||||
Returns a new Response which is a copy of this Response.
|
||||
|
|
|
|||
|
|
@ -173,7 +173,7 @@ class Downloader:
|
|||
return response
|
||||
dfd.addCallback(_downloaded)
|
||||
|
||||
# 3. After response arrives, remove the request from transferring
|
||||
# 3. After response arrives, remove the request from transferring
|
||||
# state to free up the transferring slot so it can be used by the
|
||||
# following requests (perhaps those which came from the downloader
|
||||
# middleware itself)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Download handlers for http and https schemes"""
|
||||
|
||||
import ipaddress
|
||||
import logging
|
||||
import re
|
||||
import warnings
|
||||
|
|
@ -373,7 +374,13 @@ class ScrapyAgent:
|
|||
def _cb_bodyready(self, txresponse, request):
|
||||
# deliverBody hangs for responses without body
|
||||
if txresponse.length == 0:
|
||||
return txresponse, b'', None, None
|
||||
return {
|
||||
"txresponse": txresponse,
|
||||
"body": b"",
|
||||
"flags": None,
|
||||
"certificate": None,
|
||||
"ip_address": None,
|
||||
}
|
||||
|
||||
maxsize = request.meta.get('download_maxsize', self._maxsize)
|
||||
warnsize = request.meta.get('download_warnsize', self._warnsize)
|
||||
|
|
@ -409,12 +416,17 @@ class ScrapyAgent:
|
|||
return d
|
||||
|
||||
def _cb_bodydone(self, result, request, url):
|
||||
txresponse, body, flags, certificate = result
|
||||
status = int(txresponse.code)
|
||||
headers = Headers(txresponse.headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=body)
|
||||
return respcls(url=url, status=status, headers=headers, body=body,
|
||||
flags=flags, certificate=certificate)
|
||||
headers = Headers(result["txresponse"].headers.getAllRawHeaders())
|
||||
respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
|
||||
return respcls(
|
||||
url=url,
|
||||
status=int(result["txresponse"].code),
|
||||
headers=headers,
|
||||
body=result["body"],
|
||||
flags=result["flags"],
|
||||
certificate=result["certificate"],
|
||||
ip_address=result["ip_address"],
|
||||
)
|
||||
|
||||
|
||||
@implementer(IBodyProducer)
|
||||
|
|
@ -449,12 +461,16 @@ class _ResponseReader(protocol.Protocol):
|
|||
self._reached_warnsize = False
|
||||
self._bytes_received = 0
|
||||
self._certificate = None
|
||||
self._ip_address = None
|
||||
|
||||
def connectionMade(self):
|
||||
if self._certificate is None:
|
||||
with suppress(AttributeError):
|
||||
self._certificate = ssl.Certificate(self.transport._producer.getPeerCertificate())
|
||||
|
||||
if self._ip_address is None:
|
||||
self._ip_address = ipaddress.ip_address(self.transport._producer.getPeer().host)
|
||||
|
||||
def dataReceived(self, bodyBytes):
|
||||
# This maybe called several times after cancel was called with buffered data.
|
||||
if self._finished.called:
|
||||
|
|
@ -486,16 +502,34 @@ class _ResponseReader(protocol.Protocol):
|
|||
|
||||
body = self._bodybuf.getvalue()
|
||||
if reason.check(ResponseDone):
|
||||
self._finished.callback((self._txresponse, body, None, self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": None,
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(PotentialDataLoss):
|
||||
self._finished.callback((self._txresponse, body, ['partial'], self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["partial"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
|
||||
if not self._fail_on_dataloss:
|
||||
self._finished.callback((self._txresponse, body, ['dataloss'], self._certificate))
|
||||
self._finished.callback({
|
||||
"txresponse": self._txresponse,
|
||||
"body": body,
|
||||
"flags": ["dataloss"],
|
||||
"certificate": self._certificate,
|
||||
"ip_address": self._ip_address,
|
||||
})
|
||||
return
|
||||
|
||||
elif not self._fail_on_dataloss_warned:
|
||||
|
|
|
|||
|
|
@ -17,7 +17,8 @@ from scrapy.utils.trackref import object_ref
|
|||
|
||||
class Response(object_ref):
|
||||
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None, request=None, certificate=None):
|
||||
def __init__(self, url, status=200, headers=None, body=b'', flags=None,
|
||||
request=None, certificate=None, ip_address=None):
|
||||
self.headers = Headers(headers or {})
|
||||
self.status = int(status)
|
||||
self._set_body(body)
|
||||
|
|
@ -25,6 +26,7 @@ class Response(object_ref):
|
|||
self.request = request
|
||||
self.flags = [] if flags is None else list(flags)
|
||||
self.certificate = certificate
|
||||
self.ip_address = ip_address
|
||||
|
||||
@property
|
||||
def cb_kwargs(self):
|
||||
|
|
@ -87,7 +89,8 @@ class Response(object_ref):
|
|||
"""Create a new Response with the same attributes except for those
|
||||
given new values.
|
||||
"""
|
||||
for x in ['url', 'status', 'headers', 'body', 'request', 'flags', 'certificate']:
|
||||
for x in ['url', 'status', 'headers', 'body',
|
||||
'request', 'flags', 'certificate', 'ip_address']:
|
||||
kwargs.setdefault(x, getattr(self, x))
|
||||
cls = kwargs.pop('cls', self.__class__)
|
||||
return cls(*args, **kwargs)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
from urllib.parse import urlparse
|
||||
|
||||
from twisted.internet import reactor
|
||||
from twisted.names.client import createResolver
|
||||
|
||||
from scrapy import Spider, Request
|
||||
from scrapy.crawler import CrawlerRunner
|
||||
from scrapy.utils.log import configure_logging
|
||||
|
||||
from tests.mockserver import MockServer, MockDNSServer
|
||||
|
||||
|
||||
class LocalhostSpider(Spider):
|
||||
name = "localhost_spider"
|
||||
|
||||
def start_requests(self):
|
||||
yield Request(self.url)
|
||||
|
||||
def parse(self, response):
|
||||
netloc = urlparse(response.url).netloc
|
||||
self.logger.info("Host: %s" % netloc.split(":")[0])
|
||||
self.logger.info("Type: %s" % type(response.ip_address))
|
||||
self.logger.info("IP address: %s" % response.ip_address)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
with MockServer() as mock_http_server, MockDNSServer() as mock_dns_server:
|
||||
port = urlparse(mock_http_server.http_address).port
|
||||
url = "http://not.a.real.domain:{port}/echo".format(port=port)
|
||||
|
||||
servers = [(mock_dns_server.host, mock_dns_server.port)]
|
||||
reactor.installResolver(createResolver(servers=servers))
|
||||
|
||||
configure_logging()
|
||||
runner = CrawlerRunner()
|
||||
d = runner.crawl(LocalhostSpider, url=url)
|
||||
d.addBoth(lambda _: reactor.stop())
|
||||
reactor.run()
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
import argparse
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
|
|
@ -6,18 +7,19 @@ from subprocess import Popen, PIPE
|
|||
from urllib.parse import urlencode
|
||||
|
||||
from OpenSSL import SSL
|
||||
from twisted.web.server import Site, NOT_DONE_YET
|
||||
from twisted.web.resource import Resource
|
||||
from twisted.internet import defer, reactor, ssl
|
||||
from twisted.internet.task import deferLater
|
||||
from twisted.names import dns, error
|
||||
from twisted.names.server import DNSServerFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper, Resource
|
||||
from twisted.web.server import GzipEncoderFactory, NOT_DONE_YET, Site
|
||||
from twisted.web.static import File
|
||||
from twisted.web.test.test_webclient import PayloadResource
|
||||
from twisted.web.server import GzipEncoderFactory
|
||||
from twisted.web.resource import EncodingResourceWrapper
|
||||
from twisted.web.util import redirectTo
|
||||
from twisted.internet import reactor, ssl
|
||||
from twisted.internet.task import deferLater
|
||||
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
|
||||
def getarg(request, name, default=None, type=None):
|
||||
|
|
@ -198,12 +200,10 @@ class Root(Resource):
|
|||
return b'Scrapy mock HTTP server\n'
|
||||
|
||||
|
||||
class MockServer():
|
||||
class MockServer:
|
||||
|
||||
def __enter__(self):
|
||||
from scrapy.utils.test import get_testenv
|
||||
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver'],
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'http'],
|
||||
stdout=PIPE, env=get_testenv())
|
||||
http_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
https_address = self.proc.stdout.readline().strip().decode('ascii')
|
||||
|
|
@ -224,11 +224,45 @@ class MockServer():
|
|||
return host + path
|
||||
|
||||
|
||||
class MockDNSResolver:
|
||||
"""
|
||||
Implements twisted.internet.interfaces.IResolver partially
|
||||
"""
|
||||
|
||||
def _resolve(self, name):
|
||||
record = dns.Record_A(address=b"127.0.0.1")
|
||||
answer = dns.RRHeader(name=name, payload=record)
|
||||
return [answer], [], []
|
||||
|
||||
def query(self, query, timeout=None):
|
||||
if query.type == dns.A:
|
||||
return defer.succeed(self._resolve(query.name.name))
|
||||
return defer.fail(error.DomainError())
|
||||
|
||||
def lookupAllRecords(self, name, timeout=None):
|
||||
return defer.succeed(self._resolve(name))
|
||||
|
||||
|
||||
class MockDNSServer:
|
||||
|
||||
def __enter__(self):
|
||||
self.proc = Popen([sys.executable, '-u', '-m', 'tests.mockserver', '-t', 'dns'],
|
||||
stdout=PIPE, env=get_testenv())
|
||||
host, port = self.proc.stdout.readline().strip().decode('ascii').split(":")
|
||||
self.host = host
|
||||
self.port = int(port)
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_value, traceback):
|
||||
self.proc.kill()
|
||||
self.proc.communicate()
|
||||
|
||||
|
||||
def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string=None):
|
||||
factory = ssl.DefaultOpenSSLContextFactory(
|
||||
os.path.join(os.path.dirname(__file__), keyfile),
|
||||
os.path.join(os.path.dirname(__file__), certfile),
|
||||
)
|
||||
os.path.join(os.path.dirname(__file__), keyfile),
|
||||
os.path.join(os.path.dirname(__file__), certfile),
|
||||
)
|
||||
if cipher_string:
|
||||
ctx = factory.getContext()
|
||||
# disabling TLS1.2+ because it unconditionally enables some strong ciphers
|
||||
|
|
@ -238,19 +272,34 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
|
|||
|
||||
|
||||
if __name__ == "__main__":
|
||||
root = Root()
|
||||
factory = Site(root)
|
||||
httpPort = reactor.listenTCP(0, factory)
|
||||
contextFactory = ssl_context_factory()
|
||||
httpsPort = reactor.listenSSL(0, factory, contextFactory)
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("-t", "--type", type=str, choices=("http", "dns"), default="http")
|
||||
args = parser.parse_args()
|
||||
|
||||
def print_listening():
|
||||
httpHost = httpPort.getHost()
|
||||
httpsHost = httpsPort.getHost()
|
||||
httpAddress = 'http://%s:%d' % (httpHost.host, httpHost.port)
|
||||
httpsAddress = 'https://%s:%d' % (httpsHost.host, httpsHost.port)
|
||||
print(httpAddress)
|
||||
print(httpsAddress)
|
||||
if args.type == "http":
|
||||
root = Root()
|
||||
factory = Site(root)
|
||||
httpPort = reactor.listenTCP(0, factory)
|
||||
contextFactory = ssl_context_factory()
|
||||
httpsPort = reactor.listenSSL(0, factory, contextFactory)
|
||||
|
||||
def print_listening():
|
||||
httpHost = httpPort.getHost()
|
||||
httpsHost = httpsPort.getHost()
|
||||
httpAddress = "http://%s:%d" % (httpHost.host, httpHost.port)
|
||||
httpsAddress = "https://%s:%d" % (httpsHost.host, httpsHost.port)
|
||||
print(httpAddress)
|
||||
print(httpsAddress)
|
||||
|
||||
elif args.type == "dns":
|
||||
clients = [MockDNSResolver()]
|
||||
factory = DNSServerFactory(clients=clients)
|
||||
protocol = dns.DNSDatagramProtocol(controller=factory)
|
||||
listener = reactor.listenUDP(0, protocol)
|
||||
|
||||
def print_listening():
|
||||
host = listener.getHost()
|
||||
print("%s:%s" % (host.host, host.port))
|
||||
|
||||
reactor.callWhenRunning(print_listening)
|
||||
reactor.run()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import json
|
||||
import logging
|
||||
import sys
|
||||
from ipaddress import IPv4Address
|
||||
from socket import gethostbyname
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pytest import mark
|
||||
from testfixtures import LogCapture
|
||||
|
|
@ -436,3 +439,21 @@ with multiples lines
|
|||
self.assertIsInstance(cert, Certificate)
|
||||
self.assertEqual(cert.getSubject().commonName, b"localhost")
|
||||
self.assertEqual(cert.getIssuer().commonName, b"localhost")
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_dns_server_ip_address_none(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url('/status?n=200')
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
ip_address = crawler.spider.meta['responses'][0].ip_address
|
||||
self.assertIsNone(ip_address)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_dns_server_ip_address(self):
|
||||
crawler = self.runner.create_crawler(SingleRequestSpider)
|
||||
url = self.mockserver.url('/echo?body=test')
|
||||
expected_netloc, _ = urlparse(url).netloc.split(':')
|
||||
yield crawler.crawl(seed=url, mockserver=self.mockserver)
|
||||
ip_address = crawler.spider.meta['responses'][0].ip_address
|
||||
self.assertIsInstance(ip_address, IPv4Address)
|
||||
self.assertEqual(str(ip_address), gethostbyname(expected_netloc))
|
||||
|
|
|
|||
|
|
@ -274,9 +274,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
|
|||
self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", str(log))
|
||||
|
||||
|
||||
class CrawlerProcessSubprocess(unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
|
||||
|
||||
class ScriptRunnerMixin:
|
||||
def run_script(self, script_name):
|
||||
script_path = os.path.join(self.script_dir, script_name)
|
||||
args = (sys.executable, script_path)
|
||||
|
|
@ -285,6 +283,10 @@ class CrawlerProcessSubprocess(unittest.TestCase):
|
|||
stdout, stderr = p.communicate()
|
||||
return stderr.decode('utf-8')
|
||||
|
||||
|
||||
class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
|
||||
|
||||
def test_simple(self):
|
||||
log = self.run_script('simple.py')
|
||||
self.assertIn('Spider closed (finished)', log)
|
||||
|
|
@ -332,3 +334,14 @@ class CrawlerProcessSubprocess(unittest.TestCase):
|
|||
log = self.run_script("twisted_reactor_asyncio.py")
|
||||
self.assertIn("Spider closed (finished)", log)
|
||||
self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
|
||||
|
||||
|
||||
class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
|
||||
script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerRunner')
|
||||
|
||||
def test_response_ip_address(self):
|
||||
log = self.run_script("ip_address.py")
|
||||
self.assertIn("INFO: Spider closed (finished)", log)
|
||||
self.assertIn("INFO: Host: not.a.real.domain", log)
|
||||
self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
|
||||
self.assertIn("INFO: IP address: 127.0.0.1", log)
|
||||
|
|
|
|||
Loading…
Reference in New Issue