RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #3450 from wRAR/tls-logging 

Log cipher, certificate and temp key info on establishing an SSL connection
This commit is contained in:
Mikhail Korobov 2019-07-24 01:45:40 +05:00 committed by GitHub
parent c679aefe85 c6453800cd
commit 9c514b976f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 138 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
docs/topics/settings.rst
View File

@ -440,9 +440,10 @@ or even enable client-side authentication (and various other things).
which uses the platform's certificates to validate remote endpoints.
**This is only available if you use Twisted>=14.0.**
If you do use a custom ContextFactory, make sure it accepts a ``method``
parameter at init (this is the ``OpenSSL.SSL`` method mapping
:setting:`DOWNLOADER_CLIENT_TLS_METHOD`).
If you do use a custom ContextFactory, make sure its ``__init__`` method
accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping
:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging``
parameter (``bool``).
.. setting:: DOWNLOADER_CLIENT_TLS_METHOD
@ -470,6 +471,20 @@ This setting must be one of these string values:
We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13
or above (Twisted>=14.0 if you can).
.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
-------------------------------------
Default: ``False``
Setting this to ``True`` will enable DEBUG level messages about TLS connection
parameters after establishing HTTPS connections. The kind of information logged
depends on the versions of OpenSSL and pyOpenSSL.
This setting is only used for the default
:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
.. setting:: DOWNLOADER_MIDDLEWARES
DOWNLOADER_MIDDLEWARES

11
scrapy/core/downloader/contextfactory.py
View File

@ -28,9 +28,15 @@ if twisted_version >= (14, 0, 0):
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
"""
def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
self._ssl_method = method
self.tls_verbose_logging = tls_verbose_logging
@classmethod
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
def getCertificateOptions(self):
# setting verify=True will require you to provide CAs
@ -56,7 +62,8 @@ if twisted_version >= (14, 0, 0):
return self.getCertificateOptions().getContext()
def creatorForNetloc(self, hostname, port):
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
verbose_logging=self.tls_verbose_logging)
@implementer(IPolicyForHTTPS)

7
scrapy/core/downloader/handlers/http10.py
View File

@ -1,7 +1,7 @@
"""Download handlers for http and https schemes
"""
from twisted.internet import reactor
from scrapy.utils.misc import load_object
from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.python import to_unicode
@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object):
def __init__(self, settings):
self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
self._settings = settings
def download_request(self, request, spider):
"""Return a deferred for the HTTP download"""
@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object):
def _connect(self, factory):
host, port = to_unicode(factory.host), factory.port
if factory.scheme == b'https':
return reactor.connectSSL(host, port, factory,
self.ClientContextFactory())
client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)
return reactor.connectSSL(host, port, factory, client_context_factory)
else:
return reactor.connectTCP(host, port, factory)

11
scrapy/core/downloader/handlers/http11.py
View File

@ -25,7 +25,7 @@ from scrapy.http import Headers
from scrapy.responsetypes import responsetypes
from scrapy.core.downloader.webclient import _parse
from scrapy.core.downloader.tls import openssl_methods
from scrapy.utils.misc import load_object
from scrapy.utils.misc import load_object, create_instance
from scrapy.utils.python import to_bytes, to_unicode
from scrapy import twisted_version
@ -44,14 +44,15 @@ class HTTP11DownloadHandler(object):
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
# try method-aware context factory
try:
self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None,
method=self._sslMethod)
except TypeError:
# use context factory defaults
self._contextFactory = self._contextFactoryClass()
self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None)
msg = """
'%s' does not accept `method` argument (type OpenSSL.SSL method,\
e.g. OpenSSL.SSL.SSLv23_METHOD).\
Please upgrade your context factory class to handle it or ignore it.""" % (
e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\
Please upgrade your context factory class to handle them or ignore them.""" % (
settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],)
warnings.warn(msg)
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')

30
scrapy/core/downloader/tls.py
View File

@ -2,6 +2,7 @@ import logging
from OpenSSL import SSL
from scrapy import twisted_version
from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
logger = logging.getLogger(__name__)
@ -20,6 +21,7 @@ openssl_methods = {
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
}
if twisted_version >= (14, 0, 0):
# ClientTLSOptions requires a recent-enough version of Twisted.
# Not having ScrapyClientTLSOptions should not matter for older
@ -65,13 +67,39 @@ if twisted_version >= (14, 0, 0):
Same as Twisted's private _sslverify.ClientTLSOptions,
except that VerificationError, CertificateError and ValueError
exceptions are caught, so that the connection is not closed, only
logging warnings.
logging warnings. Also, HTTPS connection parameters logging is added.
"""
def __init__(self, hostname, ctx, verbose_logging=False):
super(ScrapyClientTLSOptions, self).__init__(hostname, ctx)
self.verbose_logging = verbose_logging
def _identityVerifyingInfoCallback(self, connection, where, ret):
if where & SSL_CB_HANDSHAKE_START:
set_tlsext_host_name(connection, self._hostnameBytes)
elif where & SSL_CB_HANDSHAKE_DONE:
if self.verbose_logging:
if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15
if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0
logger.debug('SSL connection to %s using protocol %s, cipher %s',
self._hostnameASCII,
connection.get_protocol_version_name(),
connection.get_cipher_name(),
)
else:
logger.debug('SSL connection to %s using cipher %s',
self._hostnameASCII,
connection.get_cipher_name(),
)
server_cert = connection.get_peer_certificate()
logger.debug('SSL connection certificate: issuer "%s", subject "%s"',
x509name_to_string(server_cert.get_issuer()),
x509name_to_string(server_cert.get_subject()),
)
key_info = get_temp_key_info(connection._ssl)
if key_info:
logger.debug('SSL temp key: %s', key_info)
try:
verifyHostname(connection, self._hostnameASCII)
except verification_errors as e:

1
scrapy/settings/default_settings.py
View File

@ -87,6 +87,7 @@ DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClien
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
# also allowing negotiation
DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
DOWNLOADER_MIDDLEWARES = {}

50
scrapy/utils/ssl.py Normal file
View File

@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
import OpenSSL._util as pyOpenSSLutil
from scrapy.utils.python import to_native_str
def ffi_buf_to_string(buf):
return to_native_str(pyOpenSSLutil.ffi.string(buf))
def x509name_to_string(x509name):
# from OpenSSL.crypto.X509Name.__repr__
result_buffer = pyOpenSSLutil.ffi.new("char[]", 512)
pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer))
return ffi_buf_to_string(result_buffer)
def get_temp_key_info(ssl_object):
if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2
return None
# adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p)
if temp_key_p == pyOpenSSLutil.ffi.NULL:
return None
temp_key = temp_key_p[0]
pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free)
key_info = []
key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key)
if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA:
key_info.append('RSA')
elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH:
key_info.append('DH')
elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC:
key_info.append('ECDH')
ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key)
pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free)
nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key))
cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
if cname == pyOpenSSLutil.ffi.NULL:
cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
key_info.append(ffi_buf_to_string(cname))
else:
key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key))
return ', '.join(key_info)

21
tests/test_downloader_handlers.py
View File

@ -8,6 +8,7 @@ try:
except ImportError:
import mock
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.protocols.policies import WrappingFactory
from twisted.python.filepath import FilePath
@ -498,6 +499,24 @@ class Http11TestCase(HttpTestCase):
class Https11TestCase(Http11TestCase):
scheme = 'https'
tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"'
@defer.inlineCallbacks
def test_tls_logging(self):
download_handler = self.download_handler_cls(Settings({
'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,
}))
try:
with LogCapture() as log_capture:
request = Request(self.getURL('file'))
d = download_handler.download_request(request, Spider('foo'))
d.addCallback(lambda r: r.body)
d.addCallback(self.assertEqual, b"0123456789")
yield d
log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message))
finally:
yield download_handler.close()
class Https11WrongHostnameTestCase(Http11TestCase):
scheme = 'https'
@ -518,6 +537,7 @@ class Https11InvalidDNSId(Https11TestCase):
super(Https11InvalidDNSId, self).setUp()
self.host = '127.0.0.1'
class Https11InvalidDNSPattern(Https11TestCase):
"""Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain."""
@ -529,6 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase):
from service_identity.exceptions import CertificateError
except ImportError:
raise unittest.SkipTest("cryptography lib is too old")
self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"'
super(Https11InvalidDNSPattern, self).setUp()