diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 617f1dcdf..85ae2a305 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -440,9 +440,10 @@ or even enable client-side authentication (and various other things). which uses the platform's certificates to validate remote endpoints. **This is only available if you use Twisted>=14.0.** -If you do use a custom ContextFactory, make sure it accepts a ``method`` -parameter at init (this is the ``OpenSSL.SSL`` method mapping -:setting:`DOWNLOADER_CLIENT_TLS_METHOD`). +If you do use a custom ContextFactory, make sure its ``__init__`` method +accepts a ``method`` parameter (this is the ``OpenSSL.SSL`` method mapping +:setting:`DOWNLOADER_CLIENT_TLS_METHOD`) and a ``tls_verbose_logging`` +parameter (``bool``). .. setting:: DOWNLOADER_CLIENT_TLS_METHOD @@ -470,6 +471,20 @@ This setting must be one of these string values: We recommend that you use PyOpenSSL>=0.13 and Twisted>=0.13 or above (Twisted>=14.0 if you can). +.. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING + +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING +------------------------------------- + +Default: ``False`` + +Setting this to ``True`` will enable DEBUG level messages about TLS connection +parameters after establishing HTTPS connections. The kind of information logged +depends on the versions of OpenSSL and pyOpenSSL. + +This setting is only used for the default +:setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`. + .. setting:: DOWNLOADER_MIDDLEWARES DOWNLOADER_MIDDLEWARES diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 783d4c383..5ac20c0bb 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -28,9 +28,15 @@ if twisted_version >= (14, 0, 0): understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ - def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs): + def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs): super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) self._ssl_method = method + self.tls_verbose_logging = tls_verbose_logging + + @classmethod + def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): + tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING') + return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs) def getCertificateOptions(self): # setting verify=True will require you to provide CAs @@ -56,7 +62,8 @@ if twisted_version >= (14, 0, 0): return self.getCertificateOptions().getContext() def creatorForNetloc(self, hostname, port): - return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext()) + return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), + verbose_logging=self.tls_verbose_logging) @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index d875fb1e4..be7298531 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -1,7 +1,7 @@ """Download handlers for http and https schemes """ from twisted.internet import reactor -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.python import to_unicode @@ -11,6 +11,7 @@ class HTTP10DownloadHandler(object): def __init__(self, settings): self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) + self._settings = settings def download_request(self, request, spider): """Return a deferred for the HTTP download""" @@ -21,7 +22,7 @@ class HTTP10DownloadHandler(object): def _connect(self, factory): host, port = to_unicode(factory.host), factory.port if factory.scheme == b'https': - return reactor.connectSSL(host, port, factory, - self.ClientContextFactory()) + client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None) + return reactor.connectSSL(host, port, factory, client_context_factory) else: return reactor.connectTCP(host, port, factory) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index cbaa36b2d..2ccab2614 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -25,7 +25,7 @@ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse from scrapy.core.downloader.tls import openssl_methods -from scrapy.utils.misc import load_object +from scrapy.utils.misc import load_object, create_instance from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version @@ -44,14 +44,15 @@ class HTTP11DownloadHandler(object): self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) # try method-aware context factory try: - self._contextFactory = self._contextFactoryClass(method=self._sslMethod) + self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None, + method=self._sslMethod) except TypeError: # use context factory defaults - self._contextFactory = self._contextFactoryClass() + self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None) msg = """ '%s' does not accept `method` argument (type OpenSSL.SSL method,\ - e.g. OpenSSL.SSL.SSLv23_METHOD).\ - Please upgrade your context factory class to handle it or ignore it.""" % ( + e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\ + Please upgrade your context factory class to handle them or ignore them.""" % ( settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index df8051182..74afb3f10 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -2,6 +2,7 @@ import logging from OpenSSL import SSL from scrapy import twisted_version +from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) @@ -20,6 +21,7 @@ openssl_methods = { METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only } + if twisted_version >= (14, 0, 0): # ClientTLSOptions requires a recent-enough version of Twisted. # Not having ScrapyClientTLSOptions should not matter for older @@ -65,13 +67,39 @@ if twisted_version >= (14, 0, 0): Same as Twisted's private _sslverify.ClientTLSOptions, except that VerificationError, CertificateError and ValueError exceptions are caught, so that the connection is not closed, only - logging warnings. + logging warnings. Also, HTTPS connection parameters logging is added. """ + def __init__(self, hostname, ctx, verbose_logging=False): + super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + self.verbose_logging = verbose_logging + def _identityVerifyingInfoCallback(self, connection, where, ret): if where & SSL_CB_HANDSHAKE_START: set_tlsext_host_name(connection, self._hostnameBytes) elif where & SSL_CB_HANDSHAKE_DONE: + if self.verbose_logging: + if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 + if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) + else: + logger.debug('SSL connection to %s using cipher %s', + self._hostnameASCII, + connection.get_cipher_name(), + ) + server_cert = connection.get_peer_certificate() + logger.debug('SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) + key_info = get_temp_key_info(connection._ssl) + if key_info: + logger.debug('SSL temp key: %s', key_info) + try: verifyHostname(connection, self._hostnameASCII) except verification_errors as e: diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d17eb3125..086adf48e 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -87,6 +87,7 @@ DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClien DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, # also allowing negotiation +DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False DOWNLOADER_MIDDLEWARES = {} diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py new file mode 100644 index 000000000..5db1608bf --- /dev/null +++ b/scrapy/utils/ssl.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +import OpenSSL._util as pyOpenSSLutil + +from scrapy.utils.python import to_native_str + + +def ffi_buf_to_string(buf): + return to_native_str(pyOpenSSLutil.ffi.string(buf)) + + +def x509name_to_string(x509name): + # from OpenSSL.crypto.X509Name.__repr__ + result_buffer = pyOpenSSLutil.ffi.new("char[]", 512) + pyOpenSSLutil.lib.X509_NAME_oneline(x509name._name, result_buffer, len(result_buffer)) + + return ffi_buf_to_string(result_buffer) + + +def get_temp_key_info(ssl_object): + if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 + return None + + # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() + temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") + pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p) + if temp_key_p == pyOpenSSLutil.ffi.NULL: + return None + + temp_key = temp_key_p[0] + pyOpenSSLutil.ffi.gc(temp_key, pyOpenSSLutil.lib.EVP_PKEY_free) + key_info = [] + key_type = pyOpenSSLutil.lib.EVP_PKEY_id(temp_key) + if key_type == pyOpenSSLutil.lib.EVP_PKEY_RSA: + key_info.append('RSA') + elif key_type == pyOpenSSLutil.lib.EVP_PKEY_DH: + key_info.append('DH') + elif key_type == pyOpenSSLutil.lib.EVP_PKEY_EC: + key_info.append('ECDH') + ec_key = pyOpenSSLutil.lib.EVP_PKEY_get1_EC_KEY(temp_key) + pyOpenSSLutil.ffi.gc(ec_key, pyOpenSSLutil.lib.EC_KEY_free) + nid = pyOpenSSLutil.lib.EC_GROUP_get_curve_name(pyOpenSSLutil.lib.EC_KEY_get0_group(ec_key)) + cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) + if cname == pyOpenSSLutil.ffi.NULL: + cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) + key_info.append(ffi_buf_to_string(cname)) + else: + key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) + key_info.append('%s bits' % pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)) + return ', '.join(key_info) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index d2151e10e..efef4192c 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -8,6 +8,7 @@ try: except ImportError: import mock +from testfixtures import LogCapture from twisted.trial import unittest from twisted.protocols.policies import WrappingFactory from twisted.python.filepath import FilePath @@ -498,6 +499,24 @@ class Http11TestCase(HttpTestCase): class Https11TestCase(Http11TestCase): scheme = 'https' + tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", subject "/C=IE/O=Scrapy/CN=localhost"' + + @defer.inlineCallbacks + def test_tls_logging(self): + download_handler = self.download_handler_cls(Settings({ + 'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True, + })) + try: + with LogCapture() as log_capture: + request = Request(self.getURL('file')) + d = download_handler.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + yield d + log_capture.check_present(('scrapy.core.downloader.tls', 'DEBUG', self.tls_log_message)) + finally: + yield download_handler.close() + class Https11WrongHostnameTestCase(Http11TestCase): scheme = 'https' @@ -518,6 +537,7 @@ class Https11InvalidDNSId(Https11TestCase): super(Https11InvalidDNSId, self).setUp() self.host = '127.0.0.1' + class Https11InvalidDNSPattern(Https11TestCase): """Connect to HTTPS hosts where the certificate are issued to an ip instead of a domain.""" @@ -529,6 +549,7 @@ class Https11InvalidDNSPattern(Https11TestCase): from service_identity.exceptions import CertificateError except ImportError: raise unittest.SkipTest("cryptography lib is too old") + self.tls_log_message = 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=127.0.0.1", subject "/C=IE/O=Scrapy/CN=127.0.0.1"' super(Https11InvalidDNSPattern, self).setUp()