diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py index 08e8cad24..17957aba4 100644 --- a/scrapy/utils/_deps_compat.py +++ b/scrapy/utils/_deps_compat.py @@ -18,8 +18,8 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0) TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0) PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING) -# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object -PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0") +# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred +PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0") # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0") diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 22e9414b8..828a99e23 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging import ssl +import warnings from typing import TYPE_CHECKING, Any, TypedDict, TypeVar import OpenSSL._util as pyOpenSSLutil @@ -9,7 +10,11 @@ import OpenSSL.SSL import OpenSSL.version from twisted.internet.ssl import CertificateOptions, TLSVersion -from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 +from scrapy.exceptions import ScrapyDeprecationWarning +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.python import to_unicode if TYPE_CHECKING: @@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None: # pyOpenSSL utils -def ffi_buf_to_string(buf: Any) -> str: +def _ffi_buf_to_string(buf: Any) -> str: return to_unicode(pyOpenSSLutil.ffi.string(buf)) -def x509name_to_string(x509name: X509Name) -> str: +def ffi_buf_to_string(buf: Any) -> str: # pragma: no cover + warnings.warn( + "ffi_buf_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return ffi_buf_to_string(buf) + + +def _x509name_to_string(x509name: X509Name) -> str: # from OpenSSL.crypto.X509Name.__repr__ + # only used on pyOpenSSL < 24.3.0 result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512) pyOpenSSLutil.lib.X509_NAME_oneline( x509name._name, result_buffer, len(result_buffer) ) - return ffi_buf_to_string(result_buffer) + return _ffi_buf_to_string(result_buffer) -def get_temp_key_info(ssl_object: Any) -> str | None: +def x509name_to_string(x509name: X509Name) -> str: # pragma: no cover + warnings.warn( + "x509name_to_string() is deprecated.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _x509name_to_string(x509name) + + +def _get_temp_key_info(ssl_object: Any) -> str | None: # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"): - # removed in cryptography 40.0.0 + # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0) return None temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None: cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid) if cname == pyOpenSSLutil.ffi.NULL: cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid) - key_info.append(ffi_buf_to_string(cname)) + key_info.append(_ffi_buf_to_string(cname)) else: - key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) + key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type))) key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits") return ", ".join(key_info) +def get_temp_key_info(ssl_object: Any) -> str | None: # pragma: no cover + warnings.warn( + "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.", + ScrapyDeprecationWarning, + stacklevel=2, + ) + return _get_temp_key_info(ssl_object) + + def get_openssl_version() -> str: system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) system_openssl = system_openssl_bytes.decode("ascii", errors="replace") @@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection) connection.get_protocol_version_name(), connection.get_cipher_name(), ) - server_cert = connection.get_peer_certificate() - if server_cert: - logger.debug( - 'SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) - key_info = get_temp_key_info(connection._ssl) + if PYOPENSSL_X509_DEPRECATED: + if server_cert := connection.get_peer_certificate(as_cryptography=True): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + server_cert.issuer.rfc4514_string(), + server_cert.subject.rfc4514_string(), + ) + else: # noqa: PLR5501 + if server_cert_pyopenssl := connection.get_peer_certificate(): + logger.debug( + 'SSL connection certificate: issuer "%s", subject "%s"', + _x509name_to_string(server_cert_pyopenssl.get_issuer()), + _x509name_to_string(server_cert_pyopenssl.get_subject()), + ) + key_info = _get_temp_key_info(connection._ssl) if key_info: logger.debug("SSL temp key: %s", key_info) diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py index 7aa656780..5c4ca7457 100644 --- a/tests/mockserver/utils.py +++ b/tests/mockserver/utils.py @@ -10,7 +10,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey from twisted.internet.ssl import CertificateOptions, ContextFactory from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP -from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY +from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED from scrapy.utils.python import to_bytes from scrapy.utils.ssl import _get_cert_options_version_kwargs @@ -29,7 +29,7 @@ def ssl_context_factory( keyfile_path = Path(__file__).parent.parent / keyfile certfile_path = Path(__file__).parent.parent / certfile - if not PYOPENSSL_WANTS_X509_PKEY: + if PYOPENSSL_X509_DEPRECATED: cert = load_pem_x509_certificate(certfile_path.read_bytes()) key = load_pem_private_key(keyfile_path.read_bytes(), password=None) else: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 223f288bf..25a200817 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -33,7 +33,10 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse -from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1 +from scrapy.utils._deps_compat import ( + PYOPENSSL_X509_DEPRECATED, + TWISTED_TLS_LIMITS_OFFBY1, +) from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider @@ -831,8 +834,15 @@ class TestHttpsBase(TestHttpBase): is_secure = True tls_log_message = ( - 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' - 'subject "/C=IE/O=Scrapy/CN=localhost"' + ( + 'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", ' + 'subject "CN=localhost,O=Scrapy,C=IE"' + ) + if PYOPENSSL_X509_DEPRECATED + else ( + 'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", ' + 'subject "/C=IE/O=Scrapy/CN=localhost"' + ) ) def test_download_conn_lost(self) -> None: # type: ignore[override] diff --git a/tox.ini b/tox.ini index 50ac7aa86..44e240e03 100644 --- a/tox.ini +++ b/tox.ini @@ -82,7 +82,7 @@ deps = ptpython==3.0.32 # newer ones require newer Python ipython==8.39.0 - pyOpenSSL==26.2.0 + pyOpenSSL==26.3.0 pytest==9.0.3 socksio==1.0.0 types-Pygments==2.20.0.20260508