Fix deprecation warnings with pyOpenSSL 26.3.0. (#7619)

scrapy/utils/_deps_compat.py

@@ -18,8 +18,8 @@ TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
 TWISTED_TLS_LIMITS_OFFBY1 = TWISTED_VERSION < TxVersion("twisted", 26, 4, 0)
 
 PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
-# SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object
-PYOPENSSL_WANTS_X509_PKEY = PYOPENSSL_VERSION < Version("24.3.0")
+# pyOpenSSL X.509 APIs are deprecated and cryptography-based ones are preferred
+PYOPENSSL_X509_DEPRECATED = PYOPENSSL_VERSION >= Version("24.3.0")
 # SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
 PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
 

scrapy/utils/ssl.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import ssl
+import warnings
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar
 
 import OpenSSL._util as pyOpenSSLutil
@@ -9,7 +10,11 @@ import OpenSSL.SSL
 import OpenSSL.version
 from twisted.internet.ssl import CertificateOptions, TLSVersion
 
-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._deps_compat import (
+    PYOPENSSL_X509_DEPRECATED,
+    TWISTED_TLS_LIMITS_OFFBY1,
+)
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
@@ -116,23 +121,42 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
 # pyOpenSSL utils
 
 
-def ffi_buf_to_string(buf: Any) -> str:
+def _ffi_buf_to_string(buf: Any) -> str:
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
 
-def x509name_to_string(x509name: X509Name) -> str:
+def ffi_buf_to_string(buf: Any) -> str:  # pragma: no cover
+    warnings.warn(
+        "ffi_buf_to_string() is deprecated.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return ffi_buf_to_string(buf)
+
+
+def _x509name_to_string(x509name: X509Name) -> str:
     # from OpenSSL.crypto.X509Name.__repr__
+    # only used on pyOpenSSL < 24.3.0
     result_buffer: Any = pyOpenSSLutil.ffi.new("char[]", 512)
     pyOpenSSLutil.lib.X509_NAME_oneline(
         x509name._name, result_buffer, len(result_buffer)
     )
-    return ffi_buf_to_string(result_buffer)
+    return _ffi_buf_to_string(result_buffer)
 
 
-def get_temp_key_info(ssl_object: Any) -> str | None:
+def x509name_to_string(x509name: X509Name) -> str:  # pragma: no cover
+    warnings.warn(
+        "x509name_to_string() is deprecated.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return _x509name_to_string(x509name)
+
+
+def _get_temp_key_info(ssl_object: Any) -> str | None:
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     if not hasattr(pyOpenSSLutil.lib, "SSL_get_server_tmp_key"):
-        # removed in cryptography 40.0.0
+        # removed in cryptography 40.0.0 (required starting from pyOpenSSL 23.1.0)
         return None
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
     if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):
@@ -157,13 +181,22 @@ def get_temp_key_info(ssl_object: Any) -> str | None:
         cname = pyOpenSSLutil.lib.EC_curve_nid2nist(nid)
         if cname == pyOpenSSLutil.ffi.NULL:
             cname = pyOpenSSLutil.lib.OBJ_nid2sn(nid)
-        key_info.append(ffi_buf_to_string(cname))
+        key_info.append(_ffi_buf_to_string(cname))
     else:
-        key_info.append(ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
+        key_info.append(_ffi_buf_to_string(pyOpenSSLutil.lib.OBJ_nid2sn(key_type)))
     key_info.append(f"{pyOpenSSLutil.lib.EVP_PKEY_bits(temp_key)} bits")
     return ", ".join(key_info)
 
 
+def get_temp_key_info(ssl_object: Any) -> str | None:  # pragma: no cover
+    warnings.warn(
+        "get_temp_key_info() is deprecated. It's also a no-op with cryptography 40.0.0+.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    return _get_temp_key_info(ssl_object)
+
+
 def get_openssl_version() -> str:
     system_openssl_bytes = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION)
     system_openssl = system_openssl_bytes.decode("ascii", errors="replace")
@@ -177,14 +210,21 @@ def _log_ssl_conn_debug_info(hostname: str, connection: OpenSSL.SSL.Connection)
         connection.get_protocol_version_name(),
         connection.get_cipher_name(),
     )
-    server_cert = connection.get_peer_certificate()
-    if server_cert:
-        logger.debug(
-            'SSL connection certificate: issuer "%s", subject "%s"',
-            x509name_to_string(server_cert.get_issuer()),
-            x509name_to_string(server_cert.get_subject()),
-        )
-    key_info = get_temp_key_info(connection._ssl)
+    if PYOPENSSL_X509_DEPRECATED:
+        if server_cert := connection.get_peer_certificate(as_cryptography=True):
+            logger.debug(
+                'SSL connection certificate: issuer "%s", subject "%s"',
+                server_cert.issuer.rfc4514_string(),
+                server_cert.subject.rfc4514_string(),
+            )
+    else:  # noqa: PLR5501
+        if server_cert_pyopenssl := connection.get_peer_certificate():
+            logger.debug(
+                'SSL connection certificate: issuer "%s", subject "%s"',
+                _x509name_to_string(server_cert_pyopenssl.get_issuer()),
+                _x509name_to_string(server_cert_pyopenssl.get_subject()),
+            )
+    key_info = _get_temp_key_info(connection._ssl)
     if key_info:
         logger.debug("SSL temp key: %s", key_info)
 

tests/mockserver/utils.py

@@ -10,7 +10,7 @@ from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
 from twisted.internet.ssl import CertificateOptions, ContextFactory
 
 from scrapy.core.downloader.tls import _TWISTED_VERSION_MAP
-from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
+from scrapy.utils._deps_compat import PYOPENSSL_X509_DEPRECATED
 from scrapy.utils.python import to_bytes
 from scrapy.utils.ssl import _get_cert_options_version_kwargs
 
@@ -29,7 +29,7 @@ def ssl_context_factory(
     keyfile_path = Path(__file__).parent.parent / keyfile
     certfile_path = Path(__file__).parent.parent / certfile
 
-    if not PYOPENSSL_WANTS_X509_PKEY:
+    if PYOPENSSL_X509_DEPRECATED:
         cert = load_pem_x509_certificate(certfile_path.read_bytes())
         key = load_pem_private_key(keyfile_path.read_bytes(), password=None)
     else:

tests/test_downloader_handlers_http_base.py

@@ -33,7 +33,10 @@ from scrapy.exceptions import (
     UnsupportedURLSchemeError,
 )
 from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
-from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
+from scrapy.utils._deps_compat import (
+    PYOPENSSL_X509_DEPRECATED,
+    TWISTED_TLS_LIMITS_OFFBY1,
+)
 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
@@ -831,8 +834,15 @@ class TestHttpsBase(TestHttpBase):
     is_secure = True
 
     tls_log_message = (
-        'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
-        'subject "/C=IE/O=Scrapy/CN=localhost"'
+        (
+            'SSL connection certificate: issuer "CN=localhost,O=Scrapy,C=IE", '
+            'subject "CN=localhost,O=Scrapy,C=IE"'
+        )
+        if PYOPENSSL_X509_DEPRECATED
+        else (
+            'SSL connection certificate: issuer "/C=IE/O=Scrapy/CN=localhost", '
+            'subject "/C=IE/O=Scrapy/CN=localhost"'
+        )
     )
 
     def test_download_conn_lost(self) -> None:  # type: ignore[override]

tox.ini

@@ -82,7 +82,7 @@ deps =
     ptpython==3.0.32
     # newer ones require newer Python
     ipython==8.39.0
-    pyOpenSSL==26.2.0
+    pyOpenSSL==26.3.0
     pytest==9.0.3
     socksio==1.0.0
     types-Pygments==2.20.0.20260508