From 93ad6a4bc2fd2d453a37961b817a8a2a85f589c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:39:44 +0400 Subject: [PATCH 1/3] Simplify code for modern pyOpenSSL. --- scrapy/core/downloader/tls.py | 8 ++++---- scrapy/utils/ssl.py | 14 ++------------ tests/mockserver.py | 3 +-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 7d67a426f..65028d21f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -17,10 +17,10 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only - METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only + METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only } diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index ea4dde882..98efd91c7 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,14 +1,9 @@ -import OpenSSL +import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil from scrapy.utils.python import to_unicode -# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. -# Using the binding directly, as this code does, requires cryptography 2.4. -SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) - - def ffi_buf_to_string(buf): return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -22,9 +17,6 @@ def x509name_to_string(x509name): def get_temp_key_info(ssl_object): - if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 - return None - # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -55,7 +47,5 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version( - OpenSSL.SSL.SSLEAY_VERSION - ).decode('ascii', errors='replace') + system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/tests/mockserver.py b/tests/mockserver.py index 6d2d95692..4fd3adce7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -19,7 +19,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 from scrapy.utils.test import get_testenv @@ -350,7 +349,7 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if cipher_string: ctx = factory.getContext() # disabling TLS1.3 because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From fb52918d23b37cd2581aa39ebbd422b017ec8051 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:46:55 +0400 Subject: [PATCH 2/3] Set OP_LEGACY_SERVER_CONNECT to support some old servers when using OpenSSL 3. --- scrapy/core/downloader/contextfactory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 4abde2238..bc6ad34d8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -61,7 +61,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname=None, port=None): - return self.getCertificateOptions().getContext() + ctx = self.getCertificateOptions().getContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx def creatorForNetloc(self, hostname, port): return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), From 43ab8bd16acdabc8be64b421b030fb6e1751e47b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:51:07 +0400 Subject: [PATCH 3/3] Roll back the get_openssl_version() type change. --- scrapy/utils/ssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 98efd91c7..ce211bf9b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -47,5 +47,7 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) + system_openssl = OpenSSL.SSL.SSLeay_version( + OpenSSL.SSL.SSLEAY_VERSION + ).decode('ascii', errors='replace') return f'{OpenSSL.version.__version__} ({system_openssl})'