From 93ad6a4bc2fd2d453a37961b817a8a2a85f589c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:39:44 +0400 Subject: [PATCH 1/8] Simplify code for modern pyOpenSSL. --- scrapy/core/downloader/tls.py | 8 ++++---- scrapy/utils/ssl.py | 14 ++------------ tests/mockserver.py | 3 +-- 3 files changed, 7 insertions(+), 18 deletions(-) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 7d67a426f..65028d21f 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -17,10 +17,10 @@ METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only - METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only - METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: SSL.TLSv1_1_METHOD, # TLS 1.1 only + METHOD_TLSv12: SSL.TLSv1_2_METHOD, # TLS 1.2 only } diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index ea4dde882..98efd91c7 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -1,14 +1,9 @@ -import OpenSSL +import OpenSSL.SSL import OpenSSL._util as pyOpenSSLutil from scrapy.utils.python import to_unicode -# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL. -# Using the binding directly, as this code does, requires cryptography 2.4. -SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0) - - def ffi_buf_to_string(buf): return to_unicode(pyOpenSSLutil.ffi.string(buf)) @@ -22,9 +17,6 @@ def x509name_to_string(x509name): def get_temp_key_info(ssl_object): - if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'): # requires OpenSSL 1.0.2 - return None - # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key() temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **") if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p): @@ -55,7 +47,5 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version( - OpenSSL.SSL.SSLEAY_VERSION - ).decode('ascii', errors='replace') + system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) return f'{OpenSSL.version.__version__} ({system_openssl})' diff --git a/tests/mockserver.py b/tests/mockserver.py index 6d2d95692..4fd3adce7 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -19,7 +19,6 @@ from twisted.web.static import File from twisted.web.util import redirectTo from scrapy.utils.python import to_bytes, to_unicode -from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3 from scrapy.utils.test import get_testenv @@ -350,7 +349,7 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c if cipher_string: ctx = factory.getContext() # disabling TLS1.3 because it unconditionally enables some strong ciphers - ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3) + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3) ctx.set_cipher_list(to_bytes(cipher_string)) return factory From fb52918d23b37cd2581aa39ebbd422b017ec8051 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:46:55 +0400 Subject: [PATCH 2/8] Set OP_LEGACY_SERVER_CONNECT to support some old servers when using OpenSSL 3. --- scrapy/core/downloader/contextfactory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 4abde2238..bc6ad34d8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -61,7 +61,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() def getContext(self, hostname=None, port=None): - return self.getCertificateOptions().getContext() + ctx = self.getCertificateOptions().getContext() + ctx.set_options(0x4) # OP_LEGACY_SERVER_CONNECT + return ctx def creatorForNetloc(self, hostname, port): return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), From 43ab8bd16acdabc8be64b421b030fb6e1751e47b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 13 Jan 2023 20:51:07 +0400 Subject: [PATCH 3/8] Roll back the get_openssl_version() type change. --- scrapy/utils/ssl.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py index 98efd91c7..ce211bf9b 100644 --- a/scrapy/utils/ssl.py +++ b/scrapy/utils/ssl.py @@ -47,5 +47,7 @@ def get_temp_key_info(ssl_object): def get_openssl_version(): - system_openssl = OpenSSL.SSL.SSLeay_version(OpenSSL.SSL.SSLEAY_VERSION) + system_openssl = OpenSSL.SSL.SSLeay_version( + OpenSSL.SSL.SSLEAY_VERSION + ).decode('ascii', errors='replace') return f'{OpenSSL.version.__version__} ({system_openssl})' From caaeb235a08a250a9438c97259f2a74ae2ca8bbd Mon Sep 17 00:00:00 2001 From: Serhii A Date: Tue, 17 Jan 2023 13:52:41 +0200 Subject: [PATCH 4/8] =?UTF-8?q?scrapy.utils.console.DEFAULT=5FPYTHON=5FSHE?= =?UTF-8?q?LLS:=20OrderedDict=20=E2=86=92=20dict=20(#5795)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scrapy/utils/console.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py index 1bc0bd45f..4828c7767 100644 --- a/scrapy/utils/console.py +++ b/scrapy/utils/console.py @@ -1,5 +1,4 @@ from functools import wraps -from collections import OrderedDict def _embed_ipython_shell(namespace={}, banner=''): @@ -63,12 +62,12 @@ def _embed_standard_shell(namespace={}, banner=''): return wrapper -DEFAULT_PYTHON_SHELLS = OrderedDict([ - ('ptpython', _embed_ptpython_shell), - ('ipython', _embed_ipython_shell), - ('bpython', _embed_bpython_shell), - ('python', _embed_standard_shell), -]) +DEFAULT_PYTHON_SHELLS = { + 'ptpython': _embed_ptpython_shell, + 'ipython': _embed_ipython_shell, + 'bpython': _embed_bpython_shell, + 'python': _embed_standard_shell, +} def get_shell_embed_func(shells=None, known_shells=None): From f449ee53778b038f9ecb1feccb2fa9baa40e1f56 Mon Sep 17 00:00:00 2001 From: Tobias Mayr Date: Thu, 19 Jan 2023 18:44:55 +0000 Subject: [PATCH 5/8] Fix SMTP STARTTLS for Twisted >= 21.2.0 (#5406) --- scrapy/mail.py | 28 +++++++++++++++++++++------- tests/test_mail.py | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/scrapy/mail.py b/scrapy/mail.py index 2a25ccd44..b8cc28335 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,7 +12,9 @@ from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from twisted.python.versions import Version from twisted.internet import defer, ssl +from twisted import version as twisted_version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -126,16 +128,11 @@ class MailSender: 'mailattachs': nattachs, 'mailerr': errstr}) def _sendmail(self, to_addrs, msg): - # Import twisted.mail here because it is not available in python3 from twisted.internet import reactor - from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory( - self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, - heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls, - ) - factory.noisy = False + + factory = self._create_sender_factory(to_addrs, msg, d) if self.smtpssl: reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory()) @@ -143,3 +140,20 @@ class MailSender: reactor.connectTCP(self.smtphost, self.smtpport, factory) return d + + def _create_sender_factory(self, to_addrs, msg, d): + from twisted.mail.smtp import ESMTPSenderFactory + + factory_keywords = { + 'heloFallback': True, + 'requireAuthentication': False, + 'requireTransportSecurity': self.smtptls + } + + # Newer versions of twisted require the hostname to use STARTTLS + if twisted_version >= Version('twisted', 21, 2, 0): + factory_keywords['hostname'] = self.smtphost + + factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords) + factory.noisy = False + return factory diff --git a/tests/test_mail.py b/tests/test_mail.py index 9b248fbfa..fd02020ee 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -4,6 +4,11 @@ import unittest from io import BytesIO from email.charset import Charset +from twisted.internet._sslverify import ClientTLSOptions +from twisted.internet.ssl import ClientContextFactory +from twisted.python.versions import Version +from twisted.internet import defer +from twisted import version as twisted_version from scrapy.mail import MailSender @@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(text.get_charset(), Charset('utf-8')) self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body) + def test_create_sender_factory_with_host(self): + mailsender = MailSender(debug=False, smtphost='smtp.testhost.com') + + factory = mailsender._create_sender_factory(to_addrs=['test@scrapy.org'], msg='test', d=defer.Deferred()) + + context = factory.buildProtocol('test@scrapy.org').context + if twisted_version >= Version('twisted', 21, 2, 0): + self.assertIsInstance(context, ClientTLSOptions) + else: + self.assertIsInstance(context, ClientContextFactory) + if __name__ == "__main__": unittest.main() From 8270df754d5caa7e8115432923197f09b4ebc78f Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 07:55:16 -0300 Subject: [PATCH 6/8] Set `FEED_EXPORT_ENCODING='utf-8'` in the default template --- docs/topics/feed-exports.rst | 5 +++++ scrapy/templates/project/module/settings.py.tmpl | 1 + 2 files changed, 6 insertions(+) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index a620e2c04..7b662f34d 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -515,6 +515,11 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. +.. versionchanged:: 2.8 + The :command:`startproject` command now sets this setting to + ``utf-8`` in the generated + ``settings.py`` file. + .. setting:: FEED_EXPORT_FIELDS FEED_EXPORT_FIELDS diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index bbf60982c..2f6df5abc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' +FEED_EXPORT_ENCODING = 'utf-8' From 973f0cf5678adcec36aaf5d0ceb860198f34de4a Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 08:23:05 -0300 Subject: [PATCH 7/8] fix: line break --- docs/topics/feed-exports.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 7b662f34d..8775a99d0 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -517,8 +517,7 @@ Use ``utf-8`` if you want UTF-8 for JSON too. .. versionchanged:: 2.8 The :command:`startproject` command now sets this setting to - ``utf-8`` in the generated - ``settings.py`` file. + ``utf-8`` in the generated ``settings.py`` file. .. setting:: FEED_EXPORT_FIELDS From b6118480299384c88d194bf09ad60b67a36395ed Mon Sep 17 00:00:00 2001 From: Laerte Pereira Date: Fri, 20 Jan 2023 08:33:35 -0300 Subject: [PATCH 8/8] fix(docs): Change `versionchanged` value --- docs/topics/feed-exports.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index 8775a99d0..8f96b1154 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -515,7 +515,7 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. -.. versionchanged:: 2.8 +.. versionchanged:: VERSION The :command:`startproject` command now sets this setting to ``utf-8`` in the generated ``settings.py`` file.