diff --git a/docs/contributing.rst b/docs/contributing.rst index f7f1218c8..12f0e8d21 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -188,8 +188,9 @@ We use `pre-commit`_ to automatically address simple code issues before every commit. Before you start writing a patch: -#. `Install pre-commit `_. -#. On the root of your local clone of the Scrapy repository, run the following command: + #. `Install pre-commit `_. + #. On the root of your local clone of the Scrapy repository, run the following command: + .. code-block:: bash pre-commit install diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst index a620e2c04..8f96b1154 100644 --- a/docs/topics/feed-exports.rst +++ b/docs/topics/feed-exports.rst @@ -515,6 +515,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons. Use ``utf-8`` if you want UTF-8 for JSON too. +.. versionchanged:: VERSION + The :command:`startproject` command now sets this setting to + ``utf-8`` in the generated ``settings.py`` file. + .. setting:: FEED_EXPORT_FIELDS FEED_EXPORT_FIELDS diff --git a/scrapy/mail.py b/scrapy/mail.py index fbde9c547..fa1e55f1f 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -12,7 +12,9 @@ from email.mime.text import MIMEText from email.utils import formatdate from io import BytesIO +from twisted.python.versions import Version from twisted.internet import defer, ssl +from twisted import version as twisted_version from scrapy.utils.misc import arg_to_iter from scrapy.utils.python import to_bytes @@ -165,24 +167,12 @@ class MailSender: ) def _sendmail(self, to_addrs, msg): - # Import twisted.mail here because it is not available in python3 from twisted.internet import reactor - from twisted.mail.smtp import ESMTPSenderFactory msg = BytesIO(msg) d = defer.Deferred() - factory = ESMTPSenderFactory( - self.smtpuser, - self.smtppass, - self.mailfrom, - to_addrs, - msg, - d, - heloFallback=True, - requireAuthentication=False, - requireTransportSecurity=self.smtptls, - ) - factory.noisy = False + + factory = self._create_sender_factory(to_addrs, msg, d) if self.smtpssl: reactor.connectSSL( @@ -192,3 +182,28 @@ class MailSender: reactor.connectTCP(self.smtphost, self.smtpport, factory) return d + + def _create_sender_factory(self, to_addrs, msg, d): + from twisted.mail.smtp import ESMTPSenderFactory + + factory_keywords = { + "heloFallback": True, + "requireAuthentication": False, + "requireTransportSecurity": self.smtptls, + } + + # Newer versions of twisted require the hostname to use STARTTLS + if twisted_version >= Version("twisted", 21, 2, 0): + factory_keywords["hostname"] = self.smtphost + + factory = ESMTPSenderFactory( + self.smtpuser, + self.smtppass, + self.mailfrom, + to_addrs, + msg, + d, + **factory_keywords + ) + factory.noisy = False + return factory diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl index bbf60982c..2f6df5abc 100644 --- a/scrapy/templates/project/module/settings.py.tmpl +++ b/scrapy/templates/project/module/settings.py.tmpl @@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True # Set settings whose default value is deprecated to a future-proof value REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7' TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor' +FEED_EXPORT_ENCODING = 'utf-8' diff --git a/tests/test_http_response.py b/tests/test_http_response.py index 8c422bb4f..beb0b712b 100644 --- a/tests/test_http_response.py +++ b/tests/test_http_response.py @@ -519,6 +519,7 @@ class TextResponseTest(BaseResponseTest): # Inferring encoding from body also cache decoded body as sideeffect, # this test tries to ensure that calling response.encoding and # response.text in indistinct order doesn't affect final + # response.text in indistinct order doesn't affect final # values for encoding and decoded body. url = "http://example.com" body = b"\xef\xbb\xbfWORD" diff --git a/tests/test_mail.py b/tests/test_mail.py index c78980d57..0ee0400cd 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -4,6 +4,11 @@ import unittest from io import BytesIO from email.charset import Charset +from twisted.internet._sslverify import ClientTLSOptions +from twisted.internet.ssl import ClientContextFactory +from twisted.python.versions import Version +from twisted.internet import defer +from twisted import version as twisted_version from scrapy.mail import MailSender @@ -147,6 +152,19 @@ class MailSenderTest(unittest.TestCase): self.assertEqual(text.get_charset(), Charset("utf-8")) self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body) + def test_create_sender_factory_with_host(self): + mailsender = MailSender(debug=False, smtphost="smtp.testhost.com") + + factory = mailsender._create_sender_factory( + to_addrs=["test@scrapy.org"], msg="test", d=defer.Deferred() + ) + + context = factory.buildProtocol("test@scrapy.org").context + if twisted_version >= Version("twisted", 21, 2, 0): + self.assertIsInstance(context, ClientTLSOptions) + else: + self.assertIsInstance(context, ClientContextFactory) + if __name__ == "__main__": unittest.main()