Merge remote-tracking branch 'upstream/master' into add-black-formatter

docs/contributing.rst

@@ -188,8 +188,9 @@ We use `pre-commit`_ to automatically address simple code issues before every
 commit.
 
 Before you start writing a patch:
-#.  `Install pre-commit <https://pre-commit.com/#installation>`_.
-#.  On the root of your local clone of the Scrapy repository, run the following command:
+  #.  `Install pre-commit <https://pre-commit.com/#installation>`_.
+  #.  On the root of your local clone of the Scrapy repository, run the following command:
+
     .. code-block:: bash
       
       pre-commit install

docs/topics/feed-exports.rst

@@ -515,6 +515,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
 
 Use ``utf-8`` if you want UTF-8 for JSON too.
 
+.. versionchanged:: VERSION
+   The :command:`startproject` command now sets this setting to
+   ``utf-8`` in the generated ``settings.py`` file.
+
 .. setting:: FEED_EXPORT_FIELDS
 
 FEED_EXPORT_FIELDS

scrapy/mail.py

@@ -12,7 +12,9 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
 
+from twisted.python.versions import Version
 from twisted.internet import defer, ssl
+from twisted import version as twisted_version
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -165,24 +167,12 @@ class MailSender:
         )
 
     def _sendmail(self, to_addrs, msg):
-        # Import twisted.mail here because it is not available in python3
         from twisted.internet import reactor
-        from twisted.mail.smtp import ESMTPSenderFactory
 
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(
-            self.smtpuser,
-            self.smtppass,
-            self.mailfrom,
-            to_addrs,
-            msg,
-            d,
-            heloFallback=True,
-            requireAuthentication=False,
-            requireTransportSecurity=self.smtptls,
-        )
-        factory.noisy = False
+
+        factory = self._create_sender_factory(to_addrs, msg, d)
 
         if self.smtpssl:
             reactor.connectSSL(
@@ -192,3 +182,28 @@ class MailSender:
             reactor.connectTCP(self.smtphost, self.smtpport, factory)
 
         return d
+
+    def _create_sender_factory(self, to_addrs, msg, d):
+        from twisted.mail.smtp import ESMTPSenderFactory
+
+        factory_keywords = {
+            "heloFallback": True,
+            "requireAuthentication": False,
+            "requireTransportSecurity": self.smtptls,
+        }
+
+        # Newer versions of twisted require the hostname to use STARTTLS
+        if twisted_version >= Version("twisted", 21, 2, 0):
+            factory_keywords["hostname"] = self.smtphost
+
+        factory = ESMTPSenderFactory(
+            self.smtpuser,
+            self.smtppass,
+            self.mailfrom,
+            to_addrs,
+            msg,
+            d,
+            **factory_keywords
+        )
+        factory.noisy = False
+        return factory

scrapy/templates/project/module/settings.py.tmpl

@@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
 TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+FEED_EXPORT_ENCODING = 'utf-8'

tests/test_http_response.py

@@ -519,6 +519,7 @@ class TextResponseTest(BaseResponseTest):
         # Inferring encoding from body also cache decoded body as sideeffect,
         # this test tries to ensure that calling response.encoding and
         # response.text in indistinct order doesn't affect final
+        # response.text in indistinct order doesn't affect final
         # values for encoding and decoded body.
         url = "http://example.com"
         body = b"\xef\xbb\xbfWORD"

tests/test_mail.py

@@ -4,6 +4,11 @@ import unittest
 from io import BytesIO
 from email.charset import Charset
 
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
+from twisted.internet import defer
+from twisted import version as twisted_version
 from scrapy.mail import MailSender
 
 
@@ -147,6 +152,19 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(text.get_charset(), Charset("utf-8"))
         self.assertEqual(attach.get_payload(decode=True).decode("utf-8"), body)
 
+    def test_create_sender_factory_with_host(self):
+        mailsender = MailSender(debug=False, smtphost="smtp.testhost.com")
+
+        factory = mailsender._create_sender_factory(
+            to_addrs=["test@scrapy.org"], msg="test", d=defer.Deferred()
+        )
+
+        context = factory.buildProtocol("test@scrapy.org").context
+        if twisted_version >= Version("twisted", 21, 2, 0):
+            self.assertIsInstance(context, ClientTLSOptions)
+        else:
+            self.assertIsInstance(context, ClientContextFactory)
+
 
 if __name__ == "__main__":
     unittest.main()