Fix SMTP STARTTLS for Twisted >= 21.2.0 (#5406)

scrapy/mail.py

@@ -12,7 +12,9 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
 
+from twisted.python.versions import Version
 from twisted.internet import defer, ssl
+from twisted import version as twisted_version
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -126,16 +128,11 @@ class MailSender:
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
-        # Import twisted.mail here because it is not available in python3
         from twisted.internet import reactor
-        from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(
-            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
-            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
-        )
-        factory.noisy = False
+
+        factory = self._create_sender_factory(to_addrs, msg, d)
 
         if self.smtpssl:
             reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
@@ -143,3 +140,20 @@ class MailSender:
             reactor.connectTCP(self.smtphost, self.smtpport, factory)
 
         return d
+
+    def _create_sender_factory(self, to_addrs, msg, d):
+        from twisted.mail.smtp import ESMTPSenderFactory
+
+        factory_keywords = {
+            'heloFallback': True,
+            'requireAuthentication': False,
+            'requireTransportSecurity': self.smtptls
+        }
+
+        # Newer versions of twisted require the hostname to use STARTTLS
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            factory_keywords['hostname'] = self.smtphost
+
+        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)
+        factory.noisy = False
+        return factory

tests/test_mail.py

@@ -4,6 +4,11 @@ import unittest
 from io import BytesIO
 from email.charset import Charset
 
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
+from twisted.internet import defer
+from twisted import version as twisted_version
 from scrapy.mail import MailSender
 
 
@@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(text.get_charset(), Charset('utf-8'))
         self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
 
+    def test_create_sender_factory_with_host(self):
+        mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')
+
+        factory = mailsender._create_sender_factory(to_addrs=['test@scrapy.org'], msg='test', d=defer.Deferred())
+
+        context = factory.buildProtocol('test@scrapy.org').context
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            self.assertIsInstance(context, ClientTLSOptions)
+        else:
+            self.assertIsInstance(context, ClientContextFactory)
+
 
 if __name__ == "__main__":
     unittest.main()