Merge branch 'scrapy:master' into path_object_error_#5739

docs/topics/feed-exports.rst

@@ -515,6 +515,10 @@ which uses safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
 
 Use ``utf-8`` if you want UTF-8 for JSON too.
 
+.. versionchanged:: VERSION
+   The :command:`startproject` command now sets this setting to
+   ``utf-8`` in the generated ``settings.py`` file.
+
 .. setting:: FEED_EXPORT_FIELDS
 
 FEED_EXPORT_FIELDS

scrapy/core/downloader/contextfactory.py

@@ -61,7 +61,9 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
     def getContext(self, hostname=None, port=None):
-        return self.getCertificateOptions().getContext()
+        ctx = self.getCertificateOptions().getContext()
+        ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
+        return ctx
 
     def creatorForNetloc(self, hostname, port):
         return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),

scrapy/core/downloader/tls.py

@@ -17,10 +17,10 @@ METHOD_TLSv12 = 'TLSv1.2'
 
 
 openssl_methods = {
-    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
-    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
-    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
+    METHOD_TLS: SSL.SSLv23_METHOD,      # protocol negotiation (recommended)
+    METHOD_TLSv10: SSL.TLSv1_METHOD,    # TLS 1.0 only
+    METHOD_TLSv11: SSL.TLSv1_1_METHOD,  # TLS 1.1 only
+    METHOD_TLSv12: SSL.TLSv1_2_METHOD,  # TLS 1.2 only
 }
 
 

scrapy/mail.py

@@ -12,7 +12,9 @@ from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
 
+from twisted.python.versions import Version
 from twisted.internet import defer, ssl
+from twisted import version as twisted_version
 
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
@@ -126,16 +128,11 @@ class MailSender:
                       'mailattachs': nattachs, 'mailerr': errstr})
 
     def _sendmail(self, to_addrs, msg):
-        # Import twisted.mail here because it is not available in python3
         from twisted.internet import reactor
-        from twisted.mail.smtp import ESMTPSenderFactory
         msg = BytesIO(msg)
         d = defer.Deferred()
-        factory = ESMTPSenderFactory(
-            self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d,
-            heloFallback=True, requireAuthentication=False, requireTransportSecurity=self.smtptls,
-        )
-        factory.noisy = False
+
+        factory = self._create_sender_factory(to_addrs, msg, d)
 
         if self.smtpssl:
             reactor.connectSSL(self.smtphost, self.smtpport, factory, ssl.ClientContextFactory())
@@ -143,3 +140,20 @@ class MailSender:
             reactor.connectTCP(self.smtphost, self.smtpport, factory)
 
         return d
+
+    def _create_sender_factory(self, to_addrs, msg, d):
+        from twisted.mail.smtp import ESMTPSenderFactory
+
+        factory_keywords = {
+            'heloFallback': True,
+            'requireAuthentication': False,
+            'requireTransportSecurity': self.smtptls
+        }
+
+        # Newer versions of twisted require the hostname to use STARTTLS
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            factory_keywords['hostname'] = self.smtphost
+
+        factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, to_addrs, msg, d, **factory_keywords)
+        factory.noisy = False
+        return factory

scrapy/templates/project/module/settings.py.tmpl

@@ -90,3 +90,4 @@ ROBOTSTXT_OBEY = True
 # Set settings whose default value is deprecated to a future-proof value
 REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
 TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
+FEED_EXPORT_ENCODING = 'utf-8'

scrapy/utils/console.py

@@ -1,5 +1,4 @@
 from functools import wraps
-from collections import OrderedDict
 
 
 def _embed_ipython_shell(namespace={}, banner=''):
@@ -63,12 +62,12 @@ def _embed_standard_shell(namespace={}, banner=''):
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = OrderedDict([
-    ('ptpython', _embed_ptpython_shell),
-    ('ipython', _embed_ipython_shell),
-    ('bpython', _embed_bpython_shell),
-    ('python', _embed_standard_shell),
-])
+DEFAULT_PYTHON_SHELLS = {
+    'ptpython': _embed_ptpython_shell,
+    'ipython': _embed_ipython_shell,
+    'bpython': _embed_bpython_shell,
+    'python': _embed_standard_shell,
+}
 
 
 def get_shell_embed_func(shells=None, known_shells=None):

scrapy/utils/ssl.py

@@ -1,14 +1,9 @@
-import OpenSSL
+import OpenSSL.SSL
 import OpenSSL._util as pyOpenSSLutil
 
 from scrapy.utils.python import to_unicode
 
 
-# The OpenSSL symbol is present since 1.1.1 but it's not currently supported in any version of pyOpenSSL.
-# Using the binding directly, as this code does, requires cryptography 2.4.
-SSL_OP_NO_TLSv1_3 = getattr(pyOpenSSLutil.lib, 'SSL_OP_NO_TLSv1_3', 0)
-
-
 def ffi_buf_to_string(buf):
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
 
@@ -22,9 +17,6 @@ def x509name_to_string(x509name):
 
 
 def get_temp_key_info(ssl_object):
-    if not hasattr(pyOpenSSLutil.lib, 'SSL_get_server_tmp_key'):  # requires OpenSSL 1.0.2
-        return None
-
     # adapted from OpenSSL apps/s_cb.c::ssl_print_tmp_key()
     temp_key_p = pyOpenSSLutil.ffi.new("EVP_PKEY **")
     if not pyOpenSSLutil.lib.SSL_get_server_tmp_key(ssl_object, temp_key_p):

tests/mockserver.py

@@ -19,7 +19,6 @@ from twisted.web.static import File
 from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.ssl import SSL_OP_NO_TLSv1_3
 from scrapy.utils.test import get_testenv
 
 
@@ -350,7 +349,7 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
     if cipher_string:
         ctx = factory.getContext()
         # disabling TLS1.3 because it unconditionally enables some strong ciphers
-        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL_OP_NO_TLSv1_3)
+        ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3)
         ctx.set_cipher_list(to_bytes(cipher_string))
     return factory
 

tests/test_mail.py

@@ -4,6 +4,11 @@ import unittest
 from io import BytesIO
 from email.charset import Charset
 
+from twisted.internet._sslverify import ClientTLSOptions
+from twisted.internet.ssl import ClientContextFactory
+from twisted.python.versions import Version
+from twisted.internet import defer
+from twisted import version as twisted_version
 from scrapy.mail import MailSender
 
 
@@ -121,6 +126,17 @@ class MailSenderTest(unittest.TestCase):
         self.assertEqual(text.get_charset(), Charset('utf-8'))
         self.assertEqual(attach.get_payload(decode=True).decode('utf-8'), body)
 
+    def test_create_sender_factory_with_host(self):
+        mailsender = MailSender(debug=False, smtphost='smtp.testhost.com')
+
+        factory = mailsender._create_sender_factory(to_addrs=['test@scrapy.org'], msg='test', d=defer.Deferred())
+
+        context = factory.buildProtocol('test@scrapy.org').context
+        if twisted_version >= Version('twisted', 21, 2, 0):
+            self.assertIsInstance(context, ClientTLSOptions)
+        else:
+            self.assertIsInstance(context, ClientContextFactory)
+
 
 if __name__ == "__main__":
     unittest.main()