Merge pull request #2558 from scrapy/twisted-17-fix

Fixed compatibility with twisted 17+

scrapy/core/downloader/tls.py

@@ -1,6 +1,8 @@
 import logging
 from OpenSSL import SSL
 
+from scrapy import twisted_version
+
 
 logger = logging.getLogger(__name__)
 
@@ -18,11 +20,17 @@ openssl_methods = {
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
 
-# ClientTLSOptions requires a recent-enough version of Twisted
-try:
+if twisted_version >= (14, 0, 0):
+    # ClientTLSOptions requires a recent-enough version of Twisted.
+    # Not having ScrapyClientTLSOptions should not matter for older
+    # Twisted versions because it is not used in the fallback
+    # ScrapyClientContextFactory.
 
     # taken from twisted/twisted/internet/_sslverify.py
+
     try:
+        # XXX: this try-except is not needed in Twisted 17.0.0+ because
+        # it requires pyOpenSSL 0.16+.
         from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
     except ImportError:
         SSL_CB_HANDSHAKE_START = 0x10
@@ -30,10 +38,17 @@ try:
 
     from twisted.internet.ssl import AcceptableCiphers
     from twisted.internet._sslverify import (ClientTLSOptions,
-                                             _maybeSetHostNameIndication,
                                              verifyHostname,
                                              VerificationError)
 
+    if twisted_version < (17, 0, 0):
+        from twisted.internet._sslverify import _maybeSetHostNameIndication
+        set_tlsext_host_name = _maybeSetHostNameIndication
+    else:
+        def set_tlsext_host_name(connection, hostNameBytes):
+            connection.set_tlsext_host_name(hostNameBytes)
+
+
     class ScrapyClientTLSOptions(ClientTLSOptions):
         """
         SSL Client connection creator ignoring certificate verification errors
@@ -46,7 +61,7 @@ try:
 
         def _identityVerifyingInfoCallback(self, connection, where, ret):
             if where & SSL_CB_HANDSHAKE_START:
-                _maybeSetHostNameIndication(connection, self._hostnameBytes)
+                set_tlsext_host_name(connection, self._hostnameBytes)
             elif where & SSL_CB_HANDSHAKE_DONE:
                 try:
                     verifyHostname(connection, self._hostnameASCII)
@@ -62,8 +77,3 @@ try:
                             self._hostnameASCII, repr(e)))
 
     DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
-
-except ImportError:
-    # ImportError should not matter for older Twisted versions
-    # as the above is not used in the fallback ScrapyClientContextFactory
-    pass

tox.ini

@@ -21,16 +21,16 @@ passenv =
 commands =
     py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
 
-[testenv:precise]
+[testenv:trusty]
 basepython = python2.7
 deps =
     pyOpenSSL==0.13
-    lxml==2.3.2
-    Twisted==11.1.0
-    boto==2.2.2
-    Pillow<2.0
+    lxml==3.3.3
+    Twisted==13.2.0
+    boto==2.20.1
+    Pillow==2.3.0
     cssselect==0.9.1
-    zope.interface==3.6.1
+    zope.interface==4.0.5
     -rtests/requirements.txt
 
 [testenv:jessie]