Catch VerificationError but keep the rest of ClientTLSOptions

scrapy/core/downloader/contextfactory.py

@@ -1,21 +1,19 @@
 from OpenSSL import SSL
 from twisted.internet.ssl import ClientContextFactory
 
-
 try:
 
     from zope.interface.declarations import implementer
 
     # the following should be available from Twisted 14.0.0
-    from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
-    from twisted.internet._sslverify import ClientTLSOptions
+    from twisted.internet.ssl import (optionsForClientTLS,
+                                      CertificateOptions,
+                                      platformTrust)
+
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
-
-    class ScrapyClientTLSOptions(ClientTLSOptions):
-        def _identityVerifyingInfoCallback(self, connection, where, ret):
-            pass
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
 
 
     @implementer(IPolicyForHTTPS)

scrapy/core/downloader/tls.py

@@ -1,6 +1,9 @@
+import logging
 from OpenSSL import SSL
 
 
+logger = logging.getLogger(__name__)
+
 METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
@@ -14,3 +17,36 @@ openssl_methods = {
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
 }
+
+# ClientTLSOptions requires a recent-enough version of Twisted
+try:
+
+    # taken from twisted/twisted/internet/_sslverify.py
+    try:
+        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
+    except ImportError:
+        SSL_CB_HANDSHAKE_START = 0x10
+        SSL_CB_HANDSHAKE_DONE = 0x20
+
+    from twisted.internet._sslverify import (ClientTLSOptions,
+                                             _maybeSetHostNameIndication,
+                                             verifyHostname,
+                                             VerificationError)
+
+    class ScrapyClientTLSOptions(ClientTLSOptions):
+        # same as Twisted's ClientTLSOptions,
+        # except that VerificationError is caught
+        # and doesn't close the connection
+        def _identityVerifyingInfoCallback(self, connection, where, ret):
+            if where & SSL_CB_HANDSHAKE_START:
+                _maybeSetHostNameIndication(connection, self._hostnameBytes)
+            elif where & SSL_CB_HANDSHAKE_DONE:
+                try:
+                    verifyHostname(connection, self._hostnameASCII)
+                except VerificationError as e:
+                    logger.warning(e)
+
+except ImportError:
+    # ImportError should not matter for older Twisted versions
+    # as the above is not used in the fallback ScrapyClientContextFactory
+    pass