diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 3e147ae36..b643d935b 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,21 +1,19 @@ from OpenSSL import SSL from twisted.internet.ssl import ClientContextFactory - try: from zope.interface.declarations import implementer # the following should be available from Twisted 14.0.0 - from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust - from twisted.internet._sslverify import ClientTLSOptions + from twisted.internet.ssl import (optionsForClientTLS, + CertificateOptions, + platformTrust) + from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS - - class ScrapyClientTLSOptions(ClientTLSOptions): - def _identityVerifyingInfoCallback(self, connection, where, ret): - pass + from scrapy.core.downloader.tls import ScrapyClientTLSOptions @implementer(IPolicyForHTTPS) diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 64ebb0714..2cf8ba869 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,6 +1,9 @@ +import logging from OpenSSL import SSL +logger = logging.getLogger(__name__) + METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' @@ -14,3 +17,36 @@ openssl_methods = { METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only } + +# ClientTLSOptions requires a recent-enough version of Twisted +try: + + # taken from twisted/twisted/internet/_sslverify.py + try: + from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START + except ImportError: + SSL_CB_HANDSHAKE_START = 0x10 + SSL_CB_HANDSHAKE_DONE = 0x20 + + from twisted.internet._sslverify import (ClientTLSOptions, + _maybeSetHostNameIndication, + verifyHostname, + VerificationError) + + class ScrapyClientTLSOptions(ClientTLSOptions): + # same as Twisted's ClientTLSOptions, + # except that VerificationError is caught + # and doesn't close the connection + def _identityVerifyingInfoCallback(self, connection, where, ret): + if where & SSL_CB_HANDSHAKE_START: + _maybeSetHostNameIndication(connection, self._hostnameBytes) + elif where & SSL_CB_HANDSHAKE_DONE: + try: + verifyHostname(connection, self._hostnameASCII) + except VerificationError as e: + logger.warning(e) + +except ImportError: + # ImportError should not matter for older Twisted versions + # as the above is not used in the fallback ScrapyClientContextFactory + pass