diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5ac20c0bb..127a246f5 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,112 +1,85 @@ from OpenSSL import SSL -from twisted.internet.ssl import ClientContextFactory +from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust +from twisted.web.client import BrowserLikePolicyForHTTPS +from twisted.web.iweb import IPolicyForHTTPS +from zope.interface.declarations import implementer -from scrapy import twisted_version - -if twisted_version >= (14, 0, 0): - - from zope.interface.declarations import implementer - - from twisted.internet.ssl import (optionsForClientTLS, - CertificateOptions, - platformTrust) - from twisted.web.client import BrowserLikePolicyForHTTPS - from twisted.web.iweb import IPolicyForHTTPS - - from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS +from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS - @implementer(IPolicyForHTTPS) - class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): - """ - Non-peer-certificate verifying HTTPS context factory +@implementer(IPolicyForHTTPS) +class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): + """ + Non-peer-certificate verifying HTTPS context factory - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation - 'A TLS/SSL connection established with [this method] may - understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' - """ + 'A TLS/SSL connection established with [this method] may + understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' + """ - def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs): - super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) - self._ssl_method = method - self.tls_verbose_logging = tls_verbose_logging + def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs): + super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) + self._ssl_method = method + self.tls_verbose_logging = tls_verbose_logging - @classmethod - def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): - tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING') - return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs) + @classmethod + def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs): + tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING') + return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs) - def getCertificateOptions(self): - # setting verify=True will require you to provide CAs - # to verify against; in other words: it's not that simple + def getCertificateOptions(self): + # setting verify=True will require you to provide CAs + # to verify against; in other words: it's not that simple - # backward-compatible SSL/TLS method: - # - # * this will respect `method` attribute in often recommended - # `ScrapyClientContextFactory` subclass - # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) - # - # * getattr() for `_ssl_method` attribute for context factories - # not calling super(..., self).__init__ - return CertificateOptions(verify=False, - method=getattr(self, 'method', - getattr(self, '_ssl_method', None)), - fixBrokenPeers=True, - acceptableCiphers=DEFAULT_CIPHERS) + # backward-compatible SSL/TLS method: + # + # * this will respect `method` attribute in often recommended + # `ScrapyClientContextFactory` subclass + # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) + # + # * getattr() for `_ssl_method` attribute for context factories + # not calling super(..., self).__init__ + return CertificateOptions(verify=False, + method=getattr(self, 'method', + getattr(self, '_ssl_method', None)), + fixBrokenPeers=True, + acceptableCiphers=DEFAULT_CIPHERS) - # kept for old-style HTTP/1.0 downloader context twisted calls, - # e.g. connectSSL() - def getContext(self, hostname=None, port=None): - return self.getCertificateOptions().getContext() + # kept for old-style HTTP/1.0 downloader context twisted calls, + # e.g. connectSSL() + def getContext(self, hostname=None, port=None): + return self.getCertificateOptions().getContext() - def creatorForNetloc(self, hostname, port): - return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), - verbose_logging=self.tls_verbose_logging) + def creatorForNetloc(self, hostname, port): + return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(), + verbose_logging=self.tls_verbose_logging) - @implementer(IPolicyForHTTPS) - class BrowserLikeContextFactory(ScrapyClientContextFactory): - """ - Twisted-recommended context factory for web clients. +@implementer(IPolicyForHTTPS) +class BrowserLikeContextFactory(ScrapyClientContextFactory): + """ + Twisted-recommended context factory for web clients. - Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + "The default is to use a BrowserLikePolicyForHTTPS, + so unless you have special requirements you can leave this as-is." - creatorForNetloc() is the same as BrowserLikePolicyForHTTPS - except this context factory allows setting the TLS/SSL method to use. + creatorForNetloc() is the same as BrowserLikePolicyForHTTPS + except this context factory allows setting the TLS/SSL method to use. - Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) - which allows TLS protocol negotiation. - """ - def creatorForNetloc(self, hostname, port): + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation. + """ + def creatorForNetloc(self, hostname, port): - # trustRoot set to platformTrust() will use the platform's root CAs. - # - # This means that a website like https://www.cacert.org will be rejected - # by default, since CAcert.org CA certificate is seldom shipped. - return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=platformTrust(), - extraCertificateOptions={ - 'method': self._ssl_method, - }) - -else: - - class ScrapyClientContextFactory(ClientContextFactory): - "A SSL context factory which is more permissive against SSL bugs." - # see https://github.com/scrapy/scrapy/issues/82 - # and https://github.com/scrapy/scrapy/issues/26 - # and https://github.com/scrapy/scrapy/issues/981 - - def __init__(self, method=SSL.SSLv23_METHOD): - self.method = method - - def getContext(self, hostname=None, port=None): - ctx = ClientContextFactory.getContext(self) - # Enable all workarounds to SSL bugs as documented by - # https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html - ctx.set_options(SSL.OP_ALL) - return ctx + # trustRoot set to platformTrust() will use the platform's root CAs. + # + # This means that a website like https://www.cacert.org will be rejected + # by default, since CAcert.org CA certificate is seldom shipped. + return optionsForClientTLS(hostname.decode("ascii"), + trustRoot=platformTrust(), + extraCertificateOptions={ + 'method': self._ssl_method, + }) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 2ccab2614..e72052afc 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -13,12 +13,7 @@ from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss -from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed -try: - from twisted.web.client import URI -except ImportError: - from twisted.web.client import _URI as URI +from twisted.web.client import Agent, ResponseDone, HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -27,7 +22,7 @@ from scrapy.core.downloader.webclient import _parse from scrapy.core.downloader.tls import openssl_methods from scrapy.utils.misc import load_object, create_instance from scrapy.utils.python import to_bytes, to_unicode -from scrapy import twisted_version + logger = logging.getLogger(__name__) @@ -44,11 +39,19 @@ class HTTP11DownloadHandler(object): self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) # try method-aware context factory try: - self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None, - method=self._sslMethod) + self._contextFactory = create_instance( + self._contextFactoryClass, + settings=settings, + crawler=None, + method=self._sslMethod, + ) except TypeError: # use context factory defaults - self._contextFactory = create_instance(self._contextFactoryClass, settings=settings, crawler=None) + self._contextFactory = create_instance( + self._contextFactoryClass, + settings=settings, + crawler=None, + ) msg = """ '%s' does not accept `method` argument (type OpenSSL.SSL method,\ e.g. OpenSSL.SSL.SSLv23_METHOD) and/or `tls_verbose_logging` argument.\ @@ -62,10 +65,13 @@ class HTTP11DownloadHandler(object): def download_request(self, request, spider): """Return a deferred for the HTTP download""" - agent = ScrapyAgent(contextFactory=self._contextFactory, pool=self._pool, + agent = ScrapyAgent( + contextFactory=self._contextFactory, + pool=self._pool, maxsize=getattr(spider, 'download_maxsize', self._default_maxsize), warnsize=getattr(spider, 'download_warnsize', self._default_warnsize), - fail_on_dataloss=self._fail_on_dataloss) + fail_on_dataloss=self._fail_on_dataloss, + ) return agent.download_request(request) def close(self): @@ -104,11 +110,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): _responseMatcher = re.compile(br'HTTP/1\.. (?P\d{3})(?P.{,32})') - def __init__(self, reactor, host, port, proxyConf, contextFactory, - timeout=30, bindAddress=None): + def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf - super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, - proxyPort, timeout, bindAddress) + super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() self._tunneledHost = host self._tunneledPort = port @@ -117,8 +121,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, - self._proxyAuthHeader) + tunnelReq = tunnel_request_data(self._tunneledHost, self._tunneledPort, self._proxyAuthHeader) protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse @@ -141,16 +144,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocol.dataReceived = self._protocolDataReceived respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer) if respm and int(respm.group('status')) == 200: - try: - # this sets proper Server Name Indication extension - # but is only available for Twisted>=14.0 - sslOptions = self._contextFactory.creatorForNetloc( - self._tunneledHost, self._tunneledPort) - except AttributeError: - # fall back to non-SNI SSL context factory - sslOptions = self._contextFactory - self._protocol.transport.startTLS(sslOptions, - self._protocolFactory) + # set proper Server Name Indication extension + sslOptions = self._contextFactory.creatorForNetloc(self._tunneledHost, self._tunneledPort) + self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: if respm: @@ -168,8 +164,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def connect(self, protocolFactory): self._protocolFactory = protocolFactory - connectDeferred = super(TunnelingTCP4ClientEndpoint, - self).connect(protocolFactory) + connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) connectDeferred.addCallback(self.requestTunnel) connectDeferred.addErrback(self.connectFailed) return self._tunnelReadyDeferred @@ -206,42 +201,46 @@ class TunnelingAgent(Agent): def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): - super(TunnelingAgent, self).__init__(reactor, contextFactory, - connectTimeout, bindAddress, pool) + super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) self._proxyConf = proxyConf self._contextFactory = contextFactory - if twisted_version >= (15, 0, 0): - def _getEndpoint(self, uri): - return TunnelingTCP4ClientEndpoint( - self._reactor, uri.host, uri.port, self._proxyConf, - self._contextFactory, self._endpointFactory._connectTimeout, - self._endpointFactory._bindAddress) - else: - def _getEndpoint(self, scheme, host, port): - return TunnelingTCP4ClientEndpoint( - self._reactor, host, port, self._proxyConf, - self._contextFactory, self._connectTimeout, - self._bindAddress) + def _getEndpoint(self, uri): + return TunnelingTCP4ClientEndpoint( + reactor=self._reactor, + host=uri.host, + port=uri.port, + proxyConf=self._proxyConf, + contextFactory=self._contextFactory, + timeout=self._endpointFactory._connectTimeout, + bindAddress=self._endpointFactory._bindAddress, + ) - def _requestWithEndpoint(self, key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath): + def _requestWithEndpoint(self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath): # proxy host and port are required for HTTP pool `key` # otherwise, same remote host connection request could reuse # a cached tunneled connection to a different proxy key = key + self._proxyConf - return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI, - headers, bodyProducer, requestPath) + return super(TunnelingAgent, self)._requestWithEndpoint( + key=key, + endpoint=endpoint, + method=method, + parsedURI=parsedURI, + headers=headers, + bodyProducer=bodyProducer, + requestPath=requestPath, + ) class ScrapyProxyAgent(Agent): - def __init__(self, reactor, proxyURI, - connectTimeout=None, bindAddress=None, pool=None): - super(ScrapyProxyAgent, self).__init__(reactor, - connectTimeout=connectTimeout, - bindAddress=bindAddress, - pool=pool) + def __init__(self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None): + super(ScrapyProxyAgent, self).__init__( + reactor=reactor, + connectTimeout=connectTimeout, + bindAddress=bindAddress, + pool=pool, + ) self._proxyURI = URI.fromBytes(proxyURI) def request(self, method, uri, headers=None, bodyProducer=None): @@ -250,16 +249,15 @@ class ScrapyProxyAgent(Agent): """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - if twisted_version >= (15, 0, 0): - proxyEndpoint = self._getEndpoint(self._proxyURI) - else: - proxyEndpoint = self._getEndpoint(self._proxyURI.scheme, - self._proxyURI.host, - self._proxyURI.port) - key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) - return self._requestWithEndpoint(key, proxyEndpoint, method, - URI.fromBytes(uri), headers, - bodyProducer, uri) + return self._requestWithEndpoint( + key=("http-proxy", self._proxyURI.host, self._proxyURI.port), + endpoint=self._getEndpoint(self._proxyURI), + method=method, + parsedURI=URI.fromBytes(uri), + headers=headers, + bodyProducer=bodyProducer, + requestPath=uri, + ) class ScrapyAgent(object): @@ -287,18 +285,33 @@ class ScrapyAgent(object): scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams - if scheme == b'https' and not omitConnectTunnel: - proxyConf = (proxyHost, proxyPort, - request.headers.get(b'Proxy-Authorization', None)) - return self._TunnelingAgent(reactor, proxyConf, - contextFactory=self._contextFactory, connectTimeout=timeout, - bindAddress=bindaddress, pool=self._pool) + if scheme == b'https' and not omitConnectTunnel: + proxyAuth = request.headers.get(b'Proxy-Authorization', None) + proxyConf = (proxyHost, proxyPort, proxyAuth) + return self._TunnelingAgent( + reactor=reactor, + proxyConf=proxyConf, + contextFactory=self._contextFactory, + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) else: - return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'), - connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) + return self._ProxyAgent( + reactor=reactor, + proxyURI=to_bytes(proxy, encoding='ascii'), + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) - return self._Agent(reactor, contextFactory=self._contextFactory, - connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) + return self._Agent( + reactor=reactor, + contextFactory=self._contextFactory, + connectTimeout=timeout, + bindAddress=bindaddress, + pool=self._pool, + ) def download_request(self, request): timeout = request.meta.get('download_timeout') or self._connectTimeout @@ -329,8 +342,7 @@ class ScrapyAgent(object): else: bodyproducer = None start_time = time() - d = agent.request( - method, to_bytes(url, encoding='ascii'), headers, bodyproducer) + d = agent.request(method, to_bytes(url, encoding='ascii'), headers, bodyproducer) # set download latency d.addCallback(self._cb_latency, request, start_time) # response body is ready to be consumed @@ -385,8 +397,9 @@ class ScrapyAgent(object): txresponse._transport._producer.abortConnection() d = defer.Deferred(_cancel) - txresponse.deliverBody(_ResponseReader( - d, txresponse, request, maxsize, warnsize, fail_on_dataloss)) + txresponse.deliverBody( + _ResponseReader(d, txresponse, request, maxsize, warnsize, fail_on_dataloss) + ) # save response for timeouts self._txresponse = txresponse @@ -421,22 +434,20 @@ class _RequestBodyProducer(object): class _ResponseReader(protocol.Protocol): - def __init__(self, finished, txresponse, request, maxsize, warnsize, - fail_on_dataloss): + def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss): self._finished = finished self._txresponse = txresponse self._request = request self._bodybuf = BytesIO() - self._maxsize = maxsize - self._warnsize = warnsize + self._maxsize = maxsize + self._warnsize = warnsize self._fail_on_dataloss = fail_on_dataloss self._fail_on_dataloss_warned = False self._reached_warnsize = False self._bytes_received = 0 def dataReceived(self, bodyBytes): - # This maybe called several times after cancel was called with buffered - # data. + # This maybe called several times after cancel was called with buffered data. if self._finished.called: return @@ -449,8 +460,7 @@ class _ResponseReader(protocol.Protocol): {'bytes': self._bytes_received, 'maxsize': self._maxsize, 'request': self._request}) - # Clear buffer earlier to avoid keeping data in memory for a long - # time. + # Clear buffer earlier to avoid keeping data in memory for a long time. self._bodybuf.truncate(0) self._finished.cancel() diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 74afb3f10..4ed482058 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -1,5 +1,9 @@ import logging + from OpenSSL import SSL +from service_identity.exceptions import CertificateError +from twisted.internet._sslverify import ClientTLSOptions, verifyHostname, VerificationError +from twisted.internet.ssl import AcceptableCiphers from scrapy import twisted_version from scrapy.utils.ssl import x509name_to_string, get_temp_key_info @@ -7,12 +11,14 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info logger = logging.getLogger(__name__) + METHOD_SSLv3 = 'SSLv3' METHOD_TLS = 'TLS' METHOD_TLSv10 = 'TLSv1.0' METHOD_TLSv11 = 'TLSv1.1' METHOD_TLSv12 = 'TLSv1.2' + openssl_methods = { METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) @@ -22,95 +28,65 @@ openssl_methods = { } -if twisted_version >= (14, 0, 0): - # ClientTLSOptions requires a recent-enough version of Twisted. - # Not having ScrapyClientTLSOptions should not matter for older - # Twisted versions because it is not used in the fallback - # ScrapyClientContextFactory. - - # taken from twisted/twisted/internet/_sslverify.py - - try: - # XXX: this try-except is not needed in Twisted 17.0.0+ because - # it requires pyOpenSSL 0.16+. - from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START - except ImportError: - SSL_CB_HANDSHAKE_START = 0x10 - SSL_CB_HANDSHAKE_DONE = 0x20 - - from twisted.internet.ssl import AcceptableCiphers - from twisted.internet._sslverify import (ClientTLSOptions, - verifyHostname, - VerificationError) - try: - # XXX: this import would fail on Debian jessie with system installed - # service_identity library, due to lack of cryptography.x509 dependency - # See https://github.com/pyca/service_identity/issues/21 - from service_identity.exceptions import CertificateError - verification_errors = (CertificateError, VerificationError) - except ImportError: - verification_errors = VerificationError - - if twisted_version < (17, 0, 0): - from twisted.internet._sslverify import _maybeSetHostNameIndication - set_tlsext_host_name = _maybeSetHostNameIndication - else: - def set_tlsext_host_name(connection, hostNameBytes): - connection.set_tlsext_host_name(hostNameBytes) +if twisted_version < (17, 0, 0): + from twisted.internet._sslverify import _maybeSetHostNameIndication as set_tlsext_host_name +else: + def set_tlsext_host_name(connection, hostNameBytes): + connection.set_tlsext_host_name(hostNameBytes) - class ScrapyClientTLSOptions(ClientTLSOptions): - """ - SSL Client connection creator ignoring certificate verification errors - (for genuinely invalid certificates or bugs in verification code). +class ScrapyClientTLSOptions(ClientTLSOptions): + """ + SSL Client connection creator ignoring certificate verification errors + (for genuinely invalid certificates or bugs in verification code). - Same as Twisted's private _sslverify.ClientTLSOptions, - except that VerificationError, CertificateError and ValueError - exceptions are caught, so that the connection is not closed, only - logging warnings. Also, HTTPS connection parameters logging is added. - """ + Same as Twisted's private _sslverify.ClientTLSOptions, + except that VerificationError, CertificateError and ValueError + exceptions are caught, so that the connection is not closed, only + logging warnings. Also, HTTPS connection parameters logging is added. + """ - def __init__(self, hostname, ctx, verbose_logging=False): - super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) - self.verbose_logging = verbose_logging + def __init__(self, hostname, ctx, verbose_logging=False): + super(ScrapyClientTLSOptions, self).__init__(hostname, ctx) + self.verbose_logging = verbose_logging - def _identityVerifyingInfoCallback(self, connection, where, ret): - if where & SSL_CB_HANDSHAKE_START: - set_tlsext_host_name(connection, self._hostnameBytes) - elif where & SSL_CB_HANDSHAKE_DONE: - if self.verbose_logging: - if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 - if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 - logger.debug('SSL connection to %s using protocol %s, cipher %s', - self._hostnameASCII, - connection.get_protocol_version_name(), - connection.get_cipher_name(), - ) - else: - logger.debug('SSL connection to %s using cipher %s', - self._hostnameASCII, - connection.get_cipher_name(), - ) - server_cert = connection.get_peer_certificate() - logger.debug('SSL connection certificate: issuer "%s", subject "%s"', - x509name_to_string(server_cert.get_issuer()), - x509name_to_string(server_cert.get_subject()), - ) - key_info = get_temp_key_info(connection._ssl) - if key_info: - logger.debug('SSL temp key: %s', key_info) + def _identityVerifyingInfoCallback(self, connection, where, ret): + if where & SSL.SSL_CB_HANDSHAKE_START: + set_tlsext_host_name(connection, self._hostnameBytes) + elif where & SSL.SSL_CB_HANDSHAKE_DONE: + if self.verbose_logging: + if hasattr(connection, 'get_cipher_name'): # requires pyOPenSSL 0.15 + if hasattr(connection, 'get_protocol_version_name'): # requires pyOPenSSL 16.0.0 + logger.debug('SSL connection to %s using protocol %s, cipher %s', + self._hostnameASCII, + connection.get_protocol_version_name(), + connection.get_cipher_name(), + ) + else: + logger.debug('SSL connection to %s using cipher %s', + self._hostnameASCII, + connection.get_cipher_name(), + ) + server_cert = connection.get_peer_certificate() + logger.debug('SSL connection certificate: issuer "%s", subject "%s"', + x509name_to_string(server_cert.get_issuer()), + x509name_to_string(server_cert.get_subject()), + ) + key_info = get_temp_key_info(connection._ssl) + if key_info: + logger.debug('SSL temp key: %s', key_info) - try: - verifyHostname(connection, self._hostnameASCII) - except verification_errors as e: - logger.warning( - 'Remote certificate is not valid for hostname "{}"; {}'.format( - self._hostnameASCII, e)) + try: + verifyHostname(connection, self._hostnameASCII) + except (CertificateError, VerificationError) as e: + logger.warning( + 'Remote certificate is not valid for hostname "{}"; {}'.format( + self._hostnameASCII, e)) - except ValueError as e: - logger.warning( - 'Ignoring error while verifying certificate ' - 'from host "{}" (exception: {})'.format( - self._hostnameASCII, repr(e))) + except ValueError as e: + logger.warning( + 'Ignoring error while verifying certificate ' + 'from host "{}" (exception: {})'.format( + self._hostnameASCII, repr(e))) - DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT') +DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')