mirror of https://github.com/scrapy/scrapy.git
Remove fallback ScrapyClientContextFactory class (used in Twisted < 14.0.0)
16.0.0 is currently the minimum supported version
This commit is contained in:
Eugenio Lacuesta
parent
a940a80f58
commit
3164543ed1
|
|
@ -1,112 +1,85 @@
|
|||
from OpenSSL import SSL
|
||||
from twisted.internet.ssl import ClientContextFactory
|
||||
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
from zope.interface.declarations import implementer
|
||||
|
||||
from scrapy import twisted_version
|
||||
|
||||
if twisted_version >= (14, 0, 0):
|
||||
|
||||
from zope.interface.declarations import implementer
|
||||
|
||||
from twisted.internet.ssl import (optionsForClientTLS,
|
||||
CertificateOptions,
|
||||
platformTrust)
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
|
||||
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
|
||||
from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||
"""
|
||||
Non-peer-certificate verifying HTTPS context factory
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||
"""
|
||||
Non-peer-certificate verifying HTTPS context factory
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation
|
||||
|
||||
'A TLS/SSL connection established with [this method] may
|
||||
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
|
||||
"""
|
||||
'A TLS/SSL connection established with [this method] may
|
||||
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
|
||||
"""
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
|
||||
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
|
||||
self._ssl_method = method
|
||||
self.tls_verbose_logging = tls_verbose_logging
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, *args, **kwargs):
|
||||
super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
|
||||
self._ssl_method = method
|
||||
self.tls_verbose_logging = tls_verbose_logging
|
||||
|
||||
@classmethod
|
||||
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
|
||||
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
|
||||
return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
|
||||
@classmethod
|
||||
def from_settings(cls, settings, method=SSL.SSLv23_METHOD, *args, **kwargs):
|
||||
tls_verbose_logging = settings.getbool('DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING')
|
||||
return cls(method=method, tls_verbose_logging=tls_verbose_logging, *args, **kwargs)
|
||||
|
||||
def getCertificateOptions(self):
|
||||
# setting verify=True will require you to provide CAs
|
||||
# to verify against; in other words: it's not that simple
|
||||
def getCertificateOptions(self):
|
||||
# setting verify=True will require you to provide CAs
|
||||
# to verify against; in other words: it's not that simple
|
||||
|
||||
# backward-compatible SSL/TLS method:
|
||||
#
|
||||
# * this will respect `method` attribute in often recommended
|
||||
# `ScrapyClientContextFactory` subclass
|
||||
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
|
||||
#
|
||||
# * getattr() for `_ssl_method` attribute for context factories
|
||||
# not calling super(..., self).__init__
|
||||
return CertificateOptions(verify=False,
|
||||
method=getattr(self, 'method',
|
||||
getattr(self, '_ssl_method', None)),
|
||||
fixBrokenPeers=True,
|
||||
acceptableCiphers=DEFAULT_CIPHERS)
|
||||
# backward-compatible SSL/TLS method:
|
||||
#
|
||||
# * this will respect `method` attribute in often recommended
|
||||
# `ScrapyClientContextFactory` subclass
|
||||
# (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
|
||||
#
|
||||
# * getattr() for `_ssl_method` attribute for context factories
|
||||
# not calling super(..., self).__init__
|
||||
return CertificateOptions(verify=False,
|
||||
method=getattr(self, 'method',
|
||||
getattr(self, '_ssl_method', None)),
|
||||
fixBrokenPeers=True,
|
||||
acceptableCiphers=DEFAULT_CIPHERS)
|
||||
|
||||
# kept for old-style HTTP/1.0 downloader context twisted calls,
|
||||
# e.g. connectSSL()
|
||||
def getContext(self, hostname=None, port=None):
|
||||
return self.getCertificateOptions().getContext()
|
||||
# kept for old-style HTTP/1.0 downloader context twisted calls,
|
||||
# e.g. connectSSL()
|
||||
def getContext(self, hostname=None, port=None):
|
||||
return self.getCertificateOptions().getContext()
|
||||
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
|
||||
verbose_logging=self.tls_verbose_logging)
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext(),
|
||||
verbose_logging=self.tls_verbose_logging)
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
||||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
||||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
|
||||
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
Quoting https://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
# trustRoot set to platformTrust() will use the platform's root CAs.
|
||||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
# by default, since CAcert.org CA certificate is seldom shipped.
|
||||
return optionsForClientTLS(hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={
|
||||
'method': self._ssl_method,
|
||||
})
|
||||
|
||||
else:
|
||||
|
||||
class ScrapyClientContextFactory(ClientContextFactory):
|
||||
"A SSL context factory which is more permissive against SSL bugs."
|
||||
# see https://github.com/scrapy/scrapy/issues/82
|
||||
# and https://github.com/scrapy/scrapy/issues/26
|
||||
# and https://github.com/scrapy/scrapy/issues/981
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD):
|
||||
self.method = method
|
||||
|
||||
def getContext(self, hostname=None, port=None):
|
||||
ctx = ClientContextFactory.getContext(self)
|
||||
# Enable all workarounds to SSL bugs as documented by
|
||||
# https://www.openssl.org/docs/manmaster/man3/SSL_CTX_set_options.html
|
||||
ctx.set_options(SSL.OP_ALL)
|
||||
return ctx
|
||||
# trustRoot set to platformTrust() will use the platform's root CAs.
|
||||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
# by default, since CAcert.org CA certificate is seldom shipped.
|
||||
return optionsForClientTLS(hostname.decode("ascii"),
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={
|
||||
'method': self._ssl_method,
|
||||
})
|
||||
|
|
|
|||
|
|
@ -10,12 +10,14 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
METHOD_SSLv3 = 'SSLv3'
|
||||
METHOD_TLS = 'TLS'
|
||||
METHOD_TLSv10 = 'TLSv1.0'
|
||||
METHOD_TLSv11 = 'TLSv1.1'
|
||||
METHOD_TLSv12 = 'TLSv1.2'
|
||||
|
||||
|
||||
openssl_methods = {
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
|
||||
|
|
@ -25,11 +27,6 @@ openssl_methods = {
|
|||
}
|
||||
|
||||
|
||||
# ClientTLSOptions requires a recent-enough version of Twisted (14.0.0+)
|
||||
# Not having ScrapyClientTLSOptions should not matter for older
|
||||
# Twisted versions because it is not used in the fallback
|
||||
# ScrapyClientContextFactory.
|
||||
|
||||
try:
|
||||
# XXX: this import would fail on Debian jessie with system installed
|
||||
# service_identity library, due to lack of cryptography.x509 dependency
|
||||
|
|
|
|||
Loading…
Reference in New Issue