mirror of https://github.com/scrapy/scrapy.git
Backward compatibility for HTTP/10 context factory
New DOWNLOADER_CLIENT_TLS_METHOD setting to configure TLS method
This commit is contained in:
Paul Tremberth
parent
406b9a06fe
commit
57990fba7a
|
|
@ -7,47 +7,79 @@ if twisted_version >= (14, 0, 0):
|
|||
|
||||
from zope.interface.declarations import implementer
|
||||
|
||||
from twisted.internet.ssl import optionsForClientTLS
|
||||
from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
|
||||
from twisted.internet._sslverify import ClientTLSOptions
|
||||
from twisted.web.client import BrowserLikePolicyForHTTPS
|
||||
from twisted.web.iweb import IPolicyForHTTPS
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
|
||||
"""
|
||||
Using Twisted recommended context factory for twisted.web.client.Agent
|
||||
Non-peer-certificate verifying HTTPS context factory
|
||||
|
||||
Quoting:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation
|
||||
|
||||
See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html
|
||||
'A TLS/SSL connection established with [this method] may
|
||||
understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
|
||||
"""
|
||||
|
||||
def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
|
||||
super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs)
|
||||
self._ssl_method = method
|
||||
|
||||
def getCertificateOptions(self):
|
||||
# setting verify=True will require you to provide CAs
|
||||
# to verify against; in other words: it's not that simple
|
||||
return CertificateOptions(verify=False, method=self._ssl_method)
|
||||
|
||||
# kept for old-style HTTP/1.0 downloader context twisted calls,
|
||||
# e.g. connectSSL()
|
||||
def getContext(self, hostname=None, port=None):
|
||||
return self.getCertificateOptions().getContext()
|
||||
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
return ClientTLSOptions(hostname.decode("ascii"), self.getContext())
|
||||
|
||||
|
||||
@implementer(IPolicyForHTTPS)
|
||||
class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
|
||||
class BrowserLikeContextFactory(ScrapyClientContextFactory):
|
||||
"""
|
||||
Twisted-recommended context factory for web clients.
|
||||
|
||||
openssl_method = SSL.SSLv23_METHOD
|
||||
Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
|
||||
"The default is to use a BrowserLikePolicyForHTTPS,
|
||||
so unless you have special requirements you can leave this as-is."
|
||||
|
||||
creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
|
||||
except this context factory allows setting the TLS/SSL method to use.
|
||||
|
||||
Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
|
||||
which allows TLS protocol negotiation.
|
||||
"""
|
||||
def creatorForNetloc(self, hostname, port):
|
||||
|
||||
# trustRoot set to platformTrust() will use the platform's root CAs.
|
||||
#
|
||||
# This means that a website like https://www.cacert.org will be rejected
|
||||
# by default, since CAcert.org CA certificate is seldom shipped.
|
||||
return optionsForClientTLS(hostname.decode("ascii"),
|
||||
trustRoot=self._trustRoot,
|
||||
trustRoot=platformTrust(),
|
||||
extraCertificateOptions={
|
||||
'method': self.openssl_method
|
||||
'method': self._ssl_method,
|
||||
})
|
||||
|
||||
|
||||
else:
|
||||
|
||||
class OpenSSLMethodContextFactory(ClientContextFactory):
|
||||
class ScrapyClientContextFactory(ClientContextFactory):
|
||||
"A SSL context factory which is more permissive against SSL bugs."
|
||||
# see https://github.com/scrapy/scrapy/issues/82
|
||||
# and https://github.com/scrapy/scrapy/issues/26
|
||||
# and https://github.com/scrapy/scrapy/issues/981
|
||||
openssl_method = SSL.SSLv23_METHOD
|
||||
|
||||
def __init__(self):
|
||||
self.method = self.openssl_method
|
||||
def __init__(self, method=SSL.SSLv23_METHOD):
|
||||
self.method = method
|
||||
|
||||
def getContext(self, hostname=None, port=None):
|
||||
ctx = ClientContextFactory.getContext(self)
|
||||
|
|
@ -57,13 +89,3 @@ else:
|
|||
if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
|
||||
ClientTLSOptions(hostname, ctx)
|
||||
return ctx
|
||||
|
||||
ScrapyClientContextFactory = OpenSSLMethodContextFactory
|
||||
|
||||
|
||||
class SSLv3ContextFactory(OpenSSLMethodContextFactory):
|
||||
openssl_method = SSL.SSLv3_METHOD
|
||||
|
||||
|
||||
class TLSv1ContextFactory(OpenSSLMethodContextFactory):
|
||||
openssl_method = SSL.TLSv1_METHOD
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
|
|||
from scrapy.http import Headers
|
||||
from scrapy.responsetypes import responsetypes
|
||||
from scrapy.core.downloader.webclient import _parse
|
||||
from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.python import to_bytes, to_unicode
|
||||
from scrapy import twisted_version
|
||||
|
|
@ -31,8 +32,16 @@ class HTTP11DownloadHandler(object):
|
|||
self._pool = HTTPConnectionPool(reactor, persistent=True)
|
||||
self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
|
||||
self._pool._factory.noisy = False
|
||||
|
||||
self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
|
||||
self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
|
||||
self._contextFactory = self._contextFactoryClass()
|
||||
# try method-aware context factory
|
||||
try:
|
||||
self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
|
||||
except TypeError:
|
||||
# use defaults
|
||||
self._contextFactory = self._contextFactoryClass()
|
||||
|
||||
self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
|
||||
self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
|
||||
self._disconnect_timeout = 1
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
from OpenSSL import SSL
|
||||
|
||||
|
||||
METHOD_SSLv3 = 'SSLv3'
|
||||
METHOD_TLS = 'TLS'
|
||||
METHOD_TLSv10 = 'TLSv1.0'
|
||||
METHOD_TLSv11 = 'TLSv1.1'
|
||||
METHOD_TLSv12 = 'TLSv1.2'
|
||||
|
||||
openssl_methods = {
|
||||
METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended)
|
||||
METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended)
|
||||
METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only
|
||||
METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only
|
||||
METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only
|
||||
}
|
||||
|
|
@ -83,6 +83,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
|
|||
|
||||
DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
|
||||
DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
|
||||
DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
|
||||
# also allowing negotiation
|
||||
|
||||
DOWNLOADER_MIDDLEWARES = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,52 @@
|
|||
-----BEGIN PRIVATE KEY-----
|
||||
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI
|
||||
AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK
|
||||
jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl
|
||||
qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi
|
||||
AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr
|
||||
gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9
|
||||
BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+
|
||||
CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd
|
||||
hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p
|
||||
Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1
|
||||
Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO
|
||||
V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+
|
||||
Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0
|
||||
pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf
|
||||
WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL
|
||||
RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt
|
||||
FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz
|
||||
3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H
|
||||
mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du
|
||||
IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ
|
||||
fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO
|
||||
H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+
|
||||
WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf
|
||||
L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI
|
||||
IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM
|
||||
YDXg6cfx9R5QePjvZohlItOu
|
||||
-----END PRIVATE KEY-----
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM
|
||||
BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD
|
||||
Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV
|
||||
BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa
|
||||
c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx
|
||||
NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX
|
||||
BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG
|
||||
A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl
|
||||
c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk
|
||||
bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y
|
||||
wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG
|
||||
qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z
|
||||
jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv
|
||||
re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE
|
||||
dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu
|
||||
zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw
|
||||
7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI
|
||||
cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B
|
||||
eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw
|
||||
CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj
|
||||
tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/
|
||||
gU7u1w==
|
||||
-----END CERTIFICATE-----
|
||||
|
|
@ -206,8 +206,8 @@ class MockServer():
|
|||
|
||||
def ssl_context_factory():
|
||||
return ssl.DefaultOpenSSLContextFactory(
|
||||
os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
|
||||
os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
|
||||
os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
|
||||
os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue