From 30a27effa9b20eb5cf410e00840a56286a2e8d92 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 19 Feb 2016 01:54:23 +0100 Subject: [PATCH 01/10] Use best practices for TLS connections when using Twisted>=14.0 --- scrapy/core/downloader/contextfactory.py | 88 ++++++++++++++++++------ 1 file changed, 66 insertions(+), 22 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 0b39b89d8..347b74648 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,28 +1,72 @@ from OpenSSL import SSL from twisted.internet.ssl import ClientContextFactory -try: - # available since twisted 14.0 - from twisted.internet._sslverify import ClientTLSOptions -except ImportError: - ClientTLSOptions = None + +from scrapy import twisted_version + +if twisted_version >= (14, 0, 0): + + from twisted.web.client import BrowserLikePolicyForHTTPS + from twisted.internet.ssl import optionsForClientTLS + + class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): + """ + Using Twisted recommended context factory for twisted.web.client.Agent + + Quoting: + "The default is to use a BrowserLikePolicyForHTTPS, + so unless you have special requirements you can leave this as-is." + + See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html + """ -class ScrapyClientContextFactory(ClientContextFactory): - "A SSL context factory which is more permissive against SSL bugs." - # see https://github.com/scrapy/scrapy/issues/82 - # and https://github.com/scrapy/scrapy/issues/26 - # and https://github.com/scrapy/scrapy/issues/981 + class OpenSSLMethodContextFactory(ScrapyClientContextFactory): - def __init__(self): - # see this issue on why we use TLSv1_METHOD by default - # https://github.com/scrapy/scrapy/issues/194 - self.method = SSL.TLSv1_METHOD + openssl_method = SSL.SSLv23_METHOD - def getContext(self, hostname=None, port=None): - ctx = ClientContextFactory.getContext(self) - # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html - ctx.set_options(SSL.OP_ALL) - if hostname and ClientTLSOptions is not None: # workaround for TLS SNI - ClientTLSOptions(hostname, ctx) - return ctx + def creatorForNetloc(self, hostname, port): + return optionsForClientTLS(hostname.decode("ascii"), + trustRoot=self._trustRoot, + extraCertificateOptions={ + 'method': self.openssl_method + }) + + +else: + + class OpenSSLMethodContextFactory(ClientContextFactory): + "A SSL context factory which is more permissive against SSL bugs." + # see https://github.com/scrapy/scrapy/issues/82 + # and https://github.com/scrapy/scrapy/issues/26 + # and https://github.com/scrapy/scrapy/issues/981 + openssl_method = SSL.SSLv23_METHOD + + def __init__(self): + self.method = self.openssl_method + + def getContext(self, hostname=None, port=None): + ctx = ClientContextFactory.getContext(self) + # Enable all workarounds to SSL bugs as documented by + # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + ctx.set_options(SSL.OP_ALL) + if hostname and ClientTLSOptions is not None: # workaround for TLS SNI + ClientTLSOptions(hostname, ctx) + return ctx + + ScrapyClientContextFactory = OpenSSLMethodContextFactory + + +class SSLv3ContextFactory(OpenSSLMethodContextFactory): + openssl_method = SSL.SSLv3_METHOD + + +class TLSv1ContextFactory(OpenSSLMethodContextFactory): + openssl_method = SSL.TLSv1_METHOD + + +class TLSv11ContextFactory(OpenSSLMethodContextFactory): + openssl_method = SSL.TLSv1_1_METHOD + + +class TLSv12ContextFactory(OpenSSLMethodContextFactory): + openssl_method = SSL.TLSv1_2_METHOD From 45f972cb3f664705fefc932f34a1e88134819fb4 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 19 Feb 2016 02:31:57 +0100 Subject: [PATCH 02/10] Implement IPolicyForHTTPS --- scrapy/core/downloader/contextfactory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 347b74648..4d90d17f8 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -5,9 +5,11 @@ from scrapy import twisted_version if twisted_version >= (14, 0, 0): - from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.internet.ssl import optionsForClientTLS + from twisted.web.client import BrowserLikePolicyForHTTPS + from twisted.web.iweb import IPolicyForHTTPS + @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ Using Twisted recommended context factory for twisted.web.client.Agent @@ -20,6 +22,7 @@ if twisted_version >= (14, 0, 0): """ + @implementer(IPolicyForHTTPS) class OpenSSLMethodContextFactory(ScrapyClientContextFactory): openssl_method = SSL.SSLv23_METHOD From 406b9a06fe67db09b53041724a126d59f11cb519 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Fri, 19 Feb 2016 02:56:26 +0100 Subject: [PATCH 03/10] Add missing import for implementer Also remove TLSv1.1 and TLSv1.2 method: these are available only from pyOpenSSL 0.14 https://github.com/pyca/pyopenssl/releases/tag/v0.14a1 --- scrapy/core/downloader/contextfactory.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 4d90d17f8..28e8a0e53 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -5,6 +5,8 @@ from scrapy import twisted_version if twisted_version >= (14, 0, 0): + from zope.interface.declarations import implementer + from twisted.internet.ssl import optionsForClientTLS from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @@ -65,11 +67,3 @@ class SSLv3ContextFactory(OpenSSLMethodContextFactory): class TLSv1ContextFactory(OpenSSLMethodContextFactory): openssl_method = SSL.TLSv1_METHOD - - -class TLSv11ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.TLSv1_1_METHOD - - -class TLSv12ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.TLSv1_2_METHOD From 57990fba7a12a83517456487979df6461a347328 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 20 Feb 2016 01:32:21 +0100 Subject: [PATCH 04/10] Backward compatibility for HTTP/10 context factory New DOWNLOADER_CLIENT_TLS_METHOD setting to configure TLS method --- scrapy/core/downloader/contextfactory.py | 70 +++++++++++++++-------- scrapy/core/downloader/handlers/http11.py | 11 +++- scrapy/core/downloader/tls.py | 16 ++++++ scrapy/settings/default_settings.py | 2 + tests/keys/server.pem | 52 +++++++++++++++++ tests/mockserver.py | 4 +- 6 files changed, 128 insertions(+), 27 deletions(-) create mode 100644 scrapy/core/downloader/tls.py create mode 100644 tests/keys/server.pem diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 28e8a0e53..da4dd8f35 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -7,47 +7,79 @@ if twisted_version >= (14, 0, 0): from zope.interface.declarations import implementer - from twisted.internet.ssl import optionsForClientTLS + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust + from twisted.internet._sslverify import ClientTLSOptions from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ - Using Twisted recommended context factory for twisted.web.client.Agent + Non-peer-certificate verifying HTTPS context factory - Quoting: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation - See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html + 'A TLS/SSL connection established with [this method] may + understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ + def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs): + super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs) + self._ssl_method = method + + def getCertificateOptions(self): + # setting verify=True will require you to provide CAs + # to verify against; in other words: it's not that simple + return CertificateOptions(verify=False, method=self._ssl_method) + + # kept for old-style HTTP/1.0 downloader context twisted calls, + # e.g. connectSSL() + def getContext(self, hostname=None, port=None): + return self.getCertificateOptions().getContext() + + def creatorForNetloc(self, hostname, port): + return ClientTLSOptions(hostname.decode("ascii"), self.getContext()) + @implementer(IPolicyForHTTPS) - class OpenSSLMethodContextFactory(ScrapyClientContextFactory): + class BrowserLikeContextFactory(ScrapyClientContextFactory): + """ + Twisted-recommended context factory for web clients. - openssl_method = SSL.SSLv23_METHOD + Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + "The default is to use a BrowserLikePolicyForHTTPS, + so unless you have special requirements you can leave this as-is." + creatorForNetloc() is the same as BrowserLikePolicyForHTTPS + except this context factory allows setting the TLS/SSL method to use. + + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation. + """ def creatorForNetloc(self, hostname, port): + + # trustRoot set to platformTrust() will use the platform's root CAs. + # + # This means that a website like https://www.cacert.org will be rejected + # by default, since CAcert.org CA certificate is seldom shipped. return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=self._trustRoot, + trustRoot=platformTrust(), extraCertificateOptions={ - 'method': self.openssl_method + 'method': self._ssl_method, }) else: - class OpenSSLMethodContextFactory(ClientContextFactory): + class ScrapyClientContextFactory(ClientContextFactory): "A SSL context factory which is more permissive against SSL bugs." # see https://github.com/scrapy/scrapy/issues/82 # and https://github.com/scrapy/scrapy/issues/26 # and https://github.com/scrapy/scrapy/issues/981 - openssl_method = SSL.SSLv23_METHOD - def __init__(self): - self.method = self.openssl_method + def __init__(self, method=SSL.SSLv23_METHOD): + self.method = method def getContext(self, hostname=None, port=None): ctx = ClientContextFactory.getContext(self) @@ -57,13 +89,3 @@ else: if hostname and ClientTLSOptions is not None: # workaround for TLS SNI ClientTLSOptions(hostname, ctx) return ctx - - ScrapyClientContextFactory = OpenSSLMethodContextFactory - - -class SSLv3ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.SSLv3_METHOD - - -class TLSv1ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.TLSv1_METHOD diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index ad3285a32..522bac16f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -18,6 +18,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse +from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version @@ -31,8 +32,16 @@ class HTTP11DownloadHandler(object): self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False + + self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - self._contextFactory = self._contextFactoryClass() + # try method-aware context factory + try: + self._contextFactory = self._contextFactoryClass(method=self._sslMethod) + except TypeError: + # use defaults + self._contextFactory = self._contextFactoryClass() + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._disconnect_timeout = 1 diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py new file mode 100644 index 000000000..121ee83ac --- /dev/null +++ b/scrapy/core/downloader/tls.py @@ -0,0 +1,16 @@ +from OpenSSL import SSL + + +METHOD_SSLv3 = 'SSLv3' +METHOD_TLS = 'TLS' +METHOD_TLSv10 = 'TLSv1.0' +METHOD_TLSv11 = 'TLSv1.1' +METHOD_TLSv12 = 'TLSv1.2' + +openssl_methods = { + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only + METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only +} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 44e74dc61..7a4f640ce 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -83,6 +83,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' +DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, + # also allowing negotiation DOWNLOADER_MIDDLEWARES = {} diff --git a/tests/keys/server.pem b/tests/keys/server.pem new file mode 100644 index 000000000..8356f4a49 --- /dev/null +++ b/tests/keys/server.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI +AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK +jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl +qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi +AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr +gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9 +BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+ +CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd +hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p +Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1 +Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO +V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+ +Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0 +pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf +WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL +RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt +FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz +3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H +mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du +IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ +fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO +H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+ +WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf +L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI +IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM +YDXg6cfx9R5QePjvZohlItOu +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM +BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD +Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV +BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa +c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx +NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX +BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG +A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl +c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk +bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y +wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG +qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z +jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv +re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE +dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu +zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw +7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI +cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B +eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw +CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj +tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/ +gU7u1w== +-----END CERTIFICATE----- diff --git a/tests/mockserver.py b/tests/mockserver.py index 365ec81fd..087ad958f 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -206,8 +206,8 @@ class MockServer(): def ssl_context_factory(): return ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + os.path.join(os.path.dirname(__file__), 'keys/server.pem'), + os.path.join(os.path.dirname(__file__), 'keys/server.pem'), ) From 49c757f208eb41e637171543b5b337f4a280387a Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 20 Feb 2016 01:58:59 +0100 Subject: [PATCH 05/10] Fix import for Ubuntu 12.04 precise (Twisted 11.1.0) Revert test server certificate change --- scrapy/core/downloader/contextfactory.py | 10 ++--- tests/keys/server.pem | 52 ------------------------ tests/mockserver.py | 4 +- 3 files changed, 5 insertions(+), 61 deletions(-) delete mode 100644 tests/keys/server.pem diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index da4dd8f35..5e6ad57a1 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -1,12 +1,11 @@ from OpenSSL import SSL from twisted.internet.ssl import ClientContextFactory -from scrapy import twisted_version - -if twisted_version >= (14, 0, 0): +try: from zope.interface.declarations import implementer + # the following should be available from Twisted 14.0.0 from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust from twisted.internet._sslverify import ClientTLSOptions from twisted.web.client import BrowserLikePolicyForHTTPS @@ -69,8 +68,7 @@ if twisted_version >= (14, 0, 0): 'method': self._ssl_method, }) - -else: +except ImportError: class ScrapyClientContextFactory(ClientContextFactory): "A SSL context factory which is more permissive against SSL bugs." @@ -86,6 +84,4 @@ else: # Enable all workarounds to SSL bugs as documented by # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html ctx.set_options(SSL.OP_ALL) - if hostname and ClientTLSOptions is not None: # workaround for TLS SNI - ClientTLSOptions(hostname, ctx) return ctx diff --git a/tests/keys/server.pem b/tests/keys/server.pem deleted file mode 100644 index 8356f4a49..000000000 --- a/tests/keys/server.pem +++ /dev/null @@ -1,52 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI -AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK -jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl -qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi -AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr -gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9 -BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+ -CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd -hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p -Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1 -Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO -V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+ -Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0 -pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf -WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL -RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt -FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz -3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H -mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du -IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ -fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO -H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+ -WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf -L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI -IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM -YDXg6cfx9R5QePjvZohlItOu ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM -BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD -Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV -BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa -c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx -NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX -BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG -A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl -c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk -bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y -wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG -qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z -jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv -re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE -dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu -zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw -7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI -cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B -eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw -CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj -tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/ -gU7u1w== ------END CERTIFICATE----- diff --git a/tests/mockserver.py b/tests/mockserver.py index 087ad958f..365ec81fd 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -206,8 +206,8 @@ class MockServer(): def ssl_context_factory(): return ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), 'keys/server.pem'), - os.path.join(os.path.dirname(__file__), 'keys/server.pem'), + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), ) From 3f946b5278cb964ce1dab1b6e8a3a74e16d00fca Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Sat, 20 Feb 2016 02:07:45 +0100 Subject: [PATCH 06/10] Fix super() call --- scrapy/core/downloader/contextfactory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 5e6ad57a1..afc52c354 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -24,7 +24,7 @@ try: """ def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs): - super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs) + super(ScrapyClientContextFactory, self).__init__(*args, **kwargs) self._ssl_method = method def getCertificateOptions(self): From 095495e99ce327d0fc6cf2064f8a6d871d2f7dd1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 Feb 2016 01:24:58 +0100 Subject: [PATCH 07/10] Backward-compatibility for common Scrapy context factory patterns --- scrapy/core/downloader/contextfactory.py | 13 ++++++++++++- scrapy/core/downloader/tls.py | 4 ++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index afc52c354..09fcd491a 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -30,7 +30,18 @@ try: def getCertificateOptions(self): # setting verify=True will require you to provide CAs # to verify against; in other words: it's not that simple - return CertificateOptions(verify=False, method=self._ssl_method) + + # backward-compatible SSL/TLS method: + # + # * this will respect `method` attribute in often recommended + # `ScrapyClientContextFactory` subclass + # (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133) + # + # * getattr() for `_ssl_method` attribute for context factories + # not calling super(..., self).__init__ + return CertificateOptions(verify=False, + method=getattr(self, 'method', + getattr(self, '_ssl_method', None))) # kept for old-style HTTP/1.0 downloader context twisted calls, # e.g. connectSSL() diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py index 121ee83ac..64ebb0714 100644 --- a/scrapy/core/downloader/tls.py +++ b/scrapy/core/downloader/tls.py @@ -8,8 +8,8 @@ METHOD_TLSv11 = 'TLSv1.1' METHOD_TLSv12 = 'TLSv1.2' openssl_methods = { - METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) - METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only From c9890d5f7662dc2673696ceb80870c5318746495 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 Feb 2016 01:26:04 +0100 Subject: [PATCH 08/10] Add warning for context factories not accepting `method` param --- scrapy/core/downloader/handlers/http11.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 522bac16f..1fe31b206 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -4,6 +4,7 @@ import re import logging from io import BytesIO from time import time +import warnings from six.moves.urllib.parse import urldefrag from zope.interface import implementer @@ -18,7 +19,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse -from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS +from scrapy.core.downloader.tls import openssl_methods from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version @@ -39,9 +40,12 @@ class HTTP11DownloadHandler(object): try: self._contextFactory = self._contextFactoryClass(method=self._sslMethod) except TypeError: - # use defaults + # use context factory defaults self._contextFactory = self._contextFactoryClass() - + warnings.warn(""" + You are using a context factory class that does not accept the `method` argument + (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD). + Please upgrade your context factory class to handle or ignore it.""") self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._disconnect_timeout = 1 From c29a1b98b3b49a961c67ac64fa303e0265dab2d1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 Feb 2016 16:01:18 +0100 Subject: [PATCH 09/10] Make warning message a bit nicer in logs --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1fe31b206..40ad2d814 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -42,10 +42,11 @@ class HTTP11DownloadHandler(object): except TypeError: # use context factory defaults self._contextFactory = self._contextFactoryClass() - warnings.warn(""" + msg = """ You are using a context factory class that does not accept the `method` argument (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD). - Please upgrade your context factory class to handle or ignore it.""") + Please upgrade your context factory class to handle or ignore it.""" + warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._disconnect_timeout = 1 From 0336c2504af45a87d2aae4c1ed45039c0e0ea183 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 24 Feb 2016 16:42:25 +0100 Subject: [PATCH 10/10] Use context factory class name in warning message --- scrapy/core/downloader/handlers/http11.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 40ad2d814..00c09c75c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -43,9 +43,10 @@ class HTTP11DownloadHandler(object): # use context factory defaults self._contextFactory = self._contextFactoryClass() msg = """ - You are using a context factory class that does not accept the `method` argument - (type OpenSSL.SSL method, e.g. OpenSSL.SSL.SSLv23_METHOD). - Please upgrade your context factory class to handle or ignore it.""" + '%s' does not accept `method` argument (type OpenSSL.SSL method,\ + e.g. OpenSSL.SSL.SSLv23_METHOD).\ + Please upgrade your context factory class to handle it or ignore it.""" % ( + settings['DOWNLOADER_CLIENTCONTEXTFACTORY'],) warnings.warn(msg) self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')