diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py index 28e8a0e53..da4dd8f35 100644 --- a/scrapy/core/downloader/contextfactory.py +++ b/scrapy/core/downloader/contextfactory.py @@ -7,47 +7,79 @@ if twisted_version >= (14, 0, 0): from zope.interface.declarations import implementer - from twisted.internet.ssl import optionsForClientTLS + from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust + from twisted.internet._sslverify import ClientTLSOptions from twisted.web.client import BrowserLikePolicyForHTTPS from twisted.web.iweb import IPolicyForHTTPS @implementer(IPolicyForHTTPS) class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS): """ - Using Twisted recommended context factory for twisted.web.client.Agent + Non-peer-certificate verifying HTTPS context factory - Quoting: - "The default is to use a BrowserLikePolicyForHTTPS, - so unless you have special requirements you can leave this as-is." + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation - See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html + 'A TLS/SSL connection established with [this method] may + understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.' """ + def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs): + super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs) + self._ssl_method = method + + def getCertificateOptions(self): + # setting verify=True will require you to provide CAs + # to verify against; in other words: it's not that simple + return CertificateOptions(verify=False, method=self._ssl_method) + + # kept for old-style HTTP/1.0 downloader context twisted calls, + # e.g. connectSSL() + def getContext(self, hostname=None, port=None): + return self.getCertificateOptions().getContext() + + def creatorForNetloc(self, hostname, port): + return ClientTLSOptions(hostname.decode("ascii"), self.getContext()) + @implementer(IPolicyForHTTPS) - class OpenSSLMethodContextFactory(ScrapyClientContextFactory): + class BrowserLikeContextFactory(ScrapyClientContextFactory): + """ + Twisted-recommended context factory for web clients. - openssl_method = SSL.SSLv23_METHOD + Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html: + "The default is to use a BrowserLikePolicyForHTTPS, + so unless you have special requirements you can leave this as-is." + creatorForNetloc() is the same as BrowserLikePolicyForHTTPS + except this context factory allows setting the TLS/SSL method to use. + + Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD) + which allows TLS protocol negotiation. + """ def creatorForNetloc(self, hostname, port): + + # trustRoot set to platformTrust() will use the platform's root CAs. + # + # This means that a website like https://www.cacert.org will be rejected + # by default, since CAcert.org CA certificate is seldom shipped. return optionsForClientTLS(hostname.decode("ascii"), - trustRoot=self._trustRoot, + trustRoot=platformTrust(), extraCertificateOptions={ - 'method': self.openssl_method + 'method': self._ssl_method, }) else: - class OpenSSLMethodContextFactory(ClientContextFactory): + class ScrapyClientContextFactory(ClientContextFactory): "A SSL context factory which is more permissive against SSL bugs." # see https://github.com/scrapy/scrapy/issues/82 # and https://github.com/scrapy/scrapy/issues/26 # and https://github.com/scrapy/scrapy/issues/981 - openssl_method = SSL.SSLv23_METHOD - def __init__(self): - self.method = self.openssl_method + def __init__(self, method=SSL.SSLv23_METHOD): + self.method = method def getContext(self, hostname=None, port=None): ctx = ClientContextFactory.getContext(self) @@ -57,13 +89,3 @@ else: if hostname and ClientTLSOptions is not None: # workaround for TLS SNI ClientTLSOptions(hostname, ctx) return ctx - - ScrapyClientContextFactory = OpenSSLMethodContextFactory - - -class SSLv3ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.SSLv3_METHOD - - -class TLSv1ContextFactory(OpenSSLMethodContextFactory): - openssl_method = SSL.TLSv1_METHOD diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index ad3285a32..522bac16f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -18,6 +18,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ from scrapy.http import Headers from scrapy.responsetypes import responsetypes from scrapy.core.downloader.webclient import _parse +from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS from scrapy.utils.misc import load_object from scrapy.utils.python import to_bytes, to_unicode from scrapy import twisted_version @@ -31,8 +32,16 @@ class HTTP11DownloadHandler(object): self._pool = HTTPConnectionPool(reactor, persistent=True) self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN') self._pool._factory.noisy = False + + self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')] self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) - self._contextFactory = self._contextFactoryClass() + # try method-aware context factory + try: + self._contextFactory = self._contextFactoryClass(method=self._sslMethod) + except TypeError: + # use defaults + self._contextFactory = self._contextFactoryClass() + self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE') self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE') self._disconnect_timeout = 1 diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py new file mode 100644 index 000000000..121ee83ac --- /dev/null +++ b/scrapy/core/downloader/tls.py @@ -0,0 +1,16 @@ +from OpenSSL import SSL + + +METHOD_SSLv3 = 'SSLv3' +METHOD_TLS = 'TLS' +METHOD_TLSv10 = 'TLSv1.0' +METHOD_TLSv11 = 'TLSv1.1' +METHOD_TLSv12 = 'TLSv1.2' + +openssl_methods = { + METHOD_TLS: SSL.SSLv23_METHOD, # protocol negotiation (recommended) + METHOD_SSLv3: SSL.SSLv3_METHOD, # SSL 3 (NOT recommended) + METHOD_TLSv10: SSL.TLSv1_METHOD, # TLS 1.0 only + METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5), # TLS 1.1 only + METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6), # TLS 1.2 only +} diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 44e74dc61..7a4f640ce 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -83,6 +83,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader' DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' +DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform, + # also allowing negotiation DOWNLOADER_MIDDLEWARES = {} diff --git a/tests/keys/server.pem b/tests/keys/server.pem new file mode 100644 index 000000000..8356f4a49 --- /dev/null +++ b/tests/keys/server.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI +AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK +jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl +qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi +AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr +gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9 +BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+ +CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd +hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p +Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1 +Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO +V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+ +Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0 +pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf +WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL +RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt +FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz +3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H +mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du +IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ +fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO +H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+ +WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf +L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI +IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM +YDXg6cfx9R5QePjvZohlItOu +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM +BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD +Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV +BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa +c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx +NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX +BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG +A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl +c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk +bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y +wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG +qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z +jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv +re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE +dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu +zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw +7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI +cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B +eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw +CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj +tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/ +gU7u1w== +-----END CERTIFICATE----- diff --git a/tests/mockserver.py b/tests/mockserver.py index 365ec81fd..087ad958f 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -206,8 +206,8 @@ class MockServer(): def ssl_context_factory(): return ssl.DefaultOpenSSLContextFactory( - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), - os.path.join(os.path.dirname(__file__), 'keys/cert.pem'), + os.path.join(os.path.dirname(__file__), 'keys/server.pem'), + os.path.join(os.path.dirname(__file__), 'keys/server.pem'), )