From 9a8edf2bf1172e8eec282a70ad46a9dacff76d62 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 27 Sep 2018 19:52:29 +0500 Subject: [PATCH] Tests for setting SSL ciphers. --- tests/mockserver.py | 10 +++++++ tests/test_downloader_handlers.py | 43 +++++++++++++++++++++++++- tests/test_webclient.py | 50 ++++++++++++++++++++++++++++++- 3 files changed, 101 insertions(+), 2 deletions(-) diff --git a/tests/mockserver.py b/tests/mockserver.py index 3fa4bc0f0..8be8a36bb 100644 --- a/tests/mockserver.py +++ b/tests/mockserver.py @@ -3,6 +3,8 @@ import sys, time, random, os, json from six.moves.urllib.parse import urlencode from subprocess import Popen, PIPE +from OpenSSL import SSL + from twisted.web.server import Site, NOT_DONE_YET from twisted.web.resource import Resource from twisted.web.static import File @@ -222,6 +224,14 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c ) +def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'): + factory = ssl_context_factory(keyfile, certfile) + ctx = factory.getContext() + ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2) + ctx.set_cipher_list(cipher_string) + return factory + + if __name__ == "__main__": root = Root() factory = Site(root) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index efef4192c..5df2ffe45 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto from scrapy.utils.python import to_bytes from scrapy.exceptions import NotConfigured -from tests.mockserver import MockServer, ssl_context_factory, Echo +from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factory from tests.spiders import SingleRequestSpider @@ -553,6 +553,47 @@ class Https11InvalidDNSPattern(Https11TestCase): super(Https11InvalidDNSPattern, self).setUp() +class Https11BadCiphers(unittest.TestCase): + scheme = 'https' + download_handler_cls = HTTP11DownloadHandler + + keyfile = 'keys/localhost.key' + certfile = 'keys/localhost.crt' + + def setUp(self): + self.tmpname = self.mktemp() + os.mkdir(self.tmpname) + FilePath(self.tmpname).child("file").setContent(b"0123456789") + r = static.File(self.tmpname) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.host = 'localhost' + self.port = reactor.listenSSL( + 0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'), + interface=self.host) + self.portno = self.port.getHost().port + self.download_handler = self.download_handler_cls( + Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})) + self.download_request = self.download_handler.download_request + + @defer.inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + if hasattr(self.download_handler, 'close'): + yield self.download_handler.close() + shutil.rmtree(self.tmpname) + + def getURL(self, path): + return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path) + + def test_download(self): + request = Request(self.getURL('file')) + d = self.download_request(request, Spider('foo')) + d.addCallback(lambda r: r.body) + d.addCallback(self.assertEqual, b"0123456789") + return d + + class Http11MockServerTestCase(unittest.TestCase): """HTTP 1.1 test case with MockServer""" diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 766329b57..2ebe075ab 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -8,15 +8,18 @@ import shutil from twisted.trial import unittest from twisted.web import server, static, util, resource -from twisted.internet import reactor, defer +from twisted.internet import reactor, defer, ssl from twisted.test.proto_helpers import StringTransport from twisted.python.filepath import FilePath from twisted.protocols.policies import WrappingFactory from twisted.internet.defer import inlineCallbacks from scrapy.core.downloader import webclient as client +from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory from scrapy.http import Request, Headers +from scrapy.settings import Settings from scrapy.utils.python import to_bytes, to_unicode +from tests.mockserver import ssl_context_factory, broken_ssl_context_factory def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): @@ -363,3 +366,48 @@ class WebClientTestCase(unittest.TestCase): self.assertEqual(content_encoding, EncodingResource.out_encoding) self.assertEqual( response.body.decode(content_encoding), to_unicode(original_body)) + + +class WebClientSSLTestCase(unittest.TestCase): + context_factory = None + + def _listen(self, site): + return reactor.listenSSL( + 0, site, + contextFactory=self.context_factory or ssl_context_factory(), + interface="127.0.0.1") + + def getURL(self, path): + return "https://127.0.0.1:%d/%s" % (self.portno, path) + + def setUp(self): + self.tmpname = self.mktemp() + os.mkdir(self.tmpname) + FilePath(self.tmpname).child("file").setContent(b"0123456789") + r = static.File(self.tmpname) + r.putChild(b"payload", PayloadResource()) + self.site = server.Site(r, timeout=None) + self.wrapper = WrappingFactory(self.site) + self.port = self._listen(self.wrapper) + self.portno = self.port.getHost().port + + @inlineCallbacks + def tearDown(self): + yield self.port.stopListening() + shutil.rmtree(self.tmpname) + + def testPayload(self): + s = "0123456789" * 10 + return getPage(self.getURL("payload"), body=s).addCallback( + self.assertEqual, to_bytes(s)) + + +class WebClientBrokenSSLTestCase(WebClientSSLTestCase): + context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA') + + def testPayload(self): + s = "0123456789" * 10 + settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}) + return getPage(self.getURL("payload"), body=s, + contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual, + to_bytes(s))