Tests for setting SSL ciphers.

tests/mockserver.py

@@ -3,6 +3,8 @@ import sys, time, random, os, json
 from six.moves.urllib.parse import urlencode
 from subprocess import Popen, PIPE
 
+from OpenSSL import SSL
+
 from twisted.web.server import Site, NOT_DONE_YET
 from twisted.web.resource import Resource
 from twisted.web.static import File
@@ -222,6 +224,14 @@ def ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.c
          )
 
 
+def broken_ssl_context_factory(keyfile='keys/localhost.key', certfile='keys/localhost.crt', cipher_string='DEFAULT'):
+    factory = ssl_context_factory(keyfile, certfile)
+    ctx = factory.getContext()
+    ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_2)
+    ctx.set_cipher_list(cipher_string)
+    return factory
+
+
 if __name__ == "__main__":
     root = Root()
     factory = Site(root)

tests/test_downloader_handlers.py

@@ -39,7 +39,7 @@ from scrapy.utils.test import get_crawler, skip_if_no_boto
 from scrapy.utils.python import to_bytes
 from scrapy.exceptions import NotConfigured
 
-from tests.mockserver import MockServer, ssl_context_factory, Echo
+from tests.mockserver import MockServer, ssl_context_factory, Echo, broken_ssl_context_factory
 from tests.spiders import SingleRequestSpider
 
 
@@ -553,6 +553,47 @@ class Https11InvalidDNSPattern(Https11TestCase):
         super(Https11InvalidDNSPattern, self).setUp()
 
 
+class Https11BadCiphers(unittest.TestCase):
+    scheme = 'https'
+    download_handler_cls = HTTP11DownloadHandler
+
+    keyfile = 'keys/localhost.key'
+    certfile = 'keys/localhost.crt'
+
+    def setUp(self):
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.host = 'localhost'
+        self.port = reactor.listenSSL(
+            0, self.wrapper, broken_ssl_context_factory(self.keyfile, self.certfile, cipher_string='CAMELLIA256-SHA'),
+            interface=self.host)
+        self.portno = self.port.getHost().port
+        self.download_handler = self.download_handler_cls(
+            Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))
+        self.download_request = self.download_handler.download_request
+
+    @defer.inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        if hasattr(self.download_handler, 'close'):
+            yield self.download_handler.close()
+        shutil.rmtree(self.tmpname)
+
+    def getURL(self, path):
+        return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
+
+    def test_download(self):
+        request = Request(self.getURL('file'))
+        d = self.download_request(request, Spider('foo'))
+        d.addCallback(lambda r: r.body)
+        d.addCallback(self.assertEqual, b"0123456789")
+        return d
+
+
 class Http11MockServerTestCase(unittest.TestCase):
     """HTTP 1.1 test case with MockServer"""
 

tests/test_webclient.py

@@ -8,15 +8,18 @@ import shutil
 
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
-from twisted.internet import reactor, defer
+from twisted.internet import reactor, defer, ssl
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
 from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import webclient as client
+from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
 from scrapy.http import Request, Headers
+from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes, to_unicode
+from tests.mockserver import ssl_context_factory, broken_ssl_context_factory
 
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
@@ -363,3 +366,48 @@ class WebClientTestCase(unittest.TestCase):
         self.assertEqual(content_encoding, EncodingResource.out_encoding)
         self.assertEqual(
             response.body.decode(content_encoding), to_unicode(original_body))
+
+
+class WebClientSSLTestCase(unittest.TestCase):
+    context_factory = None
+
+    def _listen(self, site):
+        return reactor.listenSSL(
+            0, site,
+            contextFactory=self.context_factory or ssl_context_factory(),
+            interface="127.0.0.1")
+
+    def getURL(self, path):
+        return "https://127.0.0.1:%d/%s" % (self.portno, path)
+
+    def setUp(self):
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
+        r.putChild(b"payload", PayloadResource())
+        self.site = server.Site(r, timeout=None)
+        self.wrapper = WrappingFactory(self.site)
+        self.port = self._listen(self.wrapper)
+        self.portno = self.port.getHost().port
+
+    @inlineCallbacks
+    def tearDown(self):
+        yield self.port.stopListening()
+        shutil.rmtree(self.tmpname)
+
+    def testPayload(self):
+        s = "0123456789" * 10
+        return getPage(self.getURL("payload"), body=s).addCallback(
+            self.assertEqual, to_bytes(s))
+
+
+class WebClientBrokenSSLTestCase(WebClientSSLTestCase):
+    context_factory = broken_ssl_context_factory(cipher_string='CAMELLIA256-SHA')
+
+    def testPayload(self):
+        s = "0123456789" * 10
+        settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})
+        return getPage(self.getURL("payload"), body=s,
+                       contextFactory=ScrapyClientContextFactory(settings=settings)).addCallback(self.assertEqual,
+                                                                                                 to_bytes(s))