Allow overriding ClientContextFactory and enable SSL bug workarounds by default. refs #82

scrapy/core/downloader/handlers/http.py

@@ -8,10 +8,9 @@ from scrapy.conf import settings
 from scrapy import optional_features
 
 ssl_supported = 'ssl' in optional_features
-if ssl_supported:
-    from twisted.internet.ssl import ClientContextFactory
 
 HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
+ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
 
 
 class HttpDownloadHandler(object):

scrapy/core/downloader/webclient.py

@@ -9,6 +9,7 @@ from twisted.internet import defer
 from scrapy.http import Headers
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.responsetypes import responsetypes
+from scrapy import optional_features
 
 
 def _parsed_url_args(parsed):
@@ -135,3 +136,19 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
     def gotHeaders(self, headers):
         self.headers_time = time()
         self.response_headers = headers
+
+
+
+if 'ssl' in optional_features:
+    from twisted.internet.ssl import ClientContextFactory
+    from OpenSSL import SSL
+else:
+    ClientContextFactory = object
+
+
+class ScrapyClientContextFactory(ClientContextFactory):
+
+    def getContext(self):
+        ctx = ClientContextFactory.getContext(self)
+        ctx.set_options(SSL.OP_ALL)
+        return ctx

scrapy/settings/default_settings.py

@@ -64,6 +64,7 @@ DOWNLOAD_TIMEOUT = 180      # 3mins
 DOWNLOADER_DEBUG = False
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
+DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory'
 
 DOWNLOADER_MIDDLEWARES = {}