diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py index 61391c626..ff0beb321 100644 --- a/scrapy/core/downloader/handlers/http.py +++ b/scrapy/core/downloader/handlers/http.py @@ -8,10 +8,9 @@ from scrapy.conf import settings from scrapy import optional_features ssl_supported = 'ssl' in optional_features -if ssl_supported: - from twisted.internet.ssl import ClientContextFactory HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY']) +ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY']) class HttpDownloadHandler(object): diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 6579475a9..985cc4ae5 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -9,6 +9,7 @@ from twisted.internet import defer from scrapy.http import Headers from scrapy.utils.httpobj import urlparse_cached from scrapy.responsetypes import responsetypes +from scrapy import optional_features def _parsed_url_args(parsed): @@ -135,3 +136,24 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): def gotHeaders(self, headers): self.headers_time = time() self.response_headers = headers + + + +if 'ssl' in optional_features: + from twisted.internet.ssl import ClientContextFactory + from OpenSSL import SSL +else: + ClientContextFactory = object + + +class ScrapyClientContextFactory(ClientContextFactory): + "A SSL context factory which is more permissive against SSL bugs." + # see https://github.com/scrapy/scrapy/issues/82 + # and https://github.com/scrapy/scrapy/issues/26 + + def getContext(self): + ctx = ClientContextFactory.getContext(self) + # Enable all workarounds to SSL bugs as documented by + # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + ctx.set_options(SSL.OP_ALL) + return ctx diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 7e4363507..278ce8570 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -64,6 +64,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' +DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory' DOWNLOADER_MIDDLEWARES = {}