From 0a261700862c496dfabab8a5adc05d841bd95c05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Gra=C3=B1a?= Date: Tue, 15 May 2012 11:27:04 -0300 Subject: [PATCH] move ssl context factory to its own module and implement a non-ssl version that warns about pyopenssl support --- scrapy/core/downloader/contextfactory.py | 20 ++++++++++++++++++++ scrapy/core/downloader/webclient.py | 24 +----------------------- scrapy/settings/default_settings.py | 2 +- 3 files changed, 22 insertions(+), 24 deletions(-) create mode 100644 scrapy/core/downloader/contextfactory.py diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py new file mode 100644 index 000000000..e20830c71 --- /dev/null +++ b/scrapy/core/downloader/contextfactory.py @@ -0,0 +1,20 @@ +from OpenSSL import SSL +from twisted.internet.ssl import ClientContextFactory + + +class ScrapyClientContextFactory(ClientContextFactory): + "A SSL context factory which is more permissive against SSL bugs." + # see https://github.com/scrapy/scrapy/issues/82 + # and https://github.com/scrapy/scrapy/issues/26 + + def __init__(self): + # see this issue on why we use TLSv1_METHOD by default + # https://github.com/scrapy/scrapy/issues/194 + self.method = SSL.TLSv1_METHOD + + def getContext(self, hostname=None, port=None): + ctx = ClientContextFactory.getContext(self) + # Enable all workarounds to SSL bugs as documented by + # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + ctx.set_options(SSL.OP_ALL) + return ctx diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ef17871de..ea69dc5ca 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -79,7 +79,7 @@ class ScrapyHTTPPageGetter(HTTPClient): class ScrapyHTTPClientFactory(HTTPClientFactory): """Scrapy implementation of the HTTPClientFactory overwriting the - serUrl method to make use of our Url object that cache the parse + serUrl method to make use of our Url object that cache the parse result. """ @@ -137,25 +137,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.headers_time = time() self.response_headers = headers - - -class ScrapyClientContextFactory(ClientContextFactory): - "A SSL context factory which is more permissive against SSL bugs." - # see https://github.com/scrapy/scrapy/issues/82 - # and https://github.com/scrapy/scrapy/issues/26 - -<<<<<<< HEAD - def __init__(self): - # see this issue on why we use TLSv1_METHOD by default - # https://github.com/scrapy/scrapy/issues/194 - self.method = SSL.TLSv1_METHOD - - def getContext(self): -======= - def getContext(self, hostname, port): ->>>>>>> add http connection pool and custom ssl context factory - ctx = ClientContextFactory.getContext(self) - # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html - ctx.set_options(SSL.OP_ALL) - return ctx diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d11e29bdd..2eb15f96c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -63,7 +63,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' -DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory' +DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_MIDDLEWARES = {}