diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py new file mode 100644 index 000000000..e20830c71 --- /dev/null +++ b/scrapy/core/downloader/contextfactory.py @@ -0,0 +1,20 @@ +from OpenSSL import SSL +from twisted.internet.ssl import ClientContextFactory + + +class ScrapyClientContextFactory(ClientContextFactory): + "A SSL context factory which is more permissive against SSL bugs." + # see https://github.com/scrapy/scrapy/issues/82 + # and https://github.com/scrapy/scrapy/issues/26 + + def __init__(self): + # see this issue on why we use TLSv1_METHOD by default + # https://github.com/scrapy/scrapy/issues/194 + self.method = SSL.TLSv1_METHOD + + def getContext(self, hostname=None, port=None): + ctx = ClientContextFactory.getContext(self) + # Enable all workarounds to SSL bugs as documented by + # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html + ctx.set_options(SSL.OP_ALL) + return ctx diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index ef17871de..ea69dc5ca 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -79,7 +79,7 @@ class ScrapyHTTPPageGetter(HTTPClient): class ScrapyHTTPClientFactory(HTTPClientFactory): """Scrapy implementation of the HTTPClientFactory overwriting the - serUrl method to make use of our Url object that cache the parse + serUrl method to make use of our Url object that cache the parse result. """ @@ -137,25 +137,3 @@ class ScrapyHTTPClientFactory(HTTPClientFactory): self.headers_time = time() self.response_headers = headers - - -class ScrapyClientContextFactory(ClientContextFactory): - "A SSL context factory which is more permissive against SSL bugs." - # see https://github.com/scrapy/scrapy/issues/82 - # and https://github.com/scrapy/scrapy/issues/26 - -<<<<<<< HEAD - def __init__(self): - # see this issue on why we use TLSv1_METHOD by default - # https://github.com/scrapy/scrapy/issues/194 - self.method = SSL.TLSv1_METHOD - - def getContext(self): -======= - def getContext(self, hostname, port): ->>>>>>> add http connection pool and custom ssl context factory - ctx = ClientContextFactory.getContext(self) - # Enable all workarounds to SSL bugs as documented by - # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html - ctx.set_options(SSL.OP_ALL) - return ctx diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index d11e29bdd..2eb15f96c 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -63,7 +63,7 @@ DOWNLOAD_TIMEOUT = 180 # 3mins DOWNLOADER_DEBUG = False DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory' -DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.webclient.ScrapyClientContextFactory' +DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory' DOWNLOADER_MIDDLEWARES = {}