From 5cb071b4662805e16d607471aad9f19dc9eda314 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Mon, 18 Apr 2016 18:30:01 +0200 Subject: [PATCH 1/2] Set SNI properly when using CONNECT --- scrapy/core/downloader/handlers/http11.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9cb659e67..4a0612cdf 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -122,7 +122,10 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """ self._protocol.dataReceived = self._protocolDataReceived if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): - self._protocol.transport.startTLS(self._contextFactory, + # this set proper Server Name Indication extension + sslOptions = self._contextFactory.creatorForNetloc( + self._tunneledHost, self._tunneledPort) + self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: From 352d0d36eec12e5dcdb3d4e99b7a5e9a16d09e9f Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 19 Apr 2016 10:41:13 +0200 Subject: [PATCH 2/2] Fall back to no-SNi context factory is Twisted<14 is used --- scrapy/core/downloader/handlers/http11.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 4a0612cdf..1634a2a4b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -122,9 +122,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """ self._protocol.dataReceived = self._protocolDataReceived if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): - # this set proper Server Name Indication extension - sslOptions = self._contextFactory.creatorForNetloc( - self._tunneledHost, self._tunneledPort) + try: + # this sets proper Server Name Indication extension + # but is only available for Twisted>=14.0 + sslOptions = self._contextFactory.creatorForNetloc( + self._tunneledHost, self._tunneledPort) + except AttributeError: + # fall back to non-SNI SSL context factory + sslOptions = self._contextFactory self._protocol.transport.startTLS(sslOptions, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol)