From 7d3a529dbc3575768f9da271f598928a5d2b7080 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Wed, 6 Apr 2016 20:02:50 +0200 Subject: [PATCH] Fix HTTP Pool key for HTTPS proxy tunneled connections (CONNECT method) --- scrapy/core/downloader/handlers/http11.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1634a2a4b..88c6b9480 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -135,7 +135,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._tunnelReadyDeferred.callback(self._protocol) else: self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel.')) + TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % ( + self._host, self._port))) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -201,6 +202,14 @@ class TunnelingAgent(Agent): self._contextFactory, self._connectTimeout, self._bindAddress) + def _requestWithEndpoint(self, key, endpoint, method, parsedURI, + headers, bodyProducer, requestPath): + # proxy host and port are required for HTTP pool `key` + # otherwise, same remote host connection request could reuse + # a cached tunneled connection to a different proxy + key = key + self._proxyConf + return super(TunnelingAgent, self)._requestWithEndpoint(key, endpoint, method, parsedURI, + headers, bodyProducer, requestPath) class ScrapyAgent(object):