From c8dc158697453b610f168f97ce94945892163674 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Tue, 30 May 2017 17:22:23 +0200 Subject: [PATCH 1/3] Use HTTP pool and proper endpoint key for ProxyAgent --- scrapy/core/downloader/handlers/http11.py | 32 +++++++++++++++++++---- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 9bfdd803c..a4b077b57 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,7 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed + HTTPConnectionPool, ResponseFailed, URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -228,10 +228,33 @@ class TunnelingAgent(Agent): headers, bodyProducer, requestPath) +class ScrapyProxyAgent(Agent): + + def __init__(self, reactor, proxyURI, + connectTimeout=None, bindAddress=None, pool=None): + super(ScrapyProxyAgent, self).__init__(reactor, + connectTimeout=connectTimeout, + bindAddress=bindAddress, + pool=pool) + self._proxyURI = URI.fromBytes(proxyURI) + + def request(self, method, uri, headers=None, bodyProducer=None): + """ + Issue a new request via the configured proxy. + """ + # Cache *all* connections under the same key, since we are only + # connecting to a single destination, the proxy: + proxyEndpoint = self._getEndpoint(self._proxyURI) + key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) + return self._requestWithEndpoint(key, proxyEndpoint, method, + URI.fromBytes(uri), headers, + bodyProducer, uri) + + class ScrapyAgent(object): _Agent = Agent - _ProxyAgent = ProxyAgent + _ProxyAgent = ScrapyProxyAgent _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None, @@ -260,9 +283,8 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, - timeout=timeout, bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + return self._ProxyAgent(reactor, proxyURI=proxy, + connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) From e162c1ff40bf74802c1a9f7b93fdbb56a5ff4a13 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 11:58:17 +0200 Subject: [PATCH 2/3] Pass proxy URI to ProxyAgent as bytes --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index a4b077b57..7b77d82da 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -283,7 +283,7 @@ class ScrapyAgent(object): contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: - return self._ProxyAgent(reactor, proxyURI=proxy, + return self._ProxyAgent(reactor, proxyURI=to_bytes(proxy, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) return self._Agent(reactor, contextFactory=self._contextFactory, From 6b092c66809ec11245d03063961ca64e488580e1 Mon Sep 17 00:00:00 2001 From: Paul Tremberth Date: Thu, 1 Jun 2017 12:29:01 +0200 Subject: [PATCH 3/3] Handle Twisted versions before 15.0 --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 7b77d82da..216671f82 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -14,7 +14,11 @@ from twisted.web.iweb import IBodyProducer, UNKNOWN_LENGTH from twisted.internet.error import TimeoutError from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.client import Agent, ProxyAgent, ResponseDone, \ - HTTPConnectionPool, ResponseFailed, URI + HTTPConnectionPool, ResponseFailed +try: + from twisted.web.client import URI +except ImportError: + from twisted.web.client import _URI as URI from twisted.internet.endpoints import TCP4ClientEndpoint from scrapy.http import Headers @@ -244,7 +248,12 @@ class ScrapyProxyAgent(Agent): """ # Cache *all* connections under the same key, since we are only # connecting to a single destination, the proxy: - proxyEndpoint = self._getEndpoint(self._proxyURI) + if twisted_version >= (15, 0, 0): + proxyEndpoint = self._getEndpoint(self._proxyURI) + else: + proxyEndpoint = self._getEndpoint(self._proxyURI.scheme, + self._proxyURI.host, + self._proxyURI.port) key = ("http-proxy", self._proxyURI.host, self._proxyURI.port) return self._requestWithEndpoint(key, proxyEndpoint, method, URI.fromBytes(uri), headers,