From ae28c7d69847ca3e0ed4febd7df22d3c69ee49de Mon Sep 17 00:00:00 2001 From: duendex Date: Wed, 25 Sep 2013 12:19:03 -0300 Subject: [PATCH 01/14] Adds the functionality to do HTTPS downloads behind proxies using an HTTP CONNECT. --- scrapy/core/downloader/handlers/http11.py | 107 ++++++++++++++++++++-- 1 file changed, 99 insertions(+), 8 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 04dd4952c..d55d95242 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -5,10 +5,10 @@ from cStringIO import StringIO from urlparse import urldefrag from zope.interface import implements -from twisted.internet import defer, reactor, protocol +from twisted.internet import defer, reactor, protocol, ssl from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer -from twisted.internet.error import TimeoutError +from twisted.internet.error import TimeoutError, SSLError from twisted.web.http import PotentialDataLoss from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ HTTPConnectionPool, TCP4ClientEndpoint @@ -37,12 +37,96 @@ class HTTP11DownloadHandler(object): return self._pool.closeCachedConnections() +class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): + """An endpoint that tunnels through proxies to allow HTTPS downloads. To + accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The + HTTP CONNECT is always sent when using this endpoint, I think this could + be improved as the CONNECT will be redundant if the connection associated + with this endpoint comes from the pool and a CONNECT has already been issued + for it. + """ + + def __init__(self, reactor, host, port, proxyHost, proxyPort, + contextFactory, timeout=30, bindAddress=None): + super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, + proxyPort, timeout, bindAddress) + self._tunnelReadyDeferred = defer.Deferred() + # Although we will connect to the proxy, we need the host and port of + # the destination server in order to send the HTTP CONNECT. + self._tunneledHost = host + self._tunneledPort = port + self._contextFactory = contextFactory + + def requestTunnel(self, protocol): + """Asks the proxy to open a tunnel.""" + # Ask for the proxy to open the tunnel. + protocol.transport.write('CONNECT %s:%s HTTP/1.1\n\n' % + (self._tunneledHost, self._tunneledPort)) + # This hack is not so nice. Substitute the dataReceived method + # temporarily to intercept the response from the proxy. + self._protocolDataReceived = protocol.dataReceived + protocol.dataReceived = self.processProxyResponse + # Store the protocol because we will have to pass it when triggering the + # deferred returned in the connect method. + self._protocol = protocol + return protocol + + def processProxyResponse(self, bytes): + """Processes the response from the proxy. If the tunnel is successfully + created, notifies the client that we are ready to send requests. + """ + if bytes.find('200 Connection established') > 0: + # The tunnel is ready, switch transport to TLS. + self._protocol.transport.startTLS(self._contextFactory, + self._protocolFactory) + # Restore the protocol dataReceived method. + self._protocol.dataReceived = self._protocolDataReceived + # Trigger the callback with the protocol as the value. + self._tunnelReadyDeferred.callback(self._protocol) + else: + # Not sure if this is the best way to handle this error. + raise SSLError + + def connect(self, protocolFactory): + # Store the protocol factory as we will need it to switch to TLS. + self._protocolFactory = protocolFactory + connectDeferred = super(TunnelingTCP4ClientEndpoint, + self).connect(protocolFactory) + # Add a callback to open the tunnel when the connection is ready. + connectDeferred.addCallback(self.requestTunnel) + # Return a deferred that will be triggered when the tunnel is ready. + return self._tunnelReadyDeferred + + +class TunnelingAgent(Agent): + """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS + downloads. It may look strange that we have chosen to subclass Agent and not + ProxyAgent but consider that after the tunnel is opened the proxy is + transparent to the client; thus the agent should behave like there is no + proxy involved. + """ + + def __init__(self, reactor, proxyHost, proxyPort, contextFactory=None, + connectTimeout=None, bindAddress=None, pool=None): + super(TunnelingAgent, self).__init__(reactor, contextFactory, + connectTimeout, bindAddress, pool) + self._proxyHost = proxyHost + self._proxyPort = proxyPort + + def _getEndpoint(self, scheme, host, port): + return TunnelingTCP4ClientEndpoint(self._reactor, host, port, + self._proxyHost, self._proxyPort, self._contextFactory, + self._connectTimeout, self._bindAddress) + + class ScrapyAgent(object): _Agent = Agent _ProxyAgent = ProxyAgent + _TunnelingAgent = TunnelingAgent - def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None): + def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, + pool=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -52,15 +136,22 @@ class ScrapyAgent(object): bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - scheme, _, host, port, _ = _parse(proxy) - endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, - bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + _, _, proxyHost, proxyPort, _ = _parse(proxy) + scheme = _parse(request.url)[0] + if scheme == 'https': + # We need to tunnel the proxy using an HTTP CONNECT. + return self._TunnelingAgent(reactor, proxyHost, proxyPort, + contextFactory=self._contextFactory, connectTimeout=timeout, + bindAddress=bindaddress, pool=self._pool) + else: + endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, + timeout=timeout, bindAddress=bindaddress) + return self._ProxyAgent(endpoint) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) - def download_request(self, request): + def download_request(self, request): timeout = request.meta.get('download_timeout') or self._connectTimeout agent = self._get_agent(request, timeout) From 36e4fc37857175acd8c2a54f5c45ede4ea2d51b9 Mon Sep 17 00:00:00 2001 From: duendex Date: Wed, 25 Sep 2013 12:59:11 -0300 Subject: [PATCH 02/14] Removed some trailing spaces that I left. --- scrapy/core/downloader/handlers/http11.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index d55d95242..8a5a9d5f1 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -151,7 +151,7 @@ class ScrapyAgent(object): return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) - def download_request(self, request): + def download_request(self, request): timeout = request.meta.get('download_timeout') or self._connectTimeout agent = self._get_agent(request, timeout) From 58a98b0c04b7dd10720b4310de1aef5765570885 Mon Sep 17 00:00:00 2001 From: duendex Date: Thu, 26 Sep 2013 11:20:41 -0300 Subject: [PATCH 03/14] Improved error handling. --- scrapy/core/downloader/handlers/http11.py | 50 ++++++++++++++++++----- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 8a5a9d5f1..417e6e156 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -39,8 +39,10 @@ class HTTP11DownloadHandler(object): class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To - accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The - HTTP CONNECT is always sent when using this endpoint, I think this could + accomplish that, this endpoint sends an HTTP CONNECT to the proxy. If the + proxy fails to open the tunnel this endpoint will behave as a + L{TCP4ClientEndpoint}. + The HTTP CONNECT is always sent when using this endpoint, I think this could be improved as the CONNECT will be redundant if the connection associated with this endpoint comes from the pool and a CONNECT has already been issued for it. @@ -75,27 +77,53 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. """ + # Restore the protocol dataReceived method. + self._protocol.dataReceived = self._protocolDataReceived if bytes.find('200 Connection established') > 0: # The tunnel is ready, switch transport to TLS. self._protocol.transport.startTLS(self._contextFactory, self._protocolFactory) - # Restore the protocol dataReceived method. - self._protocol.dataReceived = self._protocolDataReceived # Trigger the callback with the protocol as the value. self._tunnelReadyDeferred.callback(self._protocol) else: - # Not sure if this is the best way to handle this error. - raise SSLError + # The proxy could not open the tunnel and will drop the connection; + # (we need to check if this is common to every proxy). In order to + # allow the client to send the request and get a response from the + # proxy, we will intercept the connectionLost message and restore + # the connection. + self._protocolConnectionLost = self._protocol.connectionLost + self._protocol.connectionLost = self.connectionLost - def connect(self, protocolFactory): + def connectFailed(self, reason): + """Propagates the errback to the appropriate deferred.""" + self._tunnelReadyDeferred.errback(reason) + + def connectionLost(self, reason): + """Restores the connection to the proxy server but does not request + for it to open a tunnel. + """ + # Restore and call the protocol connection lost method. + self._protocol.connectionLost = self._protocolConnectionLost + self._protocol.connectionLost(reason) + # Restore the connection to the proxy but don't open the tunnel. + self.connect(self._protocolFactory, False) + + def connect(self, protocolFactory, openTunnel=True): # Store the protocol factory as we will need it to switch to TLS. self._protocolFactory = protocolFactory connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) - # Add a callback to open the tunnel when the connection is ready. - connectDeferred.addCallback(self.requestTunnel) - # Return a deferred that will be triggered when the tunnel is ready. - return self._tunnelReadyDeferred + if openTunnel: + # Add a callback to open the tunnel when the connection is ready. + connectDeferred.addCallback(self.requestTunnel) + connectDeferred.addErrback(self.connectFailed) + # Return a deferred that will be triggered when the tunnel is ready. + return self._tunnelReadyDeferred + else: + def cbCallback(protocol): + self._tunnelReadyDeferred.callback(protocol) + connectDeferred.addCallback(cbCallback) + return connectDeferred class TunnelingAgent(Agent): From 628bfbcc3e0945edd3ad5df733f96c5f12a82449 Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 30 Sep 2013 10:18:57 -0300 Subject: [PATCH 04/14] Raises a custom TunnelError when the tunnel cannot be opened. Removed unnecesary comments. --- scrapy/core/downloader/handlers/http11.py | 90 ++++++++--------------- 1 file changed, 30 insertions(+), 60 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 417e6e156..f11c7ce7b 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -3,12 +3,13 @@ from time import time from cStringIO import StringIO from urlparse import urldefrag +from re import match from zope.interface import implements -from twisted.internet import defer, reactor, protocol, ssl +from twisted.internet import defer, reactor, protocol from twisted.web.http_headers import Headers as TxHeaders from twisted.web.iweb import IBodyProducer -from twisted.internet.error import TimeoutError, SSLError +from twisted.internet.error import TimeoutError from twisted.web.http import PotentialDataLoss from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \ HTTPConnectionPool, TCP4ClientEndpoint @@ -37,93 +38,64 @@ class HTTP11DownloadHandler(object): return self._pool.closeCachedConnections() +class TunnelError(Exception): + """An HTTP CONNECT tunnel could not be established by the proxy.""" + + class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): """An endpoint that tunnels through proxies to allow HTTPS downloads. To - accomplish that, this endpoint sends an HTTP CONNECT to the proxy. If the - proxy fails to open the tunnel this endpoint will behave as a - L{TCP4ClientEndpoint}. + accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The HTTP CONNECT is always sent when using this endpoint, I think this could be improved as the CONNECT will be redundant if the connection associated with this endpoint comes from the pool and a CONNECT has already been issued for it. """ - def __init__(self, reactor, host, port, proxyHost, proxyPort, - contextFactory, timeout=30, bindAddress=None): + def __init__(self, reactor, host, port, proxyConf, contextFactory, + timeout=30, bindAddress=None): + proxyHost, proxyPort = proxyConf super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() - # Although we will connect to the proxy, we need the host and port of - # the destination server in order to send the HTTP CONNECT. self._tunneledHost = host self._tunneledPort = port self._contextFactory = contextFactory def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - # Ask for the proxy to open the tunnel. - protocol.transport.write('CONNECT %s:%s HTTP/1.1\n\n' % - (self._tunneledHost, self._tunneledPort)) - # This hack is not so nice. Substitute the dataReceived method - # temporarily to intercept the response from the proxy. + tunnelReq = 'CONNECT %s:%s HTTP/1.1\n\n' % (self._tunneledHost, + self._tunneledPort) + protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse - # Store the protocol because we will have to pass it when triggering the - # deferred returned in the connect method. self._protocol = protocol return protocol def processProxyResponse(self, bytes): """Processes the response from the proxy. If the tunnel is successfully - created, notifies the client that we are ready to send requests. + created, notifies the client that we are ready to send requests. If not + raises a TunnelError. """ - # Restore the protocol dataReceived method. self._protocol.dataReceived = self._protocolDataReceived - if bytes.find('200 Connection established') > 0: - # The tunnel is ready, switch transport to TLS. + if match('HTTP/1\.. 200', bytes): self._protocol.transport.startTLS(self._contextFactory, self._protocolFactory) - # Trigger the callback with the protocol as the value. self._tunnelReadyDeferred.callback(self._protocol) else: - # The proxy could not open the tunnel and will drop the connection; - # (we need to check if this is common to every proxy). In order to - # allow the client to send the request and get a response from the - # proxy, we will intercept the connectionLost message and restore - # the connection. - self._protocolConnectionLost = self._protocol.connectionLost - self._protocol.connectionLost = self.connectionLost + self._tunnelReadyDeferred.errback( + TunnelError('Could not open CONNECT tunnel.')) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" self._tunnelReadyDeferred.errback(reason) - def connectionLost(self, reason): - """Restores the connection to the proxy server but does not request - for it to open a tunnel. - """ - # Restore and call the protocol connection lost method. - self._protocol.connectionLost = self._protocolConnectionLost - self._protocol.connectionLost(reason) - # Restore the connection to the proxy but don't open the tunnel. - self.connect(self._protocolFactory, False) - - def connect(self, protocolFactory, openTunnel=True): - # Store the protocol factory as we will need it to switch to TLS. + def connect(self, protocolFactory): self._protocolFactory = protocolFactory connectDeferred = super(TunnelingTCP4ClientEndpoint, self).connect(protocolFactory) - if openTunnel: - # Add a callback to open the tunnel when the connection is ready. - connectDeferred.addCallback(self.requestTunnel) - connectDeferred.addErrback(self.connectFailed) - # Return a deferred that will be triggered when the tunnel is ready. - return self._tunnelReadyDeferred - else: - def cbCallback(protocol): - self._tunnelReadyDeferred.callback(protocol) - connectDeferred.addCallback(cbCallback) - return connectDeferred + connectDeferred.addCallback(self.requestTunnel) + connectDeferred.addErrback(self.connectFailed) + return self._tunnelReadyDeferred class TunnelingAgent(Agent): @@ -134,17 +106,16 @@ class TunnelingAgent(Agent): proxy involved. """ - def __init__(self, reactor, proxyHost, proxyPort, contextFactory=None, + def __init__(self, reactor, proxyConf, contextFactory=None, connectTimeout=None, bindAddress=None, pool=None): super(TunnelingAgent, self).__init__(reactor, contextFactory, connectTimeout, bindAddress, pool) - self._proxyHost = proxyHost - self._proxyPort = proxyPort + self._proxyConf = proxyConf def _getEndpoint(self, scheme, host, port): return TunnelingTCP4ClientEndpoint(self._reactor, host, port, - self._proxyHost, self._proxyPort, self._contextFactory, - self._connectTimeout, self._bindAddress) + self._proxyConf, self._contextFactory, self._connectTimeout, + self._bindAddress) class ScrapyAgent(object): @@ -153,8 +124,7 @@ class ScrapyAgent(object): _ProxyAgent = ProxyAgent _TunnelingAgent = TunnelingAgent - def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, - pool=None): + def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None): self._contextFactory = contextFactory self._connectTimeout = connectTimeout self._bindAddress = bindAddress @@ -167,8 +137,8 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, _ = _parse(proxy) scheme = _parse(request.url)[0] if scheme == 'https': - # We need to tunnel the proxy using an HTTP CONNECT. - return self._TunnelingAgent(reactor, proxyHost, proxyPort, + proxyConf = (proxyHost, proxyPort) + return self._TunnelingAgent(reactor, proxyConf, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) else: From 7f053cc1d27d565c8a44f4135949087d3197e18f Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 30 Sep 2013 10:43:25 -0300 Subject: [PATCH 05/14] Adds support for proxy authentication when openning a CONNECT tunnel. --- scrapy/core/downloader/handlers/http11.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index f11c7ce7b..0187b6f3c 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -53,7 +53,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): - proxyHost, proxyPort = proxyConf + proxyHost, proxyPort, self._proxyAuthHeader = proxyConf super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, proxyPort, timeout, bindAddress) self._tunnelReadyDeferred = defer.Deferred() @@ -63,8 +63,12 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): def requestTunnel(self, protocol): """Asks the proxy to open a tunnel.""" - tunnelReq = 'CONNECT %s:%s HTTP/1.1\n\n' % (self._tunneledHost, - self._tunneledPort) + tunnelReq = 'CONNECT %s:%s HTTP/1.1\n' % (self._tunneledHost, + self._tunneledPort) + if self._proxyAuthHeader: + tunnelReq += 'Proxy-Authorization: %s \n\n' % self._proxyAuthHeader + else: + tunnelReq += '\n' protocol.transport.write(tunnelReq) self._protocolDataReceived = protocol.dataReceived protocol.dataReceived = self.processProxyResponse @@ -137,7 +141,8 @@ class ScrapyAgent(object): _, _, proxyHost, proxyPort, _ = _parse(proxy) scheme = _parse(request.url)[0] if scheme == 'https': - proxyConf = (proxyHost, proxyPort) + proxyConf = (proxyHost, proxyPort, + request.headers.get('Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) From 23c3288a6d07834687f607ac95ce5c082f061bfa Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 30 Sep 2013 12:55:48 -0300 Subject: [PATCH 06/14] Adds the option to omit the usage of a CONNECT tunnel by adding the noconnect parameter to the URL of the proxy. --- scrapy/core/downloader/handlers/http11.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 0187b6f3c..1985e1251 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -138,9 +138,10 @@ class ScrapyAgent(object): bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, _ = _parse(proxy) + _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - if scheme == 'https': + skipConnectTunnel = proxyParams.find('noconnect') >= 0 + if scheme == 'https' and not skipConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get('Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, From 88bec496f2b1a1e83b2a55dc1155988ebc7c08f0 Mon Sep 17 00:00:00 2001 From: duendex Date: Wed, 2 Oct 2013 15:03:42 -0300 Subject: [PATCH 07/14] The response matching re is now compiled once at module load time. --- scrapy/core/downloader/handlers/http11.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 1985e1251..de3cc8756 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,9 +1,10 @@ """Download handlers for http and https schemes""" +import re + from time import time from cStringIO import StringIO from urlparse import urldefrag -from re import match from zope.interface import implements from twisted.internet import defer, reactor, protocol @@ -51,6 +52,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ + _responseMatcher = re.compile('HTTP/1\.. 200') + def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): proxyHost, proxyPort, self._proxyAuthHeader = proxyConf @@ -81,7 +84,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): raises a TunnelError. """ self._protocol.dataReceived = self._protocolDataReceived - if match('HTTP/1\.. 200', bytes): + if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): self._protocol.transport.startTLS(self._contextFactory, self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) @@ -140,8 +143,8 @@ class ScrapyAgent(object): if proxy: _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] - skipConnectTunnel = proxyParams.find('noconnect') >= 0 - if scheme == 'https' and not skipConnectTunnel: + omitConnectTunnel = proxyParams.find('noconnect') >= 0 + if scheme == 'https' and not omitConnectTunnel: proxyConf = (proxyHost, proxyPort, request.headers.get('Proxy-Authorization', None)) return self._TunnelingAgent(reactor, proxyConf, From d69ba7c1aede3b8f5e683695957777f2567d5067 Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 2 Dec 2013 16:35:35 -0200 Subject: [PATCH 08/14] Changed the proxy tests to use libmproxy instead of starting mitmdump as a separate process. --- scrapy/tests/test_proxy_connect.py | 96 ++++++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 scrapy/tests/test_proxy_connect.py diff --git a/scrapy/tests/test_proxy_connect.py b/scrapy/tests/test_proxy_connect.py new file mode 100644 index 000000000..6d995e8e6 --- /dev/null +++ b/scrapy/tests/test_proxy_connect.py @@ -0,0 +1,96 @@ +import os +import subprocess +import time + +from threading import Thread +from libmproxy import controller, proxy +from netlib import http_auth + +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.utils.test import get_crawler, get_testlog +from scrapy.tests.spiders import SimpleSpider +from scrapy.tests.mockserver import MockServer + + +def docrawl(spider, settings=None): + crawler = get_crawler(settings) + crawler.configure() + crawler.crawl(spider) + return crawler.start() + + + +class HTTPSProxy(controller.Master, Thread): + + def __init__(self, port): + password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + server = proxy.ProxyServer(proxy.ProxyConfig( + authenticator = authenticator, + cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")), + port) + Thread.__init__(self) + controller.Master.__init__(self, server) + + +class ProxyConnectTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self._oldenv = os.environ.copy() + self._proxy = HTTPSProxy(8888) + self._proxy.start() + os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + self._proxy.shutdown() + os.environ = self._oldenv + + @defer.inlineCallbacks + def test_https_connect_tunnel(self): + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(200) + + @defer.inlineCallbacks + def test_https_noconnect(self): + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(200) + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + @defer.inlineCallbacks + def test_https_connect_tunnel_error(self): + spider = SimpleSpider("https://localhost:99999/status?n=200") + yield docrawl(spider) + self._assert_got_tunnel_error() + + @defer.inlineCallbacks + def test_https_tunnel_auth_error(self): + os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + # The proxy returns a 407 error code but it does not reach the client; + # he just sees a TunnelError. + self._assert_got_tunnel_error() + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + @defer.inlineCallbacks + def test_https_noconnect_auth_error(self): + os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(407) + + def _assert_got_response_code(self, code): + log = get_testlog() + self.assertEqual(log.count('Crawled (%d)' % code), 1) + + def _assert_got_tunnel_error(self): + log = get_testlog() + self.assertEqual(log.count('TunnelError'), 1) From 02bab270e83e9436a55915c52cd63e1209412573 Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 2 Dec 2013 20:25:12 -0200 Subject: [PATCH 09/14] Added mitmproxy as a requirement. --- tox.ini | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tox.ini b/tox.ini index 7503d53bd..b8ee702e4 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ deps = django # Only required to run tests mock + mitmproxy commands = trial scrapy @@ -29,6 +30,7 @@ deps = django==1.3.1 cssselect==0.9.1 mock==1.0.1 + mitmproxy=0.9.2 [testenv:trunk] basepython = python2.7 @@ -48,6 +50,7 @@ deps = w3lib>=1.5 # Only required to run tests mock>=1.0.1 + mitmproxy>=0.9.2 commands = trial scrapy From 247b330f08291cb65dad147557b60c67b74fdbaf Mon Sep 17 00:00:00 2001 From: duendex Date: Mon, 2 Dec 2013 21:34:35 -0200 Subject: [PATCH 10/14] Corrected typo in tox.ini --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index b8ee702e4..e10012e05 100644 --- a/tox.ini +++ b/tox.ini @@ -30,7 +30,7 @@ deps = django==1.3.1 cssselect==0.9.1 mock==1.0.1 - mitmproxy=0.9.2 + mitmproxy==0.9.2 [testenv:trunk] basepython = python2.7 From f8dea74948698d26fcd7e4eafe67f556b11f0b06 Mon Sep 17 00:00:00 2001 From: duendex Date: Tue, 3 Dec 2013 03:09:06 -0200 Subject: [PATCH 11/14] Added a delay to wait for the proxy to start. --- scrapy/tests/test_proxy_connect.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scrapy/tests/test_proxy_connect.py b/scrapy/tests/test_proxy_connect.py index 6d995e8e6..13e8f184d 100644 --- a/scrapy/tests/test_proxy_connect.py +++ b/scrapy/tests/test_proxy_connect.py @@ -42,6 +42,8 @@ class ProxyConnectTestCase(TestCase): self._oldenv = os.environ.copy() self._proxy = HTTPSProxy(8888) self._proxy.start() + # Wait for the proxy to start. + time.sleep(1.0) os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' From 500490ee73c85897723b3060c9f1fecf8b516b99 Mon Sep 17 00:00:00 2001 From: duendex Date: Tue, 3 Dec 2013 03:10:16 -0200 Subject: [PATCH 12/14] Corrected a test that used a dummy URL that unpurposedly had an https scheme and failed with PR 397. --- scrapy/tests/test_downloader_handlers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index c71bfbcb8..614bac037 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -207,10 +207,10 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'https://example.com') + self.assertEquals(response.body, 'http://example.com') http_proxy = self.getURL('') - request = Request('https://example.com', meta={'proxy': http_proxy}) + request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, BaseSpider('foo')).addCallback(_test) def test_download_without_proxy(self): From 6427d60fd8967bc430c0e50415d120080663a65b Mon Sep 17 00:00:00 2001 From: duendex Date: Tue, 3 Dec 2013 11:45:02 -0200 Subject: [PATCH 13/14] Fixed the location of the certificate required by libmproxy. --- scrapy/tests/keys/mitmproxy-ca.pem | 32 ++++++++++++++++++++++++++++++ scrapy/tests/test_proxy_connect.py | 4 +++- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 scrapy/tests/keys/mitmproxy-ca.pem diff --git a/scrapy/tests/keys/mitmproxy-ca.pem b/scrapy/tests/keys/mitmproxy-ca.pem new file mode 100644 index 000000000..08004feca --- /dev/null +++ b/scrapy/tests/keys/mitmproxy-ca.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt +auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha +QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB +AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF +QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol +Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w +T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll +KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj +58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc +1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT +9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB +dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn +Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT +CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX +DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt +aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u +YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp +w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk +Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB +/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG +CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB +gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG ++EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n +MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D +y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z +e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 +n95u +-----END CERTIFICATE----- diff --git a/scrapy/tests/test_proxy_connect.py b/scrapy/tests/test_proxy_connect.py index 13e8f184d..ee9576f71 100644 --- a/scrapy/tests/test_proxy_connect.py +++ b/scrapy/tests/test_proxy_connect.py @@ -26,9 +26,11 @@ class HTTPSProxy(controller.Master, Thread): def __init__(self, port): password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'keys', 'mitmproxy-ca.pem') server = proxy.ProxyServer(proxy.ProxyConfig( authenticator = authenticator, - cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")), + cacert = cert_path), port) Thread.__init__(self) controller.Master.__init__(self, server) From 8ada8f5f36b3d2b4dd92dc7bb4822c909e2b6d03 Mon Sep 17 00:00:00 2001 From: duendex Date: Tue, 3 Dec 2013 12:55:44 -0200 Subject: [PATCH 14/14] Added a test case to ensure that passing the noconnect paramenter avoids trigerring the creation of a connect tunnel when downloading from a site with https scheme. --- scrapy/tests/test_downloader_handlers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index 614bac037..14a26fe33 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -213,6 +213,16 @@ class HttpProxyTestCase(unittest.TestCase): request = Request('http://example.com', meta={'proxy': http_proxy}) return self.download_request(request, BaseSpider('foo')).addCallback(_test) + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEquals(response.status, 200) + self.assertEquals(response.url, request.url) + self.assertEquals(response.body, 'https://example.com') + + http_proxy = '%s?noconnect' % self.getURL('') + request = Request('https://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) + def test_download_without_proxy(self): def _test(response): self.assertEquals(response.status, 200)