diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 04dd4952c..de3cc8756 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -1,5 +1,7 @@ """Download handlers for http and https schemes""" +import re + from time import time from cStringIO import StringIO from urlparse import urldefrag @@ -37,10 +39,97 @@ class HTTP11DownloadHandler(object): return self._pool.closeCachedConnections() +class TunnelError(Exception): + """An HTTP CONNECT tunnel could not be established by the proxy.""" + + +class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): + """An endpoint that tunnels through proxies to allow HTTPS downloads. To + accomplish that, this endpoint sends an HTTP CONNECT to the proxy. + The HTTP CONNECT is always sent when using this endpoint, I think this could + be improved as the CONNECT will be redundant if the connection associated + with this endpoint comes from the pool and a CONNECT has already been issued + for it. + """ + + _responseMatcher = re.compile('HTTP/1\.. 200') + + def __init__(self, reactor, host, port, proxyConf, contextFactory, + timeout=30, bindAddress=None): + proxyHost, proxyPort, self._proxyAuthHeader = proxyConf + super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost, + proxyPort, timeout, bindAddress) + self._tunnelReadyDeferred = defer.Deferred() + self._tunneledHost = host + self._tunneledPort = port + self._contextFactory = contextFactory + + def requestTunnel(self, protocol): + """Asks the proxy to open a tunnel.""" + tunnelReq = 'CONNECT %s:%s HTTP/1.1\n' % (self._tunneledHost, + self._tunneledPort) + if self._proxyAuthHeader: + tunnelReq += 'Proxy-Authorization: %s \n\n' % self._proxyAuthHeader + else: + tunnelReq += '\n' + protocol.transport.write(tunnelReq) + self._protocolDataReceived = protocol.dataReceived + protocol.dataReceived = self.processProxyResponse + self._protocol = protocol + return protocol + + def processProxyResponse(self, bytes): + """Processes the response from the proxy. If the tunnel is successfully + created, notifies the client that we are ready to send requests. If not + raises a TunnelError. + """ + self._protocol.dataReceived = self._protocolDataReceived + if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): + self._protocol.transport.startTLS(self._contextFactory, + self._protocolFactory) + self._tunnelReadyDeferred.callback(self._protocol) + else: + self._tunnelReadyDeferred.errback( + TunnelError('Could not open CONNECT tunnel.')) + + def connectFailed(self, reason): + """Propagates the errback to the appropriate deferred.""" + self._tunnelReadyDeferred.errback(reason) + + def connect(self, protocolFactory): + self._protocolFactory = protocolFactory + connectDeferred = super(TunnelingTCP4ClientEndpoint, + self).connect(protocolFactory) + connectDeferred.addCallback(self.requestTunnel) + connectDeferred.addErrback(self.connectFailed) + return self._tunnelReadyDeferred + + +class TunnelingAgent(Agent): + """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS + downloads. It may look strange that we have chosen to subclass Agent and not + ProxyAgent but consider that after the tunnel is opened the proxy is + transparent to the client; thus the agent should behave like there is no + proxy involved. + """ + + def __init__(self, reactor, proxyConf, contextFactory=None, + connectTimeout=None, bindAddress=None, pool=None): + super(TunnelingAgent, self).__init__(reactor, contextFactory, + connectTimeout, bindAddress, pool) + self._proxyConf = proxyConf + + def _getEndpoint(self, scheme, host, port): + return TunnelingTCP4ClientEndpoint(self._reactor, host, port, + self._proxyConf, self._contextFactory, self._connectTimeout, + self._bindAddress) + + class ScrapyAgent(object): _Agent = Agent _ProxyAgent = ProxyAgent + _TunnelingAgent = TunnelingAgent def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None): self._contextFactory = contextFactory @@ -52,10 +141,19 @@ class ScrapyAgent(object): bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - scheme, _, host, port, _ = _parse(proxy) - endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout, - bindAddress=bindaddress) - return self._ProxyAgent(endpoint) + _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + scheme = _parse(request.url)[0] + omitConnectTunnel = proxyParams.find('noconnect') >= 0 + if scheme == 'https' and not omitConnectTunnel: + proxyConf = (proxyHost, proxyPort, + request.headers.get('Proxy-Authorization', None)) + return self._TunnelingAgent(reactor, proxyConf, + contextFactory=self._contextFactory, connectTimeout=timeout, + bindAddress=bindaddress, pool=self._pool) + else: + endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort, + timeout=timeout, bindAddress=bindaddress) + return self._ProxyAgent(endpoint) return self._Agent(reactor, contextFactory=self._contextFactory, connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool) diff --git a/scrapy/tests/keys/mitmproxy-ca.pem b/scrapy/tests/keys/mitmproxy-ca.pem new file mode 100644 index 000000000..08004feca --- /dev/null +++ b/scrapy/tests/keys/mitmproxy-ca.pem @@ -0,0 +1,32 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICWwIBAAKBgQDKLbznLxS7HSWvrmGcvVS6eQvjEWD705/csvnk/WtqAPfQMJKt +auFBxzPt6RT60SHtj/2FKt2gqsiE6cNINxGN6fGYD7HtaM5HXRVPUKJaMipJwHha +QivjIZoueraY/MtlyCkpp6dmMnHEpGY7OzwMyh1eCBHQ2JYx6VEzbks9ewIDAQAB +AoGAMpS2ye/Rc+6a2xT5fskvRWe7PZe/d8E+IWz1cACmuuJ7HS7Jw3EV4esAZukF +QqrHnjOD7akHwYZ4nCgPnyWH0lLx/4TIXE5QeLPFrhKOsSLCyhlCwNVJAdcOrDol +Qh2694Dsd4gAy5o6TA02cBpqArnbAUERX46bHBZRA+ths8ECQQD6r1Ls+bTBR52w +T3rPPhYj7EsXp40MJt0pLf1kjf+EH1bxsUqnxLawwo/lLE9omU73DFnfrAflk2Ll +KUPCjjYpAkEAznchXk2ITeRcClrBNA+1Izpb5yG1qkfc79u/CEVDDOvt7RO/89Oj +58R3pKTyffoo34fBdJz8GYDsmOeiyJEjAwJANpSHrJrtlQt/tMyJQ6gT7/xZmSvc +1OF9U6L0wbj9AgpExtjAFWkKEdA6vj34iCChBb8FrmJpUb3WUWi7nReTiQJAFyIT +9Av93LRcd7CJezrTUdolF/WX9DdPEvTtJ5ETHSyGIQ0Yccph0AMcYK82mFTiJYGB +dH5uZLEkUVGK1KwmXwJAGWLdYiQyQitRWdoURcLb4OZ2gF3+7PASgFilI8YuoYhn +Rl2Va3UtErPKJeMg2dTH18PuXykQMsQR1+rPxf1WSA== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICnzCCAgigAwIBAgIGDI2K/EOjMA0GCSqGSIb3DQEBBQUAMCgxEjAQBgNVBAMT +CW1pdG1wcm94eTESMBAGA1UEChMJbWl0bXByb3h5MB4XDTEzMDkyNjE0MzYxMVoX +DTE1MDkxNjE0MzYxMVowKDESMBAGA1UEAxMJbWl0bXByb3h5MRIwEAYDVQQKEwlt +aXRtcHJveHkwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBAMotvOcvFLsdJa+u +YZy9VLp5C+MRYPvTn9yy+eT9a2oA99Awkq1q4UHHM+3pFPrRIe2P/YUq3aCqyITp +w0g3EY3p8ZgPse1ozkddFU9QoloyKknAeFpCK+Mhmi56tpj8y2XIKSmnp2YyccSk +Zjs7PAzKHV4IEdDYljHpUTNuSz17AgMBAAGjgdMwgdAwDwYDVR0TAQH/BAUwAwEB +/zAUBglghkgBhvhCAQEBAf8EBAMCAgQwewYDVR0lAQH/BHEwbwYIKwYBBQUHAwEG +CCsGAQUFBwMCBggrBgEFBQcDBAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQB +gjcCARYGCisGAQQBgjcKAwEGCisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG ++EIEATALBgNVHQ8EBAMCAQYwHQYDVR0OBBYEFJBEfawVwhEHHW6rS8nvZFlJ582n +MA0GCSqGSIb3DQEBBQUAA4GBAHGl28Ip2CWS/MibCaFztLDxGiMBT4MW2yI2hf3D +y9g1o7ra/fSEFdIc849xXyCsGWSkMsbDML272rCH4K73MUBxxkJm46AIyRVH1z2Z +e96u4py1wNT8cznY15phr8pn36snlaHaYa+JcwGINMdSOk1VPHv6gqSC/vgUCgF1 +n95u +-----END CERTIFICATE----- diff --git a/scrapy/tests/test_downloader_handlers.py b/scrapy/tests/test_downloader_handlers.py index c71bfbcb8..14a26fe33 100644 --- a/scrapy/tests/test_downloader_handlers.py +++ b/scrapy/tests/test_downloader_handlers.py @@ -207,9 +207,19 @@ class HttpProxyTestCase(unittest.TestCase): def _test(response): self.assertEquals(response.status, 200) self.assertEquals(response.url, request.url) - self.assertEquals(response.body, 'https://example.com') + self.assertEquals(response.body, 'http://example.com') http_proxy = self.getURL('') + request = Request('http://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, BaseSpider('foo')).addCallback(_test) + + def test_download_with_proxy_https_noconnect(self): + def _test(response): + self.assertEquals(response.status, 200) + self.assertEquals(response.url, request.url) + self.assertEquals(response.body, 'https://example.com') + + http_proxy = '%s?noconnect' % self.getURL('') request = Request('https://example.com', meta={'proxy': http_proxy}) return self.download_request(request, BaseSpider('foo')).addCallback(_test) diff --git a/scrapy/tests/test_proxy_connect.py b/scrapy/tests/test_proxy_connect.py new file mode 100644 index 000000000..ee9576f71 --- /dev/null +++ b/scrapy/tests/test_proxy_connect.py @@ -0,0 +1,100 @@ +import os +import subprocess +import time + +from threading import Thread +from libmproxy import controller, proxy +from netlib import http_auth + +from twisted.internet import defer +from twisted.trial.unittest import TestCase +from scrapy.utils.test import get_crawler, get_testlog +from scrapy.tests.spiders import SimpleSpider +from scrapy.tests.mockserver import MockServer + + +def docrawl(spider, settings=None): + crawler = get_crawler(settings) + crawler.configure() + crawler.crawl(spider) + return crawler.start() + + + +class HTTPSProxy(controller.Master, Thread): + + def __init__(self, port): + password_manager = http_auth.PassManSingleUser('scrapy', 'scrapy') + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") + cert_path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'keys', 'mitmproxy-ca.pem') + server = proxy.ProxyServer(proxy.ProxyConfig( + authenticator = authenticator, + cacert = cert_path), + port) + Thread.__init__(self) + controller.Master.__init__(self, server) + + +class ProxyConnectTestCase(TestCase): + + def setUp(self): + self.mockserver = MockServer() + self.mockserver.__enter__() + self._oldenv = os.environ.copy() + self._proxy = HTTPSProxy(8888) + self._proxy.start() + # Wait for the proxy to start. + time.sleep(1.0) + os.environ['http_proxy'] = 'http://scrapy:scrapy@localhost:8888' + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + def tearDown(self): + self.mockserver.__exit__(None, None, None) + self._proxy.shutdown() + os.environ = self._oldenv + + @defer.inlineCallbacks + def test_https_connect_tunnel(self): + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(200) + + @defer.inlineCallbacks + def test_https_noconnect(self): + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888?noconnect' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(200) + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + @defer.inlineCallbacks + def test_https_connect_tunnel_error(self): + spider = SimpleSpider("https://localhost:99999/status?n=200") + yield docrawl(spider) + self._assert_got_tunnel_error() + + @defer.inlineCallbacks + def test_https_tunnel_auth_error(self): + os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + # The proxy returns a 407 error code but it does not reach the client; + # he just sees a TunnelError. + self._assert_got_tunnel_error() + os.environ['https_proxy'] = 'http://scrapy:scrapy@localhost:8888' + + @defer.inlineCallbacks + def test_https_noconnect_auth_error(self): + os.environ['https_proxy'] = 'http://wrong:wronger@localhost:8888?noconnect' + spider = SimpleSpider("https://localhost:8999/status?n=200") + yield docrawl(spider) + self._assert_got_response_code(407) + + def _assert_got_response_code(self, code): + log = get_testlog() + self.assertEqual(log.count('Crawled (%d)' % code), 1) + + def _assert_got_tunnel_error(self): + log = get_testlog() + self.assertEqual(log.count('TunnelError'), 1) diff --git a/tox.ini b/tox.ini index 7503d53bd..e10012e05 100644 --- a/tox.ini +++ b/tox.ini @@ -15,6 +15,7 @@ deps = django # Only required to run tests mock + mitmproxy commands = trial scrapy @@ -29,6 +30,7 @@ deps = django==1.3.1 cssselect==0.9.1 mock==1.0.1 + mitmproxy==0.9.2 [testenv:trunk] basepython = python2.7 @@ -48,6 +50,7 @@ deps = w3lib>=1.5 # Only required to run tests mock>=1.0.1 + mitmproxy>=0.9.2 commands = trial scrapy