diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 88c6b9480..f07397a4f 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): for it. """ - _responseMatcher = re.compile(b'HTTP/1\.. 200') + _responseMatcher = re.compile(b'HTTP/1\.. (?P\d{3})(?P.{,32})') def __init__(self, reactor, host, port, proxyConf, contextFactory, timeout=30, bindAddress=None): @@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocol = protocol return protocol - def processProxyResponse(self, bytes): + def processProxyResponse(self, rcvd_bytes): """Processes the response from the proxy. If the tunnel is successfully created, notifies the client that we are ready to send requests. If not raises a TunnelError. """ self._protocol.dataReceived = self._protocolDataReceived - if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes): + respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes) + if respm and int(respm.group('status')) == 200: try: # this sets proper Server Name Indication extension # but is only available for Twisted>=14.0 @@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint): self._protocolFactory) self._tunnelReadyDeferred.callback(self._protocol) else: + if respm: + extra = {'status': int(respm.group('status')), + 'reason': respm.group('reason').strip()} + else: + extra = rcvd_bytes[:32] self._tunnelReadyDeferred.errback( - TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % ( - self._host, self._port))) + TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % ( + self._host, self._port, extra))) def connectFailed(self, reason): """Propagates the errback to the appropriate deferred.""" @@ -157,17 +163,15 @@ def tunnel_request_data(host, port, proxy_auth_header=None): >>> from scrapy.utils.python import to_native_str as s >>> s(tunnel_request_data("example.com", 8080)) - 'CONNECT example.com:8080 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n' >>> s(tunnel_request_data("example.com", 8080, b"123")) - 'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n' + 'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n' >>> s(tunnel_request_data(b"example.com", "8090")) - 'CONNECT example.com:8090 HTTP/1.1\r\n\r\n' + 'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n' """ - tunnel_req = ( - b'CONNECT ' + - to_bytes(host, encoding='ascii') + b':' + - to_bytes(str(port)) + - b' HTTP/1.1\r\n') + host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port)) + tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n' + tunnel_req += b'Host: ' + host_value + b'\r\n' if proxy_auth_header: tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n' tunnel_req += b'\r\n' diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 45a806f2e..c63c64d86 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -401,7 +401,13 @@ class UriResource(resource.Resource): return self def render(self, request): - return request.uri + # Note: this is an ugly hack for CONNECT request timeout test. + # Returning some data here fail SSL/TLS handshake + # ToDo: implement proper HTTPS proxy tests, not faking them. + if request.method != b'CONNECT': + return request.uri + else: + return b'' class HttpProxyTestCase(unittest.TestCase):