RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2069 from redapple/https-connect-host 

[MRG] Add "Host" header in CONNECT requests to HTTPS proxies
This commit is contained in:
Mikhail Korobov 2016-07-06 21:43:41 +06:00 committed by GitHub
parent 1737c15826 15d0c89159
commit 759a555d28
2 changed files with 24 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
scrapy/core/downloader/handlers/http11.py
View File

@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
for it.
"""
_responseMatcher = re.compile(b'HTTP/1\.. 200')
_responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
def __init__(self, reactor, host, port, proxyConf, contextFactory,
timeout=30, bindAddress=None):
@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocol = protocol
return protocol
def processProxyResponse(self, bytes):
def processProxyResponse(self, rcvd_bytes):
"""Processes the response from the proxy. If the tunnel is successfully
created, notifies the client that we are ready to send requests. If not
raises a TunnelError.
"""
self._protocol.dataReceived = self._protocolDataReceived
if TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
if respm and int(respm.group('status')) == 200:
try:
# this sets proper Server Name Indication extension
# but is only available for Twisted>=14.0
@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
self._protocolFactory)
self._tunnelReadyDeferred.callback(self._protocol)
else:
if respm:
extra = {'status': int(respm.group('status')),
'reason': respm.group('reason').strip()}
else:
extra = rcvd_bytes[:32]
self._tunnelReadyDeferred.errback(
TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
self._host, self._port)))
TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
self._host, self._port, extra)))
def connectFailed(self, reason):
"""Propagates the errback to the appropriate deferred."""
@ -157,17 +163,15 @@ def tunnel_request_data(host, port, proxy_auth_header=None):
>>> from scrapy.utils.python import to_native_str as s
>>> s(tunnel_request_data("example.com", 8080))
'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
>>> s(tunnel_request_data("example.com", 8080, b"123"))
'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'
'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
>>> s(tunnel_request_data(b"example.com", "8090"))
'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'
'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
"""
tunnel_req = (
b'CONNECT ' +
to_bytes(host, encoding='ascii') + b':' +
to_bytes(str(port)) +
b' HTTP/1.1\r\n')
host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
tunnel_req += b'Host: ' + host_value + b'\r\n'
if proxy_auth_header:
tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
tunnel_req += b'\r\n'

8
tests/test_downloader_handlers.py
View File

@ -401,7 +401,13 @@ class UriResource(resource.Resource):
return self
def render(self, request):
return request.uri
# Note: this is an ugly hack for CONNECT request timeout test.
# Returning some data here fail SSL/TLS handshake
# ToDo: implement proper HTTPS proxy tests, not faking them.
if request.method != b'CONNECT':
return request.uri
else:
return b''
class HttpProxyTestCase(unittest.TestCase):