From d9343463cb69d01a43ed12d8931bbcd86d2aae09 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Tue, 21 Jun 2016 13:26:32 +0200
Subject: [PATCH] Add "Host" header in CONNECT requests to HTTPS proxies

---
 scrapy/core/downloader/handlers/http11.py | 33 +++++++++++++----------
 1 file changed, 19 insertions(+), 14 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 88c6b9480..d02027dd1 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -93,7 +93,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     for it.
     """
 
-    _responseMatcher = re.compile(b'HTTP/1\.. 200')
+    _responseMatcher = re.compile(b'HTTP/1\.. (?P<status>\d{3})(?P<reason>.{,32})')
 
     def __init__(self, reactor, host, port, proxyConf, contextFactory,
                  timeout=30, bindAddress=None):
@@ -115,13 +115,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         self._protocol = protocol
         return protocol
 
-    def processProxyResponse(self, bytes):
+    def processProxyResponse(self, rcvd_bytes):
         """Processes the response from the proxy. If the tunnel is successfully
         created, notifies the client that we are ready to send requests. If not
         raises a TunnelError.
         """
         self._protocol.dataReceived = self._protocolDataReceived
-        if  TunnelingTCP4ClientEndpoint._responseMatcher.match(bytes):
+        respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(rcvd_bytes)
+        if respm and int(respm.group('status')) == 200:
             try:
                 # this sets proper Server Name Indication extension
                 # but is only available for Twisted>=14.0
@@ -134,9 +135,14 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
                                               self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
+            if respm:
+                extra = {'status': int(respm.group('status')),
+                         'reason': respm.group('reason').strip()}
+            else:
+                extra = rcvd_bytes[:32]
             self._tunnelReadyDeferred.errback(
-                TunnelError('Could not open CONNECT tunnel with proxy %s:%s' % (
-                    self._host, self._port)))
+                TunnelError('Could not open CONNECT tunnel with proxy %s:%s [%r]' % (
+                    self._host, self._port, extra)))
 
     def connectFailed(self, reason):
         """Propagates the errback to the appropriate deferred."""
@@ -151,23 +157,22 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
-def tunnel_request_data(host, port, proxy_auth_header=None):
+def tunnel_request_data(host, port, proxy_auth_header=None, host_header=True):
     r"""
     Return binary content of a CONNECT request.
 
     >>> from scrapy.utils.python import to_native_str as s
     >>> s(tunnel_request_data("example.com", 8080))
-    'CONNECT example.com:8080 HTTP/1.1\r\n\r\n'
+    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
     >>> s(tunnel_request_data("example.com", 8080, b"123"))
-    'CONNECT example.com:8080 HTTP/1.1\r\nProxy-Authorization: 123\r\n\r\n'
+    'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
     >>> s(tunnel_request_data(b"example.com", "8090"))
-    'CONNECT example.com:8090 HTTP/1.1\r\n\r\n'
+    'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
     """
-    tunnel_req = (
-        b'CONNECT ' +
-        to_bytes(host, encoding='ascii') + b':' +
-        to_bytes(str(port)) +
-        b' HTTP/1.1\r\n')
+    host_value = to_bytes(host, encoding='ascii') + b':' + to_bytes(str(port))
+    tunnel_req = b'CONNECT ' + host_value + b' HTTP/1.1\r\n'
+    if host_header:
+        tunnel_req += b'Host: ' + host_value + b'\r\n'
     if proxy_auth_header:
         tunnel_req += b'Proxy-Authorization: ' + proxy_auth_header + b'\r\n'
     tunnel_req += b'\r\n'