Adds the functionality to do HTTPS downloads behind proxies using an

HTTP CONNECT.

scrapy/core/downloader/handlers/http11.py

@@ -5,10 +5,10 @@ from cStringIO import StringIO
 from urlparse import urldefrag
 
 from zope.interface import implements
-from twisted.internet import defer, reactor, protocol
+from twisted.internet import defer, reactor, protocol, ssl
 from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import IBodyProducer
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError, SSLError
 from twisted.web.http import PotentialDataLoss
 from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
     HTTPConnectionPool, TCP4ClientEndpoint
@@ -37,12 +37,96 @@ class HTTP11DownloadHandler(object):
         return self._pool.closeCachedConnections()
 
 
+class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
+    """An endpoint that tunnels through proxies to allow HTTPS downloads. To
+    accomplish that, this endpoint sends an HTTP CONNECT to the proxy. The
+    HTTP CONNECT is always sent when using this endpoint, I think this could
+    be improved as the CONNECT will be redundant if the connection associated
+    with this endpoint comes from the pool and a CONNECT has already been issued
+    for it.
+    """
+
+    def __init__(self, reactor, host, port, proxyHost, proxyPort,
+                contextFactory, timeout=30, bindAddress=None):
+        super(TunnelingTCP4ClientEndpoint, self).__init__(reactor, proxyHost,
+            proxyPort, timeout, bindAddress)
+        self._tunnelReadyDeferred = defer.Deferred()
+        # Although we will connect to the proxy, we need the host and port of
+        # the destination server in order to send the HTTP CONNECT.
+        self._tunneledHost = host
+        self._tunneledPort = port
+        self._contextFactory = contextFactory
+
+    def requestTunnel(self, protocol):
+        """Asks the proxy to open a tunnel."""
+        # Ask for the proxy to open the tunnel.
+        protocol.transport.write('CONNECT %s:%s HTTP/1.1\n\n' %
+                                 (self._tunneledHost, self._tunneledPort))
+        # This hack is not so nice. Substitute the dataReceived method
+        # temporarily to intercept the response from the proxy.
+        self._protocolDataReceived = protocol.dataReceived
+        protocol.dataReceived = self.processProxyResponse
+        # Store the protocol because we will have to pass it when triggering the
+        # deferred returned in the connect method.
+        self._protocol = protocol
+        return protocol
+
+    def processProxyResponse(self, bytes):
+        """Processes the response from the proxy. If the tunnel is successfully
+        created, notifies the client that we are ready to send requests.
+        """
+        if bytes.find('200 Connection established') > 0:
+            # The tunnel is ready, switch transport to TLS.
+            self._protocol.transport.startTLS(self._contextFactory,
+                                              self._protocolFactory)
+            # Restore the protocol dataReceived method.
+            self._protocol.dataReceived = self._protocolDataReceived
+            # Trigger the callback with the protocol as the value.
+            self._tunnelReadyDeferred.callback(self._protocol)
+        else:
+            # Not sure if this is the best way to handle this error.
+            raise SSLError
+
+    def connect(self, protocolFactory):
+        # Store the protocol factory as we will need it to switch to TLS.
+        self._protocolFactory = protocolFactory
+        connectDeferred = super(TunnelingTCP4ClientEndpoint,
+                                self).connect(protocolFactory)
+        # Add a callback to open the tunnel when the connection is ready.
+        connectDeferred.addCallback(self.requestTunnel)
+        # Return a deferred that will be triggered when the tunnel is ready.
+        return self._tunnelReadyDeferred
+
+
+class TunnelingAgent(Agent):
+    """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
+    downloads. It may look strange that we have chosen to subclass Agent and not
+    ProxyAgent but consider that after the tunnel is opened the proxy is
+    transparent to the client; thus the agent should behave like there is no
+    proxy involved.
+    """
+
+    def __init__(self, reactor, proxyHost, proxyPort, contextFactory=None,
+                 connectTimeout=None, bindAddress=None, pool=None):
+        super(TunnelingAgent, self).__init__(reactor, contextFactory,
+            connectTimeout, bindAddress, pool)
+        self._proxyHost = proxyHost
+        self._proxyPort = proxyPort
+
+    def _getEndpoint(self, scheme, host, port):
+        return TunnelingTCP4ClientEndpoint(self._reactor, host, port,
+            self._proxyHost, self._proxyPort, self._contextFactory,
+            self._connectTimeout, self._bindAddress)
+
+
 class ScrapyAgent(object):
 
     _Agent = Agent
     _ProxyAgent = ProxyAgent
+    _TunnelingAgent = TunnelingAgent
 
-    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None):
+    def __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None,
+                 pool=None):
         self._contextFactory = contextFactory
         self._connectTimeout = connectTimeout
         self._bindAddress = bindAddress
@@ -52,15 +136,22 @@ class ScrapyAgent(object):
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
         proxy = request.meta.get('proxy')
         if proxy:
-            scheme, _, host, port, _ = _parse(proxy)
-            endpoint = TCP4ClientEndpoint(reactor, host, port, timeout=timeout,
-                                          bindAddress=bindaddress)
-            return self._ProxyAgent(endpoint)
+            _, _, proxyHost, proxyPort, _ = _parse(proxy)
+            scheme = _parse(request.url)[0]
+            if  scheme == 'https':
+                # We need to tunnel the proxy using an HTTP CONNECT.
+                return self._TunnelingAgent(reactor, proxyHost, proxyPort,
+                    contextFactory=self._contextFactory, connectTimeout=timeout,
+                    bindAddress=bindaddress, pool=self._pool)
+            else:
+                endpoint = TCP4ClientEndpoint(reactor, proxyHost, proxyPort,
+                    timeout=timeout, bindAddress=bindaddress)
+                return self._ProxyAgent(endpoint)
 
         return self._Agent(reactor, contextFactory=self._contextFactory,
             connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool)
 
-    def download_request(self, request):
+    def download_request(self, request):    
         timeout = request.meta.get('download_timeout') or self._connectTimeout
         agent = self._get_agent(request, timeout)