Merge pull request #4649 from ajaymittur28/http-proxy-schema

Support schema-less HTTP proxy (#4504)

scrapy/core/downloader/handlers/http11.py

@@ -7,7 +7,7 @@ import warnings
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from urllib.parse import urldefrag
+from urllib.parse import urldefrag, urlunparse
 
 from twisted.internet import defer, protocol, ssl
 from twisted.internet.endpoints import TCP4ClientEndpoint
@@ -276,7 +276,7 @@ class ScrapyAgent:
         bindaddress = request.meta.get('bindaddress') or self._bindAddress
         proxy = request.meta.get('proxy')
         if proxy:
-            _, _, proxyHost, proxyPort, proxyParams = _parse(proxy)
+            proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
             proxyHost = to_unicode(proxyHost)
             omitConnectTunnel = b'noconnect' in proxyParams
@@ -301,9 +301,13 @@ class ScrapyAgent:
                     pool=self._pool,
                 )
             else:
+                proxyScheme = proxyScheme or b'http'
+                proxyHost = to_bytes(proxyHost, encoding='ascii')
+                proxyPort = to_bytes(str(proxyPort), encoding='ascii')
+                proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', ''))
                 return self._ProxyAgent(
                     reactor=reactor,
-                    proxyURI=to_bytes(proxy, encoding='ascii'),
+                    proxyURI=to_bytes(proxyURI, encoding='ascii'),
                     connectTimeout=timeout,
                     bindAddress=bindaddress,
                     pool=self._pool,

scrapy/core/downloader/webclient.py

@@ -1,3 +1,4 @@
+import re
 from time import time
 from urllib.parse import urlparse, urlunparse, urldefrag
 
@@ -32,6 +33,8 @@ def _parse(url):
     and is ascii-only.
     """
     url = url.strip()
+    if not re.match(r'^\w+://', url):
+        url = '//' + url
     parsed = urlparse(url)
     return _parsed_url_args(parsed)
 

tests/test_downloader_handlers.py

@@ -764,6 +764,16 @@ class Http11ProxyTestCase(HttpProxyTestCase):
         timeout = yield self.assertFailure(d, error.TimeoutError)
         self.assertIn(domain, timeout.osError)
 
+    def test_download_with_proxy_without_http_scheme(self):
+        def _test(response):
+            self.assertEqual(response.status, 200)
+            self.assertEqual(response.url, request.url)
+            self.assertEqual(response.body, self.expected_http_proxy_request_body)
+
+        http_proxy = self.getURL('').replace('http://', '')
+        request = Request('http://example.com', meta={'proxy': http_proxy})
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
 
 class HttpDownloadHandlerMock: