diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 50486d13c..8a91d4c5e 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag +from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -276,7 +276,7 @@ class ScrapyAgent: bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams @@ -301,9 +301,13 @@ class ScrapyAgent: pool=self._pool, ) else: + proxyScheme = proxyScheme or b'http' + proxyHost = to_bytes(proxyHost, encoding='ascii') + proxyPort = to_bytes(str(proxyPort), encoding='ascii') + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxy, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 9524cce2b..915cb5fe3 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,3 +1,4 @@ +import re from time import time from urllib.parse import urlparse, urlunparse, urldefrag @@ -32,6 +33,8 @@ def _parse(url): and is ascii-only. """ url = url.strip() + if not re.match(r'^\w+://', url): + url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed) diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index fa7d5c8a6..9c11820e5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -764,6 +764,16 @@ class Http11ProxyTestCase(HttpProxyTestCase): timeout = yield self.assertFailure(d, error.TimeoutError) self.assertIn(domain, timeout.osError) + def test_download_with_proxy_without_http_scheme(self): + def _test(response): + self.assertEqual(response.status, 200) + self.assertEqual(response.url, request.url) + self.assertEqual(response.body, self.expected_http_proxy_request_body) + + http_proxy = self.getURL('').replace('http://', '') + request = Request('http://example.com', meta={'proxy': http_proxy}) + return self.download_request(request, Spider('foo')).addCallback(_test) + class HttpDownloadHandlerMock: