From 7fc80671a84144ad36c3d5332aa822ef3be00781 Mon Sep 17 00:00:00 2001 From: ajaymittur28 Date: Wed, 1 Jul 2020 13:32:17 +0530 Subject: [PATCH] Update schemaless URI support --- scrapy/core/downloader/handlers/http11.py | 12 ++++++++---- scrapy/core/downloader/webclient.py | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py index 73e56c87d..15de8cdbd 100644 --- a/scrapy/core/downloader/handlers/http11.py +++ b/scrapy/core/downloader/handlers/http11.py @@ -7,7 +7,7 @@ import warnings from contextlib import suppress from io import BytesIO from time import time -from urllib.parse import urldefrag, urlparse +from urllib.parse import urldefrag, urlunparse from twisted.internet import defer, protocol, ssl from twisted.internet.endpoints import TCP4ClientEndpoint @@ -255,7 +255,7 @@ class ScrapyProxyAgent(Agent): bindAddress=bindAddress, pool=pool, ) - self._proxyURI = URI.fromBytes(urlparse(proxyURI)._replace(scheme=b'http').geturl()) + self._proxyURI = URI.fromBytes(proxyURI) def request(self, method, uri, headers=None, bodyProducer=None): """ @@ -297,7 +297,7 @@ class ScrapyAgent: bindaddress = request.meta.get('bindaddress') or self._bindAddress proxy = request.meta.get('proxy') if proxy: - _, _, proxyHost, proxyPort, proxyParams = _parse(proxy) + proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy) scheme = _parse(request.url)[0] proxyHost = to_unicode(proxyHost) omitConnectTunnel = b'noconnect' in proxyParams @@ -319,9 +319,13 @@ class ScrapyAgent: pool=self._pool, ) else: + proxyScheme = b'http' if not proxyScheme else proxyScheme + proxyHost = to_bytes(proxyHost, encoding='ascii') + proxyPort = to_bytes(str(proxyPort), encoding='ascii') + proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, '', '', '')) return self._ProxyAgent( reactor=reactor, - proxyURI=to_bytes(proxy, encoding='ascii'), + proxyURI=to_bytes(proxyURI, encoding='ascii'), connectTimeout=timeout, bindAddress=bindaddress, pool=self._pool, diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py index 355045d74..af49e78ce 100644 --- a/scrapy/core/downloader/webclient.py +++ b/scrapy/core/downloader/webclient.py @@ -1,5 +1,6 @@ from time import time from urllib.parse import urlparse, urlunparse, urldefrag +from re import match from twisted.web.client import HTTPClientFactory from twisted.web.http import HTTPClient @@ -32,6 +33,8 @@ def _parse(url): and is ascii-only. """ url = url.strip() + if not match(r'^\w+://', url): + url = '//' + url parsed = urlparse(url) return _parsed_url_args(parsed)