From dd45b31fe42635ff43df62c667f1ace5f9169737 Mon Sep 17 00:00:00 2001 From: Leonid Amirov Date: Tue, 3 Nov 2015 14:32:30 +0300 Subject: [PATCH] issue GH #1550 - rewritten add_http_if_no_scheme() --- scrapy/utils/url.py | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 3eac5fb3d..0e36003ad 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -6,7 +6,7 @@ Some of the functions that used to be imported from this module have been moved to the w3lib.url module. Always import those from there instead. """ import posixpath -from urlparse import urlsplit, urlunsplit +import re from six.moves.urllib.parse import (ParseResult, urlunparse, urldefrag, urlparse, parse_qsl, urlencode, unquote) @@ -115,16 +115,10 @@ def escape_ajax(url): def add_http_if_no_scheme(url): """Add http as the default scheme if it is missing from the url.""" - parts = urlsplit(url) - scheme = parts.scheme or "http" - if parts.netloc: - netloc = parts.netloc - path = parts.path - else: - path_parts = url.split("/", 1) - netloc = path_parts[0] - path = path_parts[1] if len(path_parts) > 1 else "/" + match = re.match(r"^\w+://", url, flags=re.I) + parts = urlparse(url) + if not match: + scheme = "http:" if parts.netloc else "http://" + url = scheme + url - return urlunsplit(( - scheme, netloc, path, parts.query, parts.fragment - )) + return url