diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index e94e339de..92ebbe605 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,7 +9,7 @@ from threading import Thread from scrapy.commands import ScrapyCommand from scrapy.shell import Shell from scrapy.http import Request -from scrapy.utils.url import add_scheme_if_missing +from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider @@ -43,7 +43,7 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None if url: - url = add_scheme_if_missing(url) + url = add_http_if_no_scheme(url) spider_loader = self.crawler_process.spider_loader spidercls = DefaultSpider diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 94ec4de1b..c0934ddcf 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -111,10 +111,11 @@ def escape_ajax(url): return url return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:]) -def add_scheme_if_missing(url): +def add_http_if_no_scheme(url): + """Adds http as the default scheme if it is missing from the url""" parser = parse_url(url) - if not parser.scheme: - if not parser.netloc: - parser = parser._replace(netloc=parser.path, path='') - parser = parser._replace(scheme='http') - return parser.geturl() + if url.startswith('//'): + url = 'http:' + url + elif not parser.scheme or not parser.netloc: + url = 'http://' + url + return url diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index fae4c988b..7ccf68c7a 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,7 @@ import unittest import six from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - canonicalize_url, add_scheme_if_missing) + canonicalize_url, add_http_if_no_scheme) __doctests__ = ['scrapy.utils.url'] @@ -73,21 +73,47 @@ class UrlUtilsTest(unittest.TestCase): self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider)) self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider)) - def test_add_scheme_if_missing(self): - self.assertEqual(add_scheme_if_missing('http://www.example.com'), + def test_add_http_if_no_scheme(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com'), 'http://www.example.com') - self.assertEqual(add_scheme_if_missing('http://www.example.com/some/page.html'), + self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), 'http://www.example.com/some/page.html') - self.assertEqual(add_scheme_if_missing('http://example.com'), + self.assertEqual(add_http_if_no_scheme('http://example.com'), 'http://example.com') - self.assertEqual(add_scheme_if_missing('www.example.com'), + self.assertEqual(add_http_if_no_scheme('www.example.com'), 'http://www.example.com') - self.assertEqual(add_scheme_if_missing('example.com'), + self.assertEqual(add_http_if_no_scheme('example.com'), 'http://example.com') - self.assertEqual(add_scheme_if_missing('//example.com'), + self.assertEqual(add_http_if_no_scheme('//example.com'), 'http://example.com') - self.assertEqual(add_scheme_if_missing('https://www.example.com'), + self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') + self.assertEqual(add_http_if_no_scheme('www.example.com:80'), + 'http://www.example.com:80') + self.assertEqual(add_http_if_no_scheme('www.example.com:80/some/page.html'), + 'http://www.example.com:80/some/page.html') + self.assertEqual(add_http_if_no_scheme('http://www.example.com:80/some/page.html'), + 'http://www.example.com:80/some/page.html') + self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') + self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') + self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') + self.assertEqual(add_http_if_no_scheme('username:password@example.com/some/page.html'), + 'http://username:password@example.com/some/page.html') + self.assertEqual(add_http_if_no_scheme('http://username:password@example.com/some/page.html'), + 'http://username:password@example.com/some/page.html') + self.assertEqual(add_http_if_no_scheme('username:password@example.com:80/some/part?a=1&b=2&c=3#frag'), + 'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag') + self.assertEqual(add_http_if_no_scheme('http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag'), + 'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag') + self.assertEqual(add_http_if_no_scheme('https://www.example.com'), 'https://www.example.com') + self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), + 'ftp://www.example.com') class CanonicalizeUrlTest(unittest.TestCase):