diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 95af8586b..92ebbe605 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,6 +9,7 @@ from threading import Thread from scrapy.commands import ScrapyCommand from scrapy.shell import Shell from scrapy.http import Request +from scrapy.utils.url import add_http_if_no_scheme from scrapy.utils.spider import spidercls_for_request, DefaultSpider @@ -41,6 +42,8 @@ class Command(ScrapyCommand): def run(self, args, opts): url = args[0] if args else None + if url: + url = add_http_if_no_scheme(url) spider_loader = self.crawler_process.spider_loader spidercls = DefaultSpider diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index 99f350361..398407a64 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -110,3 +110,14 @@ def escape_ajax(url): if not frag.startswith('!'): return url return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:]) + + +def add_http_if_no_scheme(url): + """Add http as the default scheme if it is missing from the url.""" + if url.startswith('//'): + url = 'http:' + url + return url + parser = parse_url(url) + if not parser.scheme or not parser.netloc: + url = 'http://' + url + return url diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 7bf0e5b4a..314ccd30f 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -4,7 +4,7 @@ import unittest import six from scrapy.spiders import Spider from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider, - canonicalize_url) + canonicalize_url, add_http_if_no_scheme) __doctests__ = ['scrapy.utils.url'] @@ -187,5 +187,112 @@ class CanonicalizeUrlTest(unittest.TestCase): "http://foo.com/AC%2FDC/") +class AddHttpIfNoScheme(unittest.TestCase): + + def test_add_scheme(self): + self.assertEqual(add_http_if_no_scheme('www.example.com'), + 'http://www.example.com') + + def test_without_subdomain(self): + self.assertEqual(add_http_if_no_scheme('example.com'), + 'http://example.com') + + def test_path(self): + self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') + + def test_port(self): + self.assertEqual(add_http_if_no_scheme('www.example.com:80'), + 'http://www.example.com:80') + + def test_fragment(self): + self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') + + def test_query(self): + self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') + + def test_username_password(self): + self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'), + 'http://username:password@www.example.com') + + def test_complete_url(self): + self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + + def test_preserve_http(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com'), + 'http://www.example.com') + + def test_preserve_http_without_subdomain(self): + self.assertEqual(add_http_if_no_scheme('http://example.com'), + 'http://example.com') + + def test_preserve_http_path(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') + + def test_preserve_http_port(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'), + 'http://www.example.com:80') + + def test_preserve_http_fragment(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') + + def test_preserve_http_query(self): + self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') + + def test_preserve_http_username_password(self): + self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'), + 'http://username:password@www.example.com') + + def test_preserve_http_complete_url(self): + self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + + def test_protocol_relative(self): + self.assertEqual(add_http_if_no_scheme('//www.example.com'), + 'http://www.example.com') + + def test_protocol_relative_without_subdomain(self): + self.assertEqual(add_http_if_no_scheme('//example.com'), + 'http://example.com') + + def test_protocol_relative_path(self): + self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'), + 'http://www.example.com/some/page.html') + + def test_protocol_relative_port(self): + self.assertEqual(add_http_if_no_scheme('//www.example.com:80'), + 'http://www.example.com:80') + + def test_protocol_relative_fragment(self): + self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'), + 'http://www.example.com/some/page#frag') + + def test_protocol_relative_query(self): + self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'), + 'http://www.example.com/do?a=1&b=2&c=3') + + def test_protocol_relative_username_password(self): + self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'), + 'http://username:password@www.example.com') + + def test_protocol_relative_complete_url(self): + self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'), + 'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag') + + def test_preserve_https(self): + self.assertEqual(add_http_if_no_scheme('https://www.example.com'), + 'https://www.example.com') + + def test_preserve_ftp(self): + self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'), + 'ftp://www.example.com') + + if __name__ == "__main__": unittest.main()