mirror of https://github.com/scrapy/scrapy.git
Merge pull request #1498 from Preetwinder/scrapy-add_scheme
[MRG+1] #1487 add_scheme_if_missing for `scrapy shell` command
This commit is contained in:
Daniel Graña
commit
fe15f93e53
|
|
@ -9,6 +9,7 @@ from threading import Thread
|
|||
from scrapy.commands import ScrapyCommand
|
||||
from scrapy.shell import Shell
|
||||
from scrapy.http import Request
|
||||
from scrapy.utils.url import add_http_if_no_scheme
|
||||
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
|
||||
|
||||
|
||||
|
|
@ -41,6 +42,8 @@ class Command(ScrapyCommand):
|
|||
|
||||
def run(self, args, opts):
|
||||
url = args[0] if args else None
|
||||
if url:
|
||||
url = add_http_if_no_scheme(url)
|
||||
spider_loader = self.crawler_process.spider_loader
|
||||
|
||||
spidercls = DefaultSpider
|
||||
|
|
|
|||
|
|
@ -110,3 +110,14 @@ def escape_ajax(url):
|
|||
if not frag.startswith('!'):
|
||||
return url
|
||||
return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
|
||||
|
||||
|
||||
def add_http_if_no_scheme(url):
|
||||
"""Add http as the default scheme if it is missing from the url."""
|
||||
if url.startswith('//'):
|
||||
url = 'http:' + url
|
||||
return url
|
||||
parser = parse_url(url)
|
||||
if not parser.scheme or not parser.netloc:
|
||||
url = 'http://' + url
|
||||
return url
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import unittest
|
|||
import six
|
||||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
|
||||
canonicalize_url)
|
||||
canonicalize_url, add_http_if_no_scheme)
|
||||
|
||||
__doctests__ = ['scrapy.utils.url']
|
||||
|
||||
|
|
@ -187,5 +187,112 @@ class CanonicalizeUrlTest(unittest.TestCase):
|
|||
"http://foo.com/AC%2FDC/")
|
||||
|
||||
|
||||
class AddHttpIfNoScheme(unittest.TestCase):
|
||||
|
||||
def test_add_scheme(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com'),
|
||||
'http://www.example.com')
|
||||
|
||||
def test_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_http(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
|
||||
def test_preserve_http_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_preserve_http_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_preserve_http_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_preserve_http_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_preserve_http_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_preserve_http_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_preserve_http_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_protocol_relative(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com'),
|
||||
'http://www.example.com')
|
||||
|
||||
def test_protocol_relative_without_subdomain(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//example.com'),
|
||||
'http://example.com')
|
||||
|
||||
def test_protocol_relative_path(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
|
||||
'http://www.example.com/some/page.html')
|
||||
|
||||
def test_protocol_relative_port(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com:80'),
|
||||
'http://www.example.com:80')
|
||||
|
||||
def test_protocol_relative_fragment(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page#frag'),
|
||||
'http://www.example.com/some/page#frag')
|
||||
|
||||
def test_protocol_relative_query(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//www.example.com/do?a=1&b=2&c=3'),
|
||||
'http://www.example.com/do?a=1&b=2&c=3')
|
||||
|
||||
def test_protocol_relative_username_password(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com'),
|
||||
'http://username:password@www.example.com')
|
||||
|
||||
def test_protocol_relative_complete_url(self):
|
||||
self.assertEqual(add_http_if_no_scheme('//username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag'),
|
||||
'http://username:password@www.example.com:80/some/page/do?a=1&b=2&c=3#frag')
|
||||
|
||||
def test_preserve_https(self):
|
||||
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
|
||||
'https://www.example.com')
|
||||
|
||||
def test_preserve_ftp(self):
|
||||
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
|
||||
'ftp://www.example.com')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue