RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adds docstring, tests and correction

This commit is contained in:
preetwinder 2015-09-18 16:31:37 +05:30
parent c517951a48
commit 8c629eee3e
3 changed files with 44 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
scrapy/commands/shell.py
View File

@ -9,7 +9,7 @@ from threading import Thread
from scrapy.commands import ScrapyCommand
from scrapy.shell import Shell
from scrapy.http import Request
from scrapy.utils.url import add_scheme_if_missing
from scrapy.utils.url import add_http_if_no_scheme
from scrapy.utils.spider import spidercls_for_request, DefaultSpider
@ -43,7 +43,7 @@ class Command(ScrapyCommand):
def run(self, args, opts):
url = args[0] if args else None
if url:
url = add_scheme_if_missing(url)
url = add_http_if_no_scheme(url)
spider_loader = self.crawler_process.spider_loader
spidercls = DefaultSpider

13
scrapy/utils/url.py
View File

@ -111,10 +111,11 @@ def escape_ajax(url):
return url
return add_or_replace_parameter(defrag, '_escaped_fragment_', frag[1:])
def add_scheme_if_missing(url):
def add_http_if_no_scheme(url):
"""Adds http as the default scheme if it is missing from the url"""
parser = parse_url(url)
if not parser.scheme:
if not parser.netloc:
parser = parser._replace(netloc=parser.path, path='')
parser = parser._replace(scheme='http')
return parser.geturl()
if url.startswith('//'):
url = 'http:' + url
elif not parser.scheme or not parser.netloc:
url = 'http://' + url
return url

44
tests/test_utils_url.py
View File

@ -4,7 +4,7 @@ import unittest
import six
from scrapy.spiders import Spider
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
canonicalize_url, add_scheme_if_missing)
canonicalize_url, add_http_if_no_scheme)
__doctests__ = ['scrapy.utils.url']
@ -73,21 +73,47 @@ class UrlUtilsTest(unittest.TestCase):
self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
def test_add_scheme_if_missing(self):
self.assertEqual(add_scheme_if_missing('http://www.example.com'),
def test_add_http_if_no_scheme(self):
self.assertEqual(add_http_if_no_scheme('http://www.example.com'),
'http://www.example.com')
self.assertEqual(add_scheme_if_missing('http://www.example.com/some/page.html'),
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
self.assertEqual(add_scheme_if_missing('http://example.com'),
self.assertEqual(add_http_if_no_scheme('http://example.com'),
'http://example.com')
self.assertEqual(add_scheme_if_missing('www.example.com'),
self.assertEqual(add_http_if_no_scheme('www.example.com'),
'http://www.example.com')
self.assertEqual(add_scheme_if_missing('example.com'),
self.assertEqual(add_http_if_no_scheme('example.com'),
'http://example.com')
self.assertEqual(add_scheme_if_missing('//example.com'),
self.assertEqual(add_http_if_no_scheme('//example.com'),
'http://example.com')
self.assertEqual(add_scheme_if_missing('https://www.example.com'),
self.assertEqual(add_http_if_no_scheme('//www.example.com/some/page.html'),
'http://www.example.com/some/page.html')
self.assertEqual(add_http_if_no_scheme('www.example.com:80'),
'http://www.example.com:80')
self.assertEqual(add_http_if_no_scheme('www.example.com:80/some/page.html'),
'http://www.example.com:80/some/page.html')
self.assertEqual(add_http_if_no_scheme('http://www.example.com:80/some/page.html'),
'http://www.example.com:80/some/page.html')
self.assertEqual(add_http_if_no_scheme('www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
self.assertEqual(add_http_if_no_scheme('http://www.example.com/some/page#frag'),
'http://www.example.com/some/page#frag')
self.assertEqual(add_http_if_no_scheme('www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
self.assertEqual(add_http_if_no_scheme('http://www.example.com/do?a=1&b=2&c=3'),
'http://www.example.com/do?a=1&b=2&c=3')
self.assertEqual(add_http_if_no_scheme('username:password@example.com/some/page.html'),
'http://username:password@example.com/some/page.html')
self.assertEqual(add_http_if_no_scheme('http://username:password@example.com/some/page.html'),
'http://username:password@example.com/some/page.html')
self.assertEqual(add_http_if_no_scheme('username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
self.assertEqual(add_http_if_no_scheme('http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag'),
'http://username:password@example.com:80/some/part?a=1&b=2&c=3#frag')
self.assertEqual(add_http_if_no_scheme('https://www.example.com'),
'https://www.example.com')
self.assertEqual(add_http_if_no_scheme('ftp://www.example.com'),
'ftp://www.example.com')
class CanonicalizeUrlTest(unittest.TestCase):