mirror of https://github.com/scrapy/scrapy.git
Rename helper function to strip_url() + add more tests
This commit is contained in:
Paul Tremberth
parent
5dd7311cd4
commit
8864d0e8c1
|
|
@ -8,7 +8,7 @@ from scrapy import signals
|
|||
from scrapy.utils.python import to_native_str
|
||||
from scrapy.utils.httpobj import urlparse_cached
|
||||
from scrapy.utils.misc import load_object
|
||||
from scrapy.utils.url import strip_url_credentials
|
||||
from scrapy.utils.url import strip_url
|
||||
|
||||
|
||||
LOCAL_SCHEMES = ('about', 'blob', 'data', 'filesystem',)
|
||||
|
|
@ -53,7 +53,11 @@ class ReferrerPolicy(object):
|
|||
return None
|
||||
parsed_url = urlparse_cached(req_or_resp)
|
||||
if parsed_url.scheme not in self.NOREFERRER_SCHEMES:
|
||||
return strip_url_credentials(parsed_url, origin_only=origin_only)
|
||||
return strip_url(parsed_url,
|
||||
strip_credentials=True,
|
||||
strip_fragment=True,
|
||||
strip_default_port=True,
|
||||
origin_only=origin_only)
|
||||
|
||||
def origin(self, req_or_resp):
|
||||
"""Return serialized origin (scheme, host, path) for a request or response URL."""
|
||||
|
|
|
|||
|
|
@ -105,33 +105,37 @@ def guess_scheme(url):
|
|||
return add_http_if_no_scheme(url)
|
||||
|
||||
|
||||
def strip_url_credentials(url, origin_only=False, keep_fragments=False):
|
||||
def strip_url(url, strip_credentials=True, strip_default_port=True, origin_only=False, strip_fragment=True):
|
||||
|
||||
"""Strip URL string from some of its components:
|
||||
|
||||
- `strip_credentials` removes "user:password@"
|
||||
- `strip_default_port` removes ":80" (resp. ":443", ":21")
|
||||
from http:// (resp. https://, ftp://) URLs
|
||||
- `origin_only` replaces path component with "/", also dropping
|
||||
query and fragment components ; it also strips credentials
|
||||
- `strip_fragment` drops any #fragment component
|
||||
"""
|
||||
|
||||
if url is None:
|
||||
return None
|
||||
|
||||
if not isinstance(url, ParseResult):
|
||||
parsed_url = urlparse(url)
|
||||
else:
|
||||
parsed_url = url
|
||||
|
||||
netloc = parsed_url.netloc
|
||||
# strip username and password if present
|
||||
if parsed_url.username or parsed_url.password:
|
||||
if (strip_credentials or origin_only) and (parsed_url.username or parsed_url.password):
|
||||
netloc = netloc.split('@')[-1]
|
||||
|
||||
# strip standard protocol numbers
|
||||
# Note: strictly speaking, standard port numbers should only be
|
||||
# stripped when comparing origins
|
||||
if parsed_url.port:
|
||||
if (parsed_url.scheme, parsed_url.port) in (('http', 80), ('https', 443)):
|
||||
if strip_default_port and parsed_url.port:
|
||||
if (parsed_url.scheme, parsed_url.port) in (('http', 80),
|
||||
('https', 443),
|
||||
('ftp', 21)):
|
||||
netloc = netloc.replace(':{p.port}'.format(p=parsed_url), '')
|
||||
|
||||
return urlunparse((
|
||||
parsed_url.scheme,
|
||||
netloc,
|
||||
'/' if origin_only else parsed_url.path,
|
||||
'' if origin_only else parsed_url.params,
|
||||
'' if origin_only else parsed_url.query,
|
||||
'' if not keep_fragments else parsed_url.fragment
|
||||
'' if strip_fragment else parsed_url.fragment
|
||||
))
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from six.moves.urllib.parse import urlparse
|
|||
from scrapy.spiders import Spider
|
||||
from scrapy.utils.url import (url_is_from_any_domain, url_is_from_spider,
|
||||
add_http_if_no_scheme, guess_scheme,
|
||||
parse_url, strip_url_credentials)
|
||||
parse_url, strip_url)
|
||||
|
||||
__doctests__ = ['scrapy.utils.url']
|
||||
|
||||
|
|
@ -242,94 +242,158 @@ for k, args in enumerate ([
|
|||
setattr (GuessSchemeTest, t_method.__name__, t_method)
|
||||
|
||||
|
||||
class StripUrlCredentials(unittest.TestCase):
|
||||
class StripUrl(unittest.TestCase):
|
||||
|
||||
def test_noop(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html'),
|
||||
'http://www.example.com/index.html')
|
||||
|
||||
def test_noop_query_string(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
'http://www.example.com/index.html?somekey=somevalue')
|
||||
|
||||
def test_fragments(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://www.example.com/index.html?somekey=somevalue#section', keep_fragments=True),
|
||||
self.assertEqual(strip_url(
|
||||
'http://www.example.com/index.html?somekey=somevalue#section', strip_fragment=False),
|
||||
'http://www.example.com/index.html?somekey=somevalue#section')
|
||||
|
||||
def test_noop_trailing_path(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://www.example.com/'),
|
||||
'http://www.example.com/')
|
||||
def test_path(self):
|
||||
for input_url, origin, output_url in [
|
||||
('http://www.example.com/',
|
||||
False,
|
||||
'http://www.example.com/'),
|
||||
|
||||
def test_noop_trailing_path2(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://www.example.com'),
|
||||
'http://www.example.com')
|
||||
('http://www.example.com',
|
||||
False,
|
||||
'http://www.example.com'),
|
||||
|
||||
def test_trailing_path_origin(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://www.example.com', origin_only=True),
|
||||
'http://www.example.com/')
|
||||
('http://www.example.com',
|
||||
True,
|
||||
'http://www.example.com/'),
|
||||
]:
|
||||
self.assertEqual(strip_url(input_url, origin_only=origin), output_url)
|
||||
self.assertEqual(strip_url(urlparse(input_url), origin_only=origin), output_url)
|
||||
|
||||
def test_username(self):
|
||||
# username is stripped (and fragment too)
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username@www.example.com/index.html?somekey=somevalue#section'),
|
||||
'http://www.example.com/index.html?somekey=somevalue')
|
||||
def test_credentials(self):
|
||||
for i, o in [
|
||||
('http://username@www.example.com/index.html?somekey=somevalue#section',
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
def test_username_empty_pass(self):
|
||||
# same as above
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'https://username:@www.example.com/index.html?somekey=somevalue#section'),
|
||||
'https://www.example.com/index.html?somekey=somevalue')
|
||||
('https://username:@www.example.com/index.html?somekey=somevalue#section',
|
||||
'https://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
def test_username_password(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'ftp://username:password@www.example.com/index.html?somekey=somevalue#section'),
|
||||
'ftp://www.example.com/index.html?somekey=somevalue')
|
||||
('ftp://username:password@www.example.com/index.html?somekey=somevalue#section',
|
||||
'ftp://www.example.com/index.html?somekey=somevalue'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_credentials=True), o)
|
||||
self.assertEqual(strip_url(urlparse(i), strip_credentials=True), o)
|
||||
|
||||
def test_default_http_port(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username:password@www.example.com:80/index.html'),
|
||||
'http://www.example.com/index.html')
|
||||
def test_default_ports_creds_off(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html?somekey=somevalue#section',
|
||||
'http://www.example.com/index.html?somekey=somevalue'),
|
||||
|
||||
def test_non_default_http_port(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username:password@www.example.com:8080/index.html'),
|
||||
'http://www.example.com:8080/index.html')
|
||||
('http://username:password@www.example.com:8080/index.html#section',
|
||||
'http://www.example.com:8080/index.html'),
|
||||
|
||||
def test_default_https_port(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'https://username:password@www.example.com:443/index.html'),
|
||||
'https://www.example.com/index.html')
|
||||
('http://username:password@www.example.com:443/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://www.example.com:443/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
def test_non_default_https_port(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'https://username:password@www.example.com:442/index.html'),
|
||||
'https://www.example.com:442/index.html')
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://www.example.com/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i), o)
|
||||
self.assertEqual(strip_url(urlparse(i)), o)
|
||||
|
||||
def test_default_ports(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html',
|
||||
'http://username:password@www.example.com/index.html'),
|
||||
|
||||
('http://username:password@www.example.com:8080/index.html',
|
||||
'http://username:password@www.example.com:8080/index.html'),
|
||||
|
||||
('http://username:password@www.example.com:443/index.html',
|
||||
'http://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://username:password@www.example.com/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://username:password@www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://username:password@www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://username:password@www.example.com/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=True, strip_credentials=False), o)
|
||||
self.assertEqual(strip_url(urlparse(i), strip_default_port=True, strip_credentials=False), o)
|
||||
|
||||
def test_default_ports_keep(self):
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://username:password@www.example.com:80/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
('http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov#section',
|
||||
'http://username:password@www.example.com:8080/index.html?somekey=somevalue&someotherkey=sov'),
|
||||
|
||||
('http://username:password@www.example.com:443/index.html',
|
||||
'http://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://username:password@www.example.com:443/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:442/index.html',
|
||||
'https://username:password@www.example.com:442/index.html'),
|
||||
|
||||
('https://username:password@www.example.com:80/index.html',
|
||||
'https://username:password@www.example.com:80/index.html'),
|
||||
|
||||
('ftp://username:password@www.example.com:21/file.txt',
|
||||
'ftp://username:password@www.example.com:21/file.txt'),
|
||||
|
||||
('ftp://username:password@www.example.com:221/file.txt',
|
||||
'ftp://username:password@www.example.com:221/file.txt'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, strip_default_port=False, strip_credentials=False), o)
|
||||
self.assertEqual(strip_url(urlparse(i), strip_default_port=False, strip_credentials=False), o)
|
||||
|
||||
def test_origin_only(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username:password@www.example.com/index.html', origin_only=True),
|
||||
'http://www.example.com/')
|
||||
for i, o in [
|
||||
('http://username:password@www.example.com/index.html',
|
||||
'http://www.example.com/'),
|
||||
|
||||
def test_default_http_port_origin_only(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username:password@www.example.com:80/index.html', origin_only=True),
|
||||
'http://www.example.com/')
|
||||
('http://username:password@www.example.com:80/foo/bar?query=value#somefrag',
|
||||
'http://www.example.com/'),
|
||||
|
||||
def test_non_default_http_port_origin_only(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'http://username:password@www.example.com:8008/index.html', origin_only=True),
|
||||
'http://www.example.com:8008/')
|
||||
('http://username:password@www.example.com:8008/foo/bar?query=value#somefrag',
|
||||
'http://www.example.com:8008/'),
|
||||
|
||||
('https://username:password@www.example.com:443/index.html',
|
||||
'https://www.example.com/'),
|
||||
]:
|
||||
self.assertEqual(strip_url(i, origin_only=True), o)
|
||||
self.assertEqual(strip_url(urlparse(i), origin_only=True), o)
|
||||
|
||||
def test_default_https_port_origin_only(self):
|
||||
self.assertEqual(strip_url_credentials(
|
||||
'https://username:password@www.example.com:443/index.html', origin_only=True),
|
||||
'https://www.example.com/')
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
|
|
|||
Loading…
Reference in New Issue