mirror of https://github.com/scrapy/scrapy.git
added canonicalize_url function to scrapy.utils.url, along with a complete suite of tests
--HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40187
This commit is contained in:
Pablo Hoffman
parent
34355048c3
commit
30803b9e89
|
|
@ -1,5 +1,5 @@
|
|||
import unittest
|
||||
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner
|
||||
from scrapy.utils.url import url_is_from_any_domain, safe_url_string, safe_download_url, url_query_parameter, add_or_replace_parameter, url_query_cleaner, canonicalize_url
|
||||
|
||||
class UrlUtilsTest(unittest.TestCase):
|
||||
|
||||
|
|
@ -74,6 +74,73 @@ class UrlUtilsTest(unittest.TestCase):
|
|||
self.assertEqual(url_query_cleaner("product.html?id=200&foo=bar&name=wired", ['id', 'name']),
|
||||
'product.html?id=200&name=wired')
|
||||
|
||||
def test_canonicalize_url(self):
|
||||
# no query arguments
|
||||
self.assertEqual(canonicalize_url("http://www.example.com"),
|
||||
"http://www.example.com")
|
||||
|
||||
# typical usage
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"),
|
||||
"http://www.example.com/do?a=1&b=2&c=3")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?c=1&b=2&a=3"),
|
||||
"http://www.example.com/do?a=3&b=2&c=1")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?&a=1"),
|
||||
"http://www.example.com/do?a=1")
|
||||
|
||||
# sorting by argument values
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?c=3&b=5&b=2&a=50"),
|
||||
"http://www.example.com/do?a=50&b=2&b=5&c=3")
|
||||
|
||||
# using keep_blank_values
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"),
|
||||
"http://www.example.com/do?a=2")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=True),
|
||||
"http://www.example.com/do?a=2&b=")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"),
|
||||
"http://www.example.com/do?a=2")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=True),
|
||||
"http://www.example.com/do?a=2&b=&c=")
|
||||
|
||||
# spaces
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"),
|
||||
"http://www.example.com/do?a=1&q=a+space")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a+space&a=1"),
|
||||
"http://www.example.com/do?a=1&q=a+space")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?q=a%20space&a=1"),
|
||||
"http://www.example.com/do?a=1&q=a+space")
|
||||
|
||||
# normalize percent-encoding case (in paths)
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/a%a3do"),
|
||||
"http://www.example.com/a%A3do"),
|
||||
# normalize percent-encoding case (in query arguments)
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?k=b%a3"),
|
||||
"http://www.example.com/do?k=b%A3")
|
||||
|
||||
# non-ASCII percent-encoding in paths
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/a do?a=1"),
|
||||
"http://www.example.com/a%20do?a=1"),
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/a %20do?a=1"),
|
||||
"http://www.example.com/a%20%20do?a=1"),
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/a do\xc2\xa3.html?a=1"),
|
||||
"http://www.example.com/a%20do%C2%A3.html?a=1")
|
||||
# non-ASCII percent-encoding in query arguments
|
||||
self.assertEqual(canonicalize_url(u"http://www.example.com/do?price=\xa3500&a=5&z=3"),
|
||||
u"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?price=\xc2\xa3500&a=5&z=3"),
|
||||
"http://www.example.com/do?a=5&price=%C2%A3500&z=3")
|
||||
self.assertEqual(canonicalize_url("http://www.example.com/do?price(\xc2\xa3)=500&a=1"),
|
||||
"http://www.example.com/do?a=1&price%28%C2%A3%29=500")
|
||||
|
||||
# urls containing auth and ports
|
||||
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com:81/do?now=1"),
|
||||
u"http://user:pass@www.example.com:81/do?now=1")
|
||||
|
||||
# remove fragments
|
||||
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag"),
|
||||
u"http://user:pass@www.example.com/do?a=1")
|
||||
self.assertEqual(canonicalize_url(u"http://user:pass@www.example.com/do?a=1#frag", remove_fragments=False),
|
||||
u"http://user:pass@www.example.com/do?a=1#frag")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
|
|
|||
|
|
@ -124,3 +124,26 @@ def add_or_replace_parameter(url, name, new_value, sep='&'):
|
|||
next_url = url.replace(name+'='+parameter,
|
||||
name+'='+new_value)
|
||||
return next_url
|
||||
|
||||
def canonicalize_url(url, keep_blank_values=False, remove_fragments=True):
|
||||
"""Canonicalize url by applying the following procedures:
|
||||
|
||||
- sort query arguments, first by key, then by value
|
||||
- percent encode paths and query arguments. non-ASCII characters are
|
||||
percent-encoded using UTF-8 (RFC-3986)
|
||||
- normalize all spaces (in query arguments) '+' (plus symbol)
|
||||
- normalize percent encodings case (%2f -> %2F)
|
||||
- remove query arguments with blank values (unless keep_blank_values is True)
|
||||
- remove fragments (if remove_fragments is True)
|
||||
|
||||
For examples see the tests in scrapy.tests.test_utils_url
|
||||
"""
|
||||
|
||||
parts = list(urlparse.urlparse(url))
|
||||
keyvals = cgi.parse_qsl(parts[4], keep_blank_values)
|
||||
keyvals.sort()
|
||||
parts[2] = urllib.quote(urllib.unquote(parts[2]))
|
||||
parts[4] = urllib.urlencode(keyvals)
|
||||
if remove_fragments:
|
||||
parts[5] = ''
|
||||
return urlparse.urlunparse(parts)
|
||||
|
|
|
|||
Loading…
Reference in New Issue