From 8724ee1674dc4fe1f391d3cfa4b90915e123c995 Mon Sep 17 00:00:00 2001 From: Daniel Grana Date: Tue, 3 Mar 2009 15:05:50 +0000 Subject: [PATCH] utils: url canonicalization keeps blanks query parameters --HG-- extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40960 --- .../trunk/scrapy/tests/test_utils_request.py | 5 +++++ scrapy/trunk/scrapy/tests/test_utils_url.py | 11 +++++++---- scrapy/trunk/scrapy/utils/url.py | 19 ++++++++++--------- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/scrapy/trunk/scrapy/tests/test_utils_request.py b/scrapy/trunk/scrapy/tests/test_utils_request.py index 57786b1b5..2a61d06ca 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_request.py +++ b/scrapy/trunk/scrapy/tests/test_utils_request.py @@ -14,6 +14,10 @@ class UtilsRequestTest(unittest.TestCase): self.assertEqual(request_fingerprint(r1), request_fingerprint(r1)) self.assertEqual(request_fingerprint(r1), request_fingerprint(r2)) + r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199') + r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199') + self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2)) + # make sure caching is working self.assertEqual(request_fingerprint(r1), r1.cache['fingerprint']) @@ -55,6 +59,7 @@ class UtilsRequestTest(unittest.TestCase): fp2 = request_fingerprint(r2) self.assertNotEqual(fp1, fp2) + def test_request_authenticate(self): r = Request("http://www.example.com") request_authenticate(r, 'someuser', 'somepass') diff --git a/scrapy/trunk/scrapy/tests/test_utils_url.py b/scrapy/trunk/scrapy/tests/test_utils_url.py index 510ccf0a5..de4792caf 100644 --- a/scrapy/trunk/scrapy/tests/test_utils_url.py +++ b/scrapy/trunk/scrapy/tests/test_utils_url.py @@ -102,14 +102,17 @@ class UrlUtilsTest(unittest.TestCase): "http://www.example.com/do?a=50&b=2&b=5&c=3") # using keep_blank_values + self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=False), + "http://www.example.com/do?a=2") self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"), - "http://www.example.com/do?a=2") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=True), "http://www.example.com/do?a=2&b=") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"), + self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=False), "http://www.example.com/do?a=2") - self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=True), + self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"), "http://www.example.com/do?a=2&b=&c=") + + self.assertEqual(canonicalize_url(u'http://www.example.com/do?1750,4'), + 'http://www.example.com/do?1750%2C4=') # spaces self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"), diff --git a/scrapy/trunk/scrapy/utils/url.py b/scrapy/trunk/scrapy/utils/url.py index eb5e75e94..2ec8ee63f 100644 --- a/scrapy/trunk/scrapy/utils/url.py +++ b/scrapy/trunk/scrapy/utils/url.py @@ -127,7 +127,7 @@ def add_or_replace_parameter(url, name, new_value, sep='&'): name+'='+new_value) return next_url -def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): +def canonicalize_url(url, keep_blank_values=True, keep_fragments=False): """Canonicalize the given url by applying the following procedures: - sort query arguments, first by key, then by value @@ -143,12 +143,13 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False): For examples see the tests in scrapy.tests.test_utils_url """ - url = unicode_to_str(url) - scheme, netloc, path, params, query, fragment = urlparse.urlparse(url) - keyvals = cgi.parse_qsl(query, keep_blank_values) - keyvals.sort() - query = urllib.urlencode(keyvals) - path = urllib.quote(urllib.unquote(path)) - fragment = '' if not keep_fragments else fragment - return urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) + url = unicode_to_str(url) + parts = list(urlparse.urlparse(url)) + keyvals = cgi.parse_qsl(parts[4], keep_blank_values) + keyvals.sort() + parts[2] = urllib.quote(urllib.unquote(parts[2])) + parts[4] = urllib.urlencode(keyvals) + if not keep_fragments: + parts[5] = '' + return urlparse.urlunparse(parts)