utils: url canonicalization keeps blanks query parameters

--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40960

scrapy/trunk/scrapy/tests/test_utils_request.py

@@ -14,6 +14,10 @@ class UtilsRequestTest(unittest.TestCase):
         self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
         self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
 
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
+        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
+
         # make sure caching is working
         self.assertEqual(request_fingerprint(r1), r1.cache['fingerprint'])
 
@@ -55,6 +59,7 @@ class UtilsRequestTest(unittest.TestCase):
         fp2 = request_fingerprint(r2)
         self.assertNotEqual(fp1, fp2)
 
+
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, 'someuser', 'somepass')

scrapy/trunk/scrapy/tests/test_utils_url.py

@@ -102,14 +102,17 @@ class UrlUtilsTest(unittest.TestCase):
                                           "http://www.example.com/do?a=50&b=2&b=5&c=3")
 
         # using keep_blank_values
+        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=False),
+                                          "http://www.example.com/do?a=2")
         self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2"),
-                                          "http://www.example.com/do?a=2")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&a=2", keep_blank_values=True),
                                           "http://www.example.com/do?a=2&b=")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"),
+        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=False),
                                           "http://www.example.com/do?a=2")
-        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2", keep_blank_values=True),
+        self.assertEqual(canonicalize_url("http://www.example.com/do?b=&c&a=2"),
                                           "http://www.example.com/do?a=2&b=&c=")
+
+        self.assertEqual(canonicalize_url(u'http://www.example.com/do?1750,4'),
+                                           'http://www.example.com/do?1750%2C4=')
                         
         # spaces
         self.assertEqual(canonicalize_url("http://www.example.com/do?q=a space&a=1"),

scrapy/trunk/scrapy/utils/url.py

@@ -127,7 +127,7 @@ def add_or_replace_parameter(url, name, new_value, sep='&'):
                                name+'='+new_value)
     return next_url
 
-def canonicalize_url(url, keep_blank_values=False, keep_fragments=False):
+def canonicalize_url(url, keep_blank_values=True, keep_fragments=False):
     """Canonicalize the given url by applying the following procedures:
 
     - sort query arguments, first by key, then by value
@@ -143,12 +143,13 @@ def canonicalize_url(url, keep_blank_values=False, keep_fragments=False):
 
     For examples see the tests in scrapy.tests.test_utils_url
     """
-    url = unicode_to_str(url)
-    scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
-    keyvals = cgi.parse_qsl(query, keep_blank_values)
-    keyvals.sort()
-    query = urllib.urlencode(keyvals)
-    path = urllib.quote(urllib.unquote(path))
-    fragment = '' if not keep_fragments else fragment
-    return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
 
+    url = unicode_to_str(url)
+    parts = list(urlparse.urlparse(url))
+    keyvals = cgi.parse_qsl(parts[4], keep_blank_values)
+    keyvals.sort()
+    parts[2] = urllib.quote(urllib.unquote(parts[2]))
+    parts[4] = urllib.urlencode(keyvals)
+    if not keep_fragments:
+        parts[5] = ''
+    return urlparse.urlunparse(parts)