canonicalize_url(): ignore case in domain names

scrapy/tests/test_utils_url.py

@@ -274,6 +274,10 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'),
                                            'http://www.example.com/caf%E9-con-leche.htm')
 
+        # domains are case insensitive
+        self.assertEqual(canonicalize_url("http://www.EXAMPLE.com"),
+                                          "http://www.example.com")
+
     def test_path_to_file_uri(self):
         if os.name == 'nt':
             self.assertEqual(path_to_file_uri("C:\\windows\clock.avi"),

scrapy/utils/url.py

@@ -154,7 +154,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \
     query = urllib.urlencode(keyvals)
     path = urllib.quote(urllib.unquote(path))
     fragment = '' if not keep_fragments else fragment
-    return urlparse.urlunparse((scheme, netloc, path, params, query, fragment))
+    return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment))
 
 def path_to_file_uri(path):
     """Convert local filesystem path to legal File URIs as described in: