diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index fa9de55bc..ec9422cbd 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -274,6 +274,10 @@ class UrlUtilsTest(unittest.TestCase): self.assertEqual(canonicalize_url(u'http://www.example.com/caf%E9-con-leche.htm'), 'http://www.example.com/caf%E9-con-leche.htm') + # domains are case insensitive + self.assertEqual(canonicalize_url("http://www.EXAMPLE.com"), + "http://www.example.com") + def test_path_to_file_uri(self): if os.name == 'nt': self.assertEqual(path_to_file_uri("C:\\windows\clock.avi"), diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index ae010cb9e..dfb7f7bb8 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -154,7 +154,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ query = urllib.urlencode(keyvals) path = urllib.quote(urllib.unquote(path)) fragment = '' if not keep_fragments else fragment - return urlparse.urlunparse((scheme, netloc, path, params, query, fragment)) + return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment)) def path_to_file_uri(path): """Convert local filesystem path to legal File URIs as described in: