diff --git a/scrapy/tests/test_utils_url.py b/scrapy/tests/test_utils_url.py index ec27bddea..d99ce9103 100644 --- a/scrapy/tests/test_utils_url.py +++ b/scrapy/tests/test_utils_url.py @@ -60,12 +60,15 @@ class UrlUtilsTest(unittest.TestCase): def test_canonicalize_url(self): # simplest case - self.assertEqual(canonicalize_url("http://www.example.com"), - "http://www.example.com") + self.assertEqual(canonicalize_url("http://www.example.com/"), + "http://www.example.com/") # always return a str assert isinstance(canonicalize_url(u"http://www.example.com"), str) + # append missing path + self.assertEqual(canonicalize_url("http://www.example.com"), + "http://www.example.com/") # typical usage self.assertEqual(canonicalize_url("http://www.example.com/do?a=1&b=2&c=3"), "http://www.example.com/do?a=1&b=2&c=3") @@ -145,8 +148,8 @@ class UrlUtilsTest(unittest.TestCase): 'http://www.example.com/caf%E9-con-leche.htm') # domains are case insensitive - self.assertEqual(canonicalize_url("http://www.EXAMPLE.com"), - "http://www.example.com") + self.assertEqual(canonicalize_url("http://www.EXAMPLE.com/"), + "http://www.example.com/") if __name__ == "__main__": diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index cae73f398..4e834681b 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -52,7 +52,7 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False, \ keyvals = cgi.parse_qsl(query, keep_blank_values) keyvals.sort() query = urllib.urlencode(keyvals) - path = safe_url_string(urllib.unquote(path)) + path = safe_url_string(urllib.unquote(path)) or '/' fragment = '' if not keep_fragments else fragment return urlparse.urlunparse((scheme, netloc.lower(), path, params, query, fragment))