diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py index c80fc6e70..406eb5843 100644 --- a/scrapy/utils/url.py +++ b/scrapy/utils/url.py @@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions): def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'): + # IDNA encoding can fail for too long labels (>63 characters) + # or missing labels (e.g. http://.example.com) + try: + netloc = parts.netloc.encode('idna') + except UnicodeError: + netloc = parts.netloc + return ( to_native_str(parts.scheme), - to_native_str(parts.netloc.encode('idna')), + to_native_str(netloc), # default encoding for path component SHOULD be UTF-8 quote(to_bytes(parts.path, path_encoding), _safe_chars), diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py index 1fc3a3510..b4819874d 100644 --- a/tests/test_utils_url.py +++ b/tests/test_utils_url.py @@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase): # without encoding, already canonicalized URL is canonicalized identically self.assertEqual(canonicalize_url(canonicalized), canonicalized) + def test_canonicalize_url_idna_exceptions(self): + # missing DNS label + self.assertEqual( + canonicalize_url(u"http://.example.com/résumé?q=résumé"), + "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9") + + # DNS label too long + self.assertEqual( + canonicalize_url( + u"http://www.{label}.com/résumé?q=résumé".format( + label=u"example"*11)), + "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format( + label=u"example"*11)) + class AddHttpIfNoScheme(unittest.TestCase):