Merge pull request #2038 from redapple/canonicalize-idna-failures

[MRG] Do not fail on canonicalizing URLs with wrong netlocs

scrapy/utils/url.py

@@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions):
 
 
 def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
+    # IDNA encoding can fail for too long labels (>63 characters)
+    # or missing labels (e.g. http://.example.com)
+    try:
+        netloc = parts.netloc.encode('idna')
+    except UnicodeError:
+        netloc = parts.netloc
+
     return (
         to_native_str(parts.scheme),
-        to_native_str(parts.netloc.encode('idna')),
+        to_native_str(netloc),
 
         # default encoding for path component SHOULD be UTF-8
         quote(to_bytes(parts.path, path_encoding), _safe_chars),

tests/test_utils_url.py

@@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase):
             # without encoding, already canonicalized URL is canonicalized identically
             self.assertEqual(canonicalize_url(canonicalized), canonicalized)
 
+    def test_canonicalize_url_idna_exceptions(self):
+        # missing DNS label
+        self.assertEqual(
+            canonicalize_url(u"http://.example.com/résumé?q=résumé"),
+            "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+        # DNS label too long
+        self.assertEqual(
+            canonicalize_url(
+                u"http://www.{label}.com/résumé?q=résumé".format(
+                    label=u"example"*11)),
+            "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
+                    label=u"example"*11))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):