From 1aec5200bc81493623f2a4e077b4e80e104e47d5 Mon Sep 17 00:00:00 2001
From: Paul Tremberth <paul.tremberth@gmail.com>
Date: Wed, 8 Jun 2016 16:49:33 +0200
Subject: [PATCH] Do not fail on canonicalizing URLs with wrong netlocs

Fixes #2010
---
 scrapy/utils/url.py     |  9 ++++++++-
 tests/test_utils_url.py | 14 ++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index c80fc6e70..406eb5843 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -41,9 +41,16 @@ def url_has_any_extension(url, extensions):
 
 
 def _safe_ParseResult(parts, encoding='utf8', path_encoding='utf8'):
+    # IDNA encoding can fail for too long labels (>63 characters)
+    # or missing labels (e.g. http://.example.com)
+    try:
+        netloc = parts.netloc.encode('idna')
+    except UnicodeError:
+        netloc = parts.netloc
+
     return (
         to_native_str(parts.scheme),
-        to_native_str(parts.netloc.encode('idna')),
+        to_native_str(netloc),
 
         # default encoding for path component SHOULD be UTF-8
         quote(to_bytes(parts.path, path_encoding), _safe_chars),
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 1fc3a3510..b4819874d 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -265,6 +265,20 @@ class CanonicalizeUrlTest(unittest.TestCase):
             # without encoding, already canonicalized URL is canonicalized identically
             self.assertEqual(canonicalize_url(canonicalized), canonicalized)
 
+    def test_canonicalize_url_idna_exceptions(self):
+        # missing DNS label
+        self.assertEqual(
+            canonicalize_url(u"http://.example.com/résumé?q=résumé"),
+            "http://.example.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9")
+
+        # DNS label too long
+        self.assertEqual(
+            canonicalize_url(
+                u"http://www.{label}.com/résumé?q=résumé".format(
+                    label=u"example"*11)),
+            "http://www.{label}.com/r%C3%A9sum%C3%A9?q=r%C3%A9sum%C3%A9".format(
+                    label=u"example"*11))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):