Use canonicalize_url in link extraction

scrapy/linkextractors/lxmlhtml.py

@@ -9,7 +9,7 @@ from w3lib.url import canonicalize_url
 
 from scrapy.link import Link
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
+from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
 from scrapy.linkextractors import FilteringLinkExtractor
 
@@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
                 url = self.process_attr(attr_val)
                 if url is None:
                     continue
-            url = to_unicode(url, encoding=response_encoding)
+            url = canonicalize_url(url, encoding=response_encoding)
             # to fix relative links after process_value
             url = urljoin(response_url, url)
             link = Link(url, _collect_string_content(el) or u'',

tests/test_linkextractors.py

@@ -2,8 +2,6 @@ import re
 import unittest
 from warnings import catch_warnings
 
-import pytest
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
@@ -214,7 +212,7 @@ class Base:
             response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
             links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
             self.assertEqual(links,
-                             [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
+                             [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
 
         def test_restrict_xpaths_concat_in_handle_data(self):
             """html entities cause SGMLParser to call handle_data hook twice"""
@@ -506,7 +504,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
             Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
         ])
 
-    @pytest.mark.xfail
     def test_restrict_xpaths_with_html_entities(self):
         super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()