diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index fdfa92370..da525d52e 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -9,7 +9,7 @@ from w3lib.url import canonicalize_url
from scrapy.link import Link
from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
-from scrapy.utils.python import unique as unique_list, to_unicode
+from scrapy.utils.python import unique as unique_list
from scrapy.utils.response import get_base_url
from scrapy.linkextractors import FilteringLinkExtractor
@@ -66,7 +66,7 @@ class LxmlParserLinkExtractor(object):
url = self.process_attr(attr_val)
if url is None:
continue
- url = to_unicode(url, encoding=response_encoding)
+ url = canonicalize_url(url, encoding=response_encoding)
# to fix relative links after process_value
url = urljoin(response_url, url)
link = Link(url, _collect_string_content(el) or u'',
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 38fb8fb4a..e9d6c0abe 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -2,8 +2,6 @@ import re
import unittest
from warnings import catch_warnings
-import pytest
-
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
@@ -214,7 +212,7 @@ class Base:
response = HtmlResponse("http://example.org/somepage/index.html", body=html, encoding='iso8859-15')
links = self.extractor_cls(restrict_xpaths='//p').extract_links(response)
self.assertEqual(links,
- [Link(url='http://example.org/%E2%99%A5/you?c=%E2%82%AC', text=u'text')])
+ [Link(url='http://example.org/%E2%99%A5/you?c=%A4', text=u'text')])
def test_restrict_xpaths_concat_in_handle_data(self):
"""html entities cause SGMLParser to call handle_data hook twice"""
@@ -506,7 +504,6 @@ class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
Link(url='http://example.org/item2.html', text=u'Pic of a dog', nofollow=False),
])
- @pytest.mark.xfail
def test_restrict_xpaths_with_html_entities(self):
super(LxmlLinkExtractorTestCase, self).test_restrict_xpaths_with_html_entities()