diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py index 46ac39a28..75f9753e7 100644 --- a/scrapy/linkextractors/lxmlhtml.py +++ b/scrapy/linkextractors/lxmlhtml.py @@ -332,7 +332,7 @@ class LxmlLinkExtractor: unique=unique, process=process_value, strip=strip, - canonicalized=not canonicalize, + canonicalized=True, ) self.allow_res: list[re.Pattern[str]] = self._compile_regexes(allow) self.deny_res: list[re.Pattern[str]] = self._compile_regexes(deny) diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 95a6aee54..7b73a133f 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -9,6 +9,7 @@ from w3lib import __version__ as w3lib_version from scrapy.http import HtmlResponse, XmlResponse from scrapy.link import Link +from scrapy.linkextractors import lxmlhtml from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor from tests import get_testdata @@ -798,6 +799,25 @@ class Base: class TestLxmlLinkExtractor(Base.TestLinkExtractorBase): extractor_cls = LxmlLinkExtractor + def test_canonicalize_once_per_link(self, monkeypatch): + canonicalize_url = lxmlhtml.canonicalize_url + calls = [] + + def counting_canonicalize_url(url, *args, **kwargs): + calls.append(url) + return canonicalize_url(url, *args, **kwargs) + + monkeypatch.setattr(lxmlhtml, "canonicalize_url", counting_canonicalize_url) + response = HtmlResponse( + "https://example.com", + body=b"".join(b'x' % i for i in range(10)), + ) + lx = self.extractor_cls(canonicalize=True) + assert lx.extract_links(response) == [ + Link(url="https://example.com/p?a=1&b=2", text="x") + ] + assert len(calls) == 10 + def test_link_restrict_text(self): html = b""" Pic of a cat