Complete test coverage for linkextractors (#7639)

* Complete test coverage for linkextractors

* pylint: disable use-implicit-booleaness-not-comparison

pyproject.toml

@@ -230,6 +230,7 @@ disable = [
     "undefined-variable",
     "unused-argument",
     "unused-variable",
+    "use-implicit-booleaness-not-comparison",
     "useless-import-alias",  # used as a hint to mypy
     "useless-return",  # https://github.com/pylint-dev/pylint/issues/6530
     "wrong-import-position",

tests/test_linkextractors.py

@@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_version
 
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor
 from tests import get_testdata
 
 
@@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
     def test_link_allowed_is_false_with_missing_url_prefix(self):
         bad_link = Link("should_have_prefix.example")
         assert not LxmlLinkExtractor()._link_allowed(bad_link)
+
+
+class TestLxmlParserLinkExtractor:
+    def test_extract_links(self):
+        html = b'<a href="http://example.com/page.html">Link</a>'
+        response = HtmlResponse("http://example.com/", body=html)
+        lx = LxmlParserLinkExtractor()
+        assert lx.extract_links(response) == [
+            Link(url="http://example.com/page.html", text="Link", nofollow=False),
+        ]
+
+    def test_strip_false(self):
+        # With strip=False, trailing whitespace on a relative href survives urljoin
+        # and is visible to process_value (safe_url_string cleans it up afterward).
+        # Here process_value rejects URLs that still carry trailing whitespace,
+        # demonstrating the difference from strip=True.
+        def reject_trailing_whitespace(url):
+            return None if url != url.rstrip() else url
+
+        html = b'<a href="page.html   ">Link</a>'
+        response = HtmlResponse("http://example.com/", body=html)
+
+        lx_strip = LxmlParserLinkExtractor(
+            strip=True, process=reject_trailing_whitespace
+        )
+        assert lx_strip.extract_links(response) == [
+            Link(url="http://example.com/page.html", text="Link", nofollow=False),
+        ]
+
+        lx_no_strip = LxmlParserLinkExtractor(
+            strip=False, process=reject_trailing_whitespace
+        )
+        assert lx_no_strip.extract_links(response) == []

tests/test_request_cb_kwargs.py

@@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider):
             self.checks.append(kwargs["callback"] == "some_callback")
             self.crawler.stats.inc_value("boolean_checks", 3)
         elif response.url.endswith("/general_without"):
-            self.checks.append(
+            self.checks.append(kwargs == {})
-                kwargs == {}  # pylint: disable=use-implicit-booleaness-not-comparison
-            )
             self.crawler.stats.inc_value("boolean_checks")
 
     def parse_no_kwargs(self, response):

tests/test_settings/__init__.py

@@ -1,4 +1,4 @@
-# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison
+# pylint: disable=unsubscriptable-object,unsupported-membership-test
 # (too many false positives)
 
 import logging

tests/test_spider.py

@@ -22,7 +22,7 @@ class TestSpider:
     def test_base_spider(self):
         spider = self.spider_class("example.com")
         assert spider.name == "example.com"
-        assert spider.start_urls == []  # pylint: disable=use-implicit-booleaness-not-comparison
+        assert spider.start_urls == []
 
     def test_spider_args(self):
         """``__init__`` method arguments are assigned to spider attributes"""

tests/test_utils_python.py

@@ -175,7 +175,7 @@ def test_get_func_args():
     assert get_func_args(partial_f2) == ["a", "c"]
     assert get_func_args(partial_f3) == ["c"]
     assert get_func_args(cal) == ["a", "b", "c"]
-    assert get_func_args(object) == []  # pylint: disable=use-implicit-booleaness-not-comparison
+    assert get_func_args(object) == []
     assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
     assert get_func_args(" ".join, stripself=True) == ["iterable"]