diff --git a/pyproject.toml b/pyproject.toml
index 2d857dc6d..6b0c561f8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -230,6 +230,7 @@ disable = [
"undefined-variable",
"unused-argument",
"unused-variable",
+ "use-implicit-booleaness-not-comparison",
"useless-import-alias", # used as a hint to mypy
"useless-return", # https://github.com/pylint-dev/pylint/issues/6530
"wrong-import-position",
diff --git a/scrapy/__main__.py b/scrapy/__main__.py
index 697b9b1e9..17b1aa538 100644
--- a/scrapy/__main__.py
+++ b/scrapy/__main__.py
@@ -1,3 +1,4 @@
+# pragma: no file cover
from scrapy.cmdline import execute
if __name__ == "__main__":
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 15d358d2a..063b92dac 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_version
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.link import Link
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor
from tests import get_testdata
@@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
def test_link_allowed_is_false_with_missing_url_prefix(self):
bad_link = Link("should_have_prefix.example")
assert not LxmlLinkExtractor()._link_allowed(bad_link)
+
+
+class TestLxmlParserLinkExtractor:
+ def test_extract_links(self):
+ html = b'Link'
+ response = HtmlResponse("http://example.com/", body=html)
+ lx = LxmlParserLinkExtractor()
+ assert lx.extract_links(response) == [
+ Link(url="http://example.com/page.html", text="Link", nofollow=False),
+ ]
+
+ def test_strip_false(self):
+ # With strip=False, trailing whitespace on a relative href survives urljoin
+ # and is visible to process_value (safe_url_string cleans it up afterward).
+ # Here process_value rejects URLs that still carry trailing whitespace,
+ # demonstrating the difference from strip=True.
+ def reject_trailing_whitespace(url):
+ return None if url != url.rstrip() else url
+
+ html = b'Link'
+ response = HtmlResponse("http://example.com/", body=html)
+
+ lx_strip = LxmlParserLinkExtractor(
+ strip=True, process=reject_trailing_whitespace
+ )
+ assert lx_strip.extract_links(response) == [
+ Link(url="http://example.com/page.html", text="Link", nofollow=False),
+ ]
+
+ lx_no_strip = LxmlParserLinkExtractor(
+ strip=False, process=reject_trailing_whitespace
+ )
+ assert lx_no_strip.extract_links(response) == []
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index 7be9241b9..6c6a6584a 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloader
from scrapy.http.request import Request
from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
from scrapy.spiders import Spider
-from scrapy.squeues import FifoMemoryQueue
+from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.test_scheduler import MockDownloader
@@ -76,6 +76,29 @@ class TestPriorityQueue:
assert queue.pop().url == req3.url
assert not queue.close()
+ def test_init_prios_with_start_queue(self):
+ temp_dir = tempfile.mkdtemp()
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler,
+ PickleFifoDiskQueue,
+ temp_dir,
+ start_queue_cls=PickleFifoDiskQueue,
+ )
+ req = Request("https://example.org/", meta={"is_start_request": True})
+ queue.push(req)
+ startprios = queue.close()
+
+ queue2 = ScrapyPriorityQueue.from_crawler(
+ self.crawler,
+ PickleFifoDiskQueue,
+ temp_dir,
+ startprios,
+ start_queue_cls=PickleFifoDiskQueue,
+ )
+ assert len(queue2) == 1
+ assert queue2.pop().url == req.url
+ queue2.close()
+
def test_queue_push_pop_priorities(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
@@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue:
assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
+ def test_pop_prefers_slot_with_fewer_active_downloads(self):
+ downloader = self.queue._downloader_interface.downloader
+
+ req_a = Request("https://example.org/a")
+ req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
+ req_b = Request("https://example.org/b")
+ req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
+ req_c = Request("https://example.org/c")
+ req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
+
+ for req in (req_a, req_b, req_c):
+ self.queue.push(req)
+
+ downloader.increment("slot-a")
+ downloader.increment("slot-c")
+
+ popped = self.queue.pop()
+ assert popped.url == req_b.url
+
+ def test_contains(self):
+ req = Request("https://example.org/")
+ req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
+ assert "example-slot" not in self.queue
+ self.queue.push(req)
+ assert "example-slot" in self.queue
+ assert "other-slot" not in self.queue
+
@pytest.mark.parametrize(
("input_", "output"),
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 0d96e1d88..ee9d2ed51 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider):
self.checks.append(kwargs["callback"] == "some_callback")
self.crawler.stats.inc_value("boolean_checks", 3)
elif response.url.endswith("/general_without"):
- self.checks.append(
- kwargs == {} # pylint: disable=use-implicit-booleaness-not-comparison
- )
+ self.checks.append(kwargs == {})
self.crawler.stats.inc_value("boolean_checks")
def parse_no_kwargs(self, response):
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 7b3c52d65..9599ed6c9 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -1,4 +1,4 @@
-# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison
+# pylint: disable=unsubscriptable-object,unsupported-membership-test
# (too many false positives)
import logging
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 23efed77a..526cc8f23 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -22,7 +22,7 @@ class TestSpider:
def test_base_spider(self):
spider = self.spider_class("example.com")
assert spider.name == "example.com"
- assert spider.start_urls == [] # pylint: disable=use-implicit-booleaness-not-comparison
+ assert spider.start_urls == []
def test_spider_args(self):
"""``__init__`` method arguments are assigned to spider attributes"""
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 27af18bb1..6e6aa0cc4 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -94,8 +94,13 @@ class TestStatsCollector:
assert stats.get_value("test2") == 35
stats.min_value("test4", 7)
assert stats.get_value("test4") == 7
+ stats.set_stats({"replaced": "stats"})
+ assert stats.get_stats() == {"replaced": "stats"}
+ stats.clear_stats()
+ assert stats.get_stats() == {}
- def test_dummy_collector(self, crawler: Crawler) -> None:
+ def test_dummy_collector(self) -> None:
+ crawler = get_crawler(Spider, {"STATS_DUMP": False})
stats = DummyStatsCollector(crawler)
assert stats.get_stats() == {}
assert stats.get_value("anything") is None
@@ -104,9 +109,11 @@ class TestStatsCollector:
stats.inc_value("v1")
stats.max_value("v2", 100)
stats.min_value("v3", 100)
+ stats.set_stats({"key": "val"})
stats.open_spider()
stats.set_value("test", "value")
assert stats.get_stats() == {}
+ stats.close_spider()
def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:
stats = StatsCollector(crawler)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index e8fe45749..2e8047e2d 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -175,7 +175,7 @@ def test_get_func_args():
assert get_func_args(partial_f2) == ["a", "c"]
assert get_func_args(partial_f3) == ["c"]
assert get_func_args(cal) == ["a", "b", "c"]
- assert get_func_args(object) == [] # pylint: disable=use-implicit-booleaness-not-comparison
+ assert get_func_args(object) == []
assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
assert get_func_args(" ".join, stripself=True) == ["iterable"]