Merge remote-tracking branch 'origin/master' into active-downloads-maxsize

pyproject.toml

@@ -230,6 +230,7 @@ disable = [
     "undefined-variable",
     "unused-argument",
     "unused-variable",
+    "use-implicit-booleaness-not-comparison",
     "useless-import-alias",  # used as a hint to mypy
     "useless-return",  # https://github.com/pylint-dev/pylint/issues/6530
     "wrong-import-position",

scrapy/__main__.py

@@ -1,3 +1,4 @@
+# pragma: no file cover
 from scrapy.cmdline import execute
 
 if __name__ == "__main__":

tests/test_linkextractors.py

@@ -9,7 +9,7 @@ from w3lib import __version__ as w3lib_version
 
 from scrapy.http import HtmlResponse, XmlResponse
 from scrapy.link import Link
-from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor
+from scrapy.linkextractors.lxmlhtml import LxmlLinkExtractor, LxmlParserLinkExtractor
 from tests import get_testdata
 
 
@@ -837,3 +837,36 @@ class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
     def test_link_allowed_is_false_with_missing_url_prefix(self):
         bad_link = Link("should_have_prefix.example")
         assert not LxmlLinkExtractor()._link_allowed(bad_link)
+
+
+class TestLxmlParserLinkExtractor:
+    def test_extract_links(self):
+        html = b'<a href="http://example.com/page.html">Link</a>'
+        response = HtmlResponse("http://example.com/", body=html)
+        lx = LxmlParserLinkExtractor()
+        assert lx.extract_links(response) == [
+            Link(url="http://example.com/page.html", text="Link", nofollow=False),
+        ]
+
+    def test_strip_false(self):
+        # With strip=False, trailing whitespace on a relative href survives urljoin
+        # and is visible to process_value (safe_url_string cleans it up afterward).
+        # Here process_value rejects URLs that still carry trailing whitespace,
+        # demonstrating the difference from strip=True.
+        def reject_trailing_whitespace(url):
+            return None if url != url.rstrip() else url
+
+        html = b'<a href="page.html   ">Link</a>'
+        response = HtmlResponse("http://example.com/", body=html)
+
+        lx_strip = LxmlParserLinkExtractor(
+            strip=True, process=reject_trailing_whitespace
+        )
+        assert lx_strip.extract_links(response) == [
+            Link(url="http://example.com/page.html", text="Link", nofollow=False),
+        ]
+
+        lx_no_strip = LxmlParserLinkExtractor(
+            strip=False, process=reject_trailing_whitespace
+        )
+        assert lx_no_strip.extract_links(response) == []

tests/test_pqueues.py

@@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloader
 from scrapy.http.request import Request
 from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
 from scrapy.spiders import Spider
-from scrapy.squeues import FifoMemoryQueue
+from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.test import get_crawler
 from tests.test_scheduler import MockDownloader
@@ -76,6 +76,29 @@ class TestPriorityQueue:
         assert queue.pop().url == req3.url
         assert not queue.close()
 
+    def test_init_prios_with_start_queue(self):
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler,
+            PickleFifoDiskQueue,
+            temp_dir,
+            start_queue_cls=PickleFifoDiskQueue,
+        )
+        req = Request("https://example.org/", meta={"is_start_request": True})
+        queue.push(req)
+        startprios = queue.close()
+
+        queue2 = ScrapyPriorityQueue.from_crawler(
+            self.crawler,
+            PickleFifoDiskQueue,
+            temp_dir,
+            startprios,
+            start_queue_cls=PickleFifoDiskQueue,
+        )
+        assert len(queue2) == 1
+        assert queue2.pop().url == req.url
+        queue2.close()
+
     def test_queue_push_pop_priorities(self):
         temp_dir = tempfile.mkdtemp()
         queue = ScrapyPriorityQueue.from_crawler(
@@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue:
 
         assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
 
+    def test_pop_prefers_slot_with_fewer_active_downloads(self):
+        downloader = self.queue._downloader_interface.downloader
+
+        req_a = Request("https://example.org/a")
+        req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
+        req_b = Request("https://example.org/b")
+        req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
+        req_c = Request("https://example.org/c")
+        req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
+
+        for req in (req_a, req_b, req_c):
+            self.queue.push(req)
+
+        downloader.increment("slot-a")
+        downloader.increment("slot-c")
+
+        popped = self.queue.pop()
+        assert popped.url == req_b.url
+
+    def test_contains(self):
+        req = Request("https://example.org/")
+        req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
+        assert "example-slot" not in self.queue
+        self.queue.push(req)
+        assert "example-slot" in self.queue
+        assert "other-slot" not in self.queue
+
 
 @pytest.mark.parametrize(
     ("input_", "output"),

tests/test_request_cb_kwargs.py

@@ -102,9 +102,7 @@ class KeywordArgumentsSpider(MockServerSpider):
             self.checks.append(kwargs["callback"] == "some_callback")
             self.crawler.stats.inc_value("boolean_checks", 3)
         elif response.url.endswith("/general_without"):
-            self.checks.append(
-                kwargs == {}  # pylint: disable=use-implicit-booleaness-not-comparison
-            )
+            self.checks.append(kwargs == {})
             self.crawler.stats.inc_value("boolean_checks")
 
     def parse_no_kwargs(self, response):

tests/test_settings/__init__.py

@@ -1,4 +1,4 @@
-# pylint: disable=unsubscriptable-object,unsupported-membership-test,use-implicit-booleaness-not-comparison
+# pylint: disable=unsubscriptable-object,unsupported-membership-test
 # (too many false positives)
 
 import logging

tests/test_spider.py

@@ -22,7 +22,7 @@ class TestSpider:
     def test_base_spider(self):
         spider = self.spider_class("example.com")
         assert spider.name == "example.com"
-        assert spider.start_urls == []  # pylint: disable=use-implicit-booleaness-not-comparison
+        assert spider.start_urls == []
 
     def test_spider_args(self):
         """``__init__`` method arguments are assigned to spider attributes"""

tests/test_stats.py

@@ -94,8 +94,13 @@ class TestStatsCollector:
         assert stats.get_value("test2") == 35
         stats.min_value("test4", 7)
         assert stats.get_value("test4") == 7
+        stats.set_stats({"replaced": "stats"})
+        assert stats.get_stats() == {"replaced": "stats"}
+        stats.clear_stats()
+        assert stats.get_stats() == {}
 
-    def test_dummy_collector(self, crawler: Crawler) -> None:
+    def test_dummy_collector(self) -> None:
+        crawler = get_crawler(Spider, {"STATS_DUMP": False})
         stats = DummyStatsCollector(crawler)
         assert stats.get_stats() == {}
         assert stats.get_value("anything") is None
@@ -104,9 +109,11 @@ class TestStatsCollector:
         stats.inc_value("v1")
         stats.max_value("v2", 100)
         stats.min_value("v3", 100)
+        stats.set_stats({"key": "val"})
         stats.open_spider()
         stats.set_value("test", "value")
         assert stats.get_stats() == {}
+        stats.close_spider()
 
     def test_deprecated_spider_arg(self, crawler: Crawler, spider: Spider) -> None:
         stats = StatsCollector(crawler)

tests/test_utils_python.py

@@ -175,7 +175,7 @@ def test_get_func_args():
     assert get_func_args(partial_f2) == ["a", "c"]
     assert get_func_args(partial_f3) == ["c"]
     assert get_func_args(cal) == ["a", "b", "c"]
-    assert get_func_args(object) == []  # pylint: disable=use-implicit-booleaness-not-comparison
+    assert get_func_args(object) == []
     assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]
     assert get_func_args(" ".join, stripself=True) == ["iterable"]