Improve test coverage for pqueues.py (#7640)

tests/test_pqueues.py

@@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloader
 from scrapy.http.request import Request
 from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
 from scrapy.spiders import Spider
-from scrapy.squeues import FifoMemoryQueue
+from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.test import get_crawler
 from tests.test_scheduler import MockDownloader
@@ -76,6 +76,29 @@ class TestPriorityQueue:
         assert queue.pop().url == req3.url
         assert not queue.close()
 
+    def test_init_prios_with_start_queue(self):
+        temp_dir = tempfile.mkdtemp()
+        queue = ScrapyPriorityQueue.from_crawler(
+            self.crawler,
+            PickleFifoDiskQueue,
+            temp_dir,
+            start_queue_cls=PickleFifoDiskQueue,
+        )
+        req = Request("https://example.org/", meta={"is_start_request": True})
+        queue.push(req)
+        startprios = queue.close()
+
+        queue2 = ScrapyPriorityQueue.from_crawler(
+            self.crawler,
+            PickleFifoDiskQueue,
+            temp_dir,
+            startprios,
+            start_queue_cls=PickleFifoDiskQueue,
+        )
+        assert len(queue2) == 1
+        assert queue2.pop().url == req.url
+        queue2.close()
+
     def test_queue_push_pop_priorities(self):
         temp_dir = tempfile.mkdtemp()
         queue = ScrapyPriorityQueue.from_crawler(
@@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue:
 
         assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
 
+    def test_pop_prefers_slot_with_fewer_active_downloads(self):
+        downloader = self.queue._downloader_interface.downloader
+
+        req_a = Request("https://example.org/a")
+        req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
+        req_b = Request("https://example.org/b")
+        req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
+        req_c = Request("https://example.org/c")
+        req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
+
+        for req in (req_a, req_b, req_c):
+            self.queue.push(req)
+
+        downloader.increment("slot-a")
+        downloader.increment("slot-c")
+
+        popped = self.queue.pop()
+        assert popped.url == req_b.url
+
+    def test_contains(self):
+        req = Request("https://example.org/")
+        req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
+        assert "example-slot" not in self.queue
+        self.queue.push(req)
+        assert "example-slot" in self.queue
+        assert "other-slot" not in self.queue
+
 
 @pytest.mark.parametrize(
     ("input_", "output"),