From 1a402578fca4ea33038c25e7b08cc11972eff2d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 11 Mar 2025 11:07:05 +0100
Subject: [PATCH] Test the current seeding policy, including a complex scenario
 with async sleeps

---
 tests/test_engine_seeding.py | 120 +++++++++++++++++++++++++++++++++++
 1 file changed, 120 insertions(+)
 create mode 100644 tests/test_engine_seeding.py

diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py
new file mode 100644
index 000000000..29c68c8e3
--- /dev/null
+++ b/tests/test_engine_seeding.py
@@ -0,0 +1,120 @@
+from __future__ import annotations
+
+from collections import deque
+
+from twisted.internet.defer import inlineCallbacks
+from twisted.trial.unittest import TestCase
+
+from scrapy import Request, Spider, signals
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.test import get_crawler
+
+from .test_spider_yield_seeds import twisted_sleep
+
+
+class MainTestCase(TestCase):
+    @inlineCallbacks
+    def test_scheduler_priority_over_seeds_simple(self):
+        """The seeding policy is to read seeds into the scheduler while the
+        scheduler is empty, but otherwise priorize requests already in the
+        scheduler.
+
+        This test shows how, given a scheduler pre-filled with a request, that
+        request is sent before sending the first seed request.
+        """
+
+        class TestScheduler(BaseScheduler):
+            def __init__(self, *args, **kwargs):
+                self.requests = deque((Request("data:,a"),))
+
+            def enqueue_request(self, request: Request) -> bool:
+                self.requests.append(request)
+                return True
+
+            def has_pending_requests(self) -> bool:
+                return bool(self.requests)
+
+            def next_request(self) -> Request | None:
+                try:
+                    return self.requests.popleft()
+                except IndexError:
+                    return None
+
+        class TestSpider(Spider):
+            name = "test"
+            start_urls = ["data:,b"]
+
+            def parse(self, response):
+                pass
+
+        actual_urls = []
+
+        def track_url(request, spider):
+            actual_urls.append(request.url)
+
+        settings = {"SCHEDULER": TestScheduler}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+        crawler.signals.connect(track_url, signals.request_reached_downloader)
+        yield crawler.crawl()
+        assert crawler.stats.get_value("finish_reason") == "finished"
+        expected_urls = ["data:,a", "data:,b"]
+        assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
+
+    @inlineCallbacks
+    def test_scheduler_priority_over_seeds_complex(self):
+        """While the seeding policy is to read seeds into the scheduler while
+        the scheduler is empty and  otherwise priorize requests already in the
+        scheduler, this is done in a non-blocking way.
+
+        That is, if the scheduler reports having requests but yields none,
+        requests from seeds will be scheduled.
+        """
+
+        class TestScheduler(BaseScheduler):
+            def __init__(self, *args, **kwargs):
+                self.requests = deque()
+                self.stop = False
+
+            def enqueue_request(self, request: Request) -> bool:
+                self.requests.append(request)
+                return True
+
+            def has_pending_requests(self) -> bool:
+                return not self.stop
+
+            def next_request(self) -> Request | None:
+                try:
+                    return self.requests.popleft()
+                except IndexError:
+                    return None
+
+        sleep_seconds = 0.0001
+
+        class TestSpider(Spider):
+            name = "test"
+
+            async def yield_seeds(self):
+                await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
+                yield Request("data:,a")
+                await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
+                self.crawler.engine.slot.scheduler.enqueue_request(Request("data:,b"))
+                await maybe_deferred_to_future(twisted_sleep(sleep_seconds))
+                yield Request("data:,c")
+                self.crawler.engine.slot.scheduler.stop = True
+
+            def parse(self, response):
+                pass
+
+        actual_urls = []
+
+        def track_url(request, spider):
+            actual_urls.append(request.url)
+
+        settings = {"SCHEDULER": TestScheduler}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+        crawler.signals.connect(track_url, signals.request_reached_downloader)
+        yield crawler.crawl()
+        assert crawler.stats.get_value("finish_reason") == "finished"
+        expected_urls = ["data:,a", "data:,b", "data:,c"]
+        assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"