Default seeding policy: lazy → greedy

docs/news.rst

@@ -11,13 +11,16 @@ Scrapy VERSION (unreleased)
 Highlights:
 
 -   Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.yield_seeds`
-    (async)
+    (async) and changed how it is iterated by default.
 
 Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
--   In ``scrapy.core.spidermw.SpiderMiddlewareManager``,
-    ``process_start_requests()`` has been replaced by ``process_seeds()``.
+-   By default, the iteration of start requests and items no longer stops once
+    there are requests in the scheduler.
+
+    You can restore the previous behavior by setting :setting:`SEEDING_POLICY`
+    to :py:enum:mem:`~scrapy.SeedingPolicy.lazy`.
 
 -   In ``scrapy.core.engine.ExecutionEngine``:
 
@@ -33,6 +36,9 @@ Backward-incompatible changes
 
 -   The ``slot`` :ref:`telnet variable <telnet-vars>` has been removed.
 
+-   In ``scrapy.core.spidermw.SpiderMiddlewareManager``,
+    ``process_start_requests()`` has been replaced by ``process_seeds()``.
+
 Deprecations
 ~~~~~~~~~~~~
 
@@ -40,16 +46,14 @@ Deprecations
     use :meth:`~scrapy.Spider.yield_seeds` instead, or both to maintain support
     for lower Scrapy versions.
 
-    (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`,
-    :issue:`6729`)
+    (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`)
 
 -   The ``process_start_requests()`` method of :ref:`spider middlewares
     <topics-spider-middleware>` is deprecated, use
     :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_seeds` instead, or
     both to maintain support for lower Scrapy versions.
 
-    (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`,
-    :issue:`6729`)
+    (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`)
 
 New features
 ~~~~~~~~~~~~
@@ -65,8 +69,8 @@ New features
 
     (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`)
 
--   The new :setting:`SEEDING_POLICY` setting allows customizing how spider
-    start requests and items are consumed.
+-   The new :setting:`SEEDING_POLICY` setting allows customizing how start
+    requests and items are iterated.
 
     You can also override the active seeding policy from
     :meth:`Spider.yield_seeds <scrapy.Spider.yield_seeds>` and from
@@ -87,7 +91,7 @@ Bug fixes
     equivalent) no longer delays the next iteration of starting requests and
     items by up to 5 seconds.
 
-    (:issue:`6715`, :issue:`6729`)
+    (:issue:`6729`)
 
 
 .. _release-2.12.0:

docs/topics/settings.rst

@@ -1740,7 +1740,7 @@ SEEDING_POLICY
 
 .. versionadded:: VERSION
 
-Default: :py:enum:mem:`SeedingPolicy.lazy <scrapy.SeedingPolicy.lazy>`
+Default: :py:enum:mem:`SeedingPolicy.greedy <scrapy.SeedingPolicy.greedy>`
 
 Determines the way :meth:`Spider.yield_seeds <scrapy.Spider.yield_seeds>` is
 iterated.

scrapy/commands/shell.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 from threading import Thread
 from typing import TYPE_CHECKING, Any
 
+from scrapy import SeedingPolicy
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
@@ -24,9 +25,10 @@ if TYPE_CHECKING:
 class Command(ScrapyCommand):
     requires_project = False
     default_settings = {
+        "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
         "KEEP_ALIVE": True,
         "LOGSTATS_INTERVAL": 0,
-        "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
+        "SEEDING_POLICY": SeedingPolicy.lazy,
     }
 
     def syntax(self) -> str:

scrapy/settings/default_settings.py

@@ -17,6 +17,8 @@ import sys
 from importlib import import_module
 from pathlib import Path
 
+from scrapy import SeedingPolicy
+
 ADDONS = {}
 
 AJAXCRAWL_ENABLED = False
@@ -308,7 +310,7 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
 
 SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000
 
-SEEDING_POLICY = "lazy"
+SEEDING_POLICY = SeedingPolicy.greedy
 
 SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader"
 SPIDER_LOADER_WARN_ONLY = False

tests/test_crawl.py

@@ -207,7 +207,7 @@ class TestCrawl(TestCase):
 
     @defer.inlineCallbacks
     def test_yield_seeds_laziness(self):
-        settings = {"CONCURRENT_REQUESTS": 1}
+        settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"}
         crawler = get_crawler(BrokenYieldSeedsSpider, settings)
         yield crawler.crawl(mockserver=self.mockserver)
         assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(

tests/test_engine_seeding.py

@@ -31,6 +31,45 @@ class MainTestCase(TestCase):
     # eventually remove the heartbeat altogether.
     timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL
 
+    @deferred_f_from_coro_f
+    async def test_greedy(self):
+        class TestScheduler(BaseScheduler):
+            def __init__(self, *args, **kwargs):
+                self.requests = deque((Request("data:,b"),))
+
+            def enqueue_request(self, request: Request) -> bool:
+                self.requests.append(request)
+                return True
+
+            def has_pending_requests(self) -> bool:
+                return bool(self.requests)
+
+            def next_request(self) -> Request | None:
+                try:
+                    return self.requests.pop()
+                except IndexError:
+                    return None
+
+        class TestSpider(Spider):
+            name = "test"
+            start_urls = ["data:,a"]
+
+            def parse(self, response):
+                pass
+
+        actual_urls = []
+
+        def track_url(request, spider):
+            actual_urls.append(request.url)
+
+        settings = {"SCHEDULER": TestScheduler}
+        crawler = get_crawler(TestSpider, settings_dict=settings)
+        crawler.signals.connect(track_url, signals.request_reached_downloader)
+        await maybe_deferred_to_future(crawler.crawl())
+        assert crawler.stats.get_value("finish_reason") == "finished"
+        expected_urls = ["data:,a", "data:,b"]
+        assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
+
     @deferred_f_from_coro_f
     async def test_lazy(self):
         class TestScheduler(BaseScheduler):
@@ -62,7 +101,7 @@ class MainTestCase(TestCase):
         def track_url(request, spider):
             actual_urls.append(request.url)
 
-        settings = {"SCHEDULER": TestScheduler}
+        settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"}
         crawler = get_crawler(TestSpider, settings_dict=settings)
         crawler.signals.connect(track_url, signals.request_reached_downloader)
         await maybe_deferred_to_future(crawler.crawl())
@@ -115,7 +154,7 @@ class MainTestCase(TestCase):
         def track_url(request, spider):
             actual_urls.append(request.url)
 
-        settings = {"SCHEDULER": TestScheduler}
+        settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"}
         crawler = get_crawler(TestSpider, settings_dict=settings)
         crawler.signals.connect(track_url, signals.request_reached_downloader)
         await maybe_deferred_to_future(crawler.crawl())
@@ -140,50 +179,12 @@ class MainTestCase(TestCase):
         def track_url(request, spider):
             actual_urls.append(request.url)
 
-        crawler = get_crawler(TestSpider)
-        crawler.signals.connect(track_url, signals.request_reached_downloader)
-        await maybe_deferred_to_future(crawler.crawl())
-        assert crawler.stats.get_value("finish_reason") == "finished"
-        expected_urls = ["data:,a", "data:,b", "data:,c"]
-        assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
-
-    @deferred_f_from_coro_f
-    async def test_greedy(self):
-        class TestScheduler(BaseScheduler):
-            def __init__(self, *args, **kwargs):
-                self.requests = deque((Request("data:,b"),))
-
-            def enqueue_request(self, request: Request) -> bool:
-                self.requests.append(request)
-                return True
-
-            def has_pending_requests(self) -> bool:
-                return bool(self.requests)
-
-            def next_request(self) -> Request | None:
-                try:
-                    return self.requests.pop()
-                except IndexError:
-                    return None
-
-        class TestSpider(Spider):
-            name = "test"
-            start_urls = ["data:,a"]
-
-            def parse(self, response):
-                pass
-
-        actual_urls = []
-
-        def track_url(request, spider):
-            actual_urls.append(request.url)
-
-        settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "greedy"}
+        settings = {"SEEDING_POLICY": "lazy"}
         crawler = get_crawler(TestSpider, settings_dict=settings)
         crawler.signals.connect(track_url, signals.request_reached_downloader)
         await maybe_deferred_to_future(crawler.crawl())
         assert crawler.stats.get_value("finish_reason") == "finished"
-        expected_urls = ["data:,a", "data:,b"]
+        expected_urls = ["data:,a", "data:,b", "data:,c"]
         assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"
 
     @deferred_f_from_coro_f
@@ -276,7 +277,7 @@ class MainTestCase(TestCase):
         def track_url(request, spider):
             actual_urls.append(request.url)
 
-        settings = {"SCHEDULER": TestScheduler}
+        settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"}
         crawler = get_crawler(TestSpider, settings_dict=settings)
         crawler.signals.connect(track_item, signals.item_scraped)
         crawler.signals.connect(track_url, signals.request_reached_downloader)