Publish the SeedingPolicy enum

docs/conf.py

@@ -27,6 +27,7 @@ author = "Scrapy developers"
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
 
 extensions = [
+    "enum_tools.autoenum",
     "hoverxref.extension",
     "notfound.extension",
     "scrapydocs",

docs/requirements.txt

@@ -1,3 +1,4 @@
+enum-tools[sphinx]==0.12.0
 sphinx==8.1.3
 sphinx-hoverxref==1.4.2
 sphinx-notfound-page==1.0.4

docs/topics/settings.rst

@@ -1740,58 +1740,18 @@ SEEDING_POLICY
 
 .. versionadded:: VERSION
 
-Default: ``"lazy"``
+Default: :py:enum:mem:`SeedingPolicy.lazy <scrapy.SeedingPolicy.lazy>`
 
-The way :meth:`Spider.yield_seeds <scrapy.Spider.yield_seeds>` is iterated:
+Determines the way :meth:`Spider.yield_seeds <scrapy.Spider.yield_seeds>` is
+iterated.
 
--   .. _lazy-seeding:
+Its value may be defined as a member of the :class:`~scrapy.SeedingPolicy` enum
+(e.g. :py:enum:mem:`SeedingPolicy.front_load
+<scrapy.SeedingPolicy.front_load>`) or as the corresponding string (e.g.
+``"front-load"``).
 
-    ``"lazy"``: Processing scheduled requests takes priority over iterating
-    seeds.
-
-    This seeding policy aims to minimize the number of requests in the
-    scheduler at any given time, to minimize resource usage (memory or disk,
-    depending on :setting:`JOBDIR`). It is best used when seed request priority
-    is not important. Switching to :ref:`idle <idle-seeding>` may lower
-    resource usage further at the cost of also lowering crawl speed.
-
--   .. _greedy-seeding:
-
-    ``"greedy"``: Iterating seeds takes priority over processing scheduled
-    requests.
-
-    Every time a seed request is iterated, it is scheduled, and then the next
-    request from the scheduler is sent.
-
-    .. note:: That request sent may not be the schedueld seed request
-        depending on the priority of scheduled requests, on the configured
-        :setting:`SCHEDULER` and on certain scheduler settings (e.g.
-        :setting:`SCHEDULER_MEMORY_QUEUE`).
-
-    This seeding policy is best used when prioritizing seed requests is
-    important, and seed requests may be sent as they come.
-
--   .. _front-load-seeding:
-
-    ``"front-load"``: The spider does not start until all seed requests have
-    been scheduled.
-
-    This seeding policy aims to give the :ref:`scheduler <topics-scheduler>`
-    full control over request order from the start. Some custom schedulers may
-    require this seeding policy to work as designed.
-
--   .. _idle-seeding:
-
-    ``"idle"``: A single seed is read only when there are neither scheduled nor
-    on-going requests.
-
-    That is, a new seed is not read until all requests triggered by the
-    previous seed, directly or indirectly, have been processed.
-
-    This seeding policy is similar to :ref:`lazy <lazy-seeding>`, but it
-    prioritizes resource savings over crawl speed. It is functionally
-    equivalent to running your spider multiple times in a row, one per seed
-    request.
+.. autoenum:: scrapy.SeedingPolicy
+    :members:
 
 .. setting:: SPIDER_CONTRACTS
 

scrapy/__init__.py

@@ -7,6 +7,7 @@ import sys
 import warnings
 
 # Declare top-level shortcuts
+from scrapy.core._seeding import SeedingPolicy
 from scrapy.http import FormRequest, Request
 from scrapy.item import Field, Item
 from scrapy.selector import Selector
@@ -17,6 +18,7 @@ __all__ = [
     "FormRequest",
     "Item",
     "Request",
+    "SeedingPolicy",
     "Selector",
     "Spider",
     "__version__",

scrapy/core/_seeding.py

@@ -0,0 +1,65 @@
+from enum import Enum
+
+try:
+    from enum_tools.documentation import document_enum
+except ImportError:
+
+    def document_enum(func):  # type: ignore[misc]
+        return func
+else:
+    # https://github.com/domdfcoding/enum_tools/issues/29
+    import enum_tools.documentation
+
+    enum_tools.documentation.INTERACTIVE = True
+
+
+@document_enum
+class SeedingPolicy(Enum):
+    front_load = "front-load"
+    """The crawl does not start until all seed requests have been scheduled.
+
+    Aims to give the :ref:`scheduler <topics-scheduler>` full control over
+    request order from the start. Some custom schedulers may require this
+    seeding policy to work as designed.
+    """
+
+    greedy = "greedy"
+    """Iterating seeds takes priority over processing scheduled requests.
+
+    Every time a seed request is iterated, it is scheduled, and then the next
+    request from the scheduler is sent.
+
+    .. note:: That request sent may not be the scheduled seed request
+        depending on the priority of scheduled requests, on the configured
+        :setting:`SCHEDULER` and on certain scheduler settings (e.g.
+        :setting:`SCHEDULER_MEMORY_QUEUE`).
+
+    Best used when prioritizing seed requests is important.
+    """
+
+    idle = "idle"
+    """A single seed is read only when there are neither scheduled nor on-going
+    requests.
+
+    That is, a new seed is not read until all requests triggered by the
+    previous seed, directly or indirectly, have been processed.
+
+    Unlike :py:enum:mem:`lazy`, resource savings are prioritized over crawl
+    speed.
+
+    It is functionally equivalent to running a spider multiple times in a row,
+    one per seed request.
+    """
+
+    lazy = "lazy"
+    """Processing scheduled requests takes priority over iterating seeds.
+
+    Aims to minimize the number of requests in the scheduler at any given time,
+    to minimize resource usage (memory or disk, depending on
+    :setting:`JOBDIR`).
+
+    It is best used when seed request priority is not important.
+
+    Switching to :py:enum:mem:`idle` may lower resource usage further at the
+    cost of also lowering crawl speed.
+    """

scrapy/core/engine.py

@@ -8,7 +8,6 @@ For more information see docs/topics/architecture.rst
 from __future__ import annotations
 
 import logging
-from enum import Enum
 from time import time
 from typing import TYPE_CHECKING, Any, TypeVar, cast
 
@@ -25,6 +24,8 @@ from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.reactor import CallLaterOnce
 
+from ._seeding import SeedingPolicy
+
 if TYPE_CHECKING:
     from collections.abc import AsyncIterable, Callable, Generator
 
@@ -77,13 +78,6 @@ class _Slot:
             self.closing.callback(None)
 
 
-class _SeedingPolicy(Enum):
-    front_load = "front-load"
-    greedy = "greedy"
-    idle = "idle"
-    lazy = "lazy"
-
-
 class ExecutionEngine:
     _SLOT_HEARTBEAT_INTERVAL: float = 5.0
 
@@ -117,9 +111,9 @@ class ExecutionEngine:
 
     def _load_seeding_policy(self) -> None:
         try:
-            self._seeding_policy = _SeedingPolicy(self.settings["SEEDING_POLICY"])
+            self._seeding_policy = SeedingPolicy(self.settings["SEEDING_POLICY"])
         except ValueError:
-            supported_values = ", ".join(policy.value for policy in _SeedingPolicy)
+            supported_values = ", ".join(policy.value for policy in SeedingPolicy)
             raise ValueError(
                 f"The value of the SEEDING_POLICY setting "
                 f"({self.settings['SEEDING_POLICY']!r}) is not supported. "
@@ -206,7 +200,7 @@ class ExecutionEngine:
             if isinstance(seed, Request):
                 self.crawl(seed)
                 if (
-                    self._seeding_policy is not _SeedingPolicy.front_load
+                    self._seeding_policy is not SeedingPolicy.front_load
                     and not self._needs_backout()
                 ):
                     self._start_scheduled_request()
@@ -215,7 +209,7 @@ class ExecutionEngine:
                 self._slot.nextcall.schedule()
         finally:
             self._waiting_for_seed = False
-        if self._seeding_policy is _SeedingPolicy.front_load and self._seeds is None:
+        if self._seeding_policy is SeedingPolicy.front_load and self._seeds is None:
             self._slot.nextcall.schedule()
 
     @inlineCallbacks
@@ -223,7 +217,7 @@ class ExecutionEngine:
         if self._slot is None or self._slot.closing is not None or self.paused:
             return
 
-        if self._seeding_policy in {_SeedingPolicy.idle, _SeedingPolicy.lazy}:
+        if self._seeding_policy in {SeedingPolicy.idle, SeedingPolicy.lazy}:
             while not self._needs_backout():
                 if self._start_scheduled_request() is None:
                     break
@@ -231,15 +225,15 @@ class ExecutionEngine:
                 self._seeds is not None
                 and not self._needs_backout()
                 and (
-                    self._seeding_policy is not _SeedingPolicy.idle
+                    self._seeding_policy is not SeedingPolicy.idle
                     or (not self._waiting_for_seed and not self.downloader.active)
                 )
             ):
                 yield self._process_next_seed()
         else:
             assert self._seeding_policy in {
-                _SeedingPolicy.front_load,
-                _SeedingPolicy.greedy,
+                SeedingPolicy.front_load,
+                SeedingPolicy.greedy,
             }
             if self._seeds is not None:
                 if not self._needs_backout():