From 38cf129241c869f590ad7c892fb2d9d96396a160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Thu, 13 Mar 2025 02:10:20 +0100 Subject: [PATCH] Publish the SeedingPolicy enum --- docs/conf.py | 1 + docs/requirements.txt | 1 + docs/topics/settings.rst | 58 ++++++----------------------------- scrapy/__init__.py | 2 ++ scrapy/core/_seeding.py | 65 ++++++++++++++++++++++++++++++++++++++++ scrapy/core/engine.py | 26 +++++++--------- 6 files changed, 88 insertions(+), 65 deletions(-) create mode 100644 scrapy/core/_seeding.py diff --git a/docs/conf.py b/docs/conf.py index 1167ce050..6985e38fe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,7 @@ author = "Scrapy developers" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ + "enum_tools.autoenum", "hoverxref.extension", "notfound.extension", "scrapydocs", diff --git a/docs/requirements.txt b/docs/requirements.txt index 103fb08d6..63243fcf3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +enum-tools[sphinx]==0.12.0 sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 042c2ed8a..dd2fb3400 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1740,58 +1740,18 @@ SEEDING_POLICY .. versionadded:: VERSION -Default: ``"lazy"`` +Default: :py:enum:mem:`SeedingPolicy.lazy ` -The way :meth:`Spider.yield_seeds ` is iterated: +Determines the way :meth:`Spider.yield_seeds ` is +iterated. -- .. _lazy-seeding: +Its value may be defined as a member of the :class:`~scrapy.SeedingPolicy` enum +(e.g. :py:enum:mem:`SeedingPolicy.front_load +`) or as the corresponding string (e.g. +``"front-load"``). - ``"lazy"``: Processing scheduled requests takes priority over iterating - seeds. - - This seeding policy aims to minimize the number of requests in the - scheduler at any given time, to minimize resource usage (memory or disk, - depending on :setting:`JOBDIR`). It is best used when seed request priority - is not important. Switching to :ref:`idle ` may lower - resource usage further at the cost of also lowering crawl speed. - -- .. _greedy-seeding: - - ``"greedy"``: Iterating seeds takes priority over processing scheduled - requests. - - Every time a seed request is iterated, it is scheduled, and then the next - request from the scheduler is sent. - - .. note:: That request sent may not be the schedueld seed request - depending on the priority of scheduled requests, on the configured - :setting:`SCHEDULER` and on certain scheduler settings (e.g. - :setting:`SCHEDULER_MEMORY_QUEUE`). - - This seeding policy is best used when prioritizing seed requests is - important, and seed requests may be sent as they come. - -- .. _front-load-seeding: - - ``"front-load"``: The spider does not start until all seed requests have - been scheduled. - - This seeding policy aims to give the :ref:`scheduler ` - full control over request order from the start. Some custom schedulers may - require this seeding policy to work as designed. - -- .. _idle-seeding: - - ``"idle"``: A single seed is read only when there are neither scheduled nor - on-going requests. - - That is, a new seed is not read until all requests triggered by the - previous seed, directly or indirectly, have been processed. - - This seeding policy is similar to :ref:`lazy `, but it - prioritizes resource savings over crawl speed. It is functionally - equivalent to running your spider multiple times in a row, one per seed - request. +.. autoenum:: scrapy.SeedingPolicy + :members: .. setting:: SPIDER_CONTRACTS diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 256504c9c..7bb958a2d 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,6 +7,7 @@ import sys import warnings # Declare top-level shortcuts +from scrapy.core._seeding import SeedingPolicy from scrapy.http import FormRequest, Request from scrapy.item import Field, Item from scrapy.selector import Selector @@ -17,6 +18,7 @@ __all__ = [ "FormRequest", "Item", "Request", + "SeedingPolicy", "Selector", "Spider", "__version__", diff --git a/scrapy/core/_seeding.py b/scrapy/core/_seeding.py new file mode 100644 index 000000000..13f5c6d6a --- /dev/null +++ b/scrapy/core/_seeding.py @@ -0,0 +1,65 @@ +from enum import Enum + +try: + from enum_tools.documentation import document_enum +except ImportError: + + def document_enum(func): # type: ignore[misc] + return func +else: + # https://github.com/domdfcoding/enum_tools/issues/29 + import enum_tools.documentation + + enum_tools.documentation.INTERACTIVE = True + + +@document_enum +class SeedingPolicy(Enum): + front_load = "front-load" + """The crawl does not start until all seed requests have been scheduled. + + Aims to give the :ref:`scheduler ` full control over + request order from the start. Some custom schedulers may require this + seeding policy to work as designed. + """ + + greedy = "greedy" + """Iterating seeds takes priority over processing scheduled requests. + + Every time a seed request is iterated, it is scheduled, and then the next + request from the scheduler is sent. + + .. note:: That request sent may not be the scheduled seed request + depending on the priority of scheduled requests, on the configured + :setting:`SCHEDULER` and on certain scheduler settings (e.g. + :setting:`SCHEDULER_MEMORY_QUEUE`). + + Best used when prioritizing seed requests is important. + """ + + idle = "idle" + """A single seed is read only when there are neither scheduled nor on-going + requests. + + That is, a new seed is not read until all requests triggered by the + previous seed, directly or indirectly, have been processed. + + Unlike :py:enum:mem:`lazy`, resource savings are prioritized over crawl + speed. + + It is functionally equivalent to running a spider multiple times in a row, + one per seed request. + """ + + lazy = "lazy" + """Processing scheduled requests takes priority over iterating seeds. + + Aims to minimize the number of requests in the scheduler at any given time, + to minimize resource usage (memory or disk, depending on + :setting:`JOBDIR`). + + It is best used when seed request priority is not important. + + Switching to :py:enum:mem:`idle` may lower resource usage further at the + cost of also lowering crawl speed. + """ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 2b6dc0e81..171b89434 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,7 +8,6 @@ For more information see docs/topics/architecture.rst from __future__ import annotations import logging -from enum import Enum from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast @@ -25,6 +24,8 @@ from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce +from ._seeding import SeedingPolicy + if TYPE_CHECKING: from collections.abc import AsyncIterable, Callable, Generator @@ -77,13 +78,6 @@ class _Slot: self.closing.callback(None) -class _SeedingPolicy(Enum): - front_load = "front-load" - greedy = "greedy" - idle = "idle" - lazy = "lazy" - - class ExecutionEngine: _SLOT_HEARTBEAT_INTERVAL: float = 5.0 @@ -117,9 +111,9 @@ class ExecutionEngine: def _load_seeding_policy(self) -> None: try: - self._seeding_policy = _SeedingPolicy(self.settings["SEEDING_POLICY"]) + self._seeding_policy = SeedingPolicy(self.settings["SEEDING_POLICY"]) except ValueError: - supported_values = ", ".join(policy.value for policy in _SeedingPolicy) + supported_values = ", ".join(policy.value for policy in SeedingPolicy) raise ValueError( f"The value of the SEEDING_POLICY setting " f"({self.settings['SEEDING_POLICY']!r}) is not supported. " @@ -206,7 +200,7 @@ class ExecutionEngine: if isinstance(seed, Request): self.crawl(seed) if ( - self._seeding_policy is not _SeedingPolicy.front_load + self._seeding_policy is not SeedingPolicy.front_load and not self._needs_backout() ): self._start_scheduled_request() @@ -215,7 +209,7 @@ class ExecutionEngine: self._slot.nextcall.schedule() finally: self._waiting_for_seed = False - if self._seeding_policy is _SeedingPolicy.front_load and self._seeds is None: + if self._seeding_policy is SeedingPolicy.front_load and self._seeds is None: self._slot.nextcall.schedule() @inlineCallbacks @@ -223,7 +217,7 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return - if self._seeding_policy in {_SeedingPolicy.idle, _SeedingPolicy.lazy}: + if self._seeding_policy in {SeedingPolicy.idle, SeedingPolicy.lazy}: while not self._needs_backout(): if self._start_scheduled_request() is None: break @@ -231,15 +225,15 @@ class ExecutionEngine: self._seeds is not None and not self._needs_backout() and ( - self._seeding_policy is not _SeedingPolicy.idle + self._seeding_policy is not SeedingPolicy.idle or (not self._waiting_for_seed and not self.downloader.active) ) ): yield self._process_next_seed() else: assert self._seeding_policy in { - _SeedingPolicy.front_load, - _SeedingPolicy.greedy, + SeedingPolicy.front_load, + SeedingPolicy.greedy, } if self._seeds is not None: if not self._needs_backout():