From e049768987cb311326ec31d7130ac2615275faf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 12 Mar 2025 09:53:08 +0100 Subject: [PATCH] Remove seeding policy references for a minimal implementation --- docs/news.rst | 3 -- docs/topics/settings.rst | 68 ----------------------------- docs/topics/spider-middleware.rst | 3 -- scrapy/core/engine.py | 20 --------- scrapy/settings/default_settings.py | 2 - scrapy/spiders/__init__.py | 3 -- tests/test_crawl.py | 8 ++-- tests/test_engine_seeding.py | 11 +++-- 8 files changed, 9 insertions(+), 109 deletions(-) diff --git a/docs/news.rst b/docs/news.rst index 7afcbb3cf..a2046723b 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -63,9 +63,6 @@ New features (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`) -- The new :setting:`SEEDING_POLICY` setting allows customizing how spider - start requests and items are consumed. - .. _release-2.12.0: diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index e86c6acbc..5bc1ccc7d 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1733,74 +1733,6 @@ Soft limit (in bytes) for response data being processed. While the sum of the sizes of all responses being processed is above this value, Scrapy does not process new requests. -.. setting:: SEEDING_POLICY - -SEEDING_POLICY --------------- - -.. versionadded:: VERSION - -Default: ``"lazy"`` - -The way :meth:`Spider.yield_seeds ` is iterated: - -- .. _lazy-seeding: - - ``"lazy"``: Seeds are only read while the :ref:`scheduler - ` is empty and the number of ongoing requests is - lower than :setting:`CONCURRENT_REQUESTS`. - - This seeding policy aims to: - - - Maximize crawl speed by maxing out concurrent requests as often - as possible. - - - Minimize the number of requests in the scheduler at any given - time by prioritizing scheduler requests over seeds, to minimize - resource usage (memory or disk, depending on - :setting:`JOBDIR`). - - This seeding policy is best used when seed request priority is not - important. Switching to :ref:`serial ` may lower - resource usage further at the cost of also lowering crawl speed. - -- .. _front-load-seeding: - - ``"front-load"``: The spider does not start until all seeds have - been read and loaded into the scheduler. - - This seeding policy aims to give the :ref:`scheduler - ` full control over request order, at the cost of - a higher resource usage and a delayed crawl start. - - This seeding policy is best used when having all requests go - through the scheduler is more important than resource usage and - crawl speed. - -- .. _greedy-seeding: - - ``"greedy"``: While the :ref:`scheduler ` is - empty and the number of ongoing requests is lower than - :setting:`CONCURRENT_REQUESTS`, seeds are read and sent directly - (bypassing the scheduler). While the scheduler has requests, seeds - are fed into the scheduler. - - This seeding policy is similar to :ref:`front-load - `, but it bypasses the scheduler for the first - few requests to avoid delaying the crawl start. - -- .. _serial-seeding: - - ``"serial"``: A single seed is read whenever the :ref:`scheduler - ` is empty and there are no ongoing requests. - That is, a new seed is not read until all requests triggered by the - previous seed, directly or indirectly, have been processed. - - This seeding policy is similar to :ref:`lazy `, but - it prioritizes resource savings over crawl speed. It is - functionally equivalent to running the spider multiple times in a - row, one per seed request. - .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 0517a4b8e..51538a6c4 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -87,9 +87,6 @@ one or more of these methods: objects, same as :meth:`~scrapy.Spider.yield_seeds`, from *seeds* or not. - As with :meth:`~scrapy.Spider.yield_seeds`, how this method is iterated - is controlled by :setting:`SEEDING_POLICY`. - To write spider middlewares that work on Scrapy versions lower than VERSION, define also a synchronous ``process_start_requests()`` method that returns an iterable. For example: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7cdc6287f..7c9fb29f5 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -8,7 +8,6 @@ For more information see docs/topics/architecture.rst from __future__ import annotations import logging -from enum import Enum from time import time from typing import TYPE_CHECKING, Any, TypeVar, cast @@ -77,13 +76,6 @@ class _Slot: self.closing.callback(None) -class _SeedingPolicy(Enum): - lazy = "lazy" - front_load = "front-load" - greedy = "greedy" - serial = "serial" - - class ExecutionEngine: _SLOT_HEARTBEAT_INTERVAL: float = 5.0 @@ -111,20 +103,8 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._load_seeding_policy() self._seeds: AsyncIterator[Any] | None = None - def _load_seeding_policy(self) -> None: - try: - self._seeding_policy = _SeedingPolicy(self.settings["SEEDING_POLICY"]) - except ValueError: - supported_values = ", ".join(policy.value for policy in _SeedingPolicy) - raise ValueError( - f"The value of the SEEDING_POLICY setting " - f"({self.settings['SEEDING_POLICY']!r}) is not supported. " - f"Supported values: {supported_values}." - ) - def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index a08becfee..645e50301 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -308,8 +308,6 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 -SEEDING_POLICY = "lazy" - SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index a6ba14697..aef2d7d93 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -113,9 +113,6 @@ class Spider(object_ref): async def yield_seeds(self): yield {"foo": "bar"} - Use :setting:`SEEDING_POLICY` to set how :meth:`yield_seeds` is - iterated. - To write spiders that work on Scrapy versions lower than VERSION, define also a synchronous ``start_requests()`` method that returns an iterable. For example: diff --git a/tests/test_crawl.py b/tests/test_crawl.py index a45dfec60..765d2c0f6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -194,10 +194,10 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_yield_seeds_unsupported_output(self): - """Anything that is not a request or a seeding policy is assumed to be - an item, avoiding a potentially expensive call to itemadapter.is_item, - and letting instead things fail when ItemAdapter is actually used on - the corresponding non-item object.""" + """Anything that is not a request is assumed to be an item, avoiding a + potentially expensive call to itemadapter.is_item, and letting instead + things fail when ItemAdapter is actually used on the corresponding + non-item object.""" with LogCapture("scrapy", level=logging.ERROR) as log: crawler = get_crawler(YieldSeedsGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index c74dc3ca1..93e4e701d 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -16,9 +16,8 @@ from .test_spider_yield_seeds import twisted_sleep class MainTestCase(TestCase): @inlineCallbacks def test_scheduler_priority_over_seeds_simple(self): - """The seeding policy is to read seeds into the scheduler while the - scheduler is empty, but otherwise priorize requests already in the - scheduler. + """Scrapy reads seeds into the scheduler while the scheduler is empty, + but otherwise prioritizes requests already in the scheduler. This test shows how, given a scheduler pre-filled with a request, that request is sent before sending the first seed request. @@ -63,9 +62,9 @@ class MainTestCase(TestCase): @inlineCallbacks def test_scheduler_priority_over_seeds_complex(self): - """While the seeding policy is to read seeds into the scheduler while - the scheduler is empty and otherwise priorize requests already in the - scheduler, this is done in a non-blocking way. + """Although Scrapy reads seeds into the scheduler while the scheduler + is empty and otherwise prioritizes requests already in the scheduler, + this is done in a non-blocking way. That is, if the scheduler reports having requests but yields none, requests from seeds will be scheduled.