diff --git a/docs/conf.py b/docs/conf.py index 1167ce050..6985e38fe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -27,6 +27,7 @@ author = "Scrapy developers" # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration extensions = [ + "enum_tools.autoenum", "hoverxref.extension", "notfound.extension", "scrapydocs", diff --git a/docs/news.rst b/docs/news.rst index de7506e42..e210ebdb1 100644 --- a/docs/news.rst +++ b/docs/news.rst @@ -11,13 +11,16 @@ Scrapy VERSION (unreleased) Highlights: - Replaced ``start_requests`` (sync) with :meth:`~scrapy.Spider.start` - (async) + (async) and changed how it is iterated by default. Backward-incompatible changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, - ``process_start_requests()`` has been replaced by ``process_start()``. +- By default, the iteration of start requests and items no longer stops once + there are requests in the scheduler. + + You can restore the previous behavior by setting :setting:`SEEDING_POLICY` + to :py:enum:mem:`~scrapy.SeedingPolicy.lazy`. - In ``scrapy.core.engine.ExecutionEngine``: @@ -33,23 +36,24 @@ Backward-incompatible changes - The ``slot`` :ref:`telnet variable ` has been removed. +- In ``scrapy.core.spidermw.SpiderMiddlewareManager``, + ``process_start_requests()`` has been replaced by ``process_start()``. + Deprecations ~~~~~~~~~~~~ - The ``start_requests()`` method of :class:`~scrapy.Spider` is deprecated, - use :meth:`~scrapy.Spider.start` instead, or both to maintain support - for lower Scrapy versions. + use :meth:`~scrapy.Spider.start` instead, or both to maintain support for + lower Scrapy versions. - (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, - :issue:`6729`) + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) - The ``process_start_requests()`` method of :ref:`spider middlewares ` is deprecated, use - :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, or - both to maintain support for lower Scrapy versions. + :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead, + or both to maintain support for lower Scrapy versions. - (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6715`, - :issue:`6729`) + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) New features ~~~~~~~~~~~~ @@ -63,16 +67,32 @@ New features requests and items, e.g. reading them from a queue service or database using an asynchronous client, without workarounds. - (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`) + (:issue:`456`, :issue:`3477`, :issue:`4467`, :issue:`5627`, :issue:`6729`) + +- The new :setting:`SEEDING_POLICY` setting allows customizing how start + requests and items are iterated. + + You can also override the active seeding policy from + :meth:`Spider.start ` and from + :meth:`SpiderMiddleware.process_start + `. + + .. note:: Some third-party spider middlewares may need to be updated for + Scrapy VERSION support before you can use them in combination with the + ability to override the active seeding policy. + + (:issue:`740`, :issue:`1051`, :issue:`1443`, :issue:`3237`, :issue:`4467`, + :issue:`5282`, :issue:`6729`) Bug fixes ~~~~~~~~~ -- Yielding a start item (i.e. from :meth:`~scrapy.Spider.start` or an - equivalent) no longer delays the next iteration of starting requests and - items by up to 5 seconds. +- Yielding an item from :meth:`Spider.start ` or from + :meth:`SpiderMiddleware.process_start + ` no longer delays + the next iteration of starting requests and items by up to 5 seconds. - (:issue:`6715`, :issue:`6729`) + (:issue:`6729`) .. _release-2.12.0: @@ -375,9 +395,13 @@ Deprecations New features ~~~~~~~~~~~~ -- ``scrapy.Spider.start_requests`` can now yield items. +- ``scrapy.Spider.start_requests()`` can now yield items. (:issue:`5289`, :issue:`6417`) + .. note:: Some third-party spider middlewares may need to be updated for + Scrapy 2.12 support before you can use them in combination with the + ability to yield items from ``start_requests()``. + - Added a new :class:`~scrapy.http.Response` subclass, :class:`~scrapy.http.JsonResponse`, for responses with a `JSON MIME type `_. diff --git a/docs/requirements.txt b/docs/requirements.txt index 103fb08d6..63243fcf3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,3 +1,4 @@ +enum-tools[sphinx]==0.12.0 sphinx==8.1.3 sphinx-hoverxref==1.4.2 sphinx-notfound-page==1.0.4 diff --git a/docs/topics/architecture.rst b/docs/topics/architecture.rst index 4eff3cbe7..e8c510ea5 100644 --- a/docs/topics/architecture.rst +++ b/docs/topics/architecture.rst @@ -150,7 +150,7 @@ requests). Use a Spider middleware if you need to * post-process output of spider callbacks - change/add/remove requests or items; -* post-process seed requests or items; +* post-process start requests or items; * handle spider exceptions; * call errback instead of callback for some of the requests based on response content. diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst index 67de4f5f1..261100e00 100644 --- a/docs/topics/settings.rst +++ b/docs/topics/settings.rst @@ -1733,6 +1733,29 @@ Soft limit (in bytes) for response data being processed. While the sum of the sizes of all responses being processed is above this value, Scrapy does not process new requests. +.. setting:: SEEDING_POLICY + +SEEDING_POLICY +-------------- + +.. versionadded:: VERSION + +Default: :py:enum:mem:`SeedingPolicy.greedy ` + +Determines the way :meth:`Spider.start ` is +iterated. + +Its value may be defined as a member of the :class:`~scrapy.SeedingPolicy` enum +(e.g. :py:enum:mem:`SeedingPolicy.lazy `) or as a +matching string (e.g. ``"lazy"``). + +You can also override the active seeding policy from :meth:`Spider.start +` and from :meth:`SpiderMiddleware.process_start +`. + +.. autoenum:: scrapy.SeedingPolicy + :members: + .. setting:: SPIDER_CONTRACTS SPIDER_CONTRACTS diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index dc36ff6b6..79b8b1555 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -83,9 +83,24 @@ one or more of these methods: async for item_or_request in start: yield item_or_request - You may yield :class:`~scrapy.Request` or :ref:`item ` - objects, same as :meth:`~scrapy.Spider.start`, from *start* or - not. + You may yield the same type of objects as :meth:`~scrapy.Spider.start`. + + As with :meth:`~scrapy.Spider.start`, how this method is iterated by + default is controlled by :setting:`SEEDING_POLICY`. It is also possible + to yield a :class:`~scrapy.SeedingPolicy` enum or a matching string to + change the active seeding policy, for example: + + .. code-block:: python + + async def process_start(self, start): + yield "front_load" + async for item_or_request in start: + yield item_or_request + yield "idle" + + .. tip:: You can also restore the configured seeding policy by + :ref:`reading its value ` from the + :setting:`SEEDING_POLICY` setting and yielding it. To write spider middlewares that work on Scrapy versions lower than VERSION, define also a synchronous ``process_start_requests()`` method diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst index aae3bb9ba..71775247c 100644 --- a/docs/topics/spiders.rst +++ b/docs/topics/spiders.rst @@ -895,8 +895,8 @@ Combine SitemapSpider with other sources of urls: other_urls = ["http://www.example.com/about"] async def start(self): - async for seed in super().start(): - yield seed + async for item_or_request in super().start(): + yield item_or_request for url in self.other_urls: yield Request(url, self.parse_other) diff --git a/extras/qpsclient.py b/extras/qpsclient.py index f95c6010f..269b27336 100644 --- a/extras/qpsclient.py +++ b/extras/qpsclient.py @@ -35,8 +35,8 @@ class QPSSpider(Spider): self.download_delay = float(self.download_delay) async def start(self): - for seed in self.start_requests(): - yield seed + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self): url = self.benchurl diff --git a/pyproject.toml b/pyproject.toml index 84bf41a94..ed6a6ed46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -223,7 +223,9 @@ markers = [ "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", ] -filterwarnings = [] +filterwarnings = [ + "ignore::DeprecationWarning:twisted.web.static" +] [tool.ruff.lint] extend-select = [ diff --git a/scrapy/__init__.py b/scrapy/__init__.py index 256504c9c..7bb958a2d 100644 --- a/scrapy/__init__.py +++ b/scrapy/__init__.py @@ -7,6 +7,7 @@ import sys import warnings # Declare top-level shortcuts +from scrapy.core._seeding import SeedingPolicy from scrapy.http import FormRequest, Request from scrapy.item import Field, Item from scrapy.selector import Selector @@ -17,6 +18,7 @@ __all__ = [ "FormRequest", "Item", "Request", + "SeedingPolicy", "Selector", "Spider", "__version__", diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py index 3047ae396..c50c963ef 100644 --- a/scrapy/commands/shell.py +++ b/scrapy/commands/shell.py @@ -9,6 +9,7 @@ from __future__ import annotations from threading import Thread from typing import TYPE_CHECKING, Any +from scrapy import SeedingPolicy from scrapy.commands import ScrapyCommand from scrapy.http import Request from scrapy.shell import Shell @@ -24,9 +25,10 @@ if TYPE_CHECKING: class Command(ScrapyCommand): requires_project = False default_settings = { + "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", "KEEP_ALIVE": True, "LOGSTATS_INTERVAL": 0, - "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter", + "SEEDING_POLICY": SeedingPolicy.lazy, } def syntax(self) -> str: diff --git a/scrapy/core/_seeding.py b/scrapy/core/_seeding.py new file mode 100644 index 000000000..08c68951c --- /dev/null +++ b/scrapy/core/_seeding.py @@ -0,0 +1,68 @@ +from enum import Enum + +try: + from enum_tools.documentation import document_enum +except ImportError: + + def document_enum(func): # type: ignore[misc] + return func +else: + # https://github.com/domdfcoding/enum_tools/issues/29 + import enum_tools.documentation + + enum_tools.documentation.INTERACTIVE = True + + +@document_enum +class SeedingPolicy(Enum): + front_load = "front_load" + """The crawl does not start until all start requests have been scheduled. + + Aims to give the :ref:`scheduler ` full control over + request order from the start. Some custom schedulers may require this + seeding policy to work as designed. + """ + + greedy = "greedy" + """Iterating start items and requests takes priority over processing + scheduled requests. + + Every time a start request is iterated, it is scheduled, and then the next + request from the scheduler is sent. + + .. note:: That request sent may not be the scheduled start request + depending on the priority of scheduled requests, on the configured + :setting:`SCHEDULER` and on certain scheduler settings (e.g. + :setting:`SCHEDULER_MEMORY_QUEUE`). + + Best used when prioritizing start requests is important. + """ + + idle = "idle" + """A single start item or request is read only when there are neither + scheduled nor on-going requests. + + That is, a new start item or request is not read until all requests + triggered by the previous start request, directly or indirectly, have been + processed. + + Unlike :py:enum:mem:`lazy`, resource savings are prioritized over crawl + speed. + + It is functionally equivalent to running a spider multiple times in a row, + one per start request. + """ + + lazy = "lazy" + """Processing scheduled requests takes priority over iterating start items + and requests. + + Aims to minimize the number of requests in the scheduler at any given time, + to minimize resource usage (memory or disk, depending on + :setting:`JOBDIR`). + + It is best used when start request priority is not important. + + Switching to :py:enum:mem:`idle` may lower resource usage further at the + cost of also lowering crawl speed. + """ diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 6bf08a890..b87fd1b78 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -24,6 +24,8 @@ from scrapy.utils.log import failure_to_exc_info, logformatter_adapter from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce +from ._seeding import SeedingPolicy + if TYPE_CHECKING: from collections.abc import AsyncIterable, Callable, Generator @@ -41,6 +43,10 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") +class _SeedingPolicyChange(Exception): + pass + + class _Slot: def __init__( self, @@ -103,9 +109,21 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None + self._load_seeding_policy() self._start: AsyncIterable[Any] | None = None self._waiting_for_seed: bool = False + def _load_seeding_policy(self) -> None: + try: + self._seeding_policy = SeedingPolicy(self.settings["SEEDING_POLICY"]) + except ValueError: + supported_values = ", ".join(policy.value for policy in SeedingPolicy) + raise ValueError( + f"The value of the SEEDING_POLICY setting " + f"({self.settings['SEEDING_POLICY']!r}) is not supported. " + f"Supported values: {supported_values}." + ) + def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler @@ -166,18 +184,13 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - def _start_scheduled_requests(self): - while not self._needs_backout(): - if self._start_scheduled_request() is None: - break - @inlineCallbacks def _process_next_seed(self): if self._waiting_for_seed: return self._waiting_for_seed = True try: - seed = yield deferred_from_coro(self._start.__anext__()) + item_or_request = yield deferred_from_coro(self._start.__anext__()) except StopAsyncIteration: self._start = None except Exception: @@ -188,21 +201,71 @@ class ExecutionEngine: extra={"spider": self.spider}, ) else: - if isinstance(seed, Request): - self.crawl(seed) + if isinstance(item_or_request, Request): + self.crawl(item_or_request) + if ( + self._seeding_policy is not SeedingPolicy.front_load + and not self._needs_backout() + ): + self._start_scheduled_request() + elif isinstance(item_or_request, (str, SeedingPolicy)): + try: + self._seeding_policy = SeedingPolicy(item_or_request) + except ValueError: + valid_policy_strings = ", ".join( + policy.value for policy in SeedingPolicy + ) + logger.error( + f"Start value {item_or_request!r} has been ignored. " + f"Start values of {str} type must be valid seeding " + f"policies ({valid_policy_strings})." + ) + self._slot.nextcall.schedule() + else: + raise _SeedingPolicyChange else: - self.scraper.start_itemproc(seed, response=None) + self.scraper.start_itemproc(item_or_request, response=None) self._slot.nextcall.schedule() finally: self._waiting_for_seed = False + if self._seeding_policy is SeedingPolicy.front_load and self._start is None: + self._slot.nextcall.schedule() @inlineCallbacks def _start_next_requests(self) -> Generator[Deferred[Any], Any, None]: if self._slot is None or self._slot.closing is not None or self.paused: return - self._start_scheduled_requests() - if self._start is not None and not self._needs_backout(): - yield self._process_next_seed() + + try: + if self._seeding_policy in {SeedingPolicy.idle, SeedingPolicy.lazy}: + while not self._needs_backout(): + if self._start_scheduled_request() is None: + break + if ( + self._start is not None + and not self._needs_backout() + and ( + self._seeding_policy is not SeedingPolicy.idle + or (not self._waiting_for_seed and not self.downloader.active) + ) + ): + yield self._process_next_seed() + else: + assert self._seeding_policy in { + SeedingPolicy.front_load, + SeedingPolicy.greedy, + } + if self._start is not None: + if not self._needs_backout(): + yield self._process_next_seed() + else: + while not self._needs_backout(): + if self._start_scheduled_request() is None: + break + except _SeedingPolicyChange: + self._slot.nextcall.schedule() + return + if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py index 645e50301..886154bfa 100644 --- a/scrapy/settings/default_settings.py +++ b/scrapy/settings/default_settings.py @@ -17,6 +17,8 @@ import sys from importlib import import_module from pathlib import Path +from scrapy import SeedingPolicy + ADDONS = {} AJAXCRAWL_ENABLED = False @@ -308,6 +310,8 @@ SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue" SCRAPER_SLOT_MAX_ACTIVE_SIZE = 5000000 +SEEDING_POLICY = SeedingPolicy.greedy + SPIDER_LOADER_CLASS = "scrapy.spiderloader.SpiderLoader" SPIDER_LOADER_WARN_ONLY = False diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index eb88f62a4..6eba1b188 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -39,7 +39,7 @@ class Spider(object_ref): name: str custom_settings: dict[_SettingsKeyT, Any] | None = None - #: Seed URLs. See :meth:`start`. + #: Start URLs. See :meth:`start`. start_urls: list[str] def __init__(self, name: str | None = None, **kwargs: Any): @@ -113,6 +113,20 @@ class Spider(object_ref): async def start(self): yield {"foo": "bar"} + Use :setting:`SEEDING_POLICY` to set how :meth:`start` is + iterated by default. It is also + possible to yield a :class:`~scrapy.SeedingPolicy` enum or a matching + string to change the active seeding policy, for example: + + .. code-block:: python + + async def start(self): + yield "front_load" + yield Request("https://a.example") + yield Request("https://b.example") + yield self.crawler.settings["SEEDING_POLICY"] + yield Request("https://c.example") + To write spiders that work on Scrapy versions lower than VERSION, define also a synchronous ``start_requests()`` method that returns an iterable. For example: @@ -122,8 +136,8 @@ class Spider(object_ref): def start_requests(self): yield Request("https://toscrape.com/") """ - for seed in self.start_requests(): - yield seed + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Any]: if not self.start_urls and hasattr(self, "start_url"): diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index ee0058880..3e74640eb 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -30,8 +30,8 @@ class InitSpider(Spider): ) async def start(self) -> AsyncIterable[Any]: - for seed in self.start_requests(): - yield seed + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Request]: self._postinit_reqs: Iterable[Request] = super().start_requests() diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index c01b54b4a..6db10c1ab 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -54,8 +54,8 @@ class SitemapSpider(Spider): self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] async def start(self) -> AsyncIterable[Any]: - for seed in self.start_requests(): - yield seed + for item_or_request in self.start_requests(): + yield item_or_request def start_requests(self) -> Iterable[Request]: for url in self.sitemap_urls: diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 00bfe0ef9..26b16a1d4 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -512,8 +512,8 @@ class TestContractsManager(unittest.TestCase): self.visited = 0 async def start(self_): # pylint: disable=no-self-argument - for seed in self.conman.from_spider(self_, self.results): - yield seed + for item_or_request in self.conman.from_spider(self_, self.results): + yield item_or_request def parse_first(self, response): self.visited += 1 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 919c599f7..92b6238d7 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -194,7 +194,8 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_unsupported_output(self): - """Anything that is not a request is assumed to be an item, avoiding a + """Anything that is not a request, a seeding policy or a string (which + is assumed to be a seeding policy) is assumed to be an item, avoiding a potentially expensive call to itemadapter.is_item, and letting instead things fail when ItemAdapter is actually used on the corresponding non-item object.""" @@ -202,11 +203,11 @@ class TestCrawl(TestCase): crawler = get_crawler(StartGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) - assert len(log.records) == 0 + assert len(log.records) == 1 @defer.inlineCallbacks def test_start_laziness(self): - settings = {"CONCURRENT_REQUESTS": 1} + settings = {"CONCURRENT_REQUESTS": 1, "SEEDING_POLICY": "lazy"} crawler = get_crawler(BrokenStartSpider, settings) yield crawler.crawl(mockserver=self.mockserver) assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index ad0dc9c12..c52a2b8bd 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -1,28 +1,77 @@ from __future__ import annotations -from collections import deque +from collections import defaultdict, deque +from logging import ERROR -from twisted.internet.defer import inlineCallbacks +from testfixtures import LogCapture from twisted.trial.unittest import TestCase -from scrapy import Request, Spider, signals +from scrapy import Request, SeedingPolicy, Spider, signals +from scrapy.core.engine import ExecutionEngine from scrapy.core.scheduler import BaseScheduler -from scrapy.utils.defer import maybe_deferred_to_future +from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future from scrapy.utils.test import get_crawler +from .mockserver import MockServer from .test_spider_start import twisted_sleep class MainTestCase(TestCase): - @inlineCallbacks - def test_scheduler_priority_over_start_simple(self): - """Scrapy reads start() into the scheduler while the scheduler is - empty, but otherwise prioritizes requests already in the scheduler. + # If the test ends before the heartbeat, it may mean that the logic to + # re-schecule a new call of _start_next_requests under the right + # ciscumstances is not properly implemented, and the hearatbeat is working + # as a workaround for that issue. This is a performance issue and should + # be addressed. + # + # It could also happen that, on some CI runners, some tests (e.g. those + # below using a mock server) run too slow and proper handling overlaps with + # the heartbeat. If that is the case, it may be worth considering + # increasing the heartbeat time. It should be safe, since in most real live + # scenarios the heartbeat should never make a difference, and we may + # eventually remove the heartbeat altogether. + timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL - This test shows how, given a scheduler pre-filled with a request, that - request is sent before sending the first seed request. - """ + @deferred_f_from_coro_f + async def test_greedy(self): + class TestScheduler(BaseScheduler): + def __init__(self, *args, **kwargs): + self.requests = deque((Request("data:,b"),)) + def enqueue_request(self, request: Request) -> bool: + self.requests.append(request) + return True + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def next_request(self) -> Request | None: + try: + return self.requests.pop() + except IndexError: + return None + + class TestSpider(Spider): + name = "test" + start_urls = ["data:,a"] + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": TestScheduler} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_lazy(self): class TestScheduler(BaseScheduler): def __init__(self, *args, **kwargs): self.requests = deque((Request("data:,a"),)) @@ -52,23 +101,18 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) assert crawler.stats.get_value("finish_reason") == "finished" expected_urls = ["data:,a", "data:,b"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" - @inlineCallbacks - def test_scheduler_priority_over_start_complex(self): - """Although Scrapy reads start() into the scheduler while the scheduler - is empty and otherwise prioritizes requests already in the scheduler, - this is done in a non-blocking way. - - That is, if the scheduler reports having requests but yields none, - requests from start() will be scheduled. - """ + @deferred_f_from_coro_f + async def test_lazy_blocking(self): + """If the scheduler reports having requests but yields none, the lazy + policy schedules start requests.""" class TestScheduler(BaseScheduler): def __init__(self, *args, **kwargs): @@ -110,10 +154,205 @@ class MainTestCase(TestCase): def track_url(request, spider): actual_urls.append(request.url) - settings = {"SCHEDULER": TestScheduler} + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} crawler = get_crawler(TestSpider, settings_dict=settings) crawler.signals.connect(track_url, signals.request_reached_downloader) - yield crawler.crawl() + await maybe_deferred_to_future(crawler.crawl()) assert crawler.stats.get_value("finish_reason") == "finished" expected_urls = ["data:,a", "data:,b", "data:,c"] assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_lazy_start_order(self): + """By default, start requests should be sent in the order in which they + are iterated.""" + + class TestSpider(Spider): + name = "test" + start_urls = ["data:,a", "data:,b", "data:,c"] + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SEEDING_POLICY": "lazy"} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b", "data:,c"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_front_load(self): + class TestScheduler(BaseScheduler): + def __init__(self, *args, **kwargs): + self.requests = defaultdict(deque) + + def enqueue_request(self, request: Request) -> bool: + self.requests[request.priority].append(request) + return True + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def next_request(self) -> Request | None: + if not self.requests: + return None + priority = max(self.requests) + request = self.requests[priority].popleft() + if not self.requests[priority]: + del self.requests[priority] + return request + + class TestSpider(Spider): + name = "test" + + async def start(self): + yield Request("data:,b", priority=0) + yield Request("data:,a", priority=1) + + def parse(self, response): + pass + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "front_load"} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = ["data:,a", "data:,b"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + @deferred_f_from_coro_f + async def test_override(self): + class TestScheduler(BaseScheduler): + def __init__(self, *args, **kwargs): + self.requests = defaultdict(deque) + + def enqueue_request(self, request: Request) -> bool: + self.requests[request.priority].append(request) + return True + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def next_request(self) -> Request | None: + if not self.requests: + return None + priority = max(self.requests) + request = self.requests[priority].popleft() + if not self.requests[priority]: + del self.requests[priority] + return request + + class TestSpider(Spider): + name = "test" + + async def start(self): + yield "front-load" # typo + yield SeedingPolicy.front_load + yield Request("data:,b", priority=1) + yield Request("data:,a", priority=2) + yield self.crawler.settings["SEEDING_POLICY"] + yield Request("data:,c", priority=3) + + def parse(self, response): + pass + + actual_items = [] + actual_urls = [] + + def track_item(item, response, spider): + actual_items.append(item) + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "lazy"} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_item, signals.item_scraped) + crawler.signals.connect(track_url, signals.request_reached_downloader) + with LogCapture(level=ERROR) as log: + await maybe_deferred_to_future(crawler.crawl()) + assert len(log.records) == 1 + assert "must be valid seeding policies" in str(log.records[0]) + assert crawler.stats.get_value("finish_reason") == "finished" + assert not actual_items, ( + f"{actual_items=} should be empty, policies are not items" + ) + expected_urls = ["data:,a", "data:,b", "data:,c"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" + + +class MockServerTestCase(TestCase): + # See the comment on the matching line above. + timeout = ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + # If requests are too fast, test_idle will fail because the outcome will + # match that of the lazy seeding policy. + delay = 0.2 + + @classmethod + def setUpClass(cls): + cls.mockserver = MockServer() + cls.mockserver.__enter__() + + @classmethod + def tearDownClass(cls): + cls.mockserver.__exit__(None, None, None) + + @deferred_f_from_coro_f + async def test_idle(self): + def _url(id): + return self.mockserver.url(f"/delay?n={self.delay}&{id}") + + class TestScheduler(BaseScheduler): + def __init__(self, *args, **kwargs): + self.requests = deque((Request(_url("a")),)) + + def enqueue_request(self, request: Request) -> bool: + self.requests.append(request) + return True + + def has_pending_requests(self) -> bool: + return bool(self.requests) + + def next_request(self) -> Request | None: + try: + return self.requests.popleft() + except IndexError: + return None + + class TestSpider(Spider): + name = "test" + start_urls = [_url("b"), _url("d")] + queue = deque((Request(_url("c")),)) + + def parse(self, response): + try: + request = self.queue.popleft() + except IndexError: + pass + else: + yield request + + actual_urls = [] + + def track_url(request, spider): + actual_urls.append(request.url) + + settings = {"SCHEDULER": TestScheduler, "SEEDING_POLICY": "idle"} + crawler = get_crawler(TestSpider, settings_dict=settings) + crawler.signals.connect(track_url, signals.request_reached_downloader) + await maybe_deferred_to_future(crawler.crawl()) + assert crawler.stats.get_value("finish_reason") == "finished" + expected_urls = [_url(letter) for letter in "abcd"] + assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}"