diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py index 8a3cf5078..56dc1ea55 100644 --- a/scrapy/commands/check.py +++ b/scrapy/commands/check.py @@ -81,14 +81,12 @@ class Command(ScrapyCommand): spider_loader = self.crawler_process.spider_loader async def start(self): - for request in conman.from_spider(self, self._result): + for request in conman.from_spider(self, result): yield request with set_environ(SCRAPY_CHECK="true"): for spidername in args or spider_loader.list(): spidercls = spider_loader.load(spidername) - - spidercls._result = result # type: ignore[assignment,attr-defined,method-assign,return-value] spidercls.start = start # type: ignore[assignment,method-assign,return-value] tested_methods = conman.tested_methods_from_spidercls(spidercls) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 1e8d01255..b08de6149 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -180,7 +180,12 @@ class ExecutionEngine: def unpause(self) -> None: self.paused = False - async def _process_next_spider_start_yield(self): + async def _process_start_next(self): + """Processes the next item or request from Spider.start(). + + If a request, it is scheduled. If an item, it is sent to item + pipelines. + """ try: item_or_request = await self._start.__anext__() except StopAsyncIteration: @@ -201,24 +206,28 @@ class ExecutionEngine: @deferred_f_from_coro_f async def start_request_processing(self) -> None: - """Process start items and requests in an asynchronous loop. - - Items are scraped. Requests are scheduled. - """ + """Starts consuming Spider.start() output and sending scheduled + requests.""" if self._started_request_processing: raise RuntimeError("Request processing already started") self._started_request_processing = True + # Starts the processing of scheduled requests, as well as a periodic + # call to that processing method for scenarios where the scheduler + # reports having pending requests but returns none. assert self._slot is not None # typing self._slot.nextcall.schedule() self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL) + while self._start is not None: - await self._process_next_spider_start_yield() + await self._process_start_next() if not self.needs_backout(): + # Give room for the outcome of self._process_start_next() to be + # processed before continuing with the next iteration. self._slot.nextcall.schedule() await self._slot.nextcall.wait() - def _start_next_requests(self) -> None: + def _start_scheduled_requests(self) -> None: if self._slot is None or self._slot.closing is not None or self.paused: return @@ -403,7 +412,7 @@ class ExecutionEngine: raise RuntimeError(f"No free spider slot when opening {spider.name!r}") logger.info("Spider opened", extra={"spider": spider}) self.spider = spider - nextcall = CallLaterOnce(self._start_next_requests) + nextcall = CallLaterOnce(self._start_scheduled_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) self._slot = _Slot(close_if_idle, nextcall, scheduler) self._start = await maybe_deferred_to_future( diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index bace52e4e..f58d96fd7 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -8,7 +8,7 @@ from __future__ import annotations import logging from collections.abc import AsyncIterable, Callable, Iterable -from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction +from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast from warnings import warn @@ -387,20 +387,20 @@ class SpiderMiddlewareManager(MiddlewareManager): dfd2.addErrback(process_spider_exception) return dfd2 - @inlineCallbacks - def process_start( - self, spider: Spider - ) -> Generator[Deferred[Any], Any, AsyncIterable[Any] | None]: + @deferred_f_from_coro_f + async def process_start(self, spider: Spider) -> AsyncIterable[Any] | None: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_start = iter(spider.start_requests()) - sync_start = yield self._process_chain( - "process_start_requests", sync_start, spider + sync_start = await maybe_deferred_to_future( + self._process_chain("process_start_requests", sync_start, spider) ) start = as_async_generator(sync_start) else: - start = yield self._iter_seeds(spider) - start = yield self._process_chain("process_start", start) + start = spider.start() + start = await maybe_deferred_to_future( + self._process_chain("process_start", start) + ) return start def _check_deprecated_start_requests_use(self, spider: Spider): @@ -473,14 +473,6 @@ class SpiderMiddlewareManager(MiddlewareManager): f"https://docs.scrapy.org/en/VERSION/news.html" ) - @staticmethod - def _iter_seeds(spider: Spider): - fn = spider.start - if isasyncgenfunction(fn): - return fn().__aiter__() - assert iscoroutinefunction(fn) - return deferred_from_coro(fn()) - # This method is only needed until _async compatibility methods are removed. @staticmethod def _get_async_method_pair(