diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 4632644e6..dc36ff6b6 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,7 +70,7 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_start(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] + .. method:: process_start(start: AsyncIterable[Any], /) -> AsyncIterable[Any] :async: Iterate over the output of :meth:`~scrapy.Spider.start` or that @@ -79,12 +79,12 @@ one or more of these methods: .. code-block:: python - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request You may yield :class:`~scrapy.Request` or :ref:`item ` - objects, same as :meth:`~scrapy.Spider.start`, from *seeds* or + objects, same as :meth:`~scrapy.Spider.start`, from *start* or not. To write spider middlewares that work on Scrapy versions lower than @@ -93,8 +93,8 @@ one or more of these methods: .. code-block:: python - def process_start_requests(self, seeds, spider): - yield from seeds + def process_start_requests(self, start, spider): + yield from start .. method:: process_spider_input(response, spider) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 23cddf481..6bf08a890 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -103,7 +103,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._seeds: AsyncIterable[Any] | None = None + self._start: AsyncIterable[Any] | None = None self._waiting_for_seed: bool = False def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: @@ -177,13 +177,13 @@ class ExecutionEngine: return self._waiting_for_seed = True try: - seed = yield deferred_from_coro(self._seeds.__anext__()) + seed = yield deferred_from_coro(self._start.__anext__()) except StopAsyncIteration: - self._seeds = None + self._start = None except Exception: - self._seeds = None + self._start = None logger.error( - "Error while reading seeds", + "Error while reading start items and requests", exc_info=True, extra={"spider": self.spider}, ) @@ -201,7 +201,7 @@ class ExecutionEngine: if self._slot is None or self._slot.closing is not None or self.paused: return self._start_scheduled_requests() - if self._seeds is not None and not self._needs_backout(): + if self._start is not None and not self._needs_backout(): yield self._process_next_seed() if self.spider_is_idle() and self._slot.close_if_idle: self._spider_idle() @@ -289,7 +289,7 @@ class ExecutionEngine: return False if self.downloader.active: # downloader has pending requests return False - if self._seeds is not None: # not all start requests are handled + if self._start is not None: # not all start requests are handled return False return not self._slot.scheduler.has_pending_requests() @@ -380,7 +380,7 @@ class ExecutionEngine: logger.info("Spider opened", extra={"spider": spider}) nextcall = CallLaterOnce(self._start_next_requests) scheduler = build_from_crawler(self.scheduler_cls, self.crawler) - self._seeds = yield self.scraper.spidermw.process_start(spider) + self._start = yield self.scraper.spidermw.process_start(spider) self._slot = _Slot(close_if_idle, nextcall, scheduler) self.spider = spider if hasattr(scheduler, "open") and (d := scheduler.open(spider)): diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 95ce47394..d5ad4bc8b 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -394,15 +394,15 @@ class SpiderMiddlewareManager(MiddlewareManager): ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: - sync_seeds = iter(spider.start_requests()) - sync_seeds = yield self._process_chain( - "process_start_requests", sync_seeds, spider + sync_start = iter(spider.start_requests()) + sync_start = yield self._process_chain( + "process_start_requests", sync_start, spider ) - seeds = as_async_generator(sync_seeds) + start = as_async_generator(sync_start) else: - seeds = yield _maybe_sync_generator_to_async(spider.start)() - seeds = yield self._process_chain("process_start", seeds) - return seeds + start = yield _maybe_sync_generator_to_async(spider.start)() + start = yield self._process_chain("process_start", start) + return start def _check_deprecated_start_requests_use(self, spider: Spider): start_requests_cls = None diff --git a/scrapy/templates/project/module/middlewares.py.tmpl b/scrapy/templates/project/module/middlewares.py.tmpl index 59a3dca21..3f0239832 100644 --- a/scrapy/templates/project/module/middlewares.py.tmpl +++ b/scrapy/templates/project/module/middlewares.py.tmpl @@ -43,11 +43,11 @@ class ${ProjectName}SpiderMiddleware: # Should return either None or an iterable of Request or item objects. pass - async def process_start(self, seeds): - # Called with the seeds from the spider start() method or with - # the output of the maching method of an earlier spider middleware. - async for seed in seeds: - yield seed + async def process_start(self, start): + # Called with an async iterator over the spider start() method or the + # maching method of an earlier spider middleware. + async for item_or_request in start: + yield item_or_request def spider_opened(self, spider): spider.logger.info("Spider opened: %s" % spider.name) diff --git a/tests/spiders.py b/tests/spiders.py index d7fffd001..c47f2bd2b 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -319,7 +319,7 @@ class ErrorSpider(FollowAllSpider): self.raise_exception() -class BrokenYieldSeedsSpider(FollowAllSpider): +class BrokenStartSpider(FollowAllSpider): fail_before_yield = False fail_yielding = False @@ -345,12 +345,12 @@ class BrokenYieldSeedsSpider(FollowAllSpider): yield from super().parse(response) -class YieldSeedsItemSpider(FollowAllSpider): +class StartItemSpider(FollowAllSpider): async def start(self): yield {"name": "test item"} -class YieldSeedsGoodAndBadOutput(FollowAllSpider): +class StartGoodAndBadOutput(FollowAllSpider): async def start(self): yield {"a": "a"} yield Request("data:,a") @@ -384,7 +384,7 @@ class SingleRequestSpider(MetaSpider): return None -class DuplicateYieldSeedsSpider(MockServerSpider): +class DuplicateStartSpider(MockServerSpider): dont_filter = True name = "duplicatestartrequests" distinct_urls = 2 diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 85d016e44..b5876e3cc 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -34,7 +34,7 @@ from tests.spiders import ( AsyncDefDeferredMaybeWrappedSpider, AsyncDefDeferredWrappedSpider, AsyncDefSpider, - BrokenYieldSeedsSpider, + BrokenStartSpider, BytesReceivedCallbackSpider, BytesReceivedErrbackSpider, CrawlSpiderWithAsyncCallback, @@ -43,14 +43,14 @@ from tests.spiders import ( CrawlSpiderWithParseMethod, CrawlSpiderWithProcessRequestCallbackKeywordArguments, DelaySpider, - DuplicateYieldSeedsSpider, + DuplicateStartSpider, FollowAllSpider, HeadersReceivedCallbackSpider, HeadersReceivedErrbackSpider, SimpleSpider, SingleRequestSpider, - YieldSeedsGoodAndBadOutput, - YieldSeedsItemSpider, + StartGoodAndBadOutput, + StartItemSpider, ) @@ -165,7 +165,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_bug_before_yield(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -176,7 +176,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_bug_yielding(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(BrokenYieldSeedsSpider) + crawler = get_crawler(BrokenStartSpider) yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver) assert len(log.records) == 1 @@ -187,7 +187,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_items(self): with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(YieldSeedsItemSpider) + crawler = get_crawler(StartItemSpider) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 @@ -199,7 +199,7 @@ class TestCrawl(TestCase): things fail when ItemAdapter is actually used on the corresponding non-item object.""" with LogCapture("scrapy", level=logging.ERROR) as log: - crawler = get_crawler(YieldSeedsGoodAndBadOutput) + crawler = get_crawler(StartGoodAndBadOutput) yield crawler.crawl(mockserver=self.mockserver) assert len(log.records) == 0 @@ -207,7 +207,7 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_laziness(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(BrokenYieldSeedsSpider, settings) + crawler = get_crawler(BrokenStartSpider, settings) yield crawler.crawl(mockserver=self.mockserver) assert crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index( 99 @@ -216,13 +216,13 @@ class TestCrawl(TestCase): @defer.inlineCallbacks def test_start_dupes(self): settings = {"CONCURRENT_REQUESTS": 1} - crawler = get_crawler(DuplicateYieldSeedsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver ) assert crawler.spider.visited == 6 - crawler = get_crawler(DuplicateYieldSeedsSpider, settings) + crawler = get_crawler(DuplicateStartSpider, settings) yield crawler.crawl( dont_filter=False, distinct_urls=3, diff --git a/tests/test_engine.py b/tests/test_engine.py index 306a0e775..4dc77e438 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -494,11 +494,11 @@ def test_request_scheduled_signal(caplog): engine.downloader._slot_gc_loop.stop() scheduler = TestScheduler() - async def seeds(): + async def start(): return yield - engine._seeds = seeds() + engine._start = start() engine._slot = _Slot(False, Mock(), scheduler) crawler.signals.connect(signal_handler, request_scheduled) keep_request = Request("https://keep.example") diff --git a/tests/test_engine_seeding.py b/tests/test_engine_seeding.py index 03a5ac64a..ad0dc9c12 100644 --- a/tests/test_engine_seeding.py +++ b/tests/test_engine_seeding.py @@ -15,9 +15,9 @@ from .test_spider_start import twisted_sleep class MainTestCase(TestCase): @inlineCallbacks - def test_scheduler_priority_over_seeds_simple(self): - """Scrapy reads seeds into the scheduler while the scheduler is empty, - but otherwise prioritizes requests already in the scheduler. + def test_scheduler_priority_over_start_simple(self): + """Scrapy reads start() into the scheduler while the scheduler is + empty, but otherwise prioritizes requests already in the scheduler. This test shows how, given a scheduler pre-filled with a request, that request is sent before sending the first seed request. @@ -61,13 +61,13 @@ class MainTestCase(TestCase): assert actual_urls == expected_urls, f"{actual_urls=} != {expected_urls=}" @inlineCallbacks - def test_scheduler_priority_over_seeds_complex(self): - """Although Scrapy reads seeds into the scheduler while the scheduler + def test_scheduler_priority_over_start_complex(self): + """Although Scrapy reads start() into the scheduler while the scheduler is empty and otherwise prioritizes requests already in the scheduler, this is done in a non-blocking way. That is, if the scheduler reports having requests but yields none, - requests from seeds will be scheduled. + requests from start() will be scheduled. """ class TestScheduler(BaseScheduler): diff --git a/tests/test_spider.py b/tests/test_spider.py index b7ddb37c4..c3d66617d 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -460,7 +460,7 @@ class TestCrawlSpider(TestSpider): ( "scrapy.core.engine", "ERROR", - "Error while reading seeds", + "Error while reading start items and requests", ), ) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 74744548d..3d8c885d1 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -320,19 +320,19 @@ class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware): ) -class ProcessYieldSeedsSimpleMiddleware: +class ProcessStartSimpleMiddleware: def process_test_start(self, test_start, spider): yield from test_start -class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): +class TestProcessStartSimple(TestBaseAsyncSpiderMiddleware): """process_start tests for simple start""" ITEM_TYPE = (Request, dict) - MW_SIMPLE = ProcessYieldSeedsSimpleMiddleware + MW_SIMPLE = ProcessStartSimpleMiddleware @inlineCallbacks - def _get_processed_seeds(self, *mw_classes): + def _get_processed_start(self, *mw_classes): class TestSpider(Spider): name = "test" @@ -353,11 +353,11 @@ class TestProcessSeedsSimple(TestBaseAsyncSpiderMiddleware): @inlineCallbacks def test_simple(self): """Simple mw""" - seeds = yield self._get_processed_seeds(self.MW_SIMPLE) - assert isasyncgen(seeds) - seed_list = yield deferred_from_coro(collect_asyncgen(seeds)) - assert len(seed_list) == self.RESULT_COUNT - assert isinstance(seed_list[0], self.ITEM_TYPE) + start = yield self._get_processed_start(self.MW_SIMPLE) + assert isasyncgen(start) + start_list = yield deferred_from_coro(collect_asyncgen(start)) + assert len(start_list) == self.RESULT_COUNT + assert isinstance(start_list[0], self.ITEM_TYPE) class UniversalMiddlewareNoSync: diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 7d5cb7cb6..6dced5126 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -17,29 +17,29 @@ ITEM_D = {"id": "d"} class AsyncioSleepSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): await sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS) - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request class NoOpSpiderMiddleware: - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request class TwistedSleepSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): await maybe_deferred_to_future(twisted_sleep(ASYNC_GEN_ERROR_MINIMUM_SECONDS)) - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request class UniversalSpiderMiddleware: - async def process_start(self, seeds): - async for seed in seeds: - yield seed + async def process_start(self, start): + async for item_or_request in start: + yield item_or_request def process_start_requests(self, start_requests, spider): raise NotImplementedError @@ -73,30 +73,30 @@ class DeprecatedWrapSpider(Spider): class ModernWrapSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): yield ITEM_A - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request yield ITEM_C class UniversalWrapSpiderMiddleware: - async def process_start(self, seeds): + async def process_start(self, start): yield ITEM_A - async for seed in seeds: - yield seed + async for item_or_request in start: + yield item_or_request yield ITEM_C - def process_start_requests(self, seeds, spider): + def process_start_requests(self, start, spider): yield ITEM_A - yield from seeds + yield from start yield ITEM_C class DeprecatedWrapSpiderMiddleware: - def process_start_requests(self, seeds, spider): + def process_start_requests(self, start, spider): yield ITEM_A - yield from seeds + yield from start yield ITEM_C