diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 2c7928201..55330da9f 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -70,7 +70,7 @@ one or more of these methods: .. class:: SpiderMiddleware - .. method:: process_seeds(seeds: AsyncIterator[Any], /) -> AsyncIterator[Any] + .. method:: process_seeds(seeds: AsyncIterable[Any], /) -> AsyncIterable[Any] :async: Iterate over the output of :meth:`~scrapy.Spider.yield_seeds` or that diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index fccf7c2d3..5ed09c77c 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,7 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import AsyncIterator + from collections.abc import AsyncIterable class Command(ScrapyCommand): @@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - async def yield_seeds(self) -> AsyncIterator[Any]: + async def yield_seeds(self) -> AsyncIterable[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" yield scrapy.Request(url, dont_filter=True) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index f0d152570..810824c7e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -22,7 +22,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: import argparse - from collections.abc import AsyncGenerator, AsyncIterator, Coroutine, Iterable + from collections.abc import AsyncGenerator, AsyncIterable, Coroutine, Iterable from twisted.python.failure import Failure @@ -258,7 +258,7 @@ class Command(BaseRunSpiderCommand): if not self.spidercls: logger.error("Unable to find spider for: %(url)s", {"url": url}) - async def yield_seeds(spider: Spider) -> AsyncIterator[Any]: + async def yield_seeds(spider: Spider) -> AsyncIterable[Any]: yield self.prepare_request(spider, Request(url), opts) if self.spidercls: diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 7c9fb29f5..471f72457 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -25,7 +25,7 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import AsyncIterator, Callable, Generator + from collections.abc import AsyncIterable, Callable, Generator from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -103,7 +103,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._seeds: AsyncIterator[Any] | None = None + self._seeds: AsyncIterable[Any] | None = None def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]: from scrapy.core.scheduler import BaseScheduler diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 280c38ccd..0affa4534 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable +from collections.abc import AsyncIterable, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine, iscoroutinefunction from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -371,7 +371,7 @@ class SpiderMiddlewareManager(MiddlewareManager): @inlineCallbacks def process_seeds( self, spider: Spider - ) -> Generator[Deferred[Any], Any, AsyncIterator[Any]]: + ) -> Generator[Deferred[Any], Any, AsyncIterable[Any]]: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_seeds = iter(spider.start_requests()) diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index aef2d7d93..5f2ac1a26 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,7 +15,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import AsyncIterator, Iterable + from collections.abc import AsyncIterable, Iterable from twisted.internet.defer import Deferred @@ -78,7 +78,7 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - async def yield_seeds(self) -> AsyncIterator[Any]: + async def yield_seeds(self) -> AsyncIterable[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. .. versionadded:: VERSION diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 5c84ae5fe..c1d38854b 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import AsyncIterator, Iterable +from collections.abc import AsyncIterable, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,7 +29,7 @@ class InitSpider(Spider): stacklevel=2, ) - async def yield_seeds(self) -> AsyncIterator[Any]: + async def yield_seeds(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 1be23421c..bc004f663 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterable, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,7 +53,7 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - async def yield_seeds(self) -> AsyncIterator[Any]: + async def yield_seeds(self) -> AsyncIterable[Any]: for seed in self.start_requests(): yield seed