From 4a23215ce34b6fdad4a9229611e21ab7f9e169be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= Date: Wed, 26 Mar 2025 21:13:18 +0100 Subject: [PATCH] Address typing issues --- scrapy/commands/bench.py | 4 ++-- scrapy/core/engine.py | 4 ++-- scrapy/core/spidermw.py | 6 +++--- scrapy/spiders/__init__.py | 4 ++-- scrapy/spiders/init.py | 4 ++-- scrapy/spiders/sitemap.py | 4 ++-- scrapy/utils/reactor.py | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py index 4cd5d640a..96bb1ae84 100644 --- a/scrapy/commands/bench.py +++ b/scrapy/commands/bench.py @@ -13,7 +13,7 @@ from scrapy.linkextractors import LinkExtractor if TYPE_CHECKING: import argparse - from collections.abc import AsyncIterable + from collections.abc import AsyncIterator class Command(ScrapyCommand): @@ -59,7 +59,7 @@ class _BenchSpider(scrapy.Spider): baseurl = "http://localhost:8998" link_extractor = LinkExtractor() - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: qargs = {"total": self.total, "show": self.show} url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}" yield scrapy.Request(url, dont_filter=True) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index b08de6149..d2fe6a813 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -29,7 +29,7 @@ from scrapy.utils.misc import build_from_crawler, load_object from scrapy.utils.reactor import CallLaterOnce if TYPE_CHECKING: - from collections.abc import AsyncIterable, Callable + from collections.abc import AsyncIterator, Callable from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler @@ -101,7 +101,7 @@ class ExecutionEngine: spider_closed_callback ) self.start_time: float | None = None - self._start: AsyncIterable[Any] | None = None + self._start: AsyncIterator[Any] | None = None self._started_request_processing = False downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"]) try: diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index f58d96fd7..f8946b221 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -7,7 +7,7 @@ See documentation in docs/topics/spider-middleware.rst from __future__ import annotations import logging -from collections.abc import AsyncIterable, Callable, Iterable +from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable from inspect import isasyncgenfunction, iscoroutine from itertools import islice from typing import TYPE_CHECKING, Any, TypeVar, Union, cast @@ -388,14 +388,14 @@ class SpiderMiddlewareManager(MiddlewareManager): return dfd2 @deferred_f_from_coro_f - async def process_start(self, spider: Spider) -> AsyncIterable[Any] | None: + async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None: self._check_deprecated_start_requests_use(spider) if self._use_start_requests: sync_start = iter(spider.start_requests()) sync_start = await maybe_deferred_to_future( self._process_chain("process_start_requests", sync_start, spider) ) - start = as_async_generator(sync_start) + start: AsyncIterator[Any] = as_async_generator(sync_start) else: start = spider.start() start = await maybe_deferred_to_future( diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 80932bc9d..acd16ee41 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,7 +15,7 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import AsyncIterable, Iterable + from collections.abc import AsyncIterator, Iterable from twisted.internet.defer import Deferred @@ -78,7 +78,7 @@ class Spider(object_ref): self.settings: BaseSettings = crawler.settings crawler.signals.connect(self.close, signals.spider_closed) - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: """Yield the initial :class:`~scrapy.Request` objects to send. .. versionadded:: VERSION diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py index 3e74640eb..ca5f08e98 100644 --- a/scrapy/spiders/init.py +++ b/scrapy/spiders/init.py @@ -1,7 +1,7 @@ from __future__ import annotations import warnings -from collections.abc import AsyncIterable, Iterable +from collections.abc import AsyncIterator, Iterable from typing import TYPE_CHECKING, Any, cast from scrapy import Request @@ -29,7 +29,7 @@ class InitSpider(Spider): stacklevel=2, ) - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: for item_or_request in self.start_requests(): yield item_or_request diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index 6db10c1ab..2813a32a0 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -4,7 +4,7 @@ import logging import re # Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation -from collections.abc import AsyncIterable, Iterable, Sequence # noqa: TC003 +from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003 from typing import TYPE_CHECKING, Any, cast from scrapy.http import Request, Response, XmlResponse @@ -53,7 +53,7 @@ class SitemapSpider(Spider): self._cbs.append((regex(r), c)) self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow] - async def start(self) -> AsyncIterable[Any]: + async def start(self) -> AsyncIterator[Any]: for item_or_request in self.start_requests(): yield item_or_request diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 184a95820..0002aef1a 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]): self._a: tuple[Any, ...] = a self._kw: dict[str, Any] = kw self._call: DelayedCall | None = None - self._deferreds = [] + self._deferreds: list[Deferred] = [] def schedule(self, delay: float = 0) -> None: from twisted.internet import reactor