From e47110f9a5a16f0628e53e16b9cb5f6a4f9721d3 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 26 Jun 2024 13:01:43 +0500 Subject: [PATCH] Add parameteres to most Deferred instances. (#6414) --- scrapy/commands/parse.py | 6 +- scrapy/core/downloader/handlers/http10.py | 3 +- scrapy/core/http2/agent.py | 26 +++++---- scrapy/core/http2/protocol.py | 18 ++++-- scrapy/core/http2/stream.py | 6 +- scrapy/core/scraper.py | 6 +- scrapy/core/spidermw.py | 7 ++- scrapy/crawler.py | 33 +++++++---- scrapy/downloadermiddlewares/robotstxt.py | 26 ++++++--- scrapy/dupefilters.py | 4 +- scrapy/extensions/feedexport.py | 16 ++--- scrapy/extensions/statsmailer.py | 2 +- scrapy/mail.py | 12 ++-- scrapy/middleware.py | 27 ++++++--- scrapy/shell.py | 10 +++- scrapy/signalmanager.py | 4 +- scrapy/spiders/__init__.py | 10 ++-- scrapy/utils/defer.py | 71 ++++++++++++----------- scrapy/utils/signal.py | 24 ++++---- scrapy/utils/test.py | 3 +- scrapy/utils/testproc.py | 4 +- 21 files changed, 190 insertions(+), 128 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index e6c5e2a47..1265aa38e 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -153,7 +153,7 @@ class Command(BaseRunSpiderCommand): @overload def iterate_spider_output(self, result: _T) -> Iterable[Any]: ... - def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]: + def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]: if inspect.isasyncgen(result): d = deferred_from_coro( collect_asyncgen(aiter_errback(result, self.handle_exception)) @@ -233,7 +233,7 @@ class Command(BaseRunSpiderCommand): response: Response, callback: Callable, cb_kwargs: Optional[Dict[str, Any]] = None, - ) -> Deferred: + ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs)) return d @@ -345,7 +345,7 @@ class Command(BaseRunSpiderCommand): def prepare_request( self, spider: Spider, request: Request, opts: argparse.Namespace ) -> Request: - def callback(response: Response, **cb_kwargs: Any) -> Deferred: + def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]: # memorize first request if not self.first_response: self.first_response = response diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py index 98f62efcf..8d7b0635c 100644 --- a/scrapy/core/downloader/handlers/http10.py +++ b/scrapy/core/downloader/handlers/http10.py @@ -10,6 +10,7 @@ from scrapy.utils.python import to_unicode if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.internet.interfaces import IConnector # typing.Self requires Python 3.11 from typing_extensions import Self @@ -45,7 +46,7 @@ class HTTP10DownloadHandler: self._connect(factory) return factory.deferred - def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred: + def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector: from twisted.internet import reactor host, port = to_unicode(factory.host), factory.port diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py index d291a5b8a..640fb7129 100644 --- a/scrapy/core/http2/agent.py +++ b/scrapy/core/http2/agent.py @@ -21,7 +21,7 @@ if TYPE_CHECKING: from twisted.internet.base import ReactorBase from twisted.internet.endpoints import HostnameEndpoint - from scrapy.http.request import Request + from scrapy.http import Request, Response from scrapy.settings import Settings from scrapy.spiders import Spider @@ -39,16 +39,18 @@ class H2ConnectionPool: self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {} # Save all requests that arrive before the connection is established - self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {} + self._pending_requests: Dict[ + ConnectionKeyT, Deque[Deferred[H2ClientProtocol]] + ] = {} def get_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: if key in self._pending_requests: # Received a request while connecting to remote # Create a deferred which will fire with the H2ClientProtocol # instance - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -63,17 +65,17 @@ class H2ConnectionPool: def _new_connection( self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint - ) -> Deferred: + ) -> Deferred[H2ClientProtocol]: self._pending_requests[key] = deque() - conn_lost_deferred: Deferred = Deferred() + conn_lost_deferred: Deferred[List[BaseException]] = Deferred() conn_lost_deferred.addCallback(self._remove_connection, key) factory = H2ClientFactory(uri, self.settings, conn_lost_deferred) conn_d = endpoint.connect(factory) conn_d.addCallback(self.put_connection, key) - d: Deferred = Deferred() + d: Deferred[H2ClientProtocol] = Deferred() self._pending_requests[key].append(d) return d @@ -141,7 +143,7 @@ class H2Agent: """ return uri.scheme, uri.host, uri.port - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: uri = URI.fromBytes(bytes(request.url, encoding="utf-8")) try: endpoint = self.get_endpoint(uri) @@ -149,9 +151,11 @@ class H2Agent: return defer.fail(Failure()) key = self.get_key(uri) - d = self._pool.get_connection(key, uri, endpoint) - d.addCallback(lambda conn: conn.request(request, spider)) - return d + d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint) + d2: Deferred[Response] = d.addCallback( + lambda conn: conn.request(request, spider) + ) + return d2 class ScrapyProxyH2Agent(H2Agent): diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py index a6809102b..8aebbaab4 100644 --- a/scrapy/core/http2/protocol.py +++ b/scrapy/core/http2/protocol.py @@ -33,7 +33,7 @@ from twisted.protocols.policies import TimeoutMixin from zope.interface import implementer from scrapy.core.http2.stream import Stream, StreamCloseReason -from scrapy.http import Request +from scrapy.http import Request, Response if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address @@ -88,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): IDLE_TIMEOUT = 240 def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: """ Arguments: @@ -99,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin): conn_lost_deferred -- Deferred fires with the reason: Failure to notify that connection was lost """ - self._conn_lost_deferred = conn_lost_deferred + self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred config = H2Configuration(client_side=True, header_encoding="utf-8") self.conn = H2Connection(config=config) @@ -215,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin): data = self.conn.data_to_send() self.transport.write(data) - def request(self, request: Request, spider: Spider) -> Deferred: + def request(self, request: Request, spider: Spider) -> Deferred[Response]: if not isinstance(request, Request): raise TypeError( f"Expected scrapy.http.Request, received {request.__class__.__qualname__}" ) stream = self._new_stream(request, spider) - d = stream.get_response() + d: Deferred[Response] = stream.get_response() # Add the stream to the request pool self._pending_request_stream_pool.append(stream) @@ -436,7 +439,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin): @implementer(IProtocolNegotiationFactory) class H2ClientFactory(Factory): def __init__( - self, uri: URI, settings: Settings, conn_lost_deferred: Deferred + self, + uri: URI, + settings: Settings, + conn_lost_deferred: Deferred[List[BaseException]], ) -> None: self.uri = uri self.settings = settings diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py index a02fbb328..d8b5cc8eb 100644 --- a/scrapy/core/http2/stream.py +++ b/scrapy/core/http2/stream.py @@ -20,7 +20,7 @@ if TYPE_CHECKING: from hpack import HeaderTuple from scrapy.core.http2.protocol import H2ClientProtocol - from scrapy.http import Request + from scrapy.http import Request, Response logger = logging.getLogger(__name__) @@ -154,7 +154,7 @@ class Stream: else: self.close(StreamCloseReason.CANCELLED) - self._deferred_response: Deferred = Deferred(_cancel) + self._deferred_response: Deferred[Response] = Deferred(_cancel) def __repr__(self) -> str: return f"Stream(id={self.stream_id!r})" @@ -180,7 +180,7 @@ class Stream: and not self.metadata["reached_warnsize"] ) - def get_response(self) -> Deferred: + def get_response(self) -> Deferred[Response]: """Simply return a Deferred which fires when response from the asynchronous request is available """ diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py index 8a9e8f687..a7d65e1e3 100644 --- a/scrapy/core/scraper.py +++ b/scrapy/core/scraper.py @@ -13,6 +13,7 @@ from typing import ( Generator, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -34,7 +35,6 @@ from scrapy.logformatter import LogFormatter from scrapy.pipelines import ItemPipelineManager from scrapy.signalmanager import SignalManager from scrapy.utils.defer import ( - DeferredListResultListT, aiter_errback, defer_fail, defer_succeed, @@ -54,7 +54,7 @@ logger = logging.getLogger(__name__) _T = TypeVar("_T") -_ParallelResult = DeferredListResultListT[Iterator[Any]] +_ParallelResult = List[Tuple[bool, Iterator[Any]]] if TYPE_CHECKING: # parameterized Deferreds require Twisted 21.7.0 @@ -374,7 +374,7 @@ class Scraper: def _itemproc_finished( self, output: Any, item: Any, response: Response, spider: Spider - ) -> Deferred: + ) -> Deferred[Any]: """ItemProcessor finished for the given ``item`` and returned ``output``""" assert self.slot is not None # typing self.slot.itemproc_size -= 1 diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py index 37a666605..c9feac29c 100644 --- a/scrapy/core/spidermw.py +++ b/scrapy/core/spidermw.py @@ -302,7 +302,10 @@ class SpiderMiddlewareManager(MiddlewareManager): recovered = MutableChain() result = self._evaluate_iterable(response, spider, result, 0, recovered) result = await maybe_deferred_to_future( - self._process_spider_output(response, spider, result) + cast( + "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]", + self._process_spider_output(response, spider, result), + ) ) if isinstance(result, AsyncIterable): return MutableAsyncChain(result, recovered) @@ -339,7 +342,7 @@ class SpiderMiddlewareManager(MiddlewareManager): def process_start_requests( self, start_requests: Iterable[Request], spider: Spider - ) -> Deferred: + ) -> Deferred[Iterable[Request]]: return self._process_chain("process_start_requests", start_requests, spider) # This method is only needed until _async compatibility methods are removed. diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 4fe5987a7..877ea5928 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -4,7 +4,18 @@ import logging import pprint import signal import warnings -from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Optional, + Set, + Type, + TypeVar, + Union, + cast, +) from twisted.internet.defer import ( Deferred, @@ -54,6 +65,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class Crawler: def __init__( @@ -140,7 +153,7 @@ class Crawler: ) @inlineCallbacks - def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]: + def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]: if self.crawling: raise RuntimeError("Crawling already taking place") if self._started: @@ -172,7 +185,7 @@ class Crawler: return ExecutionEngine(self, lambda _: self.stop()) @inlineCallbacks - def stop(self) -> Generator[Deferred, Any, None]: + def stop(self) -> Generator[Deferred[Any], Any, None]: """Starts a graceful stop of the crawler and returns a deferred that is fired when the crawler is stopped.""" if self.crawling: @@ -256,7 +269,7 @@ class CrawlerRunner: self.settings = settings self.spider_loader = self._get_spider_loader(settings) self._crawlers: Set[Crawler] = set() - self._active: Set[Deferred] = set() + self._active: Set[Deferred[None]] = set() self.bootstrap_failed = False def crawl( @@ -264,7 +277,7 @@ class CrawlerRunner: crawler_or_spidercls: Union[Type[Spider], str, Crawler], *args: Any, **kwargs: Any, - ) -> Deferred: + ) -> Deferred[None]: """ Run a crawler with the provided arguments. @@ -294,12 +307,12 @@ class CrawlerRunner: crawler = self.create_crawler(crawler_or_spidercls) return self._crawl(crawler, *args, **kwargs) - def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred: + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]: self.crawlers.add(crawler) d = crawler.crawl(*args, **kwargs) self._active.add(d) - def _done(result: Any) -> Any: + def _done(result: _T) -> _T: self.crawlers.discard(crawler) self._active.discard(d) self.bootstrap_failed |= not getattr(crawler, "spider", None) @@ -335,7 +348,7 @@ class CrawlerRunner: # temporary cast until self.spider_loader is typed return Crawler(cast(Type[Spider], spidercls), self.settings) - def stop(self) -> Deferred: + def stop(self) -> Deferred[Any]: """ Stops simultaneously all the crawling jobs taking place. @@ -344,7 +357,7 @@ class CrawlerRunner: return DeferredList([c.stop() for c in list(self.crawlers)]) @inlineCallbacks - def join(self) -> Generator[Deferred, Any, None]: + def join(self) -> Generator[Deferred[Any], Any, None]: """ join() @@ -460,7 +473,7 @@ class CrawlerProcess(CrawlerRunner): ) reactor.run(installSignalHandlers=install_signal_handlers) # blocking call - def _graceful_stop_reactor(self) -> Deferred: + def _graceful_stop_reactor(self) -> Deferred[Any]: d = self.stop() d.addBoth(self._stop_reactor) return d diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py index 70393576b..73757162f 100644 --- a/scrapy/downloadermiddlewares/robotstxt.py +++ b/scrapy/downloadermiddlewares/robotstxt.py @@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting. from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union from twisted.internet.defer import Deferred, maybeDeferred @@ -31,6 +31,8 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) +_T = TypeVar("_T") + class RobotsTxtMiddleware: DOWNLOAD_PRIORITY: int = 1000 @@ -43,7 +45,9 @@ class RobotsTxtMiddleware: "ROBOTSTXT_USER_AGENT", None ) self.crawler: Crawler = crawler - self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {} + self._parsers: Dict[ + str, Union[RobotParser, Deferred[Optional[RobotParser]], None] + ] = {} self._parserimpl: RobotParser = load_object( crawler.settings.get("ROBOTSTXT_PARSER") ) @@ -55,14 +59,18 @@ class RobotsTxtMiddleware: def from_crawler(cls, crawler: Crawler) -> Self: return cls(crawler) - def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]: + def process_request( + self, request: Request, spider: Spider + ) -> Optional[Deferred[None]]: if request.meta.get("dont_obey_robotstxt"): return None if request.url.startswith("data:") or request.url.startswith("file:"): return None - d: Deferred = maybeDeferred(self.robot_parser, request, spider) - d.addCallback(self.process_request_2, request, spider) - return d + d: Deferred[Optional[RobotParser]] = maybeDeferred( + self.robot_parser, request, spider # type: ignore[arg-type] + ) + d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider) + return d2 def process_request_2( self, rp: Optional[RobotParser], request: Request, spider: Spider @@ -86,7 +94,7 @@ class RobotsTxtMiddleware: def robot_parser( self, request: Request, spider: Spider - ) -> Union[RobotParser, Deferred, None]: + ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]: url = urlparse_cached(request) netloc = url.netloc @@ -109,9 +117,9 @@ class RobotsTxtMiddleware: parser = self._parsers[netloc] if isinstance(parser, Deferred): - d: Deferred = Deferred() + d: Deferred[Optional[RobotParser]] = Deferred() - def cb(result: Any) -> Any: + def cb(result: Optional[RobotParser]) -> Optional[RobotParser]: d.callback(result) return result diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py index ffaf783a7..40ea48510 100644 --- a/scrapy/dupefilters.py +++ b/scrapy/dupefilters.py @@ -31,10 +31,10 @@ class BaseDupeFilter: def request_seen(self, request: Request) -> bool: return False - def open(self) -> Optional[Deferred]: + def open(self) -> Optional[Deferred[None]]: pass - def close(self, reason: str) -> Optional[Deferred]: + def close(self, reason: str) -> Optional[Deferred[None]]: pass def log(self, request: Request, spider: Spider) -> None: diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py index 43c2d2815..0d7f5bfd4 100644 --- a/scrapy/extensions/feedexport.py +++ b/scrapy/extensions/feedexport.py @@ -149,7 +149,7 @@ class FeedStorageProtocol(Protocol): """Open the storage for the given spider. It must return a file-like object that will be used for the exporters""" - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: """Store the given file stream""" @@ -162,7 +162,7 @@ class BlockingFeedStorage: return NamedTemporaryFile(prefix="feed-", dir=path) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: return deferToThread(self._store_in_thread, file) def _store_in_thread(self, file: IO[bytes]) -> None: @@ -192,7 +192,7 @@ class StdoutFeedStorage: def open(self, spider: Spider) -> IO[bytes]: return self._stdout - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: pass @@ -211,7 +211,7 @@ class FileFeedStorage: dirname.mkdir(parents=True) return Path(self.path).open(self.write_mode) - def store(self, file: IO[bytes]) -> Optional[Deferred]: + def store(self, file: IO[bytes]) -> Optional[Deferred[None]]: file.close() return None @@ -483,7 +483,7 @@ _FeedSlot = create_deprecated_class( class FeedExporter: - _pending_deferreds: List[Deferred] = [] + _pending_deferreds: List[Deferred[None]] = [] @classmethod def from_crawler(cls, crawler: Crawler) -> Self: @@ -570,7 +570,7 @@ class FeedExporter: self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed) ) - def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]: + def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]: def get_file(slot_: FeedSlot) -> IO[bytes]: assert slot_.file if isinstance(slot_.file, PostProcessingManager): @@ -590,7 +590,7 @@ class FeedExporter: return None logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}" - d: Deferred = maybeDeferred(slot.storage.store, get_file(slot)) + d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot)) # type: ignore[arg-type] d.addCallback( self._handle_store_success, logmsg, spider, type(slot.storage).__name__ @@ -621,7 +621,7 @@ class FeedExporter: self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}") def _handle_store_success( - self, f: Failure, logmsg: str, spider: Spider, slot_type: str + self, result: Any, logmsg: str, spider: Spider, slot_type: str ) -> None: logger.info("Stored %s", logmsg, extra={"spider": spider}) assert self.crawler.stats diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py index e43de6f5c..cad607514 100644 --- a/scrapy/extensions/statsmailer.py +++ b/scrapy/extensions/statsmailer.py @@ -39,7 +39,7 @@ class StatsMailer: crawler.signals.connect(o.spider_closed, signal=signals.spider_closed) return o - def spider_closed(self, spider: Spider) -> Optional[Deferred]: + def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]: spider_stats = self.stats.get_stats(spider) body = "Global stats\n\n" body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items()) diff --git a/scrapy/mail.py b/scrapy/mail.py index 3ea20e831..c020732f9 100644 --- a/scrapy/mail.py +++ b/scrapy/mail.py @@ -103,7 +103,7 @@ class MailSender: mimetype: str = "text/plain", charset: Optional[str] = None, _callback: Optional[Callable[..., None]] = None, - ) -> Optional[Deferred]: + ) -> Optional[Deferred[None]]: from twisted.internet import reactor msg: MIMEBase @@ -155,7 +155,9 @@ class MailSender: ) return None - dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8")) + dfd: Deferred[Any] = self._sendmail( + rcpts, msg.as_string().encode(charset or "utf-8") + ) dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs)) dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs)) reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd) @@ -198,11 +200,11 @@ class MailSender: ) return failure - def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred: + def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]: from twisted.internet import reactor msg_io = BytesIO(msg) - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() factory = self._create_sender_factory(to_addrs, msg_io, d) @@ -216,7 +218,7 @@ class MailSender: return d def _create_sender_factory( - self, to_addrs: List[str], msg: IO[bytes], d: Deferred + self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any] ) -> ESMTPSenderFactory: from twisted.mail.smtp import ESMTPSenderFactory diff --git a/scrapy/middleware.py b/scrapy/middleware.py index ea5488ba1..2296db90e 100644 --- a/scrapy/middleware.py +++ b/scrapy/middleware.py @@ -13,6 +13,7 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, ) @@ -24,16 +25,22 @@ from scrapy.utils.misc import build_from_crawler, build_from_settings, load_obje if TYPE_CHECKING: from twisted.internet.defer import Deferred + # typing.Concatenate and typing.ParamSpec require Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Self + from typing_extensions import Concatenate, ParamSpec, Self from scrapy import Spider from scrapy.crawler import Crawler from scrapy.settings import Settings + _P = ParamSpec("_P") + logger = logging.getLogger(__name__) +_T = TypeVar("_T") +_T2 = TypeVar("_T2") + class MiddlewareManager: """Base class for implementing middleware managers""" @@ -98,16 +105,22 @@ class MiddlewareManager: if hasattr(mw, "close_spider"): self.methods["close_spider"].appendleft(mw.close_spider) - def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_parallel( + self, methodname: str, obj: _T, *args: Any + ) -> Deferred[List[_T2]]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname] + ) return process_parallel(methods, obj, *args) - def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred: - methods = cast(Iterable[Callable], self.methods[methodname]) + def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]: + methods = cast( + "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname] + ) return process_chain(methods, obj, *args) - def open_spider(self, spider: Spider) -> Deferred: + def open_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("open_spider", spider) - def close_spider(self, spider: Spider) -> Deferred: + def close_spider(self, spider: Spider) -> Deferred[List[None]]: return self._process_parallel("close_spider", spider) diff --git a/scrapy/shell.py b/scrapy/shell.py index 2c22d3d8f..b7e46274f 100644 --- a/scrapy/shell.py +++ b/scrapy/shell.py @@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst """ +from __future__ import annotations + import os import signal from typing import Any, Callable, Dict, Optional, Tuple, Union @@ -92,7 +94,9 @@ class Shell: self.vars, shells=shells, banner=self.vars.pop("banner", "") ) - def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred: + def _schedule( + self, request: Request, spider: Optional[Spider] + ) -> defer.Deferred[Any]: if is_asyncio_reactor_installed(): # set the asyncio event loop for the current thread event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"] @@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None: signal.signal(signal.SIGINT, sigint_handler) -def _request_deferred(request: Request) -> defer.Deferred: +def _request_deferred(request: Request) -> defer.Deferred[Any]: """Wrap a request inside a Deferred. This function is harmful, do not use it until you know what you are doing. @@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred: request.errback = request_errback return result - d: defer.Deferred = defer.Deferred() + d: defer.Deferred[Any] = defer.Deferred() d.addBoth(_restore_callbacks) if request.callback: d.addCallback(request.callback) diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py index 3d37b8235..b2c6dea5d 100644 --- a/scrapy/signalmanager.py +++ b/scrapy/signalmanager.py @@ -50,7 +50,9 @@ class SignalManager: kwargs.setdefault("sender", self.sender) return _signal.send_catch_log(signal, **kwargs) - def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred: + def send_catch_log_deferred( + self, signal: Any, **kwargs: Any + ) -> Deferred[List[Tuple[Any, Any]]]: """ Like :meth:`send_catch_log` but supports returning :class:`~twisted.internet.defer.Deferred` objects from signal handlers. diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 7b43f04f2..f0b0c0988 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst from __future__ import annotations import logging -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast - -from twisted.internet.defer import Deferred +from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast from scrapy import signals from scrapy.http import Request, Response @@ -19,6 +17,8 @@ from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: from collections.abc import Callable + from twisted.internet.defer import Deferred + # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 from typing_extensions import Concatenate, Self @@ -105,10 +105,10 @@ class Spider(object_ref): return url_is_from_spider(request.url, cls) @staticmethod - def close(spider: Spider, reason: str) -> Union[Deferred, None]: + def close(spider: Spider, reason: str) -> Optional[Deferred[None]]: closed = getattr(spider, "closed", None) if callable(closed): - return cast(Union[Deferred, None], closed(reason)) + return cast("Optional[Deferred[None]]", closed(reason)) return None def __repr__(self) -> str: diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 1d578e8a3..c5763a06c 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -49,13 +49,8 @@ if TYPE_CHECKING: _T = TypeVar("_T") _T2 = TypeVar("_T2") -# copied from twisted.internet.defer -_SelfResultT = TypeVar("_SelfResultT") -_DeferredListResultItemT = Tuple[bool, _SelfResultT] -DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]] - -def defer_fail(_failure: Failure) -> Deferred: +def defer_fail(_failure: Failure) -> Deferred[Any]: """Same as twisted.internet.defer.fail but delay calling errback until next reactor loop @@ -64,7 +59,7 @@ def defer_fail(_failure: Failure) -> Deferred: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() reactor.callLater(0.1, d.errback, _failure) return d @@ -78,12 +73,12 @@ def defer_succeed(result: _T) -> Deferred[_T]: """ from twisted.internet import reactor - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() reactor.callLater(0.1, d.callback, result) return d -def defer_result(result: Any) -> Deferred: +def defer_result(result: Any) -> Deferred[Any]: if isinstance(result, Deferred): return result if isinstance(result, failure.Failure): @@ -138,14 +133,14 @@ def parallel( callable: Callable[Concatenate[_T, _P], _T2], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: +) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]: """Execute a callable over the objects in the given iterable, in parallel, using no more than ``count`` concurrent calls. Taken from: https://jcalderone.livejournal.com/24285.html """ coop = Cooperator() - work = (callable(elem, *args, **named) for elem in iterable) + work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable) return DeferredList([coop.coiterate(work) for _ in range(count)]) @@ -198,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def __init__( self, aiterable: AsyncIterable[_T], - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *callable_args: _P.args, **callable_kwargs: _P.kwargs, ): self.aiterator: AsyncIterator[_T] = aiterable.__aiter__() - self.callable: Callable[Concatenate[_T, _P], _T2] = callable + self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable self.callable_args: Tuple[Any, ...] = callable_args self.callable_kwargs: Dict[str, Any] = callable_kwargs self.finished: bool = False - self.waiting_deferreds: List[Deferred] = [] + self.waiting_deferreds: List[Deferred[Any]] = [] self.anext_deferred: Optional[Deferred[_T]] = None def _callback(self, result: _T) -> None: @@ -241,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) - def __next__(self) -> Deferred: + def __next__(self) -> Deferred[Any]: # This puts a new Deferred into self.waiting_deferreds and returns it. # It also calls __anext__() if needed. if self.finished: raise StopIteration - d: Deferred = Deferred() + d: Deferred[Any] = Deferred() self.waiting_deferreds.append(d) if not self.anext_deferred: self._call_anext() @@ -256,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]): def parallel_async( async_iterable: AsyncIterable[_T], count: int, - callable: Callable[Concatenate[_T, _P], _T2], + callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]], *args: _P.args, **named: _P.kwargs, -) -> Deferred[DeferredListResultListT[Iterator[_T2]]]: - """Like parallel but for async iterators""" +) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]: + """Like ``parallel`` but for async iterators""" coop = Cooperator() - work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named) - dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)]) + work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter( + async_iterable, callable, *args, **named + ) + dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList( + [coop.coiterate(work) for _ in range(count)] + ) return dl def process_chain( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[_T]: """Return a Deferred built by chaining the given callbacks""" - d: Deferred = Deferred() + d: Deferred[_T] = Deferred() for x in callbacks: d.addCallback(x, *a, **kw) d.callback(input) @@ -307,19 +306,21 @@ def process_chain_both( def process_parallel( - callbacks: Iterable[Callable[Concatenate[_T, _P], Any]], - input: Any, + callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]], + input: _T, *a: _P.args, **kw: _P.kwargs, -) -> Deferred: +) -> Deferred[List[_T2]]: """Return a Deferred with the output of all successful calls to the given callbacks """ dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks] - d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True) - d.addCallback(lambda r: [x[1] for x in r]) - d.addErrback(lambda f: f.value.subFailure) - return d + d: Deferred[List[Tuple[bool, _T2]]] = DeferredList( + dfds, fireOnOneErrback=True, consumeErrors=True + ) + d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r]) + d2.addErrback(lambda f: f.value.subFailure) + return d2 def iter_errback( @@ -404,7 +405,7 @@ def deferred_f_from_coro_f( def maybeDeferred_coro( f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs -) -> Deferred: +) -> Deferred[Any]: """Copy of defer.maybeDeferred that also converts coroutines to Deferreds.""" try: result = f(*args, **kw) @@ -420,7 +421,7 @@ def maybeDeferred_coro( return defer.succeed(result) -def deferred_to_future(d: Deferred) -> Future: +def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """ .. versionadded:: 2.6.0 @@ -442,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future: return d.asFuture(_get_asyncio_event_loop()) -def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]: +def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]: """ .. versionadded:: 2.6.0 diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py index bb6d807ee..4310c1d56 100644 --- a/scrapy/utils/signal.py +++ b/scrapy/utils/signal.py @@ -1,5 +1,7 @@ """Helper functions for working with signals""" +from __future__ import annotations + import collections.abc import logging from typing import Any as TypingAny @@ -27,7 +29,7 @@ def send_catch_log( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny + **named: TypingAny, ) -> List[Tuple[TypingAny, TypingAny]]: """Like pydispatcher.robust.sendRobust but it also logs errors and returns Failures instead of exceptions. @@ -73,8 +75,8 @@ def send_catch_log_deferred( signal: TypingAny = Any, sender: TypingAny = Anonymous, *arguments: TypingAny, - **named: TypingAny -) -> Deferred: + **named: TypingAny, +) -> Deferred[List[Tuple[TypingAny, TypingAny]]]: """Like send_catch_log but supports returning deferreds on signal handlers. Returns a deferred that gets fired once all signal handlers deferreds were fired. @@ -92,23 +94,25 @@ def send_catch_log_deferred( dont_log = named.pop("dont_log", None) spider = named.get("spider", None) - dfds = [] + dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = [] for receiver in liveReceivers(getAllReceivers(sender, signal)): - d = maybeDeferred_coro( + d: Deferred[TypingAny] = maybeDeferred_coro( robustApply, receiver, signal=signal, sender=sender, *arguments, **named ) d.addErrback(logerror, receiver) # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html - d.addBoth( + d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth( lambda result: ( receiver, # pylint: disable=cell-var-from-loop # noqa: B023 result, ) ) - dfds.append(d) - d = DeferredList(dfds) - d.addCallback(lambda out: [x[1] for x in out]) - return d + dfds.append(d2) + dl = DeferredList(dfds) + d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback( + lambda out: [x[1] for x in out] + ) + return d3 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None: diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index fe2bfa042..30f235592 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -30,6 +30,7 @@ from scrapy.utils.boto import is_botocore_available if TYPE_CHECKING: from twisted.internet.defer import Deferred + from twisted.web.client import Response as TxResponse _T = TypeVar("_T") @@ -159,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]: return (client_mock, bucket_mock, blob_mock) -def get_web_client_agent_req(url: str) -> Deferred: +def get_web_client_agent_req(url: str) -> Deferred[TxResponse]: from twisted.internet import reactor from twisted.web.client import Agent # imports twisted.internet.reactor diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py index 8882bfc5f..bb269a9f5 100644 --- a/scrapy/utils/testproc.py +++ b/scrapy/utils/testproc.py @@ -22,7 +22,7 @@ class ProcessTest: args: Iterable[str], check_code: bool = True, settings: Optional[str] = None, - ) -> Deferred: + ) -> Deferred[TestProcessProtocol]: from twisted.internet import reactor env = os.environ.copy() @@ -49,7 +49,7 @@ class ProcessTest: class TestProcessProtocol(ProcessProtocol): def __init__(self) -> None: - self.deferred: Deferred = Deferred() + self.deferred: Deferred[TestProcessProtocol] = Deferred() self.out: bytes = b"" self.err: bytes = b"" self.exitcode: Optional[int] = None