From c43798cb9bee99ccf96047f3dfcc6debb65973d4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 2 Aug 2023 22:15:37 +0400 Subject: [PATCH] More typing for scrapy/utils/defer.py and scrapy/utils/spider.py. --- scrapy/spiderloader.py | 9 +++++---- scrapy/spiders/__init__.py | 2 +- scrapy/utils/defer.py | 24 ++++++++++++++++++++---- scrapy/utils/spider.py | 31 +++++++++++++++++++++++++++---- 4 files changed, 53 insertions(+), 13 deletions(-) diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py index ea5a26e77..13d6f9f87 100644 --- a/scrapy/spiderloader.py +++ b/scrapy/spiderloader.py @@ -1,11 +1,12 @@ import traceback import warnings from collections import defaultdict +from types import ModuleType from typing import DefaultDict, Dict, List, Tuple, Type from zope.interface import implementer -from scrapy import Spider +from scrapy import Request, Spider from scrapy.interfaces import ISpiderLoader from scrapy.settings import BaseSettings from scrapy.utils.misc import walk_modules @@ -45,12 +46,12 @@ class SpiderLoader: category=UserWarning, ) - def _load_spiders(self, module): + def _load_spiders(self, module: ModuleType) -> None: for spcls in iter_spider_classes(module): self._found[spcls.name].append((module.__name__, spcls.__name__)) self._spiders[spcls.name] = spcls - def _load_all_spiders(self): + def _load_all_spiders(self) -> None: for name in self.spider_modules: try: for module in walk_modules(name): @@ -81,7 +82,7 @@ class SpiderLoader: except KeyError: raise KeyError(f"Spider not found: {spider_name}") - def find_by_request(self, request): + def find_by_request(self, request: Request) -> List[str]: """ Return the list of spider names that can handle the given request. """ diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index 3502f8b27..388439f4f 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -82,7 +82,7 @@ class Spider(object_ref): settings.setdict(cls.custom_settings or {}, priority="spider") @classmethod - def handles_request(cls, request): + def handles_request(cls, request: Request) -> bool: return url_is_from_spider(request.url, cls) @staticmethod diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 03f026ce9..bf3c5ef5b 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -5,11 +5,13 @@ import asyncio import inspect from asyncio import Future from functools import wraps +from types import CoroutineType from typing import ( Any, AsyncGenerator, AsyncIterable, AsyncIterator, + Awaitable, Callable, Coroutine, Dict, @@ -19,8 +21,10 @@ from typing import ( List, Optional, Tuple, + TypeVar, Union, cast, + overload, ) from twisted.internet import defer @@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator): def _call_anext(self) -> None: # This starts waiting for the next result from aiterator. # If aiterator is exhausted, _errback will be called. - self.anext_deferred = cast( - Deferred, deferred_from_coro(self.aiterator.__anext__()) - ) + self.anext_deferred = deferred_from_coro(self.aiterator.__anext__()) self.anext_deferred.addCallbacks(self._callback, self._errback) def __next__(self) -> Deferred: @@ -297,7 +299,21 @@ async def aiter_errback( errback(failure.Failure(), *a, **kw) -def deferred_from_coro(o: Any) -> Any: +_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future]) +_T = TypeVar("_T") + + +@overload +def deferred_from_coro(o: _CT) -> Deferred: + ... + + +@overload +def deferred_from_coro(o: _T) -> _T: + ... + + +def deferred_from_coro(o: _T) -> Union[Deferred, _T]: """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" if isinstance(o, Deferred): return o diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 86449eeb2..3228eda49 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -1,14 +1,33 @@ +from __future__ import annotations + import inspect import logging +from types import ModuleType +from typing import ( + TYPE_CHECKING, + Any, + AsyncIterable, + Generator, + Iterable, + Optional, + Type, + Union, +) +from twisted.internet.defer import Deferred + +from scrapy import Request from scrapy.spiders import Spider from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter +if TYPE_CHECKING: + from scrapy.spiderloader import SpiderLoader + logger = logging.getLogger(__name__) -def iterate_spider_output(result): +def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]: if inspect.isasyncgen(result): return result if inspect.iscoroutine(result): @@ -18,7 +37,7 @@ def iterate_spider_output(result): return arg_to_iter(deferred_from_coro(result)) -def iter_spider_classes(module): +def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]: """Return an iterator over all spider classes defined in the given module that can be instantiated (i.e. which have name) """ @@ -37,8 +56,12 @@ def iter_spider_classes(module): def spidercls_for_request( - spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False -): + spider_loader: SpiderLoader, + request: Request, + default_spidercls: Optional[Type[Spider]] = None, + log_none: bool = False, + log_multiple: bool = False, +) -> Optional[Type[Spider]]: """Return a spider class that handles the given Request. This will look for the spiders that can handle the given request (using