More typing for scrapy/utils/defer.py and scrapy/utils/spider.py.

scrapy/spiderloader.py

@@ -1,11 +1,12 @@
 import traceback
 import warnings
 from collections import defaultdict
+from types import ModuleType
 from typing import DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
-from scrapy import Spider
+from scrapy import Request, Spider
 from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
@@ -45,12 +46,12 @@ class SpiderLoader:
                 category=UserWarning,
             )
 
-    def _load_spiders(self, module):
+    def _load_spiders(self, module: ModuleType) -> None:
         for spcls in iter_spider_classes(module):
             self._found[spcls.name].append((module.__name__, spcls.__name__))
             self._spiders[spcls.name] = spcls
 
-    def _load_all_spiders(self):
+    def _load_all_spiders(self) -> None:
         for name in self.spider_modules:
             try:
                 for module in walk_modules(name):
@@ -81,7 +82,7 @@ class SpiderLoader:
         except KeyError:
             raise KeyError(f"Spider not found: {spider_name}")
 
-    def find_by_request(self, request):
+    def find_by_request(self, request: Request) -> List[str]:
         """
         Return the list of spider names that can handle the given request.
         """

scrapy/spiders/__init__.py

@@ -82,7 +82,7 @@ class Spider(object_ref):
         settings.setdict(cls.custom_settings or {}, priority="spider")
 
     @classmethod
-    def handles_request(cls, request):
+    def handles_request(cls, request: Request) -> bool:
         return url_is_from_spider(request.url, cls)
 
     @staticmethod

scrapy/utils/defer.py

@@ -5,11 +5,13 @@ import asyncio
 import inspect
 from asyncio import Future
 from functools import wraps
+from types import CoroutineType
 from typing import (
     Any,
     AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
+    Awaitable,
     Callable,
     Coroutine,
     Dict,
@@ -19,8 +21,10 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
+    overload,
 )
 
 from twisted.internet import defer
@@ -186,9 +190,7 @@ class _AsyncCooperatorAdapter(Iterator):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = cast(
-            Deferred, deferred_from_coro(self.aiterator.__anext__())
-        )
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred:
@@ -297,7 +299,21 @@ async def aiter_errback(
             errback(failure.Failure(), *a, **kw)
 
 
-def deferred_from_coro(o: Any) -> Any:
+_CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
+_T = TypeVar("_T")
+
+
+@overload
+def deferred_from_coro(o: _CT) -> Deferred:
+    ...
+
+
+@overload
+def deferred_from_coro(o: _T) -> _T:
+    ...
+
+
+def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
         return o

scrapy/utils/spider.py

@@ -1,14 +1,33 @@
+from __future__ import annotations
+
 import inspect
 import logging
+from types import ModuleType
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterable,
+    Generator,
+    Iterable,
+    Optional,
+    Type,
+    Union,
+)
 
+from twisted.internet.defer import Deferred
+
+from scrapy import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
 
+if TYPE_CHECKING:
+    from scrapy.spiderloader import SpiderLoader
+
 logger = logging.getLogger(__name__)
 
 
-def iterate_spider_output(result):
+def iterate_spider_output(result: Any) -> Union[Iterable, AsyncIterable, Deferred]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -18,7 +37,7 @@ def iterate_spider_output(result):
     return arg_to_iter(deferred_from_coro(result))
 
 
-def iter_spider_classes(module):
+def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
@@ -37,8 +56,12 @@ def iter_spider_classes(module):
 
 
 def spidercls_for_request(
-    spider_loader, request, default_spidercls=None, log_none=False, log_multiple=False
-):
+    spider_loader: SpiderLoader,
+    request: Request,
+    default_spidercls: Optional[Type[Spider]] = None,
+    log_none: bool = False,
+    log_multiple: bool = False,
+) -> Optional[Type[Spider]]:
     """Return a spider class that handles the given Request.
 
     This will look for the spiders that can handle the given request (using