diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py index af5282553..5ec5ef6db 100644 --- a/scrapy/core/downloader/handlers/__init__.py +++ b/scrapy/core/downloader/handlers/__init__.py @@ -1,5 +1,7 @@ """Download handlers for different schemes""" +from __future__ import annotations + import logging from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast @@ -19,16 +21,16 @@ logger = logging.getLogger(__name__) class DownloadHandlers: - def __init__(self, crawler: "Crawler"): - self._crawler: "Crawler" = crawler - self._schemes: Dict[str, Union[str, Callable]] = ( + def __init__(self, crawler: Crawler): + self._crawler: Crawler = crawler + self._schemes: Dict[str, Union[str, Callable[..., Any]]] = ( {} ) # stores acceptable schemes on instancing self._handlers: Dict[str, Any] = {} # stores instanced handlers for schemes self._notconfigured: Dict[str, str] = {} # remembers failed handlers - handlers: Dict[str, Union[str, Callable]] = without_none_values( + handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values( cast( - Dict[str, Union[str, Callable]], + Dict[str, Union[str, Callable[..., Any]]], crawler.settings.getwithbase("DOWNLOAD_HANDLERS"), ) ) diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index 4cb4454e3..b342ad7a3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -86,7 +86,11 @@ class Slot: class ExecutionEngine: - def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None: + def __init__( + self, + crawler: Crawler, + spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]], + ) -> None: self.crawler: Crawler = crawler self.settings: Settings = crawler.settings self.signals: SignalManager = crawler.signals @@ -102,7 +106,9 @@ class ExecutionEngine: downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"]) self.downloader: Downloader = downloader_cls(crawler) self.scraper = Scraper(crawler) - self._spider_closed_callback: Callable = spider_closed_callback + self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = ( + spider_closed_callback + ) self.start_time: Optional[float] = None def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]: @@ -427,7 +433,7 @@ class ExecutionEngine: dfd = self.slot.close() - def log_failure(msg: str) -> Callable: + def log_failure(msg: str) -> Callable[[Failure], None]: def errback(failure: Failure) -> None: logger.error( msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider} diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 96d0dc515..77149333c 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -266,7 +266,7 @@ class Request(object_ref): return d -def _find_method(obj: Any, func: Callable) -> str: +def _find_method(obj: Any, func: Callable[..., Any]) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py index b678d1def..49f36de2d 100644 --- a/scrapy/utils/misc.py +++ b/scrapy/utils/misc.py @@ -56,7 +56,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]: return [arg] -def load_object(path: Union[str, Callable]) -> Any: +def load_object(path: Union[str, Callable[..., Any]]) -> Any: """Load an object given its absolute object path, and return it. The object can be the import path of a class, function, variable or an @@ -263,7 +263,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]: _generator_callbacks_cache = LocalWeakReferencedCache(limit=128) -def is_generator_with_return_value(callable: Callable) -> bool: +def is_generator_with_return_value(callable: Callable[..., Any]) -> bool: """ Returns True if a callable is a generator function which includes a 'return' statement with a value different than None, False otherwise @@ -300,7 +300,9 @@ def is_generator_with_return_value(callable: Callable) -> bool: return bool(_generator_callbacks_cache[callable]) -def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None: +def warn_on_generator_with_return_value( + spider: Spider, callable: Callable[..., Any] +) -> None: """ Logs a warning if a callable is a generator function and includes a 'return' statement with a value different than None diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py index fc1eb4f69..37a84a350 100644 --- a/scrapy/utils/python.py +++ b/scrapy/utils/python.py @@ -217,7 +217,7 @@ def binary_is_text(data: bytes) -> bool: return all(c not in _BINARYCHARS for c in data) -def get_func_args(func: Callable, stripself: bool = False) -> List[str]: +def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]: """Return the argument name list of a callable object""" if not callable(func): raise TypeError(f"func must be callable, got '{type(func).__name__}'") @@ -247,7 +247,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]: return args -def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: +def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]: """Returns (args, kwargs) tuple for a function >>> import re >>> get_spec(re.match) @@ -285,7 +285,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]: def equal_attributes( - obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]] + obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]] ) -> bool: """Compare two objects attributes""" # not attributes given return False by default