From 558b1d11d2f1e3063aba59d444fdb93d42a9ddb9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 1 Jul 2024 12:30:49 +0500 Subject: [PATCH] Use CallbackT for Request.callback. (#6422) --- scrapy/commands/parse.py | 16 ++++++++-------- scrapy/contracts/__init__.py | 7 ++++--- scrapy/http/request/__init__.py | 20 ++++++++++++++------ scrapy/http/response/__init__.py | 12 +++++++----- scrapy/http/response/text.py | 12 +++++++----- scrapy/spiders/__init__.py | 8 ++------ scrapy/spiders/crawl.py | 15 +++++++++------ scrapy/spiders/sitemap.py | 8 ++++---- 8 files changed, 55 insertions(+), 43 deletions(-) diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py index 1265aa38e..fbd200d88 100644 --- a/scrapy/commands/parse.py +++ b/scrapy/commands/parse.py @@ -9,7 +9,6 @@ from typing import ( TYPE_CHECKING, Any, AsyncGenerator, - Callable, Coroutine, Dict, Iterable, @@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request if TYPE_CHECKING: from twisted.python.failure import Failure + from scrapy.http.request import CallbackT from scrapy.spiders import Spider @@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand): opts: argparse.Namespace, depth: int, spider: Spider, - callback: Callable, - ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]: + callback: CallbackT, + ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]: items, requests = [], [] for x in spider_output: if is_item(x): @@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand): def run_callback( self, response: Response, - callback: Callable, + callback: CallbackT, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> Deferred[Any]: cb_kwargs = cb_kwargs or {} @@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand): def get_callback_from_rules( self, spider: Spider, response: Response - ) -> Union[Callable, str, None]: + ) -> Union[CallbackT, str, None]: if getattr(spider, "rules", None): for rule in spider.rules: # type: ignore[attr-defined] if rule.link_extractor.matches(response.url): @@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand): def scraped_data( self, args: Tuple[ - List[Any], List[Request], argparse.Namespace, int, Spider, Callable + List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT ], ) -> List[Any]: items, requests, opts, depth, spider, callback = args @@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand): spider: Spider, opts: argparse.Namespace, response: Optional[Response] = None, - ) -> Callable: - cb: Union[str, Callable, None] = None + ) -> CallbackT: + cb: Union[str, CallbackT, None] = None if response: cb = response.meta["_callback"] if not cb: diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py index 440e0dc44..a7e129948 100644 --- a/scrapy/contracts/__init__.py +++ b/scrapy/contracts/__init__.py @@ -16,6 +16,7 @@ from typing import ( Optional, Tuple, Type, + cast, ) from unittest import TestCase, TestResult @@ -62,7 +63,7 @@ class Contract: if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") return list( # pylint: disable=return-in-finally - iterate_spider_output(cb_result) + cast(Iterable[Any], iterate_spider_output(cb_result)) ) request.callback = wrapper @@ -79,7 +80,7 @@ class Contract: cb_result = cb(response, **cb_kwargs) if isinstance(cb_result, (AsyncGenerator, CoroutineType)): raise TypeError("Contracts don't support async callbacks") - output = list(iterate_spider_output(cb_result)) + output = list(cast(Iterable[Any], iterate_spider_output(cb_result))) try: results.startTest(self.testcase_post) self.post_process(output) @@ -195,7 +196,7 @@ class ContractsManager: def cb_wrapper(response: Response, **cb_kwargs: Any) -> None: try: output = cb(response, **cb_kwargs) - output = list(iterate_spider_output(output)) + output = list(cast(Iterable[Any], iterate_spider_output(output))) except Exception: case = _create_testcase(method, "callback") results.addError(case, sys.exc_info()) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 04589dd37..9381a6cb3 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -12,7 +12,6 @@ from typing import ( TYPE_CHECKING, Any, AnyStr, - Callable, Dict, Iterable, List, @@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax if TYPE_CHECKING: + from collections.abc import Callable + + from twisted.python.failure import Failure + + # typing.Concatenate requires Python 3.10 # typing.NotRequired and typing.Self require Python 3.11 - from typing_extensions import NotRequired, Self + from typing_extensions import Concatenate, NotRequired, Self + + from scrapy.http import Response + + CallbackT = Callable[Concatenate[Response, ...], Any] class VerboseCookie(TypedDict): @@ -110,7 +118,7 @@ class Request(object_ref): def __init__( self, url: str, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -119,7 +127,7 @@ class Request(object_ref): encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, flags: Optional[List[str]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: @@ -137,8 +145,8 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback: Optional[Callable] = callback - self.errback: Optional[Callable] = errback + self.callback: Optional[CallbackT] = callback + self.errback: Optional[Callable[[Failure], Any]] = errback self.cookies: CookiesT = cookies or {} self.headers: Headers = Headers(headers or {}, encoding=encoding) diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index ff3581abb..92e4852b6 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -27,7 +27,7 @@ from urllib.parse import urljoin from scrapy.exceptions import NotSupported from scrapy.http.headers import Headers -from scrapy.http.request import CookiesT, Request +from scrapy.http.request import Request from scrapy.link import Link from scrapy.utils.trackref import object_ref @@ -35,10 +35,12 @@ if TYPE_CHECKING: from ipaddress import IPv4Address, IPv6Address from twisted.internet.ssl import Certificate + from twisted.python.failure import Failure # typing.Self requires Python 3.11 from typing_extensions import Self + from scrapy.http.request import CallbackT, CookiesT from scrapy.selector import SelectorList @@ -196,7 +198,7 @@ class Response(object_ref): def follow( self, url: Union[str, Link], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -205,7 +207,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -249,7 +251,7 @@ class Response(object_ref): def follow_all( self, urls: Iterable[Union[str, Link]], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -258,7 +260,7 @@ class Response(object_ref): encoding: Optional[str] = "utf-8", priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Iterable[Request]: diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py index 0635f744f..588695002 100644 --- a/scrapy/http/response/text.py +++ b/scrapy/http/response/text.py @@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode from scrapy.utils.response import get_base_url if TYPE_CHECKING: - from scrapy.http.request import CookiesT, Request + from twisted.python.failure import Failure + + from scrapy.http.request import CallbackT, CookiesT, Request from scrapy.selector import Selector, SelectorList @@ -179,7 +181,7 @@ class TextResponse(Response): def follow( self, url: Union[str, Link, parsel.Selector], - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -188,7 +190,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, ) -> Request: @@ -232,7 +234,7 @@ class TextResponse(Response): def follow_all( self, urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None, - callback: Optional[Callable] = None, + callback: Optional[CallbackT] = None, method: str = "GET", headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, @@ -241,7 +243,7 @@ class TextResponse(Response): encoding: Optional[str] = None, priority: int = 0, dont_filter: bool = False, - errback: Optional[Callable] = None, + errback: Optional[Callable[[Failure], Any]] = None, cb_kwargs: Optional[Dict[str, Any]] = None, flags: Optional[List[str]] = None, css: Optional[str] = None, diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py index f0b0c0988..d977acd26 100644 --- a/scrapy/spiders/__init__.py +++ b/scrapy/spiders/__init__.py @@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref from scrapy.utils.url import url_is_from_spider if TYPE_CHECKING: - from collections.abc import Callable - from twisted.internet.defer import Deferred - # typing.Concatenate requires Python 3.10 # typing.Self requires Python 3.11 - from typing_extensions import Concatenate, Self + from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT from scrapy.settings import BaseSettings, _SettingsKeyT from scrapy.utils.log import SpiderLoggerAdapter - CallbackT = Callable[Concatenate[Response, ...], Any] - class Spider(object_ref): """Base class for scrapy spiders. All spiders must inherit from this diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py index 48c830d2a..2639f14b2 100644 --- a/scrapy/spiders/crawl.py +++ b/scrapy/spiders/crawl.py @@ -39,6 +39,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT _T = TypeVar("_T") @@ -73,7 +74,7 @@ class Rule: def __init__( self, link_extractor: Optional[LinkExtractor] = None, - callback: Union[Callable, str, None] = None, + callback: Union[CallbackT, str, None] = None, cb_kwargs: Optional[Dict[str, Any]] = None, follow: Optional[bool] = None, process_links: Union[ProcessLinksT, str, None] = None, @@ -81,7 +82,7 @@ class Rule: errback: Union[Callable[[Failure], Any], str, None] = None, ): self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor - self.callback: Union[Callable, str, None] = callback + self.callback: Union[CallbackT, str, None] = callback self.errback: Union[Callable[[Failure], Any], str, None] = errback self.cb_kwargs: Dict[str, Any] = cb_kwargs or {} self.process_links: Union[ProcessLinksT, str] = process_links or _identity @@ -92,7 +93,7 @@ class Rule: def _compile(self, spider: Spider) -> None: # this replaces method names with methods and we can't express this in type hints - self.callback = _get_method(self.callback, spider) + self.callback = cast("CallbackT", _get_method(self.callback, spider)) self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider)) self.process_links = cast( ProcessLinksT, _get_method(self.process_links, spider) @@ -122,7 +123,9 @@ class CrawlSpider(Spider): def parse_start_url(self, response: Response, **kwargs: Any) -> Any: return [] - def process_results(self, response: Response, results: Any) -> Any: + def process_results( + self, response: Response, results: Iterable[Any] + ) -> Iterable[Any]: return results def _build_request(self, rule_index: int, link: Link) -> Request: @@ -152,7 +155,7 @@ class CrawlSpider(Spider): rule = self._rules[cast(int, response.meta["rule"])] return self._parse_response( response, - cast(Callable, rule.callback), + cast("CallbackT", rule.callback), {**rule.cb_kwargs, **cb_kwargs}, rule.follow, ) @@ -166,7 +169,7 @@ class CrawlSpider(Spider): async def _parse_response( self, response: Response, - callback: Optional[Callable], + callback: Optional[CallbackT], cb_kwargs: Dict[str, Any], follow: bool = True, ) -> AsyncIterable[Any]: diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py index d082fbfdb..1542ef79c 100644 --- a/scrapy/spiders/sitemap.py +++ b/scrapy/spiders/sitemap.py @@ -5,7 +5,6 @@ import re from typing import ( TYPE_CHECKING, Any, - Callable, Dict, Iterable, List, @@ -27,6 +26,7 @@ if TYPE_CHECKING: from typing_extensions import Self from scrapy.crawler import Crawler + from scrapy.http.request import CallbackT logger = logging.getLogger(__name__) @@ -34,7 +34,7 @@ logger = logging.getLogger(__name__) class SitemapSpider(Spider): sitemap_urls: Sequence[str] = () sitemap_rules: Sequence[ - Tuple[Union[re.Pattern[str], str], Union[str, Callable]] + Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]] ] = [("", "parse")] sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""] sitemap_alternate_links: bool = False @@ -54,10 +54,10 @@ class SitemapSpider(Spider): def __init__(self, *a: Any, **kw: Any): super().__init__(*a, **kw) - self._cbs: List[Tuple[re.Pattern[str], Callable]] = [] + self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = [] for r, c in self.sitemap_rules: if isinstance(c, str): - c = cast(Callable, getattr(self, c)) + c = cast("CallbackT", getattr(self, c)) self._cbs.append((regex(r), c)) self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]