RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use CallbackT for Request.callback. (#6422)

This commit is contained in:
Andrey Rakhmatullin 2024-07-01 12:30:49 +05:00 committed by GitHub
parent 41e15e93e7
commit 558b1d11d2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 55 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
scrapy/commands/parse.py
View File

@ -9,7 +9,6 @@ from typing import (
TYPE_CHECKING,
Any,
AsyncGenerator,
Callable,
Coroutine,
Dict,
Iterable,
@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request
if TYPE_CHECKING:
from twisted.python.failure import Failure
from scrapy.http.request import CallbackT
from scrapy.spiders import Spider
@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand):
opts: argparse.Namespace,
depth: int,
spider: Spider,
callback: Callable,
) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
callback: CallbackT,
) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]:
items, requests = [], []
for x in spider_output:
if is_item(x):
@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand):
def run_callback(
self,
response: Response,
callback: Callable,
callback: CallbackT,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> Deferred[Any]:
cb_kwargs = cb_kwargs or {}
@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand):
def get_callback_from_rules(
self, spider: Spider, response: Response
) -> Union[Callable, str, None]:
) -> Union[CallbackT, str, None]:
if getattr(spider, "rules", None):
for rule in spider.rules: # type: ignore[attr-defined]
if rule.link_extractor.matches(response.url):
@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand):
def scraped_data(
self,
args: Tuple[
List[Any], List[Request], argparse.Namespace, int, Spider, Callable
List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT
],
) -> List[Any]:
items, requests, opts, depth, spider, callback = args
@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand):
spider: Spider,
opts: argparse.Namespace,
response: Optional[Response] = None,
) -> Callable:
cb: Union[str, Callable, None] = None
) -> CallbackT:
cb: Union[str, CallbackT, None] = None
if response:
cb = response.meta["_callback"]
if not cb:

7
scrapy/contracts/__init__.py
View File

@ -16,6 +16,7 @@ from typing import (
Optional,
Tuple,
Type,
cast,
)
from unittest import TestCase, TestResult
@ -62,7 +63,7 @@ class Contract:
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
return list( # pylint: disable=return-in-finally
iterate_spider_output(cb_result)
cast(Iterable[Any], iterate_spider_output(cb_result))
)
request.callback = wrapper
@ -79,7 +80,7 @@ class Contract:
cb_result = cb(response, **cb_kwargs)
if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
raise TypeError("Contracts don't support async callbacks")
output = list(iterate_spider_output(cb_result))
output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
try:
results.startTest(self.testcase_post)
self.post_process(output)
@ -195,7 +196,7 @@ class ContractsManager:
def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
try:
output = cb(response, **cb_kwargs)
output = list(iterate_spider_output(output))
output = list(cast(Iterable[Any], iterate_spider_output(output)))
except Exception:
case = _create_testcase(method, "callback")
results.addError(case, sys.exc_info())

20
scrapy/http/request/__init__.py
View File

@ -12,7 +12,6 @@ from typing import (
TYPE_CHECKING,
Any,
AnyStr,
Callable,
Dict,
Iterable,
List,
@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
if TYPE_CHECKING:
from collections.abc import Callable
from twisted.python.failure import Failure
# typing.Concatenate requires Python 3.10
# typing.NotRequired and typing.Self require Python 3.11
from typing_extensions import NotRequired, Self
from typing_extensions import Concatenate, NotRequired, Self
from scrapy.http import Response
CallbackT = Callable[Concatenate[Response, ...], Any]
class VerboseCookie(TypedDict):
@ -110,7 +118,7 @@ class Request(object_ref):
def __init__(
self,
url: str,
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@ -119,7 +127,7 @@ class Request(object_ref):
encoding: str = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
flags: Optional[List[str]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
) -> None:
@ -137,8 +145,8 @@ class Request(object_ref):
)
if not (callable(errback) or errback is None):
raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
self.callback: Optional[Callable] = callback
self.errback: Optional[Callable] = errback
self.callback: Optional[CallbackT] = callback
self.errback: Optional[Callable[[Failure], Any]] = errback
self.cookies: CookiesT = cookies or {}
self.headers: Headers = Headers(headers or {}, encoding=encoding)

12
scrapy/http/response/__init__.py
View File

@ -27,7 +27,7 @@ from urllib.parse import urljoin
from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
from scrapy.http.request import CookiesT, Request
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref
@ -35,10 +35,12 @@ if TYPE_CHECKING:
from ipaddress import IPv4Address, IPv6Address
from twisted.internet.ssl import Certificate
from twisted.python.failure import Failure
# typing.Self requires Python 3.11
from typing_extensions import Self
from scrapy.http.request import CallbackT, CookiesT
from scrapy.selector import SelectorList
@ -196,7 +198,7 @@ class Response(object_ref):
def follow(
self,
url: Union[str, Link],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@ -205,7 +207,7 @@ class Response(object_ref):
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@ -249,7 +251,7 @@ class Response(object_ref):
def follow_all(
self,
urls: Iterable[Union[str, Link]],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@ -258,7 +260,7 @@ class Response(object_ref):
encoding: Optional[str] = "utf-8",
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Iterable[Request]:

12
scrapy/http/response/text.py
View File

@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode
from scrapy.utils.response import get_base_url
if TYPE_CHECKING:
from scrapy.http.request import CookiesT, Request
from twisted.python.failure import Failure
from scrapy.http.request import CallbackT, CookiesT, Request
from scrapy.selector import Selector, SelectorList
@ -179,7 +181,7 @@ class TextResponse(Response):
def follow(
self,
url: Union[str, Link, parsel.Selector],
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@ -188,7 +190,7 @@ class TextResponse(Response):
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
) -> Request:
@ -232,7 +234,7 @@ class TextResponse(Response):
def follow_all(
self,
urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,
callback: Optional[Callable] = None,
callback: Optional[CallbackT] = None,
method: str = "GET",
headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
body: Optional[Union[bytes, str]] = None,
@ -241,7 +243,7 @@ class TextResponse(Response):
encoding: Optional[str] = None,
priority: int = 0,
dont_filter: bool = False,
errback: Optional[Callable] = None,
errback: Optional[Callable[[Failure], Any]] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
flags: Optional[List[str]] = None,
css: Optional[str] = None,

8
scrapy/spiders/__init__.py
View File

@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
if TYPE_CHECKING:
from collections.abc import Callable
from twisted.internet.defer import Deferred
# typing.Concatenate requires Python 3.10
# typing.Self requires Python 3.11
from typing_extensions import Concatenate, Self
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
from scrapy.settings import BaseSettings, _SettingsKeyT
from scrapy.utils.log import SpiderLoggerAdapter
CallbackT = Callable[Concatenate[Response, ...], Any]
class Spider(object_ref):
"""Base class for scrapy spiders. All spiders must inherit from this

15
scrapy/spiders/crawl.py
View File

@ -39,6 +39,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
_T = TypeVar("_T")
@ -73,7 +74,7 @@ class Rule:
def __init__(
self,
link_extractor: Optional[LinkExtractor] = None,
callback: Union[Callable, str, None] = None,
callback: Union[CallbackT, str, None] = None,
cb_kwargs: Optional[Dict[str, Any]] = None,
follow: Optional[bool] = None,
process_links: Union[ProcessLinksT, str, None] = None,
@ -81,7 +82,7 @@ class Rule:
errback: Union[Callable[[Failure], Any], str, None] = None,
):
self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor
self.callback: Union[Callable, str, None] = callback
self.callback: Union[CallbackT, str, None] = callback
self.errback: Union[Callable[[Failure], Any], str, None] = errback
self.cb_kwargs: Dict[str, Any] = cb_kwargs or {}
self.process_links: Union[ProcessLinksT, str] = process_links or _identity
@ -92,7 +93,7 @@ class Rule:
def _compile(self, spider: Spider) -> None:
# this replaces method names with methods and we can't express this in type hints
self.callback = _get_method(self.callback, spider)
self.callback = cast("CallbackT", _get_method(self.callback, spider))
self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
self.process_links = cast(
ProcessLinksT, _get_method(self.process_links, spider)
@ -122,7 +123,9 @@ class CrawlSpider(Spider):
def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
return []
def process_results(self, response: Response, results: Any) -> Any:
def process_results(
self, response: Response, results: Iterable[Any]
) -> Iterable[Any]:
return results
def _build_request(self, rule_index: int, link: Link) -> Request:
@ -152,7 +155,7 @@ class CrawlSpider(Spider):
rule = self._rules[cast(int, response.meta["rule"])]
return self._parse_response(
response,
cast(Callable, rule.callback),
cast("CallbackT", rule.callback),
{**rule.cb_kwargs, **cb_kwargs},
rule.follow,
)
@ -166,7 +169,7 @@ class CrawlSpider(Spider):
async def _parse_response(
self,
response: Response,
callback: Optional[Callable],
callback: Optional[CallbackT],
cb_kwargs: Dict[str, Any],
follow: bool = True,
) -> AsyncIterable[Any]:

8
scrapy/spiders/sitemap.py
View File

@ -5,7 +5,6 @@ import re
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
Iterable,
List,
@ -27,6 +26,7 @@ if TYPE_CHECKING:
from typing_extensions import Self
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
logger = logging.getLogger(__name__)
@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
class SitemapSpider(Spider):
sitemap_urls: Sequence[str] = ()
sitemap_rules: Sequence[
Tuple[Union[re.Pattern[str], str], Union[str, Callable]]
Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]]
] = [("", "parse")]
sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""]
sitemap_alternate_links: bool = False
@ -54,10 +54,10 @@ class SitemapSpider(Spider):
def __init__(self, *a: Any, **kw: Any):
super().__init__(*a, **kw)
self._cbs: List[Tuple[re.Pattern[str], Callable]] = []
self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = []
for r, c in self.sitemap_rules:
if isinstance(c, str):
c = cast(Callable, getattr(self, c))
c = cast("CallbackT", getattr(self, c))
self._cbs.append((regex(r), c))
self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]