From 732557e6988dd3d1df26f973e3ae651f7bcbdc5b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 31 Oct 2023 22:34:43 +0400 Subject: [PATCH] Full typing for scrapy/http/request. --- scrapy/http/common.py | 7 +++-- scrapy/http/request/__init__.py | 47 ++++++++++++++++------------- scrapy/http/request/form.py | 27 +++++++++++------ scrapy/http/request/json_request.py | 8 +++-- scrapy/http/request/rpc.py | 4 +-- 5 files changed, 55 insertions(+), 38 deletions(-) diff --git a/scrapy/http/common.py b/scrapy/http/common.py index bc8861574..a3d9d5b81 100644 --- a/scrapy/http/common.py +++ b/scrapy/http/common.py @@ -1,5 +1,8 @@ -def obsolete_setter(setter, attrname): - def newsetter(self, value): +from typing import Any, Callable, NoReturn + + +def obsolete_setter(setter: Callable, attrname: str) -> Callable[[Any, Any], NoReturn]: + def newsetter(self: Any, value: Any) -> NoReturn: c = self.__class__.__name__ msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead" raise AttributeError(msg) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 06a52f5fd..7c9a4ba95 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -9,14 +9,17 @@ from typing import ( Any, AnyStr, Callable, + Dict, Iterable, List, Mapping, + NoReturn, Optional, Tuple, Type, TypeVar, Union, + cast, ) from w3lib.url import safe_url_string @@ -32,7 +35,7 @@ from scrapy.utils.url import escape_ajax RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") -def NO_CALLBACK(*args, **kwargs): +def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn: """When assigned to the ``callback`` parameter of :class:`~scrapy.http.Request`, it indicates that the request is not meant to have a spider callback at all. @@ -92,21 +95,21 @@ class Request(object_ref): headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None, body: Optional[Union[bytes, str]] = None, cookies: Optional[Union[dict, List[dict]]] = None, - meta: Optional[dict] = None, + meta: Optional[Dict[str, Any]] = None, encoding: str = "utf-8", priority: int = 0, dont_filter: bool = False, errback: Optional[Callable] = None, flags: Optional[List[str]] = None, - cb_kwargs: Optional[dict] = None, + cb_kwargs: Optional[Dict[str, Any]] = None, ) -> None: - self._encoding = encoding # this one has to be set first - self.method = str(method).upper() + self._encoding: str = encoding # this one has to be set first + self.method: str = str(method).upper() self._set_url(url) self._set_body(body) if not isinstance(priority, int): raise TypeError(f"Request priority not an integer: {priority!r}") - self.priority = priority + self.priority: int = priority if not (callable(callback) or callback is None): raise TypeError( @@ -114,25 +117,27 @@ class Request(object_ref): ) if not (callable(errback) or errback is None): raise TypeError(f"errback must be a callable, got {type(errback).__name__}") - self.callback = callback - self.errback = errback + self.callback: Optional[Callable] = callback + self.errback: Optional[Callable] = errback - self.cookies = cookies or {} - self.headers = Headers(headers or {}, encoding=encoding) - self.dont_filter = dont_filter + self.cookies: Union[dict, List[dict]] = cookies or {} + self.headers: Headers = Headers(headers or {}, encoding=encoding) + self.dont_filter: bool = dont_filter - self._meta = dict(meta) if meta else None - self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None - self.flags = [] if flags is None else list(flags) + self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None + self._cb_kwargs: Optional[Dict[str, Any]] = ( + dict(cb_kwargs) if cb_kwargs else None + ) + self.flags: List[str] = [] if flags is None else list(flags) @property - def cb_kwargs(self) -> dict: + def cb_kwargs(self) -> Dict[str, Any]: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self) -> dict: + def meta(self) -> Dict[str, Any]: if self._meta is None: self._meta = {} return self._meta @@ -174,19 +179,19 @@ class Request(object_ref): def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> "Request": + def replace(self, *args: Any, **kwargs: Any) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) cls = kwargs.pop("cls", self.__class__) - return cls(*args, **kwargs) + return cast(Request, cls(*args, **kwargs)) @classmethod def from_curl( cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, - **kwargs, + **kwargs: Any, ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the @@ -219,7 +224,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict: + def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. @@ -244,7 +249,7 @@ class Request(object_ref): return d -def _find_method(obj, func): +def _find_method(obj: Any, func: Callable) -> str: """Helper function for Request.to_dict""" # Only instance methods contain ``__func__`` if obj and hasattr(func, "__func__"): diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 8da779b7c..0f80a0ab7 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,9 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit from lxml.html import ( @@ -24,7 +26,10 @@ from scrapy.http.response.text import TextResponse from scrapy.utils.python import is_listlike, to_bytes from scrapy.utils.response import get_base_url -FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + FormdataKVType = Tuple[str, Union[str, Iterable[str]]] FormdataType = Optional[Union[dict, List[FormdataKVType]]] @@ -33,7 +38,9 @@ FormdataType = Optional[Union[dict, List[FormdataKVType]]] class FormRequest(Request): valid_form_methods = ["GET", "POST"] - def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: + def __init__( + self, *args: Any, formdata: FormdataType = None, **kwargs: Any + ) -> None: if formdata and kwargs.get("method") is None: kwargs["method"] = "POST" @@ -54,7 +61,7 @@ class FormRequest(Request): @classmethod def from_response( - cls: Type[FormRequestTypeVar], + cls, response: TextResponse, formname: Optional[str] = None, formid: Optional[str] = None, @@ -64,8 +71,8 @@ class FormRequest(Request): dont_click: bool = False, formxpath: Optional[str] = None, formcss: Optional[str] = None, - **kwargs, - ) -> FormRequestTypeVar: + **kwargs: Any, + ) -> Self: kwargs.setdefault("encoding", response.encoding) if formcss is not None: @@ -121,12 +128,12 @@ def _get_form( if formname is not None: f = root.xpath(f'//form[@name="{formname}"]') if f: - return f[0] + return cast(FormElement, f[0]) if formid is not None: f = root.xpath(f'//form[@id="{formid}"]') if f: - return f[0] + return cast(FormElement, f[0]) # Get form element from xpath, if not found, go up if formxpath is not None: @@ -135,7 +142,7 @@ def _get_form( el = nodes[0] while True: if el.tag == "form": - return el + return cast(FormElement, el) el = el.getparent() if el is None: break @@ -147,7 +154,7 @@ def _get_form( except IndexError: raise IndexError(f"Form number {formnumber} not found in {response}") else: - return form + return cast(FormElement, form) def _get_inputs( diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index f52c0401d..1dd9e6c87 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Optional, Tuple +from typing import Any, Optional, Tuple from scrapy.http.request import Request @@ -16,7 +16,9 @@ from scrapy.http.request import Request class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + def __init__( + self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any + ) -> None: dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault("sort_keys", True) self._dumps_kwargs = dumps_kwargs @@ -42,7 +44,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> Request: + def replace(self, *args: Any, **kwargs: Any) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index 43692923b..bde860a66 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,7 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib -from typing import Optional +from typing import Any, Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -14,7 +14,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, encoding: Optional[str] = None, **kwargs): + def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any): if "body" not in kwargs and "params" in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs["body"] = xmlrpclib.dumps(**kw)