Full typing for scrapy/http/request.

scrapy/http/common.py

@@ -1,5 +1,8 @@
-def obsolete_setter(setter, attrname):
-    def newsetter(self, value):
+from typing import Any, Callable, NoReturn
+
+
+def obsolete_setter(setter: Callable, attrname: str) -> Callable[[Any, Any], NoReturn]:
+    def newsetter(self: Any, value: Any) -> NoReturn:
         c = self.__class__.__name__
         msg = f"{c}.{attrname} is not modifiable, use {c}.replace() instead"
         raise AttributeError(msg)

scrapy/http/request/__init__.py

@@ -9,14 +9,17 @@ from typing import (
     Any,
     AnyStr,
     Callable,
+    Dict,
     Iterable,
     List,
     Mapping,
+    NoReturn,
     Optional,
     Tuple,
     Type,
     TypeVar,
     Union,
+    cast,
 )
 
 from w3lib.url import safe_url_string
@@ -32,7 +35,7 @@ from scrapy.utils.url import escape_ajax
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
-def NO_CALLBACK(*args, **kwargs):
+def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
     """When assigned to the ``callback`` parameter of
     :class:`~scrapy.http.Request`, it indicates that the request is not meant
     to have a spider callback at all.
@@ -92,21 +95,21 @@ class Request(object_ref):
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
         cookies: Optional[Union[dict, List[dict]]] = None,
-        meta: Optional[dict] = None,
+        meta: Optional[Dict[str, Any]] = None,
         encoding: str = "utf-8",
         priority: int = 0,
         dont_filter: bool = False,
         errback: Optional[Callable] = None,
         flags: Optional[List[str]] = None,
-        cb_kwargs: Optional[dict] = None,
+        cb_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
-        self._encoding = encoding  # this one has to be set first
-        self.method = str(method).upper()
+        self._encoding: str = encoding  # this one has to be set first
+        self.method: str = str(method).upper()
         self._set_url(url)
         self._set_body(body)
         if not isinstance(priority, int):
             raise TypeError(f"Request priority not an integer: {priority!r}")
-        self.priority = priority
+        self.priority: int = priority
 
         if not (callable(callback) or callback is None):
             raise TypeError(
@@ -114,25 +117,27 @@ class Request(object_ref):
             )
         if not (callable(errback) or errback is None):
             raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
-        self.callback = callback
-        self.errback = errback
+        self.callback: Optional[Callable] = callback
+        self.errback: Optional[Callable] = errback
 
-        self.cookies = cookies or {}
-        self.headers = Headers(headers or {}, encoding=encoding)
-        self.dont_filter = dont_filter
+        self.cookies: Union[dict, List[dict]] = cookies or {}
+        self.headers: Headers = Headers(headers or {}, encoding=encoding)
+        self.dont_filter: bool = dont_filter
 
-        self._meta = dict(meta) if meta else None
-        self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
-        self.flags = [] if flags is None else list(flags)
+        self._meta: Optional[Dict[str, Any]] = dict(meta) if meta else None
+        self._cb_kwargs: Optional[Dict[str, Any]] = (
+            dict(cb_kwargs) if cb_kwargs else None
+        )
+        self.flags: List[str] = [] if flags is None else list(flags)
 
     @property
-    def cb_kwargs(self) -> dict:
+    def cb_kwargs(self) -> Dict[str, Any]:
         if self._cb_kwargs is None:
             self._cb_kwargs = {}
         return self._cb_kwargs
 
     @property
-    def meta(self) -> dict:
+    def meta(self) -> Dict[str, Any]:
         if self._meta is None:
             self._meta = {}
         return self._meta
@@ -174,19 +179,19 @@ class Request(object_ref):
     def copy(self) -> "Request":
         return self.replace()
 
-    def replace(self, *args, **kwargs) -> "Request":
+    def replace(self, *args: Any, **kwargs: Any) -> "Request":
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
         cls = kwargs.pop("cls", self.__class__)
-        return cls(*args, **kwargs)
+        return cast(Request, cls(*args, **kwargs))
 
     @classmethod
     def from_curl(
         cls: Type[RequestTypeVar],
         curl_command: str,
         ignore_unknown_options: bool = True,
-        **kwargs,
+        **kwargs: Any,
     ) -> RequestTypeVar:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
@@ -219,7 +224,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> dict:
+    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
@@ -244,7 +249,7 @@ class Request(object_ref):
         return d
 
 
-def _find_method(obj, func):
+def _find_method(obj: Any, func: Callable) -> str:
     """Helper function for Request.to_dict"""
     # Only instance methods contain ``__func__``
     if obj and hasattr(func, "__func__"):

scrapy/http/request/form.py

@@ -5,7 +5,9 @@ This module implements the FormRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
-from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union, cast
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 from lxml.html import (
@@ -24,7 +26,10 @@ from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import is_listlike, to_bytes
 from scrapy.utils.response import get_base_url
 
-FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
 FormdataType = Optional[Union[dict, List[FormdataKVType]]]
@@ -33,7 +38,9 @@ FormdataType = Optional[Union[dict, List[FormdataKVType]]]
 class FormRequest(Request):
     valid_form_methods = ["GET", "POST"]
 
-    def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
+    def __init__(
+        self, *args: Any, formdata: FormdataType = None, **kwargs: Any
+    ) -> None:
         if formdata and kwargs.get("method") is None:
             kwargs["method"] = "POST"
 
@@ -54,7 +61,7 @@ class FormRequest(Request):
 
     @classmethod
     def from_response(
-        cls: Type[FormRequestTypeVar],
+        cls,
         response: TextResponse,
         formname: Optional[str] = None,
         formid: Optional[str] = None,
@@ -64,8 +71,8 @@ class FormRequest(Request):
         dont_click: bool = False,
         formxpath: Optional[str] = None,
         formcss: Optional[str] = None,
-        **kwargs,
-    ) -> FormRequestTypeVar:
+        **kwargs: Any,
+    ) -> Self:
         kwargs.setdefault("encoding", response.encoding)
 
         if formcss is not None:
@@ -121,12 +128,12 @@ def _get_form(
     if formname is not None:
         f = root.xpath(f'//form[@name="{formname}"]')
         if f:
-            return f[0]
+            return cast(FormElement, f[0])
 
     if formid is not None:
         f = root.xpath(f'//form[@id="{formid}"]')
         if f:
-            return f[0]
+            return cast(FormElement, f[0])
 
     # Get form element from xpath, if not found, go up
     if formxpath is not None:
@@ -135,7 +142,7 @@ def _get_form(
             el = nodes[0]
             while True:
                 if el.tag == "form":
-                    return el
+                    return cast(FormElement, el)
                 el = el.getparent()
                 if el is None:
                     break
@@ -147,7 +154,7 @@ def _get_form(
     except IndexError:
         raise IndexError(f"Form number {formnumber} not found in {response}")
     else:
-        return form
+        return cast(FormElement, form)
 
 
 def _get_inputs(

scrapy/http/request/json_request.py

@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 import copy
 import json
 import warnings
-from typing import Optional, Tuple
+from typing import Any, Optional, Tuple
 
 from scrapy.http.request import Request
 
@@ -16,7 +16,9 @@ from scrapy.http.request import Request
 class JsonRequest(Request):
     attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
 
-    def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None:
+    def __init__(
+        self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any
+    ) -> None:
         dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
         dumps_kwargs.setdefault("sort_keys", True)
         self._dumps_kwargs = dumps_kwargs
@@ -42,7 +44,7 @@ class JsonRequest(Request):
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args, **kwargs) -> Request:
+    def replace(self, *args: Any, **kwargs: Any) -> Request:
         body_passed = kwargs.get("body", None) is not None
         data = kwargs.pop("data", None)
         data_passed = data is not None

scrapy/http/request/rpc.py

@@ -5,7 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 import xmlrpc.client as xmlrpclib
-from typing import Optional
+from typing import Any, Optional
 
 from scrapy.http.request import Request
 from scrapy.utils.python import get_func_args
@@ -14,7 +14,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 
 
 class XmlRpcRequest(Request):
-    def __init__(self, *args, encoding: Optional[str] = None, **kwargs):
+    def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
         if "body" not in kwargs and "params" in kwargs:
             kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
             kwargs["body"] = xmlrpclib.dumps(**kw)