diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 5965a1c6c..f1fdc3858 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -98,7 +98,7 @@ def get_retry_request(
{'request': request, 'retry_times': retry_times, 'reason': reason},
extra={'spider': spider}
)
- new_request = request.copy()
+ new_request: Request = request.copy()
new_request.meta['retry_times'] = retry_times
new_request.dont_filter = True
if priority_adjust is None:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index ad884feac..7672dec00 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -5,7 +5,7 @@ requests in Scrapy.
See documentation in docs/topics/request-response.rst
"""
import inspect
-from typing import Optional, Tuple
+from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union
from w3lib.url import safe_url_string
@@ -18,6 +18,9 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.url import escape_ajax
+RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
+
+
class Request(object_ref):
"""Represents an HTTP request, which is usually generated in a Spider and
executed by the Downloader, thus generating a :class:`Response`.
@@ -36,10 +39,22 @@ class Request(object_ref):
:func:`~scrapy.utils.request.request_from_dict`.
"""
- def __init__(self, url, callback=None, method='GET', headers=None, body=None,
- cookies=None, meta=None, encoding='utf-8', priority=0,
- dont_filter=False, errback=None, flags=None, cb_kwargs=None):
-
+ def __init__(
+ self,
+ url: str,
+ callback: Optional[Callable] = None,
+ method: str = "GET",
+ headers: Optional[dict] = None,
+ body: Optional[Union[bytes, str]] = None,
+ cookies: Optional[Union[dict, List[dict]]] = None,
+ meta: Optional[dict] = None,
+ encoding: str = "utf-8",
+ priority: int = 0,
+ dont_filter: bool = False,
+ errback: Optional[Callable] = None,
+ flags: Optional[List[str]] = None,
+ cb_kwargs: Optional[dict] = None,
+ ) -> None:
self._encoding = encoding # this one has to be set first
self.method = str(method).upper()
self._set_url(url)
@@ -64,23 +79,23 @@ class Request(object_ref):
self.flags = [] if flags is None else list(flags)
@property
- def cb_kwargs(self):
+ def cb_kwargs(self) -> dict:
if self._cb_kwargs is None:
self._cb_kwargs = {}
return self._cb_kwargs
@property
- def meta(self):
+ def meta(self) -> dict:
if self._meta is None:
self._meta = {}
return self._meta
- def _get_url(self):
+ def _get_url(self) -> str:
return self._url
- def _set_url(self, url):
+ def _set_url(self, url: str) -> None:
if not isinstance(url, str):
- raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}')
+ raise TypeError(f"Request url must be str, got {type(url).__name__}")
s = safe_url_string(url, self.encoding)
self._url = escape_ajax(s)
@@ -94,31 +109,27 @@ class Request(object_ref):
url = property(_get_url, obsolete_setter(_set_url, 'url'))
- def _get_body(self):
+ def _get_body(self) -> bytes:
return self._body
- def _set_body(self, body):
- if body is None:
- self._body = b''
- else:
- self._body = to_bytes(body, self.encoding)
+ def _set_body(self, body: Optional[Union[str, bytes]]) -> None:
+ self._body = b"" if body is None else to_bytes(body, self.encoding)
body = property(_get_body, obsolete_setter(_set_body, 'body'))
@property
- def encoding(self):
+ def encoding(self) -> str:
return self._encoding
- def __str__(self):
+ def __str__(self) -> str:
return f"<{self.method} {self.url}>"
__repr__ = __str__
- def copy(self):
- """Return a copy of this Request"""
+ def copy(self) -> "Request":
return self.replace()
- def replace(self, *args, **kwargs):
+ def replace(self, *args, **kwargs) -> "Request":
"""Create a new Request with the same attributes except for those given new values"""
for x in self.attributes:
kwargs.setdefault(x, getattr(self, x))
@@ -126,7 +137,9 @@ class Request(object_ref):
return cls(*args, **kwargs)
@classmethod
- def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs):
+ def from_curl(
+ cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs
+ ) -> RequestTypeVar:
"""Create a Request object from a string containing a `cURL
`_ command. It populates the HTTP method, the
URL, the headers, the cookies and the body. It accepts the same
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ef2eb3ba6..c3e112041 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,22 +5,28 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
+from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union
from urllib.parse import urljoin, urlencode
-import lxml.html
+from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement
from parsel.selector import create_root_node
from w3lib.html import strip_html5_whitespace
from scrapy.http.request import Request
+from scrapy.http.response.text import TextResponse
from scrapy.utils.python import to_bytes, is_listlike
from scrapy.utils.response import get_base_url
+FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest")
+
+FormdataType = Optional[Union[dict, List[Tuple[str, str]]]]
+
+
class FormRequest(Request):
valid_form_methods = ['GET', 'POST']
- def __init__(self, *args, **kwargs):
- formdata = kwargs.pop('formdata', None)
+ def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None:
if formdata and kwargs.get('method') is None:
kwargs['method'] = 'POST'
@@ -36,9 +42,19 @@ class FormRequest(Request):
self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr)
@classmethod
- def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
- clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
-
+ def from_response(
+ cls: Type[FormRequestTypeVar],
+ response: TextResponse,
+ formname: Optional[str] = None,
+ formid: Optional[str] = None,
+ formnumber: Optional[int] = 0,
+ formdata: FormdataType = None,
+ clickdata: Optional[dict] = None,
+ dont_click: bool = False,
+ formxpath: Optional[str] = None,
+ formcss: Optional[str] = None,
+ **kwargs,
+ ) -> FormRequestTypeVar:
kwargs.setdefault('encoding', response.encoding)
if formcss is not None:
@@ -46,7 +62,7 @@ class FormRequest(Request):
formxpath = HTMLTranslator().css_to_xpath(formcss)
form = _get_form(response, formname, formid, formnumber, formxpath)
- formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
+ formdata = _get_inputs(form, formdata, dont_click, clickdata)
url = _get_form_url(form, kwargs.pop('url', None))
method = kwargs.pop('method', form.method)
@@ -58,7 +74,7 @@ class FormRequest(Request):
return cls(url=url, method=method, formdata=formdata, **kwargs)
-def _get_form_url(form, url):
+def _get_form_url(form: FormElement, url: Optional[str]) -> str:
if url is None:
action = form.get('action')
if action is None:
@@ -67,17 +83,22 @@ def _get_form_url(form, url):
return urljoin(form.base_url, url)
-def _urlencode(seq, enc):
+def _urlencode(seq: Iterable, enc: str) -> str:
values = [(to_bytes(k, enc), to_bytes(v, enc))
for k, vs in seq
for v in (vs if is_listlike(vs) else [vs])]
return urlencode(values, doseq=True)
-def _get_form(response, formname, formid, formnumber, formxpath):
- """Find the form element """
- root = create_root_node(response.text, lxml.html.HTMLParser,
- base_url=get_base_url(response))
+def _get_form(
+ response: TextResponse,
+ formname: Optional[str],
+ formid: Optional[str],
+ formnumber: Optional[int],
+ formxpath: Optional[str],
+) -> FormElement:
+ """Find the wanted form element within the given response."""
+ root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
forms = root.xpath('//form')
if not forms:
raise ValueError(f"No