From 19c7415aae1678631d5ca115a13094b6bd70f245 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Sat, 1 May 2021 16:34:39 -0300 Subject: [PATCH 1/7] Request type hints --- scrapy/downloadermiddlewares/retry.py | 2 +- scrapy/http/request/__init__.py | 69 ++++++++++++++++----------- scrapy/utils/curl.py | 2 +- 3 files changed, 42 insertions(+), 31 deletions(-) diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py index 5965a1c6c..f1fdc3858 100644 --- a/scrapy/downloadermiddlewares/retry.py +++ b/scrapy/downloadermiddlewares/retry.py @@ -98,7 +98,7 @@ def get_retry_request( {'request': request, 'retry_times': retry_times, 'reason': reason}, extra={'spider': spider} ) - new_request = request.copy() + new_request: Request = request.copy() new_request.meta['retry_times'] = retry_times new_request.dont_filter = True if priority_adjust is None: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 498f1b052..3cce9f501 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,22 +4,38 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from typing import Callable, List, Optional, Type, TypeVar, Union + from w3lib.url import safe_url_string +from scrapy.http.common import obsolete_setter from scrapy.http.headers import Headers +from scrapy.utils.curl import curl_to_request_kwargs from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax -from scrapy.http.common import obsolete_setter -from scrapy.utils.curl import curl_to_request_kwargs + + +RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") class Request(object_ref): - - def __init__(self, url, callback=None, method='GET', headers=None, body=None, - cookies=None, meta=None, encoding='utf-8', priority=0, - dont_filter=False, errback=None, flags=None, cb_kwargs=None): - + def __init__( + self, + url: str, + callback: Optional[Callable] = None, + method: str = "GET", + headers: Optional[dict] = None, + body: Optional[Union[bytes, str]] = None, + cookies: Optional[Union[dict, List[dict]]]=None, + meta: Optional[dict] = None, + encoding: str = "utf-8", + priority: int = 0, + dont_filter: bool = False, + errback: Optional[Callable] = None, + flags: Optional[List[str]] = None, + cb_kwargs: Optional[dict] = None, + ) -> None: self._encoding = encoding # this one has to be set first self.method = str(method).upper() self._set_url(url) @@ -44,23 +60,23 @@ class Request(object_ref): self.flags = [] if flags is None else list(flags) @property - def cb_kwargs(self): + def cb_kwargs(self) -> dict: if self._cb_kwargs is None: self._cb_kwargs = {} return self._cb_kwargs @property - def meta(self): + def meta(self) -> dict: if self._meta is None: self._meta = {} return self._meta - def _get_url(self): + def _get_url(self) -> str: return self._url - def _set_url(self, url): + def _set_url(self, url: str) -> None: if not isinstance(url, str): - raise TypeError(f'Request url must be str or unicode, got {type(url).__name__}') + raise TypeError(f"Request url must be str, got {type(url).__name__}") s = safe_url_string(url, self.encoding) self._url = escape_ajax(s) @@ -74,34 +90,28 @@ class Request(object_ref): url = property(_get_url, obsolete_setter(_set_url, 'url')) - def _get_body(self): + def _get_body(self) -> bytes: return self._body - def _set_body(self, body): - if body is None: - self._body = b'' - else: - self._body = to_bytes(body, self.encoding) + def _set_body(self, body: Optional[Union[str, bytes]]) -> None: + self._body = b"" if body is None else to_bytes(body, self.encoding) body = property(_get_body, obsolete_setter(_set_body, 'body')) @property - def encoding(self): + def encoding(self) -> str: return self._encoding - def __str__(self): + def __str__(self) -> str: return f"<{self.method} {self.url}>" __repr__ = __str__ - def copy(self): - """Return a copy of this Request""" + def copy(self) -> RequestTypeVar: return self.replace() - def replace(self, *args, **kwargs): - """Create a new Request with the same attributes except for those - given new values. - """ + def replace(self, *args, **kwargs) -> RequestTypeVar: + """Create a new Request with the same attributes except for those given new values""" for x in ['url', 'method', 'headers', 'body', 'cookies', 'meta', 'flags', 'encoding', 'priority', 'dont_filter', 'callback', 'errback', 'cb_kwargs']: kwargs.setdefault(x, getattr(self, x)) @@ -109,7 +119,9 @@ class Request(object_ref): return cls(*args, **kwargs) @classmethod - def from_curl(cls, curl_command, ignore_unknown_options=True, **kwargs): + def from_curl( + cls: Type[RequestTypeVar], curl_command: str, ignore_unknown_options: bool = True, **kwargs + ) -> RequestTypeVar: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -136,8 +148,7 @@ class Request(object_ref): To translate a cURL command into a Scrapy request, you may use `curl2scrapy `_. - - """ + """ request_kwargs = curl_to_request_kwargs(curl_command, ignore_unknown_options) request_kwargs.update(kwargs) return cls(**request_kwargs) diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py index d8b3deaa1..74f82ad75 100644 --- a/scrapy/utils/curl.py +++ b/scrapy/utils/curl.py @@ -54,7 +54,7 @@ def _parse_headers_and_cookies(parsed_args): return headers, cookies -def curl_to_request_kwargs(curl_command, ignore_unknown_options=True): +def curl_to_request_kwargs(curl_command: str, ignore_unknown_options: bool = True) -> dict: """Convert a cURL command syntax to Request kwargs. :param str curl_command: string containing the curl command From 216dd37953112e360348e19eaf8cae45a52fe87a Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:16:40 -0300 Subject: [PATCH 2/7] Type hints for FormRequest --- scrapy/http/request/form.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index ef2eb3ba6..4465f40ae 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,6 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode import lxml.html @@ -12,15 +13,18 @@ from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace from scrapy.http.request import Request +from scrapy.http.response.text import TextResponse from scrapy.utils.python import to_bytes, is_listlike from scrapy.utils.response import get_base_url +FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") + + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, **kwargs): - formdata = kwargs.pop('formdata', None) + def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -36,9 +40,19 @@ class FormRequest(Request): self._set_url(self.url + ('&' if '?' in self.url else '?') + querystr) @classmethod - def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None, - clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs): - + def from_response( + cls: Type[FormRequestTypeVar], + response: TextResponse, + formname: Optional[str] = None, + formid: Optional[str] = None, + formnumber: Optional[int] = 0, + formdata: Optional[dict] = None, + clickdata: Optional[dict] = None, + dont_click: bool = False, + formxpath: Optional[str] = None, + formcss: Optional[str] = None, + **kwargs, + ) -> FormRequestTypeVar: kwargs.setdefault('encoding', response.encoding) if formcss is not None: From c594017e518af31edc24619bfc590907ba6280c7 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 11:27:21 -0300 Subject: [PATCH 3/7] Type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 4465f40ae..781e3495a 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst from typing import Optional, Type, TypeVar from urllib.parse import urljoin, urlencode -import lxml.html +from lxml.html import HTMLParser, FormElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -72,7 +72,7 @@ class FormRequest(Request): return cls(url=url, method=method, formdata=formdata, **kwargs) -def _get_form_url(form, url): +def _get_form_url(form: FormElement, url: Optional[str]) -> str: if url is None: action = form.get('action') if action is None: @@ -88,10 +88,15 @@ def _urlencode(seq, enc): return urlencode(values, doseq=True) -def _get_form(response, formname, formid, formnumber, formxpath): - """Find the form element """ - root = create_root_node(response.text, lxml.html.HTMLParser, - base_url=get_base_url(response)) +def _get_form( + response: TextResponse, + formname: Optional[str], + formid: Optional[str], + formnumber: Optional[int], + formxpath: Optional[str], +) -> FormElement: + """Find the wanted form element within the given response.""" + root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response)) forms = root.xpath('//form') if not forms: raise ValueError(f"No
element found in {response}") @@ -119,8 +124,7 @@ def _get_form(response, formname, formid, formnumber, formxpath): break raise ValueError(f'No element found with {formxpath}') - # If we get here, it means that either formname was None - # or invalid + # If we get here, it means that either formname was None or invalid if formnumber is not None: try: form = forms[formnumber] From 85f88a5710e51a3137ded72f5fdb1c3465480355 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:02:16 -0300 Subject: [PATCH 4/7] More type hints for private functions used by FormRequest --- scrapy/http/request/form.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 781e3495a..1ad878c03 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,7 +5,7 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import Optional, Type, TypeVar +from typing import List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode from lxml.html import HTMLParser, FormElement @@ -20,11 +20,13 @@ from scrapy.utils.response import get_base_url FormRequestTypeVar = TypeVar("FormRequestTypeVar", bound="FormRequest") +FormdataType = Optional[Union[dict, List[Tuple[str, str]]]] + class FormRequest(Request): valid_form_methods = ['GET', 'POST'] - def __init__(self, *args, formdata: Optional[dict] = None, **kwargs) -> None: + def __init__(self, *args, formdata: FormdataType = None, **kwargs) -> None: if formdata and kwargs.get('method') is None: kwargs['method'] = 'POST' @@ -46,7 +48,7 @@ class FormRequest(Request): formname: Optional[str] = None, formid: Optional[str] = None, formnumber: Optional[int] = 0, - formdata: Optional[dict] = None, + formdata: FormdataType = None, clickdata: Optional[dict] = None, dont_click: bool = False, formxpath: Optional[str] = None, @@ -60,7 +62,7 @@ class FormRequest(Request): formxpath = HTMLTranslator().css_to_xpath(formcss) form = _get_form(response, formname, formid, formnumber, formxpath) - formdata = _get_inputs(form, formdata, dont_click, clickdata, response) + formdata = _get_inputs(form, formdata, dont_click, clickdata) url = _get_form_url(form, kwargs.pop('url', None)) method = kwargs.pop('method', form.method) @@ -134,22 +136,27 @@ def _get_form( return form -def _get_inputs(form, formdata, dont_click, clickdata, response): +def _get_inputs( + form: FormElement, + formdata: FormdataType, + dont_click: bool, + clickdata: Optional[dict], +) -> List[Tuple[str, str]]: + """Return a list of key-value pairs for the inputs found in the given form.""" try: formdata_keys = dict(formdata or ()).keys() except (ValueError, TypeError): raise ValueError('formdata should be a dict or iterable of tuples') if not formdata: - formdata = () + formdata = [] inputs = form.xpath('descendant::textarea' '|descendant::select' '|descendant::input[not(@type) or @type[' ' not(re:test(., "^(?:submit|image|reset)$", "i"))' ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', - namespaces={ - "re": "http://exslt.org/regular-expressions"}) + namespaces={"re": "http://exslt.org/regular-expressions"}) values = [(k, '' if v is None else v) for k, v in (_value(e) for e in inputs) if k and k not in formdata_keys] @@ -160,7 +167,7 @@ def _get_inputs(form, formdata, dont_click, clickdata, response): values.append(clickable) if isinstance(formdata, dict): - formdata = formdata.items() + formdata = formdata.items() # type: ignore[assignment] values.extend((k, v) for k, v in formdata if v is not None) return values @@ -189,7 +196,7 @@ def _select_value(ele, n, v): return n, v -def _get_clickable(clickdata, form): +def _get_clickable(clickdata: Optional[dict], form: FormElement) -> Optional[Tuple[str, str]]: """ Returns the clickable element specified in clickdata, if the latter is given. If not, it returns the first @@ -201,7 +208,7 @@ def _get_clickable(clickdata, form): namespaces={"re": "http://exslt.org/regular-expressions"} )) if not clickables: - return + return None # If we don't have clickdata, we just use the first clickable element if clickdata is None: From c9fecca010a3ddddd1145f65a78dde30b5c71a72 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:25:26 -0300 Subject: [PATCH 5/7] More type hints --- scrapy/http/request/form.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py index 1ad878c03..c3e112041 100644 --- a/scrapy/http/request/form.py +++ b/scrapy/http/request/form.py @@ -5,10 +5,10 @@ This module implements the FormRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ -from typing import List, Optional, Tuple, Type, TypeVar, Union +from typing import Iterable, List, Optional, Tuple, Type, TypeVar, Union from urllib.parse import urljoin, urlencode -from lxml.html import HTMLParser, FormElement +from lxml.html import FormElement, HtmlElement, HTMLParser, SelectElement from parsel.selector import create_root_node from w3lib.html import strip_html5_whitespace @@ -83,7 +83,7 @@ def _get_form_url(form: FormElement, url: Optional[str]) -> str: return urljoin(form.base_url, url) -def _urlencode(seq, enc): +def _urlencode(seq: Iterable, enc: str) -> str: values = [(to_bytes(k, enc), to_bytes(v, enc)) for k, vs in seq for v in (vs if is_listlike(vs) else [vs])] @@ -157,9 +157,11 @@ def _get_inputs( ' and (../@checked or' ' not(re:test(., "^(?:checkbox|radio)$", "i")))]]', namespaces={"re": "http://exslt.org/regular-expressions"}) - values = [(k, '' if v is None else v) - for k, v in (_value(e) for e in inputs) - if k and k not in formdata_keys] + values = [ + (k, '' if v is None else v) + for k, v in (_value(e) for e in inputs) + if k and k not in formdata_keys + ] if not dont_click: clickable = _get_clickable(clickdata, form) @@ -173,7 +175,7 @@ def _get_inputs( return values -def _value(ele): +def _value(ele: HtmlElement): n = ele.name v = ele.value if ele.tag == 'select': @@ -181,7 +183,7 @@ def _value(ele): return n, v -def _select_value(ele, n, v): +def _select_value(ele: SelectElement, n: str, v: str): multiple = ele.multiple if v is None and not multiple: # Match browser behaviour on simple select tag without options selected @@ -192,7 +194,8 @@ def _select_value(ele, n, v): # This is a workround to bug in lxml fixed 2.3.1 # fix https://github.com/lxml/lxml/commit/57f49eed82068a20da3db8f1b18ae00c1bab8b12#L1L1139 selected_options = ele.xpath('.//option[@selected]') - v = [(o.get('value') or o.text or '').strip() for o in selected_options] + values = [(o.get('value') or o.text or '').strip() for o in selected_options] + return n, values return n, v From 479260dca012b3d03221f2e6322008448d0fb8b5 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Tue, 1 Jun 2021 12:52:46 -0300 Subject: [PATCH 6/7] Type hints for Request subclasses --- scrapy/http/request/json_request.py | 17 +++++++---------- scrapy/http/request/rpc.py | 4 ++-- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 04e80d897..dba3c3a82 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -8,9 +8,9 @@ See documentation in docs/topics/request-response.rst import copy import json import warnings -from typing import Tuple +from typing import Optional, Tuple -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar from scrapy.utils.deprecate import create_deprecated_class @@ -18,8 +18,8 @@ class JsonRequest(Request): attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",) - def __init__(self, *args, **kwargs): - dumps_kwargs = copy.deepcopy(kwargs.pop('dumps_kwargs', {})) + def __init__(self, *args, dumps_kwargs: Optional[dict] = None, **kwargs) -> None: + dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {} dumps_kwargs.setdefault('sort_keys', True) self._dumps_kwargs = dumps_kwargs @@ -29,10 +29,8 @@ class JsonRequest(Request): if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) - if 'method' not in kwargs: kwargs['method'] = 'POST' @@ -41,23 +39,22 @@ class JsonRequest(Request): self.headers.setdefault('Accept', 'application/json, text/javascript, */*; q=0.01') @property - def dumps_kwargs(self): + def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs): + def replace(self, *args, **kwargs) -> RequestTypeVar: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None if body_passed and data_passed: warnings.warn('Both body and data passed. data will be ignored') - elif not body_passed and data_passed: kwargs['body'] = self._dumps(data) return super().replace(*args, **kwargs) - def _dumps(self, data): + def _dumps(self, data: dict) -> str: """Convert to JSON """ return json.dumps(data, **self._dumps_kwargs) diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py index c70912e49..06d98cea5 100644 --- a/scrapy/http/request/rpc.py +++ b/scrapy/http/request/rpc.py @@ -5,6 +5,7 @@ This module implements the XmlRpcRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ import xmlrpc.client as xmlrpclib +from typing import Optional from scrapy.http.request import Request from scrapy.utils.python import get_func_args @@ -15,8 +16,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps) class XmlRpcRequest(Request): - def __init__(self, *args, **kwargs): - encoding = kwargs.get('encoding', None) + def __init__(self, *args, encoding: Optional[str] = None, **kwargs): if 'body' not in kwargs and 'params' in kwargs: kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs) kwargs['body'] = xmlrpclib.dumps(**kw) From ce6447731a91e32faf17564d21ddc3b88e582f50 Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Mon, 7 Jun 2021 13:25:04 -0300 Subject: [PATCH 7/7] Replace return type --- scrapy/http/request/__init__.py | 4 ++-- scrapy/http/request/json_request.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index 8f00c20b7..7672dec00 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -126,10 +126,10 @@ class Request(object_ref): __repr__ = __str__ - def copy(self) -> RequestTypeVar: + def copy(self) -> "Request": return self.replace() - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> "Request": """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index dba3c3a82..728a2a104 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -10,7 +10,7 @@ import json import warnings from typing import Optional, Tuple -from scrapy.http.request import Request, RequestTypeVar +from scrapy.http.request import Request from scrapy.utils.deprecate import create_deprecated_class @@ -42,7 +42,7 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args, **kwargs) -> RequestTypeVar: + def replace(self, *args, **kwargs) -> Request: body_passed = kwargs.get('body', None) is not None data = kwargs.pop('data', None) data_passed = data is not None