From a6cee787dd45fabba3f39dbb1752baeef649f5b7 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 11 Nov 2023 20:00:12 +0400 Subject: [PATCH 01/13] Improve type hints for copy() and replace() in Request and Response. --- .../downloadermiddlewares/httpcompression.py | 6 +-- scrapy/downloadermiddlewares/redirect.py | 1 + scrapy/http/request/__init__.py | 37 +++++++++++++++---- scrapy/http/request/json_request.py | 26 +++++++++++-- scrapy/http/response/__init__.py | 31 +++++++++++++--- 5 files changed, 81 insertions(+), 20 deletions(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index 56a58a750..d44eb933a 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -2,7 +2,7 @@ from __future__ import annotations import io import zlib -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from scrapy import Request, Spider from scrapy.crawler import Crawler @@ -74,12 +74,12 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs = dict(cls=respcls, body=decoded_body) + kwargs: Dict[str, Any] = dict(body=decoded_body) if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable kwargs["encoding"] = None - response = response.replace(**kwargs) + response = response.replace(cls=respcls, **kwargs) if not content_encoding: del response.headers["Content-Encoding"] diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py index 814b1a561..7b1401ac8 100644 --- a/scrapy/downloadermiddlewares/redirect.py +++ b/scrapy/downloadermiddlewares/redirect.py @@ -27,6 +27,7 @@ def _build_redirect_request( redirect_request = source_request.replace( url=url, **kwargs, + cls=None, cookies=None, ) if "Cookie" in redirect_request.headers: diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py index a1c5a5e51..4effc2178 100644 --- a/scrapy/http/request/__init__.py +++ b/scrapy/http/request/__init__.py @@ -4,8 +4,11 @@ requests in Scrapy. See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import inspect from typing import ( + TYPE_CHECKING, Any, AnyStr, Callable, @@ -19,7 +22,7 @@ from typing import ( Type, TypeVar, Union, - cast, + overload, ) from w3lib.url import safe_url_string @@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes from scrapy.utils.trackref import object_ref from scrapy.utils.url import escape_ajax +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + + RequestTypeVar = TypeVar("RequestTypeVar", bound="Request") @@ -173,23 +181,36 @@ class Request(object_ref): def __repr__(self) -> str: return f"<{self.method} {self.url}>" - def copy(self) -> "Request": + def copy(self) -> Self: return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> "Request": + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: """Create a new Request with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Request, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) @classmethod def from_curl( - cls: Type[RequestTypeVar], + cls, curl_command: str, ignore_unknown_options: bool = True, **kwargs: Any, - ) -> RequestTypeVar: + ) -> Self: """Create a Request object from a string containing a `cURL `_ command. It populates the HTTP method, the URL, the headers, the cookies and the body. It accepts the same @@ -221,7 +242,7 @@ class Request(object_ref): request_kwargs.update(kwargs) return cls(**request_kwargs) - def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]: + def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]: """Return a dictionary containing the Request's data. Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object. diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 1dd9e6c87..5c09835e4 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class See documentation in docs/topics/request-response.rst """ +from __future__ import annotations + import copy import json import warnings -from typing import Any, Optional, Tuple +from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload -from scrapy.http.request import Request +from scrapy.http.request import Request, RequestTypeVar + +if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self class JsonRequest(Request): @@ -44,7 +50,19 @@ class JsonRequest(Request): def dumps_kwargs(self) -> dict: return self._dumps_kwargs - def replace(self, *args: Any, **kwargs: Any) -> Request: + @overload + def replace( + self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any + ) -> RequestTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any + ) -> Request: body_passed = kwargs.get("body", None) is not None data = kwargs.pop("data", None) data_passed = data is not None @@ -54,7 +72,7 @@ class JsonRequest(Request): elif not body_passed and data_passed: kwargs["body"] = self._dumps(data) - return super().replace(*args, **kwargs) + return super().replace(*args, cls=cls, **kwargs) def _dumps(self, data: dict) -> str: """Convert to JSON""" diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py index 6eae3e8b3..e889a6460 100644 --- a/scrapy/http/response/__init__.py +++ b/scrapy/http/response/__init__.py @@ -19,8 +19,10 @@ from typing import ( Mapping, Optional, Tuple, + Type, + TypeVar, Union, - cast, + overload, ) from urllib.parse import urljoin @@ -33,9 +35,15 @@ from scrapy.link import Link from scrapy.utils.trackref import object_ref if TYPE_CHECKING: + # typing.Self requires Python 3.11 + from typing_extensions import Self + from scrapy.selector import SelectorList +ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response") + + class Response(object_ref): """An object that represents an HTTP response, which is usually downloaded (by the Downloader) and fed to the Spiders for processing. @@ -132,16 +140,29 @@ class Response(object_ref): def __repr__(self) -> str: return f"<{self.status} {self.url}>" - def copy(self) -> Response: + def copy(self) -> Self: """Return a copy of this Response""" return self.replace() - def replace(self, *args: Any, **kwargs: Any) -> Response: + @overload + def replace( + self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any + ) -> ResponseTypeVar: + ... + + @overload + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: + ... + + def replace( + self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any + ) -> Response: """Create a new Response with the same attributes except for those given new values""" for x in self.attributes: kwargs.setdefault(x, getattr(self, x)) - cls = kwargs.pop("cls", self.__class__) - return cast(Response, cls(*args, **kwargs)) + if cls is None: + cls = self.__class__ + return cls(*args, **kwargs) def urljoin(self, url: str) -> str: """Join this Response's url with a possible relative url to form an From 5d55e4f56b77168b961db15e0f03d608fad69e7d Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:15:06 +0400 Subject: [PATCH 02/13] Add mypy tests. --- tests_typing/test_http_request.mypy-testing | 66 ++++++++++++++++++++ tests_typing/test_http_response.mypy-testing | 45 +++++++++++++ tox.ini | 8 +++ 3 files changed, 119 insertions(+) create mode 100644 tests_typing/test_http_request.mypy-testing create mode 100644 tests_typing/test_http_response.mypy-testing diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing new file mode 100644 index 000000000..a306b15fe --- /dev/null +++ b/tests_typing/test_http_request.mypy-testing @@ -0,0 +1,66 @@ +import pytest + +from scrapy import Request +from scrapy.http import JsonRequest + + +class MyRequest(Request): + pass + + +class MyRequest2(Request): + pass + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=None) + Request("data:,", headers={}) + Request("data:,", headers=[]) + Request("data:,", headers={"foo": "bar"}) + Request("data:,", headers={b"foo": "bar"}) + Request("data:,", headers={"foo": b"bar"}) + Request("data:,", headers=[("foo", "bar")]) + Request("data:,", headers=[(b"foo", "bar")]) + Request("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.copy() + reveal_type(req_copy) # R: __main__.MyRequest + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + req = Request("data:,") + reveal_type(req) # R: scrapy.http.request.Request + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.Request + + req = MyRequest("data:,") + reveal_type(req) # R: __main__.MyRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: __main__.MyRequest + req_copy2 = req.replace(body=b"a", cls=MyRequest2) + reveal_type(req_copy2) # R: __main__.MyRequest2 + + +@pytest.mark.mypy_testing +def mypy_test_jsonrequest_copy_replace(): + req = JsonRequest("data:,") + reveal_type(req) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.copy() + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy = req.replace(body=b"a") + reveal_type(req_copy) # R: scrapy.http.request.json_request.JsonRequest + req_copy_my = req.replace(body=b"a", cls=MyRequest) + reveal_type(req_copy_my) # R: __main__.MyRequest diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing new file mode 100644 index 000000000..66ac6ad1d --- /dev/null +++ b/tests_typing/test_http_response.mypy-testing @@ -0,0 +1,45 @@ +import pytest + +from scrapy.http import HtmlResponse, Response, TextResponse + + +@pytest.mark.mypy_testing +def mypy_test_headers(): + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=None) + Response("data:,", headers={}) + Response("data:,", headers=[]) + Response("data:,", headers={"foo": "bar"}) + Response("data:,", headers={b"foo": "bar"}) + Response("data:,", headers={"foo": b"bar"}) + Response("data:,", headers=[("foo", "bar")]) + Response("data:,", headers=[(b"foo", "bar")]) + Response("data:,", headers=[("foo", b"bar")]) + + +@pytest.mark.mypy_testing +def mypy_test_copy(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.copy() + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + + +@pytest.mark.mypy_testing +def mypy_test_replace(): + resp = Response("data:,") + reveal_type(resp) # R: scrapy.http.response.Response + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.Response + + resp = HtmlResponse("data:,") + reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse + resp_copy = resp.replace(body=b"a") + reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse + resp_copy2 = resp.replace(body=b"a", cls=TextResponse) + reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse diff --git a/tox.ini b/tox.ini index 932c0b805..c3fa54339 100644 --- a/tox.ini +++ b/tox.ini @@ -46,6 +46,14 @@ deps = commands = mypy {posargs: scrapy tests} +[testenv:typing-tests] +deps = + {[testenv]deps} + {[testenv:typing]deps} + pytest-mypy-testing==0.1.1 +commands = + pytest {posargs: tests_typing} + [testenv:pre-commit] basepython = python3 deps = From 204d6e180a7c8bc59f188230fb001339a5a43476 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:47:52 +0400 Subject: [PATCH 03/13] Enable typing-tests in CI. --- .github/workflows/checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index d6fc0f6c5..ed1629b67 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -18,6 +18,9 @@ jobs: - python-version: 3.8 env: TOXENV: typing + - python-version: 3.8 + env: + TOXENV: typing-tests - python-version: "3.11" # Keep in sync with .readthedocs.yml env: TOXENV: docs From 8776b4a6fb64e87c7baf96ae256e04a09246e360 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sun, 12 Nov 2023 20:52:29 +0400 Subject: [PATCH 04/13] Fix env deps for typing-tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c3fa54339..21ac4c3ff 100644 --- a/tox.ini +++ b/tox.ini @@ -48,7 +48,7 @@ commands = [testenv:typing-tests] deps = - {[testenv]deps} + -rtests/requirements.txt {[testenv:typing]deps} pytest-mypy-testing==0.1.1 commands = From db5a73f7bb44704b1751a3d005f53cbcd9846415 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:02:39 +0400 Subject: [PATCH 05/13] Update the expected mypy output to match the old Python one. --- tests_typing/test_http_request.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index a306b15fe..636e6895f 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -14,7 +14,7 @@ class MyRequest2(Request): @pytest.mark.mypy_testing def mypy_test_headers(): - Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Request("data:,", headers=1) # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Request("data:,", headers=None) Request("data:,", headers={}) Request("data:,", headers=[]) From ebdea4037a38bb207f90658b9380fda7a2e3e825 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 15 Nov 2023 12:31:31 +0400 Subject: [PATCH 06/13] Update another output line. --- tests_typing/test_http_response.mypy-testing | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 66ac6ad1d..2e58b4fbc 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse @pytest.mark.mypy_testing def mypy_test_headers(): - Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None" + Response("data:,", headers=1) # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]" Response("data:,", headers=None) Response("data:,", headers={}) Response("data:,", headers=[]) From 1fab844f7dd5fe622899c41ad8a0d28dd27c5089 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 15:57:51 +0400 Subject: [PATCH 07/13] Pin the Python version for typing-tests. --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 21ac4c3ff..f0788c0af 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,7 @@ commands = mypy {posargs: scrapy tests} [testenv:typing-tests] +basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} From a72394a388a8c41ab07f4511b096d85e6de168fe Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:14:53 +0400 Subject: [PATCH 08/13] Add tests for replace() with kwargs. --- tests_typing/test_http_request.mypy-testing | 14 ++++++++++++++ tests_typing/test_http_response.mypy-testing | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing index 636e6895f..665db9088 100644 --- a/tests_typing/test_http_request.mypy-testing +++ b/tests_typing/test_http_request.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy import Request @@ -33,6 +35,9 @@ def mypy_test_copy(): req_copy = req.copy() reveal_type(req_copy) # R: scrapy.http.request.Request + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.copy() @@ -45,13 +50,22 @@ def mypy_test_replace(): reveal_type(req) # R: scrapy.http.request.Request req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: scrapy.http.request.Request + kwargs: Dict[str, Any] = {} + req_copy2 = req.replace(body=b"a", **kwargs) + reveal_type(req_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): req = MyRequest("data:,") reveal_type(req) # R: __main__.MyRequest req_copy = req.replace(body=b"a") reveal_type(req_copy) # R: __main__.MyRequest req_copy2 = req.replace(body=b"a", cls=MyRequest2) reveal_type(req_copy2) # R: __main__.MyRequest2 + kwargs: Dict[str, Any] = {} + req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs) + reveal_type(req_copy3) # R: __main__.MyRequest2 @pytest.mark.mypy_testing diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing index 2e58b4fbc..d58ac1027 100644 --- a/tests_typing/test_http_response.mypy-testing +++ b/tests_typing/test_http_response.mypy-testing @@ -1,3 +1,5 @@ +from typing import Any, Dict + import pytest from scrapy.http import HtmlResponse, Response, TextResponse @@ -24,6 +26,9 @@ def mypy_test_copy(): resp_copy = resp.copy() reveal_type(resp_copy) # R: scrapy.http.response.Response + +@pytest.mark.mypy_testing +def mypy_test_copy_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.copy() @@ -36,10 +41,19 @@ def mypy_test_replace(): reveal_type(resp) # R: scrapy.http.response.Response resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.Response + kwargs: Dict[str, Any] = {} + resp_copy2 = resp.replace(body=b"a", **kwargs) + reveal_type(resp_copy2) # R: Any + +@pytest.mark.mypy_testing +def mypy_test_replace_subclass(): resp = HtmlResponse("data:,") reveal_type(resp) # R: scrapy.http.response.html.HtmlResponse resp_copy = resp.replace(body=b"a") reveal_type(resp_copy) # R: scrapy.http.response.html.HtmlResponse resp_copy2 = resp.replace(body=b"a", cls=TextResponse) reveal_type(resp_copy2) # R: scrapy.http.response.text.TextResponse + kwargs: Dict[str, Any] = {} + resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs) + reveal_type(resp_copy3) # R: scrapy.http.response.text.TextResponse From f56b5fc39ef3b322b8d0ad17fb424440bd79da0b Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 20 Dec 2023 16:19:11 +0400 Subject: [PATCH 09/13] Bump typing deps. --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index f0788c0af..25b30d759 100644 --- a/tox.ini +++ b/tox.ini @@ -33,14 +33,14 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.6.1 - typing-extensions==4.8.0 + mypy==1.7.1 + typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 - types-Pillow==10.1.0.0 - types-Pygments==2.16.0.0 + types-Pillow==10.1.0.2 + types-Pygments==2.17.0.0 types-pyOpenSSL==23.3.0.0 - types-setuptools==68.2.0.0 + types-setuptools==69.0.0.0 # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211 w3lib >= 2.1.2 commands = From 2534a28ef032ae03e567859a498307b07ad34f64 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 25 Dec 2023 15:03:08 +0400 Subject: [PATCH 10/13] Bump mypy. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 25b30d759..8996b12a4 100644 --- a/tox.ini +++ b/tox.ini @@ -33,7 +33,7 @@ install_command = [testenv:typing] basepython = python3 deps = - mypy==1.7.1 + mypy==1.8.0 typing-extensions==4.9.0 types-attrs==19.1.0 types-lxml==2023.10.21 From 706eb8d4275be993867122e5e41c31321488309e Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 14:33:55 +0500 Subject: [PATCH 11/13] Fix a merge error. --- scrapy/downloadermiddlewares/httpcompression.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py index aebdfb3e4..2352be0fe 100644 --- a/scrapy/downloadermiddlewares/httpcompression.py +++ b/scrapy/downloadermiddlewares/httpcompression.py @@ -135,7 +135,7 @@ class HttpCompressionMiddleware: respcls = responsetypes.from_args( headers=response.headers, url=response.url, body=decoded_body ) - kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body} + kwargs: Dict[str, Any] = {"body": decoded_body} if issubclass(respcls, TextResponse): # force recalculating the encoding until we make sure the # responsetypes guessing is reliable From 032e6a091a27b406aa48293f752d4782f8cac159 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Thu, 29 Feb 2024 16:24:52 +0500 Subject: [PATCH 12/13] Reformat the new changes with new black. --- scrapy/http/request/json_request.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py index 5c09835e4..59b11c692 100644 --- a/scrapy/http/request/json_request.py +++ b/scrapy/http/request/json_request.py @@ -53,12 +53,10 @@ class JsonRequest(Request): @overload def replace( self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any - ) -> RequestTypeVar: - ... + ) -> RequestTypeVar: ... @overload - def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: - ... + def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ... def replace( self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any From 6b75d8f3b3107957f3ae381ce3882ac3778f34c4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 5 Mar 2024 22:23:48 +0500 Subject: [PATCH 13/13] Bump pytest-mypy-testing. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index c43bd73d1..7192b6808 100644 --- a/tox.ini +++ b/tox.ini @@ -47,7 +47,7 @@ basepython = python3.8 deps = -rtests/requirements.txt {[testenv:typing]deps} - pytest-mypy-testing==0.1.1 + pytest-mypy-testing==0.1.3 commands = pytest {posargs: tests_typing}