From a6cee787dd45fabba3f39dbb1752baeef649f5b7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 11 Nov 2023 20:00:12 +0400
Subject: [PATCH 01/18] Improve type hints for copy() and replace() in Request
 and Response.

---
 .../downloadermiddlewares/httpcompression.py  |  6 +--
 scrapy/downloadermiddlewares/redirect.py      |  1 +
 scrapy/http/request/__init__.py               | 37 +++++++++++++++----
 scrapy/http/request/json_request.py           | 26 +++++++++++--
 scrapy/http/response/__init__.py              | 31 +++++++++++++---
 5 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 56a58a750..d44eb933a 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import io
 import zlib
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
@@ -74,12 +74,12 @@ class HttpCompressionMiddleware:
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs = dict(cls=respcls, body=decoded_body)
+                kwargs: Dict[str, Any] = dict(body=decoded_body)
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
                     kwargs["encoding"] = None
-                response = response.replace(**kwargs)
+                response = response.replace(cls=respcls, **kwargs)
                 if not content_encoding:
                     del response.headers["Content-Encoding"]
 
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 814b1a561..7b1401ac8 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -27,6 +27,7 @@ def _build_redirect_request(
     redirect_request = source_request.replace(
         url=url,
         **kwargs,
+        cls=None,
         cookies=None,
     )
     if "Cookie" in redirect_request.headers:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a1c5a5e51..4effc2178 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,8 +4,11 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+from __future__ import annotations
+
 import inspect
 from typing import (
+    TYPE_CHECKING,
     Any,
     AnyStr,
     Callable,
@@ -19,7 +22,7 @@ from typing import (
     Type,
     TypeVar,
     Union,
-    cast,
+    overload,
 )
 
 from w3lib.url import safe_url_string
@@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
@@ -173,23 +181,36 @@ class Request(object_ref):
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 
-    def copy(self) -> "Request":
+    def copy(self) -> Self:
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> "Request":
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Request, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     @classmethod
     def from_curl(
-        cls: Type[RequestTypeVar],
+        cls,
         curl_command: str,
         ignore_unknown_options: bool = True,
         **kwargs: Any,
-    ) -> RequestTypeVar:
+    ) -> Self:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
         URL, the headers, the cookies and the body. It accepts the same
@@ -221,7 +242,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+    def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 1dd9e6c87..5c09835e4 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
+from __future__ import annotations
+
 import copy
 import json
 import warnings
-from typing import Any, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
 
-from scrapy.http.request import Request
+from scrapy.http.request import Request, RequestTypeVar
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class JsonRequest(Request):
@@ -44,7 +50,19 @@ class JsonRequest(Request):
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args: Any, **kwargs: Any) -> Request:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         body_passed = kwargs.get("body", None) is not None
         data = kwargs.pop("data", None)
         data_passed = data is not None
@@ -54,7 +72,7 @@ class JsonRequest(Request):
         elif not body_passed and data_passed:
             kwargs["body"] = self._dumps(data)
 
-        return super().replace(*args, **kwargs)
+        return super().replace(*args, cls=cls, **kwargs)
 
     def _dumps(self, data: dict) -> str:
         """Convert to JSON"""
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 6eae3e8b3..e889a6460 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -19,8 +19,10 @@ from typing import (
     Mapping,
     Optional,
     Tuple,
+    Type,
+    TypeVar,
     Union,
-    cast,
+    overload,
 )
 from urllib.parse import urljoin
 
@@ -33,9 +35,15 @@ from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
 if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
     from scrapy.selector import SelectorList
 
 
+ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
+
+
 class Response(object_ref):
     """An object that represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -132,16 +140,29 @@ class Response(object_ref):
     def __repr__(self) -> str:
         return f"<{self.status} {self.url}>"
 
-    def copy(self) -> Response:
+    def copy(self) -> Self:
         """Return a copy of this Response"""
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> Response:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
+    ) -> ResponseTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
+    ) -> Response:
         """Create a new Response with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Response, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     def urljoin(self, url: str) -> str:
         """Join this Response's url with a possible relative url to form an

From 5d55e4f56b77168b961db15e0f03d608fad69e7d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:15:06 +0400
Subject: [PATCH 02/18] Add mypy tests.

---
 tests_typing/test_http_request.mypy-testing  | 66 ++++++++++++++++++++
 tests_typing/test_http_response.mypy-testing | 45 +++++++++++++
 tox.ini                                      |  8 +++
 3 files changed, 119 insertions(+)
 create mode 100644 tests_typing/test_http_request.mypy-testing
 create mode 100644 tests_typing/test_http_response.mypy-testing

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
new file mode 100644
index 000000000..a306b15fe
--- /dev/null
+++ b/tests_typing/test_http_request.mypy-testing
@@ -0,0 +1,66 @@
+import pytest
+
+from scrapy import Request
+from scrapy.http import JsonRequest
+
+
+class MyRequest(Request):
+    pass
+
+
+class MyRequest2(Request):
+    pass
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=None)
+    Request("data:,", headers={})
+    Request("data:,", headers=[])
+    Request("data:,", headers={"foo": "bar"})
+    Request("data:,", headers={b"foo": "bar"})
+    Request("data:,", headers={"foo": b"bar"})
+    Request("data:,", headers=[("foo", "bar")])
+    Request("data:,", headers=[(b"foo", "bar")])
+    Request("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: __main__.MyRequest
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: __main__.MyRequest
+    req_copy2 = req.replace(body=b"a", cls=MyRequest2)
+    reveal_type(req_copy2)  # R: __main__.MyRequest2
+
+
+@pytest.mark.mypy_testing
+def mypy_test_jsonrequest_copy_replace():
+    req = JsonRequest("data:,")
+    reveal_type(req)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy_my = req.replace(body=b"a", cls=MyRequest)
+    reveal_type(req_copy_my)  # R: __main__.MyRequest
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
new file mode 100644
index 000000000..66ac6ad1d
--- /dev/null
+++ b/tests_typing/test_http_response.mypy-testing
@@ -0,0 +1,45 @@
+import pytest
+
+from scrapy.http import HtmlResponse, Response, TextResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=None)
+    Response("data:,", headers={})
+    Response("data:,", headers=[])
+    Response("data:,", headers={"foo": "bar"})
+    Response("data:,", headers={b"foo": "bar"})
+    Response("data:,", headers={"foo": b"bar"})
+    Response("data:,", headers=[("foo", "bar")])
+    Response("data:,", headers=[(b"foo", "bar")])
+    Response("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
+    reveal_type(resp_copy2)  # R: scrapy.http.response.text.TextResponse
diff --git a/tox.ini b/tox.ini
index 932c0b805..c3fa54339 100644
--- a/tox.ini
+++ b/tox.ini
@@ -46,6 +46,14 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+deps =
+    {[testenv]deps}
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.1
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =

From 204d6e180a7c8bc59f188230fb001339a5a43476 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:47:52 +0400
Subject: [PATCH 03/18] Enable typing-tests in CI.

---
 .github/workflows/checks.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d6fc0f6c5..ed1629b67 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs

From 8776b4a6fb64e87c7baf96ae256e04a09246e360 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:52:29 +0400
Subject: [PATCH 04/18] Fix env deps for typing-tests.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c3fa54339..21ac4c3ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,7 +48,7 @@ commands =
 
 [testenv:typing-tests]
 deps =
-    {[testenv]deps}
+    -rtests/requirements.txt
     {[testenv:typing]deps}
     pytest-mypy-testing==0.1.1
 commands =

From db5a73f7bb44704b1751a3d005f53cbcd9846415 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 15 Nov 2023 12:02:39 +0400
Subject: [PATCH 05/18] Update the expected mypy output to match the old Python
 one.

---
 tests_typing/test_http_request.mypy-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index a306b15fe..636e6895f 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -14,7 +14,7 @@ class MyRequest2(Request):
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
     Request("data:,", headers=None)
     Request("data:,", headers={})
     Request("data:,", headers=[])

From ebdea4037a38bb207f90658b9380fda7a2e3e825 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 15 Nov 2023 12:31:31 +0400
Subject: [PATCH 06/18] Update another output line.

---
 tests_typing/test_http_response.mypy-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index 66ac6ad1d..2e58b4fbc 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
     Response("data:,", headers=None)
     Response("data:,", headers={})
     Response("data:,", headers=[])

From 1fab844f7dd5fe622899c41ad8a0d28dd27c5089 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 15:57:51 +0400
Subject: [PATCH 07/18] Pin the Python version for typing-tests.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 21ac4c3ff..f0788c0af 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,6 +47,7 @@ commands =
     mypy {posargs: scrapy tests}
 
 [testenv:typing-tests]
+basepython = python3.8
 deps =
     -rtests/requirements.txt
     {[testenv:typing]deps}

From a72394a388a8c41ab07f4511b096d85e6de168fe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 16:14:53 +0400
Subject: [PATCH 08/18] Add tests for replace() with kwargs.

---
 tests_typing/test_http_request.mypy-testing  | 14 ++++++++++++++
 tests_typing/test_http_response.mypy-testing | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index 636e6895f..665db9088 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 import pytest
 
 from scrapy import Request
@@ -33,6 +35,9 @@ def mypy_test_copy():
     req_copy = req.copy()
     reveal_type(req_copy)  # R: scrapy.http.request.Request
 
+
+@pytest.mark.mypy_testing
+def mypy_test_copy_subclass():
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.copy()
@@ -45,13 +50,22 @@ def mypy_test_replace():
     reveal_type(req)  # R: scrapy.http.request.Request
     req_copy = req.replace(body=b"a")
     reveal_type(req_copy)  # R: scrapy.http.request.Request
+    kwargs: Dict[str, Any] = {}
+    req_copy2 = req.replace(body=b"a", **kwargs)
+    reveal_type(req_copy2)  # R: Any
 
+
+@pytest.mark.mypy_testing
+def mypy_test_replace_subclass():
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.replace(body=b"a")
     reveal_type(req_copy)  # R: __main__.MyRequest
     req_copy2 = req.replace(body=b"a", cls=MyRequest2)
     reveal_type(req_copy2)  # R: __main__.MyRequest2
+    kwargs: Dict[str, Any] = {}
+    req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs)
+    reveal_type(req_copy3)  # R: __main__.MyRequest2
 
 
 @pytest.mark.mypy_testing
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index 2e58b4fbc..d58ac1027 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 import pytest
 
 from scrapy.http import HtmlResponse, Response, TextResponse
@@ -24,6 +26,9 @@ def mypy_test_copy():
     resp_copy = resp.copy()
     reveal_type(resp_copy)  # R: scrapy.http.response.Response
 
+
+@pytest.mark.mypy_testing
+def mypy_test_copy_subclass():
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.copy()
@@ -36,10 +41,19 @@ def mypy_test_replace():
     reveal_type(resp)  # R: scrapy.http.response.Response
     resp_copy = resp.replace(body=b"a")
     reveal_type(resp_copy)  # R: scrapy.http.response.Response
+    kwargs: Dict[str, Any] = {}
+    resp_copy2 = resp.replace(body=b"a", **kwargs)
+    reveal_type(resp_copy2)  # R: Any
 
+
+@pytest.mark.mypy_testing
+def mypy_test_replace_subclass():
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.replace(body=b"a")
     reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
     reveal_type(resp_copy2)  # R: scrapy.http.response.text.TextResponse
+    kwargs: Dict[str, Any] = {}
+    resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs)
+    reveal_type(resp_copy3)  # R: scrapy.http.response.text.TextResponse

From f56b5fc39ef3b322b8d0ad17fb424440bd79da0b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 16:19:11 +0400
Subject: [PATCH 09/18] Bump typing deps.

---
 tox.ini | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tox.ini b/tox.ini
index f0788c0af..25b30d759 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,14 +33,14 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.6.1
-    typing-extensions==4.8.0
+    mypy==1.7.1
+    typing-extensions==4.9.0
     types-attrs==19.1.0
     types-lxml==2023.10.21
-    types-Pillow==10.1.0.0
-    types-Pygments==2.16.0.0
+    types-Pillow==10.1.0.2
+    types-Pygments==2.17.0.0
     types-pyOpenSSL==23.3.0.0
-    types-setuptools==68.2.0.0
+    types-setuptools==69.0.0.0
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =

From 2534a28ef032ae03e567859a498307b07ad34f64 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 25 Dec 2023 15:03:08 +0400
Subject: [PATCH 10/18] Bump mypy.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 25b30d759..8996b12a4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,7 +33,7 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.7.1
+    mypy==1.8.0
     typing-extensions==4.9.0
     types-attrs==19.1.0
     types-lxml==2023.10.21

From 706eb8d4275be993867122e5e41c31321488309e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 29 Feb 2024 14:33:55 +0500
Subject: [PATCH 11/18] Fix a merge error.

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index aebdfb3e4..2352be0fe 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -135,7 +135,7 @@ class HttpCompressionMiddleware:
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body}
+                kwargs: Dict[str, Any] = {"body": decoded_body}
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable

From 032e6a091a27b406aa48293f752d4782f8cac159 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 29 Feb 2024 16:24:52 +0500
Subject: [PATCH 12/18] Reformat the new changes with new black.

---
 scrapy/http/request/json_request.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 5c09835e4..59b11c692 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -53,12 +53,10 @@ class JsonRequest(Request):
     @overload
     def replace(
         self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
-    ) -> RequestTypeVar:
-        ...
+    ) -> RequestTypeVar: ...
 
     @overload
-    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
-        ...
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
 
     def replace(
         self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any

From 6b75d8f3b3107957f3ae381ce3882ac3778f34c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 5 Mar 2024 22:23:48 +0500
Subject: [PATCH 13/18] Bump pytest-mypy-testing.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c43bd73d1..7192b6808 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,7 @@ basepython = python3.8
 deps =
     -rtests/requirements.txt
     {[testenv:typing]deps}
-    pytest-mypy-testing==0.1.1
+    pytest-mypy-testing==0.1.3
 commands =
     pytest {posargs: tests_typing}
 

From 2e214210f6707181a863dbceabf2d34e767396cb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 2 Jun 2024 01:48:37 +0500
Subject: [PATCH 14/18] Add parameters to iterable generics, replace generators
 with iterables.

---
 scrapy/commands/parse.py         |  8 ++--
 scrapy/core/engine.py            |  5 ++-
 scrapy/core/scraper.py           |  6 ++-
 scrapy/core/spidermw.py          | 70 ++++++++++++++++++++------------
 scrapy/http/response/__init__.py |  3 +-
 scrapy/http/response/text.py     |  3 +-
 scrapy/utils/iterators.py        | 28 ++++++-------
 scrapy/utils/misc.py             |  6 +--
 scrapy/utils/python.py           | 32 +++++++--------
 scrapy/utils/request.py          |  5 +--
 scrapy/utils/sitemap.py          |  4 +-
 scrapy/utils/spider.py           | 13 +++---
 tests/test_commands.py           |  4 +-
 13 files changed, 103 insertions(+), 84 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2453c0d39..f916a3e75 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand):
 
     @overload
     def iterate_spider_output(
-        self, result: Union[AsyncGenerator, CoroutineType]
-    ) -> Deferred: ...
+        self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
+    ) -> Deferred[_T]: ...
 
     @overload
-    def iterate_spider_output(self, result: _T) -> Iterable: ...
+    def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
 
-    def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
+    def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b342ad7a3..dededf99d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -372,7 +372,10 @@ class ExecutionEngine:
 
     @inlineCallbacks
     def open_spider(
-        self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
+        self,
+        spider: Spider,
+        start_requests: Iterable[Request] = (),
+        close_if_idle: bool = True,
     ) -> Generator[Deferred, Any, None]:
         if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 566e6628b..3b7492838 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,6 +16,7 @@ from typing import (
     Set,
     Tuple,
     Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -47,6 +48,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
+_T = TypeVar("_T")
 QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
 
 
@@ -256,14 +258,14 @@ class Scraper:
 
     def handle_spider_output(
         self,
-        result: Union[Iterable, AsyncIterable],
+        result: Union[Iterable[_T], AsyncIterable[_T]],
         request: Request,
         response: Response,
         spider: Spider,
     ) -> Deferred:
         if not result:
             return defer_succeed(None)
-        it: Union[Iterable, AsyncIterable]
+        it: Union[Iterable[_T], AsyncIterable[_T]]
         if isinstance(result, AsyncIterable):
             it = aiter_errback(
                 result, self.handle_spider_error, request, response, spider
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 2cef2e1dd..cb1a93a68 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import (
     Any,
-    AsyncGenerator,
     AsyncIterable,
     Callable,
     Generator,
@@ -17,6 +16,7 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
 )
@@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
 logger = logging.getLogger(__name__)
 
 
+_T = TypeVar("_T")
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
@@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
         self,
         response: Response,
         spider: Spider,
-        iterable: Union[Iterable, AsyncIterable],
+        iterable: Union[Iterable[_T], AsyncIterable[_T]],
         exception_processor_index: int,
-        recover_to: Union[MutableChain, MutableAsyncChain],
-    ) -> Union[Generator, AsyncGenerator]:
-        def process_sync(iterable: Iterable) -> Generator:
+        recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
+    ) -> Union[Iterable[_T], AsyncIterable[_T]]:
+        def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
             try:
                 yield from iterable
             except Exception as ex:
-                exception_result = self._process_spider_exception(
-                    response, spider, Failure(ex), exception_processor_index
+                exception_result = cast(
+                    Union[Failure, MutableChain[_T]],
+                    self._process_spider_exception(
+                        response, spider, Failure(ex), exception_processor_index
+                    ),
                 )
                 if isinstance(exception_result, Failure):
                     raise
+                assert isinstance(recover_to, MutableChain)
                 recover_to.extend(exception_result)
 
-        async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
+        async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
             try:
                 async for r in iterable:
                     yield r
             except Exception as ex:
-                exception_result = self._process_spider_exception(
-                    response, spider, Failure(ex), exception_processor_index
+                exception_result = cast(
+                    Union[Failure, MutableAsyncChain[_T]],
+                    self._process_spider_exception(
+                        response, spider, Failure(ex), exception_processor_index
+                    ),
                 )
                 if isinstance(exception_result, Failure):
                     raise
+                assert isinstance(recover_to, MutableAsyncChain)
                 recover_to.extend(exception_result)
 
         if isinstance(iterable, AsyncIterable):
@@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
         _failure: Failure,
         start_index: int = 0,
-    ) -> Union[Failure, MutableChain]:
+    ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
         exception = _failure.value
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
@@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                dfd: Deferred = self._process_spider_output(
-                    response, spider, result, method_index + 1
+                dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
+                    self._process_spider_output(
+                        response, spider, result, method_index + 1
+                    )
                 )
                 # _process_spider_output() returns a Deferred only because of downgrading so this can be
                 # simplified when downgrading is removed.
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
-                    return cast(MutableChain, dfd.result)
+                    return cast(
+                        Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
+                    )
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture
                 msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
         self,
         response: Response,
         spider: Spider,
-        result: Union[Iterable, AsyncIterable],
+        result: Union[Iterable[_T], AsyncIterable[_T]],
         start_index: int = 0,
-    ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
+    ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        recovered: Union[MutableChain, MutableAsyncChain]
+        recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
         last_result_is_async = isinstance(result, AsyncIterable)
         if last_result_is_async:
             recovered = MutableAsyncChain()
@@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
-                exception_result = self._process_spider_exception(
+                exception_result: Union[
+                    Failure, MutableChain[_T], MutableAsyncChain[_T]
+                ] = self._process_spider_exception(
                     response, spider, Failure(ex), method_index + 1
                 )
                 if isinstance(exception_result, Failure):
@@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     async def _process_callback_output(
-        self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
-    ) -> Union[MutableChain, MutableAsyncChain]:
-        recovered: Union[MutableChain, MutableAsyncChain]
+        self,
+        response: Response,
+        spider: Spider,
+        result: Union[Iterable[_T], AsyncIterable[_T]],
+    ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
+        recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
         if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
@@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
     ) -> Deferred:
         async def process_callback_output(
-            result: Union[Iterable, AsyncIterable]
-        ) -> Union[MutableChain, MutableAsyncChain]:
+            result: Union[Iterable[_T], AsyncIterable[_T]]
+        ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
             return await self._process_callback_output(response, spider, result)
 
-        def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
+        def process_spider_exception(
+            _failure: Failure,
+        ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
             return self._process_spider_exception(response, spider, _failure)
 
-        dfd = mustbe_deferred(
+        dfd: Deferred = mustbe_deferred(
             self._process_spider_input, scrape_func, response, request, spider
         )
         dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 166c4de97..daf193f59 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -14,7 +14,6 @@ from typing import (
     AnyStr,
     Callable,
     Dict,
-    Generator,
     Iterable,
     List,
     Mapping,
@@ -242,7 +241,7 @@ class Response(object_ref):
         errback: Optional[Callable] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
-    ) -> Generator[Request, None, None]:
+    ) -> Iterable[Request]:
         """
         .. versionadded:: 2.0
 
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 44c36b682..df4d90829 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -15,7 +15,6 @@ from typing import (
     AnyStr,
     Callable,
     Dict,
-    Generator,
     Iterable,
     List,
     Mapping,
@@ -246,7 +245,7 @@ class TextResponse(Response):
         flags: Optional[List[str]] = None,
         css: Optional[str] = None,
         xpath: Optional[str] = None,
-    ) -> Generator[Request, None, None]:
+    ) -> Iterable[Request]:
         """
         A generator that produces :class:`~.Request` instances to follow all
         links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index cd6e9d04e..41a842386 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -6,8 +6,7 @@ from typing import (
     Any,
     Callable,
     Dict,
-    Generator,
-    Iterable,
+    Iterator,
     List,
     Literal,
     Optional,
@@ -22,14 +21,12 @@ from lxml import etree  # nosec
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch, to_unicode
+from scrapy.utils.python import re_rsearch
 
 logger = logging.getLogger(__name__)
 
 
-def xmliter(
-    obj: Union[Response, str, bytes], nodename: str
-) -> Generator[Selector, Any, None]:
+def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
     """Return a iterator of Selector's over all nodes of a XML document,
        given the name of the node to iterate. Useful for parsing XML feeds.
 
@@ -90,7 +87,7 @@ def xmliter_lxml(
     nodename: str,
     namespace: Optional[str] = None,
     prefix: str = "x",
-) -> Generator[Selector, Any, None]:
+) -> Iterator[Selector]:
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
     iterable = etree.iterparse(
@@ -168,7 +165,7 @@ def csviter(
     headers: Optional[List[str]] = None,
     encoding: Optional[str] = None,
     quotechar: Optional[str] = None,
-) -> Generator[Dict[str, str], Any, None]:
+) -> Iterator[Dict[str, str]]:
     """Returns an iterator of dictionaries from the given csv object
 
     obj can be:
@@ -184,10 +181,13 @@ def csviter(
     quotechar is the character used to enclosure fields on the given obj.
     """
 
-    encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
-
-    def row_to_unicode(row_: Iterable) -> List[str]:
-        return [to_unicode(field, encoding) for field in row_]
+    if encoding is not None:
+        warn(
+            "The encoding argument of csviter() is ignored and will be removed"
+            " in a future Scrapy version.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
 
     lines = StringIO(_body_or_str(obj, unicode=True))
 
@@ -200,13 +200,11 @@ def csviter(
 
     if not headers:
         try:
-            row = next(csv_r)
+            headers = next(csv_r)
         except StopIteration:
             return
-        headers = row_to_unicode(row)
 
     for row in csv_r:
-        row = row_to_unicode(row)
         if len(row) != len(headers):
             logger.warning(
                 "ignoring row %(csvlnum)d (length: %(csvrow)d, "
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 49f36de2d..3d11c1035 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -20,8 +20,8 @@ from typing import (
     Any,
     Callable,
     Deque,
-    Generator,
     Iterable,
+    Iterator,
     List,
     Optional,
     Type,
@@ -227,7 +227,7 @@ def build_from_settings(
 
 
 @contextmanager
-def set_environ(**kwargs: str) -> Generator[None, Any, None]:
+def set_environ(**kwargs: str) -> Iterator[None]:
     """Temporarily set environment variables inside the context manager and
     fully restore previous environment afterwards
     """
@@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
                 os.environ[k] = v
 
 
-def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
+def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
     """Similar to ``ast.walk``, but walks only function body and skips nested
     functions defined within the node.
     """
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 37a84a350..059d8e04d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -15,12 +15,10 @@ from itertools import chain
 from typing import (
     TYPE_CHECKING,
     Any,
-    AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
     Callable,
     Dict,
-    Generator,
     Iterable,
     Iterator,
     List,
@@ -163,7 +161,7 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
+    def _chunk_iter() -> Iterable[Tuple[str, int]]:
         offset = len(text)
         while True:
             offset -= chunk_size * 1024
@@ -351,43 +349,45 @@ else:
         gc.collect()
 
 
-class MutableChain(Iterable):
+class MutableChain(Iterable[_T]):
     """
     Thin wrapper around itertools.chain, allowing to add iterables "in-place"
     """
 
-    def __init__(self, *args: Iterable):
-        self.data = chain.from_iterable(args)
+    def __init__(self, *args: Iterable[_T]):
+        self.data: Iterator[_T] = chain.from_iterable(args)
 
-    def extend(self, *iterables: Iterable) -> None:
+    def extend(self, *iterables: Iterable[_T]) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
-    def __iter__(self) -> Iterator:
+    def __iter__(self) -> Iterator[_T]:
         return self
 
-    def __next__(self) -> Any:
+    def __next__(self) -> _T:
         return next(self.data)
 
 
-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+async def _async_chain(
+    *iterables: Union[Iterable[_T], AsyncIterable[_T]]
+) -> AsyncIterator[_T]:
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
 
 
-class MutableAsyncChain(AsyncIterable):
+class MutableAsyncChain(AsyncIterable[_T]):
     """
     Similar to MutableChain but for async iterables
     """
 
-    def __init__(self, *args: Union[Iterable, AsyncIterable]):
-        self.data = _async_chain(*args)
+    def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
+        self.data: AsyncIterator[_T] = _async_chain(*args)
 
-    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+    def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
         self.data = _async_chain(self.data, _async_chain(*iterables))
 
-    def __aiter__(self) -> AsyncIterator:
+    def __aiter__(self) -> AsyncIterator[_T]:
         return self
 
-    async def __anext__(self) -> Any:
+    async def __anext__(self) -> _T:
         return await self.data.__anext__()
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 42a6537a8..45b8008f4 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -12,7 +12,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
-    Generator,
     Iterable,
     List,
     Optional,
@@ -40,9 +39,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
-def _serialize_headers(
-    headers: Iterable[bytes], request: Request
-) -> Generator[bytes, Any, None]:
+def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
     for header in headers:
         if header in request.headers:
             yield header
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index cf429043d..7a91afe59 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
-from typing import Any, Dict, Generator, Iterator, Optional, Union
+from typing import Any, Dict, Iterable, Iterator, Optional, Union
 from urllib.parse import urljoin
 
 import lxml.etree  # nosec
@@ -42,7 +42,7 @@ class Sitemap:
 
 def sitemap_urls_from_robots(
     robots_text: str, base_url: Optional[str] = None
-) -> Generator[str, Any, None]:
+) -> Iterable[str]:
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index cbbb01d85..b05135c04 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -7,7 +7,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncGenerator,
-    Generator,
     Iterable,
     Literal,
     Optional,
@@ -34,18 +33,20 @@ _T = TypeVar("_T")
 
 # https://stackoverflow.com/questions/60222982
 @overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ...  # type: ignore[overload-overlap]
+def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ...  # type: ignore[overload-overlap]
 
 
 @overload
-def iterate_spider_output(result: CoroutineType) -> Deferred: ...
+def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
 
 
 @overload
-def iterate_spider_output(result: _T) -> Iterable: ...
+def iterate_spider_output(result: _T) -> Iterable[Any]: ...
 
 
-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
+def iterate_spider_output(
+    result: Any,
+) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
     return arg_to_iter(deferred_from_coro(result))
 
 
-def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
+def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b9d468c66..857a56b73 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -13,7 +13,7 @@ from shutil import copytree, rmtree
 from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 from tempfile import TemporaryFile, mkdtemp
 from threading import Timer
-from typing import Dict, Generator, Optional, Union
+from typing import Dict, Iterator, Optional, Union
 from unittest import skipIf
 
 from pytest import mark
@@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
         """
 
     @contextmanager
-    def _create_file(self, content, name=None) -> Generator[str, None, None]:
+    def _create_file(self, content, name=None) -> Iterator[str]:
         tmpdir = Path(self.mktemp())
         tmpdir.mkdir()
         if name:

From de146ad7cef9e3478290be021129979f69fc6d03 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:09:19 +0500
Subject: [PATCH 15/18] Bump typing deps.

---
 scrapy/extensions/httpcache.py | 5 ++---
 scrapy/http/headers.py         | 3 +--
 tox.ini                        | 9 ++++-----
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3f4af42b7..b7219bf07 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -370,12 +370,11 @@ class FilesystemCacheStorage:
         with self._open(rpath / "pickled_meta", "wb") as f:
             pickle.dump(metadata, f, protocol=4)
         with self._open(rpath / "response_headers", "wb") as f:
-            # headers_dict_to_raw() needs a better type hint
-            f.write(cast(bytes, headers_dict_to_raw(response.headers)))
+            f.write(headers_dict_to_raw(response.headers))
         with self._open(rpath / "response_body", "wb") as f:
             f.write(response.body)
         with self._open(rpath / "request_headers", "wb") as f:
-            f.write(cast(bytes, headers_dict_to_raw(request.headers)))
+            f.write(headers_dict_to_raw(request.headers))
         with self._open(rpath / "request_body", "wb") as f:
             f.write(request.body)
 
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 73aee7178..85b9229d3 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -118,8 +118,7 @@ class Headers(CaselessDict):
         ]
 
     def to_string(self) -> bytes:
-        # cast() can be removed if the headers_dict_to_raw() hint is improved
-        return cast(bytes, headers_dict_to_raw(self))
+        return headers_dict_to_raw(self)
 
     def to_unicode_dict(self) -> CaseInsensitiveDict:
         """Return headers as a CaseInsensitiveDict with str keys
diff --git a/tox.ini b/tox.ini
index 5a5e80496..023a86c5a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,18 +47,17 @@ install_command =
 basepython = python3
 deps =
     mypy==1.10.0
-    typing-extensions==4.11.0
+    typing-extensions==4.12.1
     types-lxml==2024.4.14
     types-Pygments==2.18.0.20240506
     types-pyOpenSSL==24.1.0.20240425
-    types-setuptools==69.5.0.20240518
+    types-setuptools==70.0.0.20240524
     botocore-stubs==1.34.94
-    boto3-stubs[s3]==1.34.108
+    boto3-stubs[s3]==1.34.119
     attrs >= 18.2.0
     Pillow >= 10.3.0
     pytest >= 8.2.0
-    # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
-    w3lib >= 2.1.2
+    w3lib >= 2.2.0
 commands =
     mypy {posargs: scrapy tests}
 

From 262c10d85bd34732b0c692bdc8d16375d83a178f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:11:34 +0500
Subject: [PATCH 16/18] Use typing.Coroutine instead of types.CoroutineType.

---
 scrapy/commands/parse.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index f916a3e75..ce6f4dc51 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -3,11 +3,11 @@ import functools
 import inspect
 import json
 import logging
-from types import CoroutineType
 from typing import (
     Any,
     AsyncGenerator,
     Callable,
+    Coroutine,
     Dict,
     Iterable,
     List,
@@ -140,7 +140,7 @@ class Command(BaseRunSpiderCommand):
 
     @overload
     def iterate_spider_output(
-        self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
+        self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]]
     ) -> Deferred[_T]: ...
 
     @overload

From 480a11b68bee19162cc0da59e9bed42b29bc9cfe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:48:16 +0500
Subject: [PATCH 17/18] Add mssing __future__ imports.

---
 scrapy/commands/parse.py | 2 ++
 scrapy/core/spidermw.py  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ce6f4dc51..3320a1ee4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import argparse
 import functools
 import inspect
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index cb1a93a68..58873f0d9 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,6 +4,8 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 
+from __future__ import annotations
+
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice

From 144ff6c756fa58da2bc1a85879aa6f89300030d1 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Wed, 5 Jun 2024 21:09:10 -0300
Subject: [PATCH 18/18] Document missing parts of response.json method

---
 docs/topics/dynamic-content.rst | 7 +++----
 docs/topics/selectors.rst       | 8 ++++++++
 scrapy/selector/unified.py      | 1 +
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index a0f4b4411..a99f1e222 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -115,15 +115,14 @@ Handling different response formats
 Once you have a response with the desired data, how you extract the desired
 data from it depends on the type of response:
 
--   If the response is HTML or XML, use :ref:`selectors
+-   If the response is HTML, XML or JSON, use :ref:`selectors
     <topics-selectors>` as usual.
 
--   If the response is JSON, use :func:`json.loads` to load the desired data from
-    :attr:`response.text <scrapy.http.TextResponse.text>`:
+-   If the response is JSON, use :func:`response.json()` to load the desired data:
 
     .. code-block:: python
 
-        data = json.loads(response.text)
+        data = response.json()
 
     If the desired data is inside HTML or XML code embedded within JSON data,
     you can load that HTML or XML code into a
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index e32fc2b70..0aae41cc8 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -1060,6 +1060,12 @@ Selector objects
 
           For convenience, this method can be called as ``response.css()``
 
+  .. automethod:: jmespath
+
+      .. note::
+
+          For convenience, this method can be called as ``response.jmespath()``
+
   .. automethod:: get
 
      See also: :ref:`old-extraction-api`
@@ -1092,6 +1098,8 @@ SelectorList objects
 
    .. automethod:: css
 
+   .. automethod:: jmespath
+
    .. automethod:: getall
 
       See also: :ref:`old-extraction-api`
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index e852aadc7..bfddb87cb 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref):
 
     * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
     * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+    * ``"json"`` for :class:`~scrapy.http.TextResponse` type
     * ``"html"`` for anything else
 
     Otherwise, if ``type`` is set, the selector type will be forced and no