From a6cee787dd45fabba3f39dbb1752baeef649f5b7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 11 Nov 2023 20:00:12 +0400
Subject: [PATCH 01/82] Improve type hints for copy() and replace() in Request
 and Response.

---
 .../downloadermiddlewares/httpcompression.py  |  6 +--
 scrapy/downloadermiddlewares/redirect.py      |  1 +
 scrapy/http/request/__init__.py               | 37 +++++++++++++++----
 scrapy/http/request/json_request.py           | 26 +++++++++++--
 scrapy/http/response/__init__.py              | 31 +++++++++++++---
 5 files changed, 81 insertions(+), 20 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 56a58a750..d44eb933a 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import io
 import zlib
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
@@ -74,12 +74,12 @@ class HttpCompressionMiddleware:
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs = dict(cls=respcls, body=decoded_body)
+                kwargs: Dict[str, Any] = dict(body=decoded_body)
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable
                     kwargs["encoding"] = None
-                response = response.replace(**kwargs)
+                response = response.replace(cls=respcls, **kwargs)
                 if not content_encoding:
                     del response.headers["Content-Encoding"]
 
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 814b1a561..7b1401ac8 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -27,6 +27,7 @@ def _build_redirect_request(
     redirect_request = source_request.replace(
         url=url,
         **kwargs,
+        cls=None,
         cookies=None,
     )
     if "Cookie" in redirect_request.headers:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index a1c5a5e51..4effc2178 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -4,8 +4,11 @@ requests in Scrapy.
 
 See documentation in docs/topics/request-response.rst
 """
+from __future__ import annotations
+
 import inspect
 from typing import (
+    TYPE_CHECKING,
     Any,
     AnyStr,
     Callable,
@@ -19,7 +22,7 @@ from typing import (
     Type,
     TypeVar,
     Union,
-    cast,
+    overload,
 )
 
 from w3lib.url import safe_url_string
@@ -31,6 +34,11 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
 
 
@@ -173,23 +181,36 @@ class Request(object_ref):
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 
-    def copy(self) -> "Request":
+    def copy(self) -> Self:
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> "Request":
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Request, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     @classmethod
     def from_curl(
-        cls: Type[RequestTypeVar],
+        cls,
         curl_command: str,
         ignore_unknown_options: bool = True,
         **kwargs: Any,
-    ) -> RequestTypeVar:
+    ) -> Self:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
         URL, the headers, the cookies and the body. It accepts the same
@@ -221,7 +242,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+    def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 1dd9e6c87..5c09835e4 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -5,12 +5,18 @@ This module implements the JsonRequest class which is a more convenient class
 See documentation in docs/topics/request-response.rst
 """
 
+from __future__ import annotations
+
 import copy
 import json
 import warnings
-from typing import Any, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, overload
 
-from scrapy.http.request import Request
+from scrapy.http.request import Request, RequestTypeVar
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class JsonRequest(Request):
@@ -44,7 +50,19 @@ class JsonRequest(Request):
     def dumps_kwargs(self) -> dict:
         return self._dumps_kwargs
 
-    def replace(self, *args: Any, **kwargs: Any) -> Request:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
+    ) -> RequestTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any
+    ) -> Request:
         body_passed = kwargs.get("body", None) is not None
         data = kwargs.pop("data", None)
         data_passed = data is not None
@@ -54,7 +72,7 @@ class JsonRequest(Request):
         elif not body_passed and data_passed:
             kwargs["body"] = self._dumps(data)
 
-        return super().replace(*args, **kwargs)
+        return super().replace(*args, cls=cls, **kwargs)
 
     def _dumps(self, data: dict) -> str:
         """Convert to JSON"""
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 6eae3e8b3..e889a6460 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -19,8 +19,10 @@ from typing import (
     Mapping,
     Optional,
     Tuple,
+    Type,
+    TypeVar,
     Union,
-    cast,
+    overload,
 )
 from urllib.parse import urljoin
 
@@ -33,9 +35,15 @@ from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
 if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
     from scrapy.selector import SelectorList
 
 
+ResponseTypeVar = TypeVar("ResponseTypeVar", bound="Response")
+
+
 class Response(object_ref):
     """An object that represents an HTTP response, which is usually
     downloaded (by the Downloader) and fed to the Spiders for processing.
@@ -132,16 +140,29 @@ class Response(object_ref):
     def __repr__(self) -> str:
         return f"<{self.status} {self.url}>"
 
-    def copy(self) -> Response:
+    def copy(self) -> Self:
         """Return a copy of this Response"""
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> Response:
+    @overload
+    def replace(
+        self, *args: Any, cls: Type[ResponseTypeVar], **kwargs: Any
+    ) -> ResponseTypeVar:
+        ...
+
+    @overload
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
+        ...
+
+    def replace(
+        self, *args: Any, cls: Optional[Type[Response]] = None, **kwargs: Any
+    ) -> Response:
         """Create a new Response with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
-        cls = kwargs.pop("cls", self.__class__)
-        return cast(Response, cls(*args, **kwargs))
+        if cls is None:
+            cls = self.__class__
+        return cls(*args, **kwargs)
 
     def urljoin(self, url: str) -> str:
         """Join this Response's url with a possible relative url to form an

From 5d55e4f56b77168b961db15e0f03d608fad69e7d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:15:06 +0400
Subject: [PATCH 02/82] Add mypy tests.

---
 tests_typing/test_http_request.mypy-testing  | 66 ++++++++++++++++++++
 tests_typing/test_http_response.mypy-testing | 45 +++++++++++++
 tox.ini                                      |  8 +++
 3 files changed, 119 insertions(+)
 create mode 100644 tests_typing/test_http_request.mypy-testing
 create mode 100644 tests_typing/test_http_response.mypy-testing

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
new file mode 100644
index 000000000..a306b15fe
--- /dev/null
+++ b/tests_typing/test_http_request.mypy-testing
@@ -0,0 +1,66 @@
+import pytest
+
+from scrapy import Request
+from scrapy.http import JsonRequest
+
+
+class MyRequest(Request):
+    pass
+
+
+class MyRequest2(Request):
+    pass
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=None)
+    Request("data:,", headers={})
+    Request("data:,", headers=[])
+    Request("data:,", headers={"foo": "bar"})
+    Request("data:,", headers={b"foo": "bar"})
+    Request("data:,", headers={"foo": b"bar"})
+    Request("data:,", headers=[("foo", "bar")])
+    Request("data:,", headers=[(b"foo", "bar")])
+    Request("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: __main__.MyRequest
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    req = Request("data:,")
+    reveal_type(req)  # R: scrapy.http.request.Request
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.Request
+
+    req = MyRequest("data:,")
+    reveal_type(req)  # R: __main__.MyRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: __main__.MyRequest
+    req_copy2 = req.replace(body=b"a", cls=MyRequest2)
+    reveal_type(req_copy2)  # R: __main__.MyRequest2
+
+
+@pytest.mark.mypy_testing
+def mypy_test_jsonrequest_copy_replace():
+    req = JsonRequest("data:,")
+    reveal_type(req)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.copy()
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy = req.replace(body=b"a")
+    reveal_type(req_copy)  # R: scrapy.http.request.json_request.JsonRequest
+    req_copy_my = req.replace(body=b"a", cls=MyRequest)
+    reveal_type(req_copy_my)  # R: __main__.MyRequest
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
new file mode 100644
index 000000000..66ac6ad1d
--- /dev/null
+++ b/tests_typing/test_http_response.mypy-testing
@@ -0,0 +1,45 @@
+import pytest
+
+from scrapy.http import HtmlResponse, Response, TextResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_headers():
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=None)
+    Response("data:,", headers={})
+    Response("data:,", headers=[])
+    Response("data:,", headers={"foo": "bar"})
+    Response("data:,", headers={b"foo": "bar"})
+    Response("data:,", headers={"foo": b"bar"})
+    Response("data:,", headers=[("foo", "bar")])
+    Response("data:,", headers=[(b"foo", "bar")])
+    Response("data:,", headers=[("foo", b"bar")])
+
+
+@pytest.mark.mypy_testing
+def mypy_test_copy():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.copy()
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+
+
+@pytest.mark.mypy_testing
+def mypy_test_replace():
+    resp = Response("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.Response
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.Response
+
+    resp = HtmlResponse("data:,")
+    reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy = resp.replace(body=b"a")
+    reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
+    resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
+    reveal_type(resp_copy2)  # R: scrapy.http.response.text.TextResponse
diff --git a/tox.ini b/tox.ini
index 932c0b805..c3fa54339 100644
--- a/tox.ini
+++ b/tox.ini
@@ -46,6 +46,14 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+deps =
+    {[testenv]deps}
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.1
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =

From 204d6e180a7c8bc59f188230fb001339a5a43476 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:47:52 +0400
Subject: [PATCH 03/82] Enable typing-tests in CI.

---
 .github/workflows/checks.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d6fc0f6c5..ed1629b67 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs

From 8776b4a6fb64e87c7baf96ae256e04a09246e360 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 12 Nov 2023 20:52:29 +0400
Subject: [PATCH 04/82] Fix env deps for typing-tests.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c3fa54339..21ac4c3ff 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,7 +48,7 @@ commands =
 
 [testenv:typing-tests]
 deps =
-    {[testenv]deps}
+    -rtests/requirements.txt
     {[testenv:typing]deps}
     pytest-mypy-testing==0.1.1
 commands =

From db5a73f7bb44704b1751a3d005f53cbcd9846415 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 15 Nov 2023 12:02:39 +0400
Subject: [PATCH 05/82] Update the expected mypy output to match the old Python
 one.

---
 tests_typing/test_http_request.mypy-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index a306b15fe..636e6895f 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -14,7 +14,7 @@ class MyRequest2(Request):
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
     Request("data:,", headers=None)
     Request("data:,", headers={})
     Request("data:,", headers=[])

From ebdea4037a38bb207f90658b9380fda7a2e3e825 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 15 Nov 2023 12:31:31 +0400
Subject: [PATCH 06/82] Update another output line.

---
 tests_typing/test_http_response.mypy-testing | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index 66ac6ad1d..2e58b4fbc 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -5,7 +5,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 @pytest.mark.mypy_testing
 def mypy_test_headers():
-    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Union[Mapping[str, Any], Iterable[Tuple[str, Any]], None]"
     Response("data:,", headers=None)
     Response("data:,", headers={})
     Response("data:,", headers=[])

From 1fab844f7dd5fe622899c41ad8a0d28dd27c5089 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 15:57:51 +0400
Subject: [PATCH 07/82] Pin the Python version for typing-tests.

---
 tox.ini | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tox.ini b/tox.ini
index 21ac4c3ff..f0788c0af 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,6 +47,7 @@ commands =
     mypy {posargs: scrapy tests}
 
 [testenv:typing-tests]
+basepython = python3.8
 deps =
     -rtests/requirements.txt
     {[testenv:typing]deps}

From a72394a388a8c41ab07f4511b096d85e6de168fe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 16:14:53 +0400
Subject: [PATCH 08/82] Add tests for replace() with kwargs.

---
 tests_typing/test_http_request.mypy-testing  | 14 ++++++++++++++
 tests_typing/test_http_response.mypy-testing | 14 ++++++++++++++
 2 files changed, 28 insertions(+)

diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index 636e6895f..665db9088 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 import pytest
 
 from scrapy import Request
@@ -33,6 +35,9 @@ def mypy_test_copy():
     req_copy = req.copy()
     reveal_type(req_copy)  # R: scrapy.http.request.Request
 
+
+@pytest.mark.mypy_testing
+def mypy_test_copy_subclass():
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.copy()
@@ -45,13 +50,22 @@ def mypy_test_replace():
     reveal_type(req)  # R: scrapy.http.request.Request
     req_copy = req.replace(body=b"a")
     reveal_type(req_copy)  # R: scrapy.http.request.Request
+    kwargs: Dict[str, Any] = {}
+    req_copy2 = req.replace(body=b"a", **kwargs)
+    reveal_type(req_copy2)  # R: Any
 
+
+@pytest.mark.mypy_testing
+def mypy_test_replace_subclass():
     req = MyRequest("data:,")
     reveal_type(req)  # R: __main__.MyRequest
     req_copy = req.replace(body=b"a")
     reveal_type(req_copy)  # R: __main__.MyRequest
     req_copy2 = req.replace(body=b"a", cls=MyRequest2)
     reveal_type(req_copy2)  # R: __main__.MyRequest2
+    kwargs: Dict[str, Any] = {}
+    req_copy3 = req.replace(body=b"a", cls=MyRequest2, **kwargs)
+    reveal_type(req_copy3)  # R: __main__.MyRequest2
 
 
 @pytest.mark.mypy_testing
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index 2e58b4fbc..d58ac1027 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -1,3 +1,5 @@
+from typing import Any, Dict
+
 import pytest
 
 from scrapy.http import HtmlResponse, Response, TextResponse
@@ -24,6 +26,9 @@ def mypy_test_copy():
     resp_copy = resp.copy()
     reveal_type(resp_copy)  # R: scrapy.http.response.Response
 
+
+@pytest.mark.mypy_testing
+def mypy_test_copy_subclass():
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.copy()
@@ -36,10 +41,19 @@ def mypy_test_replace():
     reveal_type(resp)  # R: scrapy.http.response.Response
     resp_copy = resp.replace(body=b"a")
     reveal_type(resp_copy)  # R: scrapy.http.response.Response
+    kwargs: Dict[str, Any] = {}
+    resp_copy2 = resp.replace(body=b"a", **kwargs)
+    reveal_type(resp_copy2)  # R: Any
 
+
+@pytest.mark.mypy_testing
+def mypy_test_replace_subclass():
     resp = HtmlResponse("data:,")
     reveal_type(resp)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy = resp.replace(body=b"a")
     reveal_type(resp_copy)  # R: scrapy.http.response.html.HtmlResponse
     resp_copy2 = resp.replace(body=b"a", cls=TextResponse)
     reveal_type(resp_copy2)  # R: scrapy.http.response.text.TextResponse
+    kwargs: Dict[str, Any] = {}
+    resp_copy3 = resp.replace(body=b"a", cls=TextResponse, **kwargs)
+    reveal_type(resp_copy3)  # R: scrapy.http.response.text.TextResponse

From f56b5fc39ef3b322b8d0ad17fb424440bd79da0b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 20 Dec 2023 16:19:11 +0400
Subject: [PATCH 09/82] Bump typing deps.

---
 tox.ini | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tox.ini b/tox.ini
index f0788c0af..25b30d759 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,14 +33,14 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.6.1
-    typing-extensions==4.8.0
+    mypy==1.7.1
+    typing-extensions==4.9.0
     types-attrs==19.1.0
     types-lxml==2023.10.21
-    types-Pillow==10.1.0.0
-    types-Pygments==2.16.0.0
+    types-Pillow==10.1.0.2
+    types-Pygments==2.17.0.0
     types-pyOpenSSL==23.3.0.0
-    types-setuptools==68.2.0.0
+    types-setuptools==69.0.0.0
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =

From 2534a28ef032ae03e567859a498307b07ad34f64 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 25 Dec 2023 15:03:08 +0400
Subject: [PATCH 10/82] Bump mypy.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 25b30d759..8996b12a4 100644
--- a/tox.ini
+++ b/tox.ini
@@ -33,7 +33,7 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.7.1
+    mypy==1.8.0
     typing-extensions==4.9.0
     types-attrs==19.1.0
     types-lxml==2023.10.21

From 706eb8d4275be993867122e5e41c31321488309e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 29 Feb 2024 14:33:55 +0500
Subject: [PATCH 11/82] Fix a merge error.

---
 scrapy/downloadermiddlewares/httpcompression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index aebdfb3e4..2352be0fe 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -135,7 +135,7 @@ class HttpCompressionMiddleware:
                 respcls = responsetypes.from_args(
                     headers=response.headers, url=response.url, body=decoded_body
                 )
-                kwargs: Dict[str, Any] = {"cls": respcls, "body": decoded_body}
+                kwargs: Dict[str, Any] = {"body": decoded_body}
                 if issubclass(respcls, TextResponse):
                     # force recalculating the encoding until we make sure the
                     # responsetypes guessing is reliable

From 032e6a091a27b406aa48293f752d4782f8cac159 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 29 Feb 2024 16:24:52 +0500
Subject: [PATCH 12/82] Reformat the new changes with new black.

---
 scrapy/http/request/json_request.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 5c09835e4..59b11c692 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -53,12 +53,10 @@ class JsonRequest(Request):
     @overload
     def replace(
         self, *args: Any, cls: Type[RequestTypeVar], **kwargs: Any
-    ) -> RequestTypeVar:
-        ...
+    ) -> RequestTypeVar: ...
 
     @overload
-    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self:
-        ...
+    def replace(self, *args: Any, cls: None = None, **kwargs: Any) -> Self: ...
 
     def replace(
         self, *args: Any, cls: Optional[Type[Request]] = None, **kwargs: Any

From 6b75d8f3b3107957f3ae381ce3882ac3778f34c4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 5 Mar 2024 22:23:48 +0500
Subject: [PATCH 13/82] Bump pytest-mypy-testing.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index c43bd73d1..7192b6808 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,7 +47,7 @@ basepython = python3.8
 deps =
     -rtests/requirements.txt
     {[testenv:typing]deps}
-    pytest-mypy-testing==0.1.1
+    pytest-mypy-testing==0.1.3
 commands =
     pytest {posargs: tests_typing}
 

From b99526b740890e63f1d05074b3e358a9ae59b77f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 19 May 2024 15:45:51 +0500
Subject: [PATCH 14/82] Full typing for scrapy/contracts.

---
 scrapy/contracts/__init__.py | 75 ++++++++++++++++++++++++------------
 scrapy/contracts/default.py  | 19 ++++-----
 2 files changed, 60 insertions(+), 34 deletions(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index d46eb7c51..b300b8457 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -3,10 +3,23 @@ import sys
 from functools import wraps
 from inspect import getmembers
 from types import CoroutineType
-from typing import AsyncGenerator, Dict, Optional, Type
-from unittest import TestCase
+from typing import (
+    Any,
+    AsyncGenerator,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+)
+from unittest import TestCase, TestResult
 
-from scrapy.http import Request
+from twisted.python.failure import Failure
+
+from scrapy import Spider
+from scrapy.http import Request, Response
 from scrapy.utils.python import get_spec
 from scrapy.utils.spider import iterate_spider_output
 
@@ -15,18 +28,20 @@ class Contract:
     """Abstract class for contracts"""
 
     request_cls: Optional[Type[Request]] = None
+    name: str
 
-    def __init__(self, method, *args):
+    def __init__(self, method: Callable, *args: Any):
         self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
         self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
-        self.args = args
+        self.args: Tuple[Any, ...] = args
 
-    def add_pre_hook(self, request, results):
+    def add_pre_hook(self, request: Request, results: TestResult) -> Request:
         if hasattr(self, "pre_process"):
             cb = request.callback
+            assert cb is not None
 
             @wraps(cb)
-            def wrapper(response, **cb_kwargs):
+            def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]:
                 try:
                     results.startTest(self.testcase_pre)
                     self.pre_process(response)
@@ -49,12 +64,13 @@ class Contract:
 
         return request
 
-    def add_post_hook(self, request, results):
+    def add_post_hook(self, request: Request, results: TestResult) -> Request:
         if hasattr(self, "post_process"):
             cb = request.callback
+            assert cb is not None
 
             @wraps(cb)
-            def wrapper(response, **cb_kwargs):
+            def wrapper(response: Response, **cb_kwargs: Any) -> List[Any]:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                     raise TypeError("Contracts don't support async callbacks")
@@ -76,18 +92,18 @@ class Contract:
 
         return request
 
-    def adjust_request_args(self, args):
+    def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
         return args
 
 
 class ContractsManager:
-    contracts: Dict[str, Contract] = {}
+    contracts: Dict[str, Type[Contract]] = {}
 
-    def __init__(self, contracts):
+    def __init__(self, contracts: Iterable[Type[Contract]]):
         for contract in contracts:
             self.contracts[contract.name] = contract
 
-    def tested_methods_from_spidercls(self, spidercls):
+    def tested_methods_from_spidercls(self, spidercls: Type[Spider]) -> List[str]:
         is_method = re.compile(r"^\s*@", re.MULTILINE).search
         methods = []
         for key, value in getmembers(spidercls):
@@ -96,21 +112,26 @@ class ContractsManager:
 
         return methods
 
-    def extract_contracts(self, method):
-        contracts = []
+    def extract_contracts(self, method: Callable) -> List[Contract]:
+        contracts: List[Contract] = []
+        assert method.__doc__ is not None
         for line in method.__doc__.split("\n"):
             line = line.strip()
 
             if line.startswith("@"):
-                name, args = re.match(r"@(\w+)\s*(.*)", line).groups()
+                m = re.match(r"@(\w+)\s*(.*)", line)
+                assert m is not None
+                name, args = m.groups()
                 args = re.split(r"\s+", args)
 
                 contracts.append(self.contracts[name](method, *args))
 
         return contracts
 
-    def from_spider(self, spider, results):
-        requests = []
+    def from_spider(
+        self, spider: Spider, results: TestResult
+    ) -> List[Optional[Request]]:
+        requests: List[Optional[Request]] = []
         for method in self.tested_methods_from_spidercls(type(spider)):
             bound_method = spider.__getattribute__(method)
             try:
@@ -121,7 +142,7 @@ class ContractsManager:
 
         return requests
 
-    def from_method(self, method, results):
+    def from_method(self, method: Callable, results: TestResult) -> Optional[Request]:
         contracts = self.extract_contracts(method)
         if contracts:
             request_cls = Request
@@ -154,14 +175,18 @@ class ContractsManager:
 
                 self._clean_req(request, method, results)
                 return request
+        return None
 
-    def _clean_req(self, request, method, results):
+    def _clean_req(
+        self, request: Request, method: Callable, results: TestResult
+    ) -> None:
         """stop the request from returning objects and records any errors"""
 
         cb = request.callback
+        assert cb is not None
 
         @wraps(cb)
-        def cb_wrapper(response, **cb_kwargs):
+        def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
             try:
                 output = cb(response, **cb_kwargs)
                 output = list(iterate_spider_output(output))
@@ -169,7 +194,7 @@ class ContractsManager:
                 case = _create_testcase(method, "callback")
                 results.addError(case, sys.exc_info())
 
-        def eb_wrapper(failure):
+        def eb_wrapper(failure: Failure) -> None:
             case = _create_testcase(method, "errback")
             exc_info = failure.type, failure.value, failure.getTracebackObject()
             results.addError(case, exc_info)
@@ -178,11 +203,11 @@ class ContractsManager:
         request.errback = eb_wrapper
 
 
-def _create_testcase(method, desc):
-    spider = method.__self__.name
+def _create_testcase(method: Callable, desc: str) -> TestCase:
+    spider = method.__self__.name  # type: ignore[attr-defined]
 
     class ContractTestCase(TestCase):
-        def __str__(_self):
+        def __str__(_self) -> str:
             return f"[{spider}] {method.__name__} ({desc})"
 
     name = f"{spider}_{method.__name__}"
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index eac702cef..71ca4168a 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -1,4 +1,5 @@
 import json
+from typing import Any, Callable, Dict, List, Optional
 
 from itemadapter import ItemAdapter, is_item
 
@@ -15,7 +16,7 @@ class UrlContract(Contract):
 
     name = "url"
 
-    def adjust_request_args(self, args):
+    def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
         args["url"] = self.args[0]
         return args
 
@@ -29,7 +30,7 @@ class CallbackKeywordArgumentsContract(Contract):
 
     name = "cb_kwargs"
 
-    def adjust_request_args(self, args):
+    def adjust_request_args(self, args: Dict[str, Any]) -> Dict[str, Any]:
         args["cb_kwargs"] = json.loads(" ".join(self.args))
         return args
 
@@ -48,14 +49,14 @@ class ReturnsContract(Contract):
     """
 
     name = "returns"
-    object_type_verifiers = {
+    object_type_verifiers: Dict[Optional[str], Callable[[Any], bool]] = {
         "request": lambda x: isinstance(x, Request),
         "requests": lambda x: isinstance(x, Request),
         "item": is_item,
         "items": is_item,
     }
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
 
         if len(self.args) not in [1, 2, 3]:
@@ -66,16 +67,16 @@ class ReturnsContract(Contract):
         self.obj_type_verifier = self.object_type_verifiers[self.obj_name]
 
         try:
-            self.min_bound = int(self.args[1])
+            self.min_bound: float = int(self.args[1])
         except IndexError:
             self.min_bound = 1
 
         try:
-            self.max_bound = int(self.args[2])
+            self.max_bound: float = int(self.args[2])
         except IndexError:
             self.max_bound = float("inf")
 
-    def post_process(self, output):
+    def post_process(self, output: List[Any]) -> None:
         occurrences = 0
         for x in output:
             if self.obj_type_verifier(x):
@@ -85,7 +86,7 @@ class ReturnsContract(Contract):
 
         if not assertion:
             if self.min_bound == self.max_bound:
-                expected = self.min_bound
+                expected = str(self.min_bound)
             else:
                 expected = f"{self.min_bound}..{self.max_bound}"
 
@@ -101,7 +102,7 @@ class ScrapesContract(Contract):
 
     name = "scrapes"
 
-    def post_process(self, output):
+    def post_process(self, output: List[Any]) -> None:
         for x in output:
             if is_item(x):
                 missing = [arg for arg in self.args if arg not in ItemAdapter(x)]

From e676cd3ce0d488f56498b766912725066bcee4d9 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Wed, 22 May 2024 07:55:53 -0300
Subject: [PATCH 15/82] docs: Remove top-level reactor imports from
 CrawlerProces/CrawlerRunner examples

---
 docs/topics/practices.rst | 35 ++++++++++++++++++++++++++++++++---
 1 file changed, 32 insertions(+), 3 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index cd359b147..7731180fe 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -92,7 +92,6 @@ reactor after ``MySpider`` has finished running.
 
 .. code-block:: python
 
-    from twisted.internet import reactor
     import scrapy
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
@@ -107,6 +106,33 @@ reactor after ``MySpider`` has finished running.
     runner = CrawlerRunner()
 
     d = runner.crawl(MySpider)
+    from twisted.internet import reactor
+
+    d.addBoth(lambda _: reactor.stop())
+    reactor.run()  # the script will block here until the crawling is finished
+
+Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
+
+.. code-block:: python
+
+    import scrapy
+    from scrapy.crawler import CrawlerRunner
+    from scrapy.utils.log import configure_logging
+
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+
+    configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
+    from scrapy.utils.reactor import install_reactor
+
+    install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+    runner = CrawlerRunner()
+    d = runner.crawl(MySpider)
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
     reactor.run()  # the script will block here until the crawling is finished
 
@@ -151,7 +177,6 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
 .. code-block:: python
 
     import scrapy
-    from twisted.internet import reactor
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -173,6 +198,8 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
+    from twisted.internet import reactor
+
     d.addBoth(lambda _: reactor.stop())
 
     reactor.run()  # the script will block here until all crawling jobs are finished
@@ -181,7 +208,7 @@ Same example but running the spiders sequentially by chaining the deferreds:
 
 .. code-block:: python
 
-    from twisted.internet import reactor, defer
+    from twisted.internet import defer
     from scrapy.crawler import CrawlerRunner
     from scrapy.utils.log import configure_logging
     from scrapy.utils.project import get_project_settings
@@ -209,6 +236,8 @@ Same example but running the spiders sequentially by chaining the deferreds:
         reactor.stop()
 
 
+    from twisted.internet import reactor
+
     crawl()
     reactor.run()  # the script will block here until the last crawl call is finished
 

From 8210fae25a9d812447df617155001b9861e0d834 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 22 May 2024 18:50:50 -0300
Subject: [PATCH 16/82] Update docs/topics/practices.rst

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 7731180fe..710be7aa2 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -111,7 +111,9 @@ reactor after ``MySpider`` has finished running.
     d.addBoth(lambda _: reactor.stop())
     reactor.run()  # the script will block here until the crawling is finished
 
-Same example but using a non-default reactor, is only necessary call ``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
+Same example but using a non-default reactor, it's only necessary call
+``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess``
+ already does this automatically.
 
 .. code-block:: python
 

From dc6a495fee41949d50178b9e46d6f41e83425ca2 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 22 May 2024 18:51:02 -0300
Subject: [PATCH 17/82] Update docs/topics/practices.rst

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 710be7aa2..cec098012 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -106,6 +106,7 @@ reactor after ``MySpider`` has finished running.
     runner = CrawlerRunner()
 
     d = runner.crawl(MySpider)
+
     from twisted.internet import reactor
 
     d.addBoth(lambda _: reactor.stop())

From 3f66b66e3f645393dbb263a1ec7ab04bdabd74b4 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Wed, 22 May 2024 22:01:55 -0300
Subject: [PATCH 18/82] fix: checks

---
 docs/topics/practices.rst | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index cec098012..aa81ceea5 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -113,8 +113,7 @@ reactor after ``MySpider`` has finished running.
     reactor.run()  # the script will block here until the crawling is finished
 
 Same example but using a non-default reactor, it's only necessary call
-``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess``
- already does this automatically.
+``install_reactor`` if you are using ``CrawlerRunner`` since ``CrawlerProcess`` already does this automatically.
 
 .. code-block:: python
 

From e143dc795228424fa98cb40e17b9993617ae61ae Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 22 May 2024 22:26:31 -0300
Subject: [PATCH 19/82] Update tests-macos.yml

---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 252176464..95016146e 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -1,4 +1,4 @@
-name: macOS
+name: macOS.
 on: [push, pull_request]
 
 concurrency:

From 9d5a0d287b69a69fe34cbe3130438fb36f1f3441 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 22 May 2024 22:27:07 -0300
Subject: [PATCH 20/82] Retrigger CI

---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 95016146e..252176464 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -1,4 +1,4 @@
-name: macOS.
+name: macOS
 on: [push, pull_request]
 
 concurrency:

From 17e623cf0cfb5c695c43ceb069026d44cb28ca21 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 23 May 2024 07:00:24 -0300
Subject: [PATCH 21/82] Update docs/topics/practices.rst

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index aa81ceea5..64b3b6e81 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -128,6 +128,7 @@ Same example but using a non-default reactor, it's only necessary call
 
 
     configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s"})
+
     from scrapy.utils.reactor import install_reactor
 
     install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")

From 8ec67ca230a69effb2e0442fb2a6c06cd6c92adf Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 23 May 2024 07:00:35 -0300
Subject: [PATCH 22/82] Update docs/topics/practices.rst

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 64b3b6e81..ee484e63f 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -134,6 +134,7 @@ Same example but using a non-default reactor, it's only necessary call
     install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
     runner = CrawlerRunner()
     d = runner.crawl(MySpider)
+
     from twisted.internet import reactor
 
     d.addBoth(lambda _: reactor.stop())

From 62c89aaf056687091235bb846ac565f7c801c359 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 23 May 2024 07:00:45 -0300
Subject: [PATCH 23/82] Update docs/topics/practices.rst

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index ee484e63f..1500011e7 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -202,6 +202,7 @@ Same example using :class:`~scrapy.crawler.CrawlerRunner`:
     runner.crawl(MySpider1)
     runner.crawl(MySpider2)
     d = runner.join()
+
     from twisted.internet import reactor
 
     d.addBoth(lambda _: reactor.stop())

From 2facdd4fb08ec3edaf1752047dd86d5b565621a1 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Sun, 26 May 2024 19:55:54 -0300
Subject: [PATCH 24/82] Add change reactor test to CrawlerRunner

---
 .flake8                               |  1 +
 scrapy/crawler.py                     |  2 ++
 tests/CrawlerRunner/change_reactor.py | 31 +++++++++++++++++++++++++++
 tests/test_crawler.py                 |  8 +++++++
 4 files changed, 42 insertions(+)
 create mode 100644 tests/CrawlerRunner/change_reactor.py

diff --git a/.flake8 b/.flake8
index 62ccad9cf..0e43b9b56 100644
--- a/.flake8
+++ b/.flake8
@@ -9,6 +9,7 @@ exclude =
 per-file-ignores =
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
+    tests/CrawlerRunner/change_reactor.py:E402
 # F401: Module imported but unused
     scrapy/__init__.py:E402
     scrapy/core/downloader/handlers/http.py:F401
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index ccfe78891..4fe5987a7 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -129,6 +129,8 @@ class Crawler:
             if is_asyncio_reactor_installed() and event_loop:
                 verify_installed_asyncio_event_loop(event_loop)
 
+            log_reactor_info()
+
         self.extensions = ExtensionManager.from_crawler(self)
         self.settings.freeze()
 
diff --git a/tests/CrawlerRunner/change_reactor.py b/tests/CrawlerRunner/change_reactor.py
new file mode 100644
index 000000000..b20aa0c7c
--- /dev/null
+++ b/tests/CrawlerRunner/change_reactor.py
@@ -0,0 +1,31 @@
+from scrapy import Spider
+from scrapy.crawler import CrawlerRunner
+from scrapy.utils.log import configure_logging
+
+
+class NoRequestsSpider(Spider):
+    name = "no_request"
+
+    custom_settings = {
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    }
+
+    def start_requests(self):
+        return []
+
+
+configure_logging({"LOG_FORMAT": "%(levelname)s: %(message)s", "LOG_LEVEL": "DEBUG"})
+
+
+from scrapy.utils.reactor import install_reactor
+
+install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+runner = CrawlerRunner()
+
+d = runner.crawl(NoRequestsSpider)
+
+from twisted.internet import reactor
+
+d.addBoth(callback=lambda _: reactor.stop())
+reactor.run()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 989208694..791ea1faa 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -926,3 +926,11 @@ class CrawlerRunnerSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("INFO: Host: not.a.real.domain", log)
         self.assertIn("INFO: Type: <class 'ipaddress.IPv4Address'>", log)
         self.assertIn("INFO: IP address: 127.0.0.1", log)
+
+    def test_change_default_reactor(self):
+        log = self.run_script("change_reactor.py")
+        self.assertIn(
+            "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+            log,
+        )
+        self.assertIn("DEBUG: Using asyncio event loop", log)

From 6cd085785028d97393f26e6fee22e6c03e5c90a8 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Sun, 26 May 2024 19:57:16 -0300
Subject: [PATCH 25/82] Move path

---
 .flake8 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.flake8 b/.flake8
index 0e43b9b56..cf1a96476 100644
--- a/.flake8
+++ b/.flake8
@@ -9,7 +9,6 @@ exclude =
 per-file-ignores =
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
-    tests/CrawlerRunner/change_reactor.py:E402
 # F401: Module imported but unused
     scrapy/__init__.py:E402
     scrapy/core/downloader/handlers/http.py:F401
@@ -17,6 +16,7 @@ per-file-ignores =
     scrapy/linkextractors/__init__.py:E402,F401
     scrapy/selector/__init__.py:F401
     scrapy/spiders/__init__.py:E402,F401
+    tests/CrawlerRunner/change_reactor.py:E402
 
     # Issues pending a review:
     scrapy/utils/url.py:F403,F405

From 9ba4dd311dd9d2a5341ee9c0c6d1b50eb44ac406 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 28 May 2024 12:27:49 +0400
Subject: [PATCH 26/82] Install typing stubs for boto3 and botocore. (#6370)

---
 scrapy/core/downloader/handlers/s3.py |  3 ++-
 scrapy/extensions/feedexport.py       | 11 +++++++----
 tox.ini                               | 13 ++++++++-----
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 9a0811a50..1a3d36f45 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -59,7 +59,8 @@ class S3DownloadHandler:
             assert aws_access_key_id is not None
             assert aws_secret_access_key is not None
             SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
-            self._signer = SignerCls(
+            # botocore.auth.BaseSigner doesn't have an __init__() with args, only subclasses do
+            self._signer = SignerCls(  # type: ignore[call-arg]
                 botocore.credentials.Credentials(
                     aws_access_key_id, aws_secret_access_key, aws_session_token
                 )
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 97f39afe7..3c2bb5593 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -238,13 +238,16 @@ class S3FeedStorage(BlockingFeedStorage):
         self.acl: Optional[str] = acl
         self.endpoint_url: Optional[str] = endpoint_url
         self.region_name: Optional[str] = region_name
+        # It can be either botocore.client.BaseClient or mypy_boto3_s3.S3Client,
+        # there seems to be no good way to infer it statically.
+        self.s3_client: Any
 
         if IS_BOTO3_AVAILABLE:
             import boto3.session
 
-            session = boto3.session.Session()
+            boto3_session = boto3.session.Session()
 
-            self.s3_client = session.client(
+            self.s3_client = boto3_session.client(
                 "s3",
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
@@ -261,9 +264,9 @@ class S3FeedStorage(BlockingFeedStorage):
 
             import botocore.session
 
-            session = botocore.session.get_session()
+            botocore_session = botocore.session.get_session()
 
-            self.s3_client = session.create_client(
+            self.s3_client = botocore_session.create_client(
                 "s3",
                 aws_access_key_id=self.access_key,
                 aws_secret_access_key=self.secret_key,
diff --git a/tox.ini b/tox.ini
index cde4243f3..5a5e80496 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,12 +48,15 @@ basepython = python3
 deps =
     mypy==1.10.0
     typing-extensions==4.11.0
-    types-attrs==19.1.0
     types-lxml==2024.4.14
-    types-Pillow==10.2.0.20240423
-    types-Pygments==2.17.0.20240310
-    types-pyOpenSSL==24.0.0.20240417
-    types-setuptools==69.5.0.20240423
+    types-Pygments==2.18.0.20240506
+    types-pyOpenSSL==24.1.0.20240425
+    types-setuptools==69.5.0.20240518
+    botocore-stubs==1.34.94
+    boto3-stubs[s3]==1.34.108
+    attrs >= 18.2.0
+    Pillow >= 10.3.0
+    pytest >= 8.2.0
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =

From 986d1ee1dd5b2efba0f787af8ee510450b4af3b4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 28 May 2024 12:37:19 +0400
Subject: [PATCH 27/82] Move CI from the decommissioned macos-11 to
 macos-latest. (#6372)

---
 .github/workflows/tests-macos.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 252176464..a297f494c 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -7,7 +7,7 @@ concurrency:
 
 jobs:
   tests:
-    runs-on: macos-11
+    runs-on: macos-latest
     strategy:
       fail-fast: false
       matrix:

From cadb0dd707fc54670cb0eab06f0af65dcaa99354 Mon Sep 17 00:00:00 2001
From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com>
Date: Tue, 28 May 2024 14:12:58 +0530
Subject: [PATCH 28/82] Fix overridable methods in MediaPipeline (#6368)

---
 scrapy/pipelines/media.py    |  26 +++---
 tests/test_pipeline_media.py | 173 ++++++++++++-----------------------
 2 files changed, 73 insertions(+), 126 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 5f6c5cb07..25e00b0ea 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import functools
 import logging
+from abc import ABC, abstractmethod
 from collections import defaultdict
 from typing import TYPE_CHECKING
 
@@ -27,7 +28,7 @@ def _DUMMY_CALLBACK(response):
     return response
 
 
-class MediaPipeline:
+class MediaPipeline(ABC):
     LOG_FAILED_RESULTS = True
 
     class SpiderInfo:
@@ -55,14 +56,6 @@ class MediaPipeline:
             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
     def _key_for_pipe(self, key, base_class_name=None, settings=None):
-        """
-        >>> MediaPipeline()._key_for_pipe("IMAGES")
-        'IMAGES'
-        >>> class MyPipe(MediaPipeline):
-        ...     pass
-        >>> MyPipe()._key_for_pipe("IMAGES", base_class_name="MediaPipeline")
-        'MYPIPE_IMAGES'
-        """
         class_name = self.__class__.__name__
         formatted_key = f"{class_name.upper()}_{key}"
         if (
@@ -192,21 +185,25 @@ class MediaPipeline:
             defer_result(result).chainDeferred(wad)
 
     # Overridable Interface
+    @abstractmethod
     def media_to_download(self, request, info, *, item=None):
         """Check request before starting download"""
-        pass
+        raise NotImplementedError()
 
+    @abstractmethod
     def get_media_requests(self, item, info):
         """Returns the media requests to download"""
-        pass
+        raise NotImplementedError()
 
+    @abstractmethod
     def media_downloaded(self, response, request, info, *, item=None):
         """Handler for success downloads"""
-        return response
+        raise NotImplementedError()
 
+    @abstractmethod
     def media_failed(self, failure, request, info):
         """Handler for failed downloads"""
-        return failure
+        raise NotImplementedError()
 
     def item_completed(self, results, item, info):
         """Called per item when all media requests has been processed"""
@@ -221,6 +218,7 @@ class MediaPipeline:
                     )
         return item
 
+    @abstractmethod
     def file_path(self, request, response=None, info=None, *, item=None):
         """Returns the path where downloaded media should be stored"""
-        pass
+        raise NotImplementedError()
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index d4dde4a40..763453551 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,4 +1,3 @@
-import io
 from typing import Optional
 
 from testfixtures import LogCapture
@@ -11,7 +10,6 @@ from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.files import FileException
-from scrapy.pipelines.images import ImagesPipeline
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
@@ -35,8 +33,26 @@ def _mocked_download_func(request, info):
     return response() if callable(response) else response
 
 
+class UserDefinedPipeline(MediaPipeline):
+
+    def media_to_download(self, request, info, *, item=None):
+        pass
+
+    def get_media_requests(self, item, info):
+        pass
+
+    def media_downloaded(self, response, request, info, *, item=None):
+        return {}
+
+    def media_failed(self, failure, request, info):
+        return failure
+
+    def file_path(self, request, response=None, info=None, *, item=None):
+        return ""
+
+
 class BaseMediaPipelineTestCase(unittest.TestCase):
-    pipeline_class = MediaPipeline
+    pipeline_class = UserDefinedPipeline
     settings = None
 
     def setUp(self):
@@ -54,54 +70,6 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
             if not name.startswith("_"):
                 disconnect_all(signal)
 
-    def test_default_media_to_download(self):
-        request = Request("http://url")
-        assert self.pipe.media_to_download(request, self.info) is None
-
-    def test_default_get_media_requests(self):
-        item = {"name": "name"}
-        assert self.pipe.get_media_requests(item, self.info) is None
-
-    def test_default_media_downloaded(self):
-        request = Request("http://url")
-        response = Response("http://url", body=b"")
-        assert self.pipe.media_downloaded(response, request, self.info) is response
-
-    def test_default_media_failed(self):
-        request = Request("http://url")
-        fail = Failure(Exception())
-        assert self.pipe.media_failed(fail, request, self.info) is fail
-
-    def test_default_item_completed(self):
-        item = {"name": "name"}
-        assert self.pipe.item_completed([], item, self.info) is item
-
-        # Check that failures are logged by default
-        fail = Failure(Exception())
-        results = [(True, 1), (False, fail)]
-
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
-
-        assert new_item is item
-        assert len(log.records) == 1
-        record = log.records[0]
-        assert record.levelname == "ERROR"
-        self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
-
-        # disable failure logging and check again
-        self.pipe.LOG_FAILED_RESULTS = False
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
-        assert new_item is item
-        assert len(log.records) == 0
-
-    @inlineCallbacks
-    def test_default_process_item(self):
-        item = {"name": "name"}
-        new_item = yield self.pipe.process_item(item, self.spider)
-        assert new_item is item
-
     def test_modify_media_request(self):
         request = Request("http://url")
         self.pipe._modify_media_request(request)
@@ -175,8 +143,38 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         context = getattr(info.downloaded[fp].value, "__context__", None)
         self.assertIsNone(context)
 
+    def test_default_item_completed(self):
+        item = {"name": "name"}
+        assert self.pipe.item_completed([], item, self.info) is item
 
-class MockedMediaPipeline(MediaPipeline):
+        # Check that failures are logged by default
+        fail = Failure(Exception())
+        results = [(True, 1), (False, fail)]
+
+        with LogCapture() as log:
+            new_item = self.pipe.item_completed(results, item, self.info)
+
+        assert new_item is item
+        assert len(log.records) == 1
+        record = log.records[0]
+        assert record.levelname == "ERROR"
+        self.assertTupleEqual(record.exc_info, failure_to_exc_info(fail))
+
+        # disable failure logging and check again
+        self.pipe.LOG_FAILED_RESULTS = False
+        with LogCapture() as log:
+            new_item = self.pipe.item_completed(results, item, self.info)
+        assert new_item is item
+        assert len(log.records) == 0
+
+    @inlineCallbacks
+    def test_default_process_item(self):
+        item = {"name": "name"}
+        new_item = yield self.pipe.process_item(item, self.spider)
+        assert new_item is item
+
+
+class MockedMediaPipeline(UserDefinedPipeline):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._mockcalled = []
@@ -232,7 +230,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         )
         item = {"requests": req}
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item["results"], [(True, rsp)])
+        self.assertEqual(new_item["results"], [(True, {})])
         self.assertEqual(
             self.pipe._mockcalled,
             [
@@ -277,7 +275,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         req2 = Request("http://url2", meta={"response": fail})
         item = {"requests": [req1, req2]}
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item["results"], [(True, rsp1), (False, fail)])
+        self.assertEqual(new_item["results"], [(True, {}), (False, fail)])
         m = self.pipe._mockcalled
         # only once
         self.assertEqual(m[0], "get_media_requests")  # first hook called
@@ -315,7 +313,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = {"requests": req1}
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(new_item["results"], [(True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, {})])
 
         # rsp2 is ignored, rsp1 must be in results because request fingerprints are the same
         req2 = Request(
@@ -325,7 +323,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
         self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
-        self.assertEqual(new_item["results"], [(True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, {})])
 
     @inlineCallbacks
     def test_results_are_cached_for_requests_of_single_item(self):
@@ -337,7 +335,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = {"requests": [req1, req2]}
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, {}), (True, {})])
 
     @inlineCallbacks
     def test_wait_if_request_is_downloading(self):
@@ -363,7 +361,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         req2 = Request(req1.url, meta={"response": rsp2_func})
         item = {"requests": [req1, req2]}
         new_item = yield self.pipe.process_item(item, self.spider)
-        self.assertEqual(new_item["results"], [(True, rsp1), (True, rsp1)])
+        self.assertEqual(new_item["results"], [(True, {}), (True, {})])
 
     @inlineCallbacks
     def test_use_media_to_download_result(self):
@@ -376,57 +374,15 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
             ["get_media_requests", "media_to_download", "item_completed"],
         )
 
-
-class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self._mockcalled = []
-
-    def get_media_requests(self, item, info):
-        item_url = item["image_urls"][0]
-        output_img = io.BytesIO()
-        img = Image.new("RGB", (60, 30), color="red")
-        img.save(output_img, format="JPEG")
-        return Request(
-            item_url,
-            meta={
-                "response": Response(item_url, status=200, body=output_img.getvalue())
-            },
+    def test_key_for_pipe(self):
+        self.assertEqual(
+            self.pipe._key_for_pipe("IMAGES", base_class_name="MediaPipeline"),
+            "MOCKEDMEDIAPIPELINE_IMAGES",
         )
 
-    def inc_stats(self, *args, **kwargs):
-        return True
-
-    def media_to_download(self, request, info):
-        self._mockcalled.append("media_to_download")
-        return super().media_to_download(request, info)
-
-    def media_downloaded(self, response, request, info):
-        self._mockcalled.append("media_downloaded")
-        return super().media_downloaded(response, request, info)
-
-    def file_downloaded(self, response, request, info):
-        self._mockcalled.append("file_downloaded")
-        return super().file_downloaded(response, request, info)
-
-    def file_path(self, request, response=None, info=None):
-        self._mockcalled.append("file_path")
-        return super().file_path(request, response, info)
-
-    def thumb_path(self, request, thumb_id, response=None, info=None):
-        self._mockcalled.append("thumb_path")
-        return super().thumb_path(request, thumb_id, response, info)
-
-    def get_images(self, response, request, info):
-        self._mockcalled.append("get_images")
-        return super().get_images(response, request, info)
-
-    def image_downloaded(self, response, request, info):
-        self._mockcalled.append("image_downloaded")
-        return super().image_downloaded(response, request, info)
-
 
 class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
+
     def _assert_request_no3xx(self, pipeline_class, settings):
         pipe = pipeline_class(settings=Settings(settings))
         request = Request("http://url")
@@ -452,18 +408,11 @@ class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):
             else:
                 self.assertNotIn(status, request.meta["handle_httpstatus_list"])
 
-    def test_standard_setting(self):
-        self._assert_request_no3xx(MediaPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
-
     def test_subclass_standard_setting(self):
-        class UserDefinedPipeline(MediaPipeline):
-            pass
 
         self._assert_request_no3xx(UserDefinedPipeline, {"MEDIA_ALLOW_REDIRECTS": True})
 
     def test_subclass_specific_setting(self):
-        class UserDefinedPipeline(MediaPipeline):
-            pass
 
         self._assert_request_no3xx(
             UserDefinedPipeline, {"USERDEFINEDPIPELINE_MEDIA_ALLOW_REDIRECTS": True}

From 0d58af86971ba54bfc1feffa88fc564ffba650f4 Mon Sep 17 00:00:00 2001
From: Fabian Schneebauer <fabian@fs-it.org>
Date: Wed, 29 May 2024 10:59:32 +0200
Subject: [PATCH 29/82] Add support for multiple referer policy tokens.

---
 scrapy/spidermiddlewares/referer.py    | 20 ++++++-----
 tests/test_spidermiddleware_referer.py | 47 ++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 9 deletions(-)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index a0b6851e5..7706c8c15 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -323,15 +323,17 @@ def _load_policy_class(
     try:
         return cast(Type[ReferrerPolicy], load_object(policy))
     except ValueError:
-        try:
-            return _policy_classes[policy.lower()]
-        except KeyError:
-            msg = f"Could not load referrer policy {policy!r}"
-            if not warning_only:
-                raise RuntimeError(msg)
-            else:
-                warnings.warn(msg, RuntimeWarning)
-                return None
+        tokens = [token.strip() for token in policy.lower().split(",")]
+        for token in tokens[::-1]:
+            if token in _policy_classes:
+                return _policy_classes[token]
+
+        msg = f"Could not load referrer policy {policy!r}"
+        if not warning_only:
+            raise RuntimeError(msg)
+        else:
+            warnings.warn(msg, RuntimeWarning)
+            return None
 
 
 class RefererMiddleware:
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index afffa87fb..5797edfbd 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -884,6 +884,53 @@ class TestSettingsPolicyByName(TestCase):
         with self.assertRaises(RuntimeError):
             RefererMiddleware(settings)
 
+    def test_multiple_policy_tokens(self):
+        # test parsing without space(s) after the comma
+        settings1 = Settings(
+            {
+                "REFERRER_POLICY": ",".join(
+                    [
+                        "some-custom-unknown-policy",
+                        POLICY_SAME_ORIGIN,
+                        POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+                        "another-custom-unknown-policy",
+                    ]
+                )
+            }
+        )
+        mw1 = RefererMiddleware(settings1)
+        self.assertEqual(mw1.default_policy, StrictOriginWhenCrossOriginPolicy)
+
+        # test parsing with space(s) after the comma
+        settings2 = Settings(
+            {
+                "REFERRER_POLICY": ",    ".join(
+                    [
+                        POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN,
+                        "another-custom-unknown-policy",
+                        POLICY_UNSAFE_URL,
+                    ]
+                )
+            }
+        )
+        mw2 = RefererMiddleware(settings2)
+        self.assertEqual(mw2.default_policy, UnsafeUrlPolicy)
+
+    def test_multiple_policy_tokens_all_invalid(self):
+        settings = Settings(
+            {
+                "REFERRER_POLICY": ",".join(
+                    [
+                        "some-custom-unknown-policy",
+                        "another-custom-unknown-policy",
+                        "yet-another-custom-unknown-policy",
+                    ]
+                )
+            }
+        )
+        with self.assertRaises(RuntimeError):
+            RefererMiddleware(settings)
+
 
 class TestPolicyHeaderPrecedence001(MixinUnsafeUrl, TestRefererMiddleware):
     settings = {"REFERRER_POLICY": "scrapy.spidermiddlewares.referer.SameOriginPolicy"}

From 62a028b99dc73b8ddddd37f986780a5a070f2938 Mon Sep 17 00:00:00 2001
From: Fabian Schneebauer <67049088+0xdeb@users.noreply.github.com>
Date: Wed, 29 May 2024 13:19:27 +0200
Subject: [PATCH 30/82] Add spec link to scrapy/spidermiddlewares/referer.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/spidermiddlewares/referer.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 7706c8c15..8af0bdf5b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -324,6 +324,7 @@ def _load_policy_class(
         return cast(Type[ReferrerPolicy], load_object(policy))
     except ValueError:
         tokens = [token.strip() for token in policy.lower().split(",")]
+        # https://www.w3.org/TR/referrer-policy/#parse-referrer-policy-from-header
         for token in tokens[::-1]:
             if token in _policy_classes:
                 return _policy_classes[token]

From b4293e8f9efac5046f92e4ebfd744be443b858b0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 10:50:36 +0400
Subject: [PATCH 31/82] Misc typing improvements. (#6384)

---
 scrapy/core/http2/agent.py                    |  4 +--
 scrapy/core/http2/protocol.py                 |  8 +++--
 scrapy/core/http2/stream.py                   |  4 +--
 .../downloadermiddlewares/httpcompression.py  | 14 +++++----
 scrapy/downloadermiddlewares/offsite.py       | 30 ++++++++++++-------
 scrapy/loader/__init__.py                     | 12 +++++++-
 scrapy/utils/benchserver.py                   | 12 ++++----
 scrapy/utils/curl.py                          | 23 +++++++++-----
 scrapy/utils/datatypes.py                     |  2 +-
 scrapy/utils/request.py                       |  2 +-
 scrapy/utils/response.py                      |  2 +-
 scrapy/utils/testsite.py                      |  4 +--
 12 files changed, 78 insertions(+), 39 deletions(-)

diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 215ea9716..935af2214 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -119,7 +119,7 @@ class H2Agent:
             self._reactor, self._context_factory, connect_timeout, bind_address
         )
 
-    def get_endpoint(self, uri: URI):
+    def get_endpoint(self, uri: URI) -> HostnameEndpoint:
         return self.endpoint_factory.endpointForURI(uri)
 
     def get_key(self, uri: URI) -> Tuple:
@@ -161,7 +161,7 @@ class ScrapyProxyH2Agent(H2Agent):
         )
         self._proxy_uri = proxy_uri
 
-    def get_endpoint(self, uri: URI):
+    def get_endpoint(self, uri: URI) -> HostnameEndpoint:
         return self.endpoint_factory.endpointForURI(self._proxy_uri)
 
     def get_key(self, uri: URI) -> Tuple:
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index bc8da50d7..8898b8118 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -22,7 +22,11 @@ from h2.events import (
 from h2.exceptions import FrameTooLargeError, H2Error
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
-from twisted.internet.interfaces import IHandshakeListener, IProtocolNegotiationFactory
+from twisted.internet.interfaces import (
+    IAddress,
+    IHandshakeListener,
+    IProtocolNegotiationFactory,
+)
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
@@ -431,7 +435,7 @@ class H2ClientFactory(Factory):
         self.settings = settings
         self.conn_lost_deferred = conn_lost_deferred
 
-    def buildProtocol(self, addr) -> H2ClientProtocol:
+    def buildProtocol(self, addr: IAddress) -> H2ClientProtocol:
         return H2ClientProtocol(self.uri, self.settings, self.conn_lost_deferred)
 
     def acceptableProtocols(self) -> List[bytes]:
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 4132fc385..224691078 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,7 +1,7 @@
 import logging
 from enum import Enum
 from io import BytesIO
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from h2.errors import ErrorCodes
 from h2.exceptions import H2Error, ProtocolError, StreamClosedError
@@ -142,7 +142,7 @@ class Stream:
             "headers": Headers({}),
         }
 
-        def _cancel(_) -> None:
+        def _cancel(_: Any) -> None:
             # Close this stream as gracefully as possible
             # If the associated request is initiated we reset this stream
             # else we directly call close() method
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 0e5e215ac..8e170a1c7 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import warnings
 from itertools import chain
 from logging import getLogger
-from typing import TYPE_CHECKING, List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 from scrapy import Request, Spider, signals
 from scrapy.crawler import Crawler
@@ -149,20 +149,24 @@ class HttpCompressionMiddleware:
 
         return response
 
-    def _handle_encoding(self, body, content_encoding, max_size):
+    def _handle_encoding(
+        self, body: bytes, content_encoding: List[bytes], max_size: int
+    ) -> Tuple[bytes, List[bytes]]:
         to_decode, to_keep = self._split_encodings(content_encoding)
         for encoding in to_decode:
             body = self._decode(body, encoding, max_size)
         return body, to_keep
 
-    def _split_encodings(self, content_encoding):
-        to_keep = [
+    def _split_encodings(
+        self, content_encoding: List[bytes]
+    ) -> Tuple[List[bytes], List[bytes]]:
+        to_keep: List[bytes] = [
             encoding.strip().lower()
             for encoding in chain.from_iterable(
                 encodings.split(b",") for encodings in content_encoding
             )
         ]
-        to_decode = []
+        to_decode: List[bytes] = []
         while to_keep:
             encoding = to_keep.pop()
             if encoding not in ACCEPTED_ENCODINGS:
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index 1e5026925..bd8dbe329 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -1,33 +1,43 @@
+from __future__ import annotations
+
 import logging
 import re
 import warnings
+from typing import TYPE_CHECKING, Set
 
-from scrapy import signals
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.httpobj import urlparse_cached
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
 class OffsiteMiddleware:
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
         return o
 
-    def __init__(self, stats):
+    def __init__(self, stats: StatsCollector):
         self.stats = stats
-        self.domains_seen = set()
+        self.domains_seen: Set[str] = set()
 
-    def spider_opened(self, spider):
-        self.host_regex = self.get_host_regex(spider)
+    def spider_opened(self, spider: Spider) -> None:
+        self.host_regex: re.Pattern[str] = self.get_host_regex(spider)
 
-    def request_scheduled(self, request, spider):
+    def request_scheduled(self, request: Request, spider: Spider) -> None:
         self.process_request(request, spider)
 
-    def process_request(self, request, spider):
+    def process_request(self, request: Request, spider: Spider) -> None:
         if request.dont_filter or self.should_follow(request, spider):
             return None
         domain = urlparse_cached(request).hostname
@@ -42,13 +52,13 @@ class OffsiteMiddleware:
         self.stats.inc_value("offsite/filtered", spider=spider)
         raise IgnoreRequest
 
-    def should_follow(self, request, spider):
+    def should_follow(self, request: Request, spider: Spider) -> bool:
         regex = self.host_regex
         # hostname can be None for wrong urls (like javascript links)
         host = urlparse_cached(request).hostname or ""
         return bool(regex.search(host))
 
-    def get_host_regex(self, spider):
+    def get_host_regex(self, spider: Spider) -> re.Pattern[str]:
         """Override this method to implement a different offsite policy"""
         allowed_domains = getattr(spider, "allowed_domains", None)
         if not allowed_domains:
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index 529fa279e..db0b4820f 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -4,8 +4,11 @@ Item Loader
 See documentation in docs/topics/loaders.rst
 """
 
+from typing import Any, Optional
+
 import itemloaders
 
+from scrapy.http import TextResponse
 from scrapy.item import Item
 from scrapy.selector import Selector
 
@@ -82,7 +85,14 @@ class ItemLoader(itemloaders.ItemLoader):
     default_item_class: type = Item
     default_selector_class = Selector
 
-    def __init__(self, item=None, selector=None, response=None, parent=None, **context):
+    def __init__(
+        self,
+        item: Any = None,
+        selector: Optional[Selector] = None,
+        response: Optional[TextResponse] = None,
+        parent: Optional[itemloaders.ItemLoader] = None,
+        **context: Any
+    ):
         if selector is None and response is not None:
             try:
                 selector = self.default_selector_class(response)
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index f6f704d4b..e9ea51aa1 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -1,21 +1,23 @@
 import random
+from typing import Any
 from urllib.parse import urlencode
 
 from twisted.web.resource import Resource
-from twisted.web.server import Site
+from twisted.web.server import Request, Site
 
 
 class Root(Resource):
     isLeaf = True
 
-    def getChild(self, name, request):
+    def getChild(self, name: str, request: Request) -> Resource:
         return self
 
-    def render(self, request):
+    def render(self, request: Request) -> bytes:
         total = _getarg(request, b"total", 100, int)
         show = _getarg(request, b"show", 10, int)
         nlist = [random.randint(1, total) for _ in range(show)]  # nosec
         request.write(b"<html><head></head><body>")
+        assert request.args is not None
         args = request.args.copy()
         for nl in nlist:
             args["n"] = nl
@@ -27,7 +29,7 @@ class Root(Resource):
         return b""
 
 
-def _getarg(request, name, default=None, type=str):
+def _getarg(request, name: bytes, default: Any = None, type=str):
     return type(request.args[name][0]) if name in request.args else default
 
 
@@ -38,7 +40,7 @@ if __name__ == "__main__":
     factory = Site(root)
     httpPort = reactor.listenTCP(8998, Site(root))
 
-    def _print_listening():
+    def _print_listening() -> None:
         httpHost = httpPort.getHost()
         print(f"Bench server at http://{httpHost.host}:{httpHost.port}")
 
diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index f5dbbd64e..c10e48511 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -2,13 +2,20 @@ import argparse
 import warnings
 from http.cookies import SimpleCookie
 from shlex import split
+from typing import Any, Dict, List, NoReturn, Optional, Sequence, Tuple, Union
 from urllib.parse import urlparse
 
 from w3lib.http import basic_auth_header
 
 
 class DataAction(argparse.Action):
-    def __call__(self, parser, namespace, values, option_string=None):
+    def __call__(
+        self,
+        parser: argparse.ArgumentParser,
+        namespace: argparse.Namespace,
+        values: Union[str, Sequence[Any], None],
+        option_string: Optional[str] = None,
+    ) -> None:
         value = str(values)
         if value.startswith("$"):
             value = value[1:]
@@ -16,7 +23,7 @@ class DataAction(argparse.Action):
 
 
 class CurlParser(argparse.ArgumentParser):
-    def error(self, message):
+    def error(self, message: str) -> NoReturn:
         error_msg = f"There was an error parsing the curl command: {message}"
         raise ValueError(error_msg)
 
@@ -42,9 +49,11 @@ for argument in safe_to_ignore_arguments:
     curl_parser.add_argument(*argument, action="store_true")
 
 
-def _parse_headers_and_cookies(parsed_args):
-    headers = []
-    cookies = {}
+def _parse_headers_and_cookies(
+    parsed_args: argparse.Namespace,
+) -> Tuple[List[Tuple[str, bytes]], Dict[str, str]]:
+    headers: List[Tuple[str, bytes]] = []
+    cookies: Dict[str, str] = {}
     for header in parsed_args.headers or ():
         name, val = header.split(":", 1)
         name = name.strip()
@@ -64,7 +73,7 @@ def _parse_headers_and_cookies(parsed_args):
 
 def curl_to_request_kwargs(
     curl_command: str, ignore_unknown_options: bool = True
-) -> dict:
+) -> Dict[str, Any]:
     """Convert a cURL command syntax to Request kwargs.
 
     :param str curl_command: string containing the curl command
@@ -98,7 +107,7 @@ def curl_to_request_kwargs(
 
     method = parsed_args.method or "GET"
 
-    result = {"method": method.upper(), "url": url}
+    result: Dict[str, Any] = {"method": method.upper(), "url": url}
 
     headers, cookies = _parse_headers_and_cookies(parsed_args)
 
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 0ba2fe4e2..b2118495f 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -110,7 +110,7 @@ class CaseInsensitiveDict(collections.UserDict):
     as keys and allows case-insensitive lookups.
     """
 
-    def __init__(self, *args, **kwargs) -> None:
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         self._keys: dict = {}
         super().__init__(*args, **kwargs)
 
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index c86f9fe39..5be80ec0f 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -138,7 +138,7 @@ class RequestFingerprinter:
     """
 
     @classmethod
-    def from_crawler(cls, crawler) -> Self:
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
     def __init__(self, crawler: Optional[Crawler] = None):
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index a0b06f75c..320059b3a 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -58,7 +58,7 @@ def response_status_message(status: Union[bytes, float, int, str]) -> str:
     return f"{status_int} {to_unicode(message)}"
 
 
-def _remove_html_comments(body):
+def _remove_html_comments(body: bytes) -> bytes:
     start = body.find(b"<!--")
     while start != -1:
         end = body.find(b"-->", start + 1)
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index de9ce992a..ca1f68116 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -15,12 +15,12 @@ class SiteTest:
         super().tearDown()
         self.site.stopListening()
 
-    def url(self, path):
+    def url(self, path: str) -> str:
         return urljoin(self.baseurl, path)
 
 
 class NoMetaRefreshRedirect(util.Redirect):
-    def render(self, request):
+    def render(self, request: server.Request) -> bytes:
         content = util.Redirect.render(self, request)
         return content.replace(
             b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'

From da42e8f124362a5087c50bca7f76dcc573e8194a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 21:11:50 +0500
Subject: [PATCH 32/82] Add parameters to typing.Dict.

---
 scrapy/core/downloader/handlers/__init__.py |  5 +++-
 scrapy/core/http2/protocol.py               |  4 +--
 scrapy/core/http2/stream.py                 |  4 +--
 scrapy/extensions/feedexport.py             |  4 ++-
 scrapy/http/request/__init__.py             |  4 +--
 scrapy/http/request/form.py                 | 30 ++++++++++++++-------
 scrapy/http/request/json_request.py         | 18 ++++++-------
 scrapy/http/response/__init__.py            |  4 +--
 scrapy/http/response/text.py                |  4 +--
 scrapy/item.py                              |  2 +-
 scrapy/logformatter.py                      | 22 ++++++++++-----
 scrapy/settings/__init__.py                 |  2 +-
 scrapy/spiders/__init__.py                  |  6 ++---
 scrapy/utils/conf.py                        |  3 ++-
 scrapy/utils/log.py                         | 13 ++++++---
 scrapy/utils/python.py                      | 10 ++++---
 scrapy/utils/request.py                     |  2 +-
 17 files changed, 85 insertions(+), 52 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index ade51ca63..af5282553 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -27,7 +27,10 @@ class DownloadHandlers:
         self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
         handlers: Dict[str, Union[str, Callable]] = without_none_values(
-            crawler.settings.getwithbase("DOWNLOAD_HANDLERS")
+            cast(
+                Dict[str, Union[str, Callable]],
+                crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
+            )
         )
         for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 8898b8118..063835b17 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -3,7 +3,7 @@ import itertools
 import logging
 from collections import deque
 from ipaddress import IPv4Address, IPv6Address
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -115,7 +115,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         # Some meta data of this connection
         # initialized when connection is successfully made
-        self.metadata: Dict = {
+        self.metadata: Dict[str, Any] = {
             # Peer certificate instance
             "certificate": None,
             # Address of the server we are connected to which
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 224691078..7c70e86db 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -110,7 +110,7 @@ class Stream:
 
         # Metadata of an HTTP/2 connection stream
         # initialized when stream is instantiated
-        self.metadata: Dict = {
+        self.metadata: Dict[str, Any] = {
             "request_content_length": (
                 0 if self._request.body is None else len(self._request.body)
             ),
@@ -131,7 +131,7 @@ class Stream:
         # Private variable used to build the response
         # this response is then converted to appropriate Response class
         # passed to the response deferred callback
-        self._response: Dict = {
+        self._response: Dict[str, Any] = {
             # Data received frame by frame from the server is appended
             # and passed to the response Deferred when completely received.
             "body": BytesIO(),
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3c2bb5593..de8a288f6 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -694,7 +694,9 @@ class FeedExporter:
         self.slots = slots
 
     def _load_components(self, setting_prefix: str) -> Dict[str, Any]:
-        conf = without_none_values(self.settings.getwithbase(setting_prefix))
+        conf = without_none_values(
+            cast(Dict[str, str], self.settings.getwithbase(setting_prefix))
+        )
         d = {}
         for k, v in conf.items():
             try:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 191b3cef4..dfb1dca89 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -97,7 +97,7 @@ class Request(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[dict, List[dict]]] = None,
+        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: str = "utf-8",
         priority: int = 0,
@@ -123,7 +123,7 @@ class Request(object_ref):
         self.callback: Optional[Callable] = callback
         self.errback: Optional[Callable] = errback
 
-        self.cookies: Union[dict, List[dict]] = cookies or {}
+        self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {}
         self.headers: Headers = Headers(headers or {}, encoding=encoding)
         self.dont_filter: bool = dont_filter
 
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 3206d79cd..ea98ed795 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -7,7 +7,17 @@ See documentation in docs/topics/request-response.rst
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Union,
+    cast,
+)
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
 from lxml.html import FormElement  # nosec
@@ -26,8 +36,9 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-FormdataKVType = Tuple[str, Union[str, Iterable[str]]]
-FormdataType = Optional[Union[dict, List[FormdataKVType]]]
+FormdataVType = Union[str, Iterable[str]]
+FormdataKVType = Tuple[str, FormdataVType]
+FormdataType = Optional[Union[Dict[str, FormdataVType], List[FormdataKVType]]]
 
 
 class FormRequest(Request):
@@ -62,7 +73,7 @@ class FormRequest(Request):
         formid: Optional[str] = None,
         formnumber: int = 0,
         formdata: FormdataType = None,
-        clickdata: Optional[dict] = None,
+        clickdata: Optional[Dict[str, Union[str, int]]] = None,
         dont_click: bool = False,
         formxpath: Optional[str] = None,
         formcss: Optional[str] = None,
@@ -156,7 +167,7 @@ def _get_inputs(
     form: FormElement,
     formdata: FormdataType,
     dont_click: bool,
-    clickdata: Optional[dict],
+    clickdata: Optional[Dict[str, Union[str, int]]],
 ) -> List[FormdataKVType]:
     """Return a list of key-value pairs for the inputs found in the given form."""
     try:
@@ -186,10 +197,8 @@ def _get_inputs(
         if clickable and clickable[0] not in formdata and not clickable[0] is None:
             values.append(clickable)
 
-    if isinstance(formdata, dict):
-        formdata = formdata.items()  # type: ignore[assignment]
-
-    values.extend((k, v) for k, v in formdata if v is not None)
+    formdata_items = formdata.items() if isinstance(formdata, dict) else formdata
+    values.extend((k, v) for k, v in formdata_items if v is not None)
     return values
 
 
@@ -216,7 +225,7 @@ def _select_value(
 
 
 def _get_clickable(
-    clickdata: Optional[dict], form: FormElement
+    clickdata: Optional[Dict[str, Union[str, int]]], form: FormElement
 ) -> Optional[Tuple[str, str]]:
     """
     Returns the clickable element specified in clickdata,
@@ -243,6 +252,7 @@ def _get_clickable(
     # because that uniquely identifies the element
     nr = clickdata.get("nr", None)
     if nr is not None:
+        assert isinstance(nr, int)
         try:
             el = list(form.inputs)[nr]
         except IndexError:
diff --git a/scrapy/http/request/json_request.py b/scrapy/http/request/json_request.py
index 1dd9e6c87..405c0b9d0 100644
--- a/scrapy/http/request/json_request.py
+++ b/scrapy/http/request/json_request.py
@@ -8,7 +8,7 @@ See documentation in docs/topics/request-response.rst
 import copy
 import json
 import warnings
-from typing import Any, Optional, Tuple
+from typing import Any, Dict, Optional, Tuple
 
 from scrapy.http.request import Request
 
@@ -17,15 +17,15 @@ class JsonRequest(Request):
     attributes: Tuple[str, ...] = Request.attributes + ("dumps_kwargs",)
 
     def __init__(
-        self, *args: Any, dumps_kwargs: Optional[dict] = None, **kwargs: Any
+        self, *args: Any, dumps_kwargs: Optional[Dict[str, Any]] = None, **kwargs: Any
     ) -> None:
         dumps_kwargs = copy.deepcopy(dumps_kwargs) if dumps_kwargs is not None else {}
         dumps_kwargs.setdefault("sort_keys", True)
-        self._dumps_kwargs = dumps_kwargs
+        self._dumps_kwargs: Dict[str, Any] = dumps_kwargs
 
         body_passed = kwargs.get("body", None) is not None
-        data = kwargs.pop("data", None)
-        data_passed = data is not None
+        data: Any = kwargs.pop("data", None)
+        data_passed: bool = data is not None
 
         if body_passed and data_passed:
             warnings.warn("Both body and data passed. data will be ignored")
@@ -41,13 +41,13 @@ class JsonRequest(Request):
         )
 
     @property
-    def dumps_kwargs(self) -> dict:
+    def dumps_kwargs(self) -> Dict[str, Any]:
         return self._dumps_kwargs
 
     def replace(self, *args: Any, **kwargs: Any) -> Request:
         body_passed = kwargs.get("body", None) is not None
-        data = kwargs.pop("data", None)
-        data_passed = data is not None
+        data: Any = kwargs.pop("data", None)
+        data_passed: bool = data is not None
 
         if body_passed and data_passed:
             warnings.warn("Both body and data passed. data will be ignored")
@@ -56,6 +56,6 @@ class JsonRequest(Request):
 
         return super().replace(*args, **kwargs)
 
-    def _dumps(self, data: dict) -> str:
+    def _dumps(self, data: Any) -> str:
         """Convert to JSON"""
         return json.dumps(data, **self._dumps_kwargs)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index d73dfce4b..14618e5e7 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -181,7 +181,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[dict, List[dict]]] = None,
+        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
@@ -234,7 +234,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[dict, List[dict]]] = None,
+        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 522ffc0d5..a83279ac8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -183,7 +183,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[dict, List[dict]]] = None,
+        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,
@@ -236,7 +236,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[dict, List[dict]]] = None,
+        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,
diff --git a/scrapy/item.py b/scrapy/item.py
index 2daea64cc..3f93809e7 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-class Field(dict):
+class Field(Dict[str, Any]):
     """Container of field metadata"""
 
 
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index d720b2f38..42a03b560 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import logging
 import os
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union
 
 from twisted.python.failure import Failure
 
@@ -26,6 +26,12 @@ DOWNLOADERRORMSG_SHORT = "Error downloading %(request)s"
 DOWNLOADERRORMSG_LONG = "Error downloading %(request)s: %(errmsg)s"
 
 
+class LogFormatterResult(TypedDict):
+    level: int
+    msg: str
+    args: Union[Dict[str, Any], Tuple[Any, ...]]
+
+
 class LogFormatter:
     """Class for generating log messages for different actions.
 
@@ -64,7 +70,9 @@ class LogFormatter:
                     }
     """
 
-    def crawled(self, request: Request, response: Response, spider: Spider) -> dict:
+    def crawled(
+        self, request: Request, response: Response, spider: Spider
+    ) -> LogFormatterResult:
         """Logs a message when the crawler finds a webpage."""
         request_flags = f" {str(request.flags)}" if request.flags else ""
         response_flags = f" {str(response.flags)}" if response.flags else ""
@@ -84,7 +92,7 @@ class LogFormatter:
 
     def scraped(
         self, item: Any, response: Union[Response, Failure], spider: Spider
-    ) -> dict:
+    ) -> LogFormatterResult:
         """Logs a message when an item is scraped by a spider."""
         src: Any
         if isinstance(response, Failure):
@@ -102,7 +110,7 @@ class LogFormatter:
 
     def dropped(
         self, item: Any, exception: BaseException, response: Response, spider: Spider
-    ) -> dict:
+    ) -> LogFormatterResult:
         """Logs a message when an item is dropped while it is passing through the item pipeline."""
         return {
             "level": logging.WARNING,
@@ -115,7 +123,7 @@ class LogFormatter:
 
     def item_error(
         self, item: Any, exception: BaseException, response: Response, spider: Spider
-    ) -> dict:
+    ) -> LogFormatterResult:
         """Logs a message when an item causes an error while it is passing
         through the item pipeline.
 
@@ -135,7 +143,7 @@ class LogFormatter:
         request: Request,
         response: Union[Response, Failure],
         spider: Spider,
-    ) -> dict:
+    ) -> LogFormatterResult:
         """Logs an error message from a spider.
 
         .. versionadded:: 2.0
@@ -155,7 +163,7 @@ class LogFormatter:
         request: Request,
         spider: Spider,
         errmsg: Optional[str] = None,
-    ) -> dict:
+    ) -> LogFormatterResult:
         """Logs a download error message from a spider (typically coming from
         the engine).
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index d270a72f4..4448b6f4b 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -411,7 +411,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         self._assert_mutability()
         if isinstance(values, str):
-            values = cast(dict, json.loads(values))
+            values = cast(Dict[_SettingsKeyT, Any], json.loads(values))
         if values is not None:
             if isinstance(values, BaseSettings):
                 for name, value in values.items():
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index bef041325..7b43f04f2 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
     from typing_extensions import Concatenate, Self
 
     from scrapy.crawler import Crawler
-    from scrapy.settings import BaseSettings
+    from scrapy.settings import BaseSettings, _SettingsKeyT
     from scrapy.utils.log import SpiderLoggerAdapter
 
     CallbackT = Callable[Concatenate[Response, ...], Any]
@@ -36,7 +36,7 @@ class Spider(object_ref):
     """
 
     name: str
-    custom_settings: Optional[dict] = None
+    custom_settings: Optional[Dict[_SettingsKeyT, Any]] = None
 
     def __init__(self, name: Optional[str] = None, **kwargs: Any):
         if name is not None:
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 641dfa4a2..c63b69995 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -16,6 +16,7 @@ from typing import (
     MutableMapping,
     Optional,
     Union,
+    cast,
 )
 
 from scrapy.exceptions import ScrapyDeprecationWarning, UsageError
@@ -173,7 +174,7 @@ def feed_process_params_from_cli(
     suitable to be used as the FEEDS setting.
     """
     valid_output_formats: Iterable[str] = without_none_values(
-        settings.getwithbase("FEED_EXPORTERS")
+        cast(Dict[str, str], settings.getwithbase("FEED_EXPORTERS"))
     ).keys()
 
     def check_valid_format(output_format: str) -> None:
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 430a91e95..cbfd170ed 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -7,6 +7,7 @@ from types import TracebackType
 from typing import (
     TYPE_CHECKING,
     Any,
+    Dict,
     List,
     MutableMapping,
     Optional,
@@ -20,7 +21,8 @@ from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.settings import Settings
+from scrapy.logformatter import LogFormatterResult
+from scrapy.settings import Settings, _SettingsKeyT
 from scrapy.utils.versions import scrapy_components_versions
 
 if TYPE_CHECKING:
@@ -86,7 +88,8 @@ DEFAULT_LOGGING = {
 
 
 def configure_logging(
-    settings: Union[Settings, dict, None] = None, install_root_handler: bool = True
+    settings: Union[Settings, Dict[_SettingsKeyT, Any], None] = None,
+    install_root_handler: bool = True,
 ) -> None:
     """
     Initialize logging defaults for Scrapy.
@@ -234,7 +237,9 @@ class LogCounterHandler(logging.Handler):
         self.crawler.stats.inc_value(sname)
 
 
-def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
+def logformatter_adapter(
+    logkws: LogFormatterResult,
+) -> Tuple[int, str, Union[Dict[str, Any], Tuple[Any, ...]]]:
     """
     Helper that takes the dictionary output from the methods in LogFormatter
     and adapts it into a tuple of positional arguments for logger.log calls,
@@ -245,7 +250,7 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
     message = logkws.get("msg") or ""
     # NOTE: This also handles 'args' being an empty dict, that case doesn't
     # play well in logger.log calls
-    args = logkws if not logkws.get("args") else logkws["args"]
+    args = cast(Dict[str, Any], logkws) if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 578cde2ac..0a50f4e1e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -42,6 +42,8 @@ if TYPE_CHECKING:
     _P = ParamSpec("_P")
 
 _T = TypeVar("_T")
+_KT = TypeVar("_KT")
+_VT = TypeVar("_VT")
 
 
 def flatten(x: Iterable) -> list:
@@ -303,14 +305,16 @@ def equal_attributes(
 
 
 @overload
-def without_none_values(iterable: Mapping) -> dict: ...
+def without_none_values(iterable: Mapping[_KT, _VT]) -> Dict[_KT, _VT]: ...
 
 
 @overload
-def without_none_values(iterable: Iterable) -> Iterable: ...
+def without_none_values(iterable: Iterable[_KT]) -> Iterable[_KT]: ...
 
 
-def without_none_values(iterable: Union[Mapping, Iterable]) -> Union[dict, Iterable]:
+def without_none_values(
+    iterable: Union[Mapping[_KT, _VT], Iterable[_KT]]
+) -> Union[Dict[_KT, _VT], Iterable[_KT]]:
     """Return a copy of ``iterable`` with all ``None`` entries removed.
 
     If ``iterable`` is a mapping, return a dictionary where all pairs that have
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 5be80ec0f..42a6537a8 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -197,7 +197,7 @@ def referer_str(request: Request) -> Optional[str]:
     return to_unicode(referrer, errors="replace")
 
 
-def request_from_dict(d: dict, *, spider: Optional[Spider] = None) -> Request:
+def request_from_dict(d: Dict[str, Any], *, spider: Optional[Spider] = None) -> Request:
     """Create a :class:`~scrapy.Request` object from a dict.
 
     If a spider is given, it will try to resolve the callbacks looking at the

From 98c755e5fbc005083a5fde810476f2de610bf912 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 21:20:22 +0500
Subject: [PATCH 33/82] Add parameters to typing.List.

---
 scrapy/core/scheduler.py              |  8 ++++----
 scrapy/downloadermiddlewares/stats.py |  6 ++++--
 scrapy/utils/asyncgen.py              | 10 +++++++---
 scrapy/utils/python.py                |  8 ++++----
 4 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index f30a5d9c9..e3b95e977 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -4,7 +4,7 @@ import json
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Type, cast
+from typing import TYPE_CHECKING, Any, List, Optional, Type, cast
 
 from twisted.internet.defer import Deferred
 
@@ -362,13 +362,13 @@ class Scheduler(BaseScheduler):
             return str(dqdir)
         return None
 
-    def _read_dqs_state(self, dqdir: str) -> list:
+    def _read_dqs_state(self, dqdir: str) -> List[int]:
         path = Path(dqdir, "active.json")
         if not path.exists():
             return []
         with path.open(encoding="utf-8") as f:
-            return cast(list, json.load(f))
+            return cast(List[int], json.load(f))
 
-    def _write_dqs_state(self, dqdir: str, state: list) -> None:
+    def _write_dqs_state(self, dqdir: str, state: List[int]) -> None:
         with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
             json.dump(state, f)
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index df30e8ca4..444702757 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Dict, Union
+from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
 from twisted.web import http
 
@@ -17,7 +17,9 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-def get_header_size(headers: Dict[str, Union[list, tuple]]) -> int:
+def get_header_size(
+    headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]
+) -> int:
     size = 0
     for key, value in headers.items():
         if isinstance(value, (list, tuple)):
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 0505db343..67c8e1a01 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,14 +1,18 @@
-from typing import AsyncGenerator, AsyncIterable, Iterable, Union
+from typing import AsyncGenerator, AsyncIterable, Iterable, List, TypeVar, Union
+
+_T = TypeVar("_T")
 
 
-async def collect_asyncgen(result: AsyncIterable) -> list:
+async def collect_asyncgen(result: AsyncIterable[_T]) -> List[_T]:
     results = []
     async for x in result:
         results.append(x)
     return results
 
 
-async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+async def as_async_generator(
+    it: Union[Iterable[_T], AsyncIterable[_T]]
+) -> AsyncGenerator[_T, None]:
     """Wraps an iterable (sync or async) into an async generator."""
     if isinstance(it, AsyncIterable):
         async for r in it:
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 0a50f4e1e..3db7acf81 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -46,7 +46,7 @@ _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
 
 
-def flatten(x: Iterable) -> list:
+def flatten(x: Iterable[Any]) -> List[Any]:
     """flatten(sequence) -> list
 
     Returns a single, flat list which contains all elements retrieved
@@ -66,7 +66,7 @@ def flatten(x: Iterable) -> list:
     return list(iflatten(x))
 
 
-def iflatten(x: Iterable) -> Iterable:
+def iflatten(x: Iterable[Any]) -> Iterable[Any]:
     """iflatten(sequence) -> iterator
 
     Similar to ``.flatten()``, but returns iterator instead"""
@@ -101,10 +101,10 @@ def is_listlike(x: Any) -> bool:
     return hasattr(x, "__iter__") and not isinstance(x, (str, bytes))
 
 
-def unique(list_: Iterable, key: Callable[[Any], Any] = lambda x: x) -> list:
+def unique(list_: Iterable[_T], key: Callable[[_T], Any] = lambda x: x) -> List[_T]:
     """efficient function to uniquify a list preserving item order"""
     seen = set()
-    result = []
+    result: List[_T] = []
     for item in list_:
         seenkey = key(item)
         if seenkey in seen:

From 4164e63725dc19bc8585abbfb0e5009f8eceefcc Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 21:23:55 +0500
Subject: [PATCH 34/82] Add parameters to typing.Tuple.

---
 scrapy/core/http2/agent.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 935af2214..999764a6e 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -20,6 +20,8 @@ from scrapy.http.request import Request
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
 
+ConnectionKeyT = Tuple[bytes, bytes, int]
+
 
 class H2ConnectionPool:
     def __init__(self, reactor: ReactorBase, settings: Settings) -> None:
@@ -28,13 +30,13 @@ class H2ConnectionPool:
 
         # Store a dictionary which is used to get the respective
         # H2ClientProtocolInstance using the  key as Tuple(scheme, hostname, port)
-        self._connections: Dict[Tuple, H2ClientProtocol] = {}
+        self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {}
 
         # Save all requests that arrive before the connection is established
-        self._pending_requests: Dict[Tuple, Deque[Deferred]] = {}
+        self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {}
 
     def get_connection(
-        self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
+        self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
     ) -> Deferred:
         if key in self._pending_requests:
             # Received a request while connecting to remote
@@ -54,7 +56,7 @@ class H2ConnectionPool:
         return self._new_connection(key, uri, endpoint)
 
     def _new_connection(
-        self, key: Tuple, uri: URI, endpoint: HostnameEndpoint
+        self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
     ) -> Deferred:
         self._pending_requests[key] = deque()
 
@@ -69,7 +71,9 @@ class H2ConnectionPool:
         self._pending_requests[key].append(d)
         return d
 
-    def put_connection(self, conn: H2ClientProtocol, key: Tuple) -> H2ClientProtocol:
+    def put_connection(
+        self, conn: H2ClientProtocol, key: ConnectionKeyT
+    ) -> H2ClientProtocol:
         self._connections[key] = conn
 
         # Now as we have established a proper HTTP/2 connection
@@ -81,7 +85,9 @@ class H2ConnectionPool:
 
         return conn
 
-    def _remove_connection(self, errors: List[BaseException], key: Tuple) -> None:
+    def _remove_connection(
+        self, errors: List[BaseException], key: ConnectionKeyT
+    ) -> None:
         self._connections.pop(key)
 
         # Call the errback of all the pending requests for this connection
@@ -122,7 +128,7 @@ class H2Agent:
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
         return self.endpoint_factory.endpointForURI(uri)
 
-    def get_key(self, uri: URI) -> Tuple:
+    def get_key(self, uri: URI) -> ConnectionKeyT:
         """
         Arguments:
             uri - URI obtained directly from request URL
@@ -164,6 +170,6 @@ class ScrapyProxyH2Agent(H2Agent):
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
         return self.endpoint_factory.endpointForURI(self._proxy_uri)
 
-    def get_key(self, uri: URI) -> Tuple:
+    def get_key(self, uri: URI) -> ConnectionKeyT:
         """We use the proxy uri instead of uri obtained from request url"""
-        return "http-proxy", self._proxy_uri.host, self._proxy_uri.port
+        return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port

From 70c56faf4847406de6eb3594758c5531610757e8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 21:41:27 +0500
Subject: [PATCH 35/82] Add parameters to typing.IO.

---
 scrapy/extensions/httpcache.py | 9 ++++++---
 scrapy/mail.py                 | 4 ++--
 scrapy/pipelines/files.py      | 2 +-
 scrapy/utils/ftp.py            | 2 +-
 scrapy/utils/misc.py           | 2 +-
 5 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index dd5bce24f..3f4af42b7 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -315,7 +315,9 @@ class FilesystemCacheStorage:
         self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
         self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
         # https://github.com/python/mypy/issues/10740
-        self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = (
+        self._open: Callable[
+            Concatenate[Union[str, os.PathLike], str, ...], IO[bytes]
+        ] = (
             gzip.open if self.use_gzip else open  # type: ignore[assignment]
         )
 
@@ -368,11 +370,12 @@ class FilesystemCacheStorage:
         with self._open(rpath / "pickled_meta", "wb") as f:
             pickle.dump(metadata, f, protocol=4)
         with self._open(rpath / "response_headers", "wb") as f:
-            f.write(headers_dict_to_raw(response.headers))
+            # headers_dict_to_raw() needs a better type hint
+            f.write(cast(bytes, headers_dict_to_raw(response.headers)))
         with self._open(rpath / "response_body", "wb") as f:
             f.write(response.body)
         with self._open(rpath / "request_headers", "wb") as f:
-            f.write(headers_dict_to_raw(request.headers))
+            f.write(cast(bytes, headers_dict_to_raw(request.headers)))
         with self._open(rpath / "request_body", "wb") as f:
             f.write(request.body)
 
diff --git a/scrapy/mail.py b/scrapy/mail.py
index fd6302550..f4ce2800c 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -97,7 +97,7 @@ class MailSender:
         subject: str,
         body: str,
         cc: Union[str, List[str], None] = None,
-        attachs: Sequence[Tuple[str, str, IO]] = (),
+        attachs: Sequence[Tuple[str, str, IO[Any]]] = (),
         mimetype: str = "text/plain",
         charset: Optional[str] = None,
         _callback: Optional[Callable[..., None]] = None,
@@ -214,7 +214,7 @@ class MailSender:
         return d
 
     def _create_sender_factory(
-        self, to_addrs: List[str], msg: IO, d: Deferred
+        self, to_addrs: List[str], msg: IO[bytes], d: Deferred
     ) -> ESMTPSenderFactory:
         from twisted.mail.smtp import ESMTPSenderFactory
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 47457f2a8..c1ce0939c 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -47,7 +47,7 @@ def _to_string(path: Union[str, PathLike]) -> str:
     return str(path)  # convert a Path object to string
 
 
-def _md5sum(file: IO) -> str:
+def _md5sum(file: IO[bytes]) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.
 
diff --git a/scrapy/utils/ftp.py b/scrapy/utils/ftp.py
index c77681a53..152f3374e 100644
--- a/scrapy/utils/ftp.py
+++ b/scrapy/utils/ftp.py
@@ -21,7 +21,7 @@ def ftp_makedirs_cwd(ftp: FTP, path: str, first_call: bool = True) -> None:
 def ftp_store_file(
     *,
     path: str,
-    file: IO,
+    file: IO[bytes],
     host: str,
     port: int,
     username: str,
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index faf52e44a..b678d1def 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -111,7 +111,7 @@ def walk_modules(path: str) -> List[ModuleType]:
     return mods
 
 
-def md5sum(file: IO) -> str:
+def md5sum(file: IO[bytes]) -> str:
     """Calculate the md5 checksum of a file-like object without reading its
     whole content in memory.
 

From 751c91e614b91827dc68cd462b907c4b9d03f071 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 21:57:14 +0500
Subject: [PATCH 36/82] Add parameters to misc generics.

---
 scrapy/core/engine.py         | 20 +++++++++++---------
 scrapy/core/http2/protocol.py |  4 ++--
 scrapy/utils/datatypes.py     |  4 ++--
 scrapy/utils/python.py        |  2 +-
 scrapy/utils/test.py          |  8 +++++---
 5 files changed, 21 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4eca03800..4cb4454e3 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -5,6 +5,8 @@ For more information see docs/topics/architecture.rst
 
 """
 
+from __future__ import annotations
+
 import logging
 from time import time
 from typing import (
@@ -51,15 +53,15 @@ class Slot:
         self,
         start_requests: Iterable[Request],
         close_if_idle: bool,
-        nextcall: CallLaterOnce,
-        scheduler: "BaseScheduler",
+        nextcall: CallLaterOnce[None],
+        scheduler: BaseScheduler,
     ) -> None:
         self.closing: Optional[Deferred] = None
         self.inprogress: Set[Request] = set()
         self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
         self.close_if_idle: bool = close_if_idle
-        self.nextcall: CallLaterOnce = nextcall
-        self.scheduler: "BaseScheduler" = scheduler
+        self.nextcall: CallLaterOnce[None] = nextcall
+        self.scheduler: BaseScheduler = scheduler
         self.heartbeat: LoopingCall = LoopingCall(nextcall.schedule)
 
     def add_request(self, request: Request) -> None:
@@ -84,8 +86,8 @@ class Slot:
 
 
 class ExecutionEngine:
-    def __init__(self, crawler: "Crawler", spider_closed_callback: Callable) -> None:
-        self.crawler: "Crawler" = crawler
+    def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None:
+        self.crawler: Crawler = crawler
         self.settings: Settings = crawler.settings
         self.signals: SignalManager = crawler.signals
         assert crawler.logformatter
@@ -94,7 +96,7 @@ class ExecutionEngine:
         self.spider: Optional[Spider] = None
         self.running: bool = False
         self.paused: bool = False
-        self.scheduler_cls: Type["BaseScheduler"] = self._get_scheduler_class(
+        self.scheduler_cls: Type[BaseScheduler] = self._get_scheduler_class(
             crawler.settings
         )
         downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
@@ -103,10 +105,10 @@ class ExecutionEngine:
         self._spider_closed_callback: Callable = spider_closed_callback
         self.start_time: Optional[float] = None
 
-    def _get_scheduler_class(self, settings: BaseSettings) -> Type["BaseScheduler"]:
+    def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:
         from scrapy.core.scheduler import BaseScheduler
 
-        scheduler_cls: Type = load_object(settings["SCHEDULER"])
+        scheduler_cls: Type[BaseScheduler] = load_object(settings["SCHEDULER"])
         if not issubclass(scheduler_cls, BaseScheduler):
             raise TypeError(
                 f"The provided scheduler class ({settings['SCHEDULER']})"
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index 063835b17..f2f1cb0b8 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -3,7 +3,7 @@ import itertools
 import logging
 from collections import deque
 from ipaddress import IPv4Address, IPv6Address
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Deque, Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -107,7 +107,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 
         # If requests are received before connection is made we keep
         # all requests in a pool and send them as the connection is made
-        self._pending_request_stream_pool: deque = deque()
+        self._pending_request_stream_pool: Deque[Stream] = deque()
 
         # Save an instance of errors raised which lead to losing the connection
         # We pass these instances to the streams ResponseFailed() failure
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index b2118495f..d06887610 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -196,8 +196,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
-    def __init__(self, seq: Sequence):
-        self.seq: Sequence = seq
+    def __init__(self, seq: Sequence[Any]):
+        self.seq: Sequence[Any] = seq
 
     def __contains__(self, item: Any) -> bool:
         return item not in self.seq
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 3db7acf81..fc1eb4f69 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -148,7 +148,7 @@ def to_bytes(
 
 
 def re_rsearch(
-    pattern: Union[str, Pattern], text: str, chunk_size: int = 1024
+    pattern: Union[str, Pattern[str]], text: str, chunk_size: int = 1024
 ) -> Optional[Tuple[int, int]]:
     """
     This function does a reverse search in a text using a regular expression
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 7a8c5c859..268d8d4be 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,7 +7,7 @@ import os
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
-from typing import Any, Coroutine, Dict, List, Optional, Tuple, Type
+from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar
 from unittest import TestCase, mock
 
 from twisted.internet.defer import Deferred
@@ -17,6 +17,8 @@ from scrapy import Spider
 from scrapy.crawler import Crawler
 from scrapy.utils.boto import is_botocore_available
 
+_T = TypeVar("_T")
+
 
 def assert_gcs_environ() -> None:
     if "GCS_PROJECT_ID" not in os.environ:
@@ -118,8 +120,8 @@ def assert_samelines(
     testcase.assertEqual(text1.splitlines(), text2.splitlines(), msg)
 
 
-def get_from_asyncio_queue(value: Any) -> Coroutine:
-    q: asyncio.Queue = asyncio.Queue()
+def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
+    q: asyncio.Queue[_T] = asyncio.Queue()
     getter = q.get()
     q.put_nowait(value)
     return getter

From 859a77ee4243f17f338072e45785383f12516308 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 31 May 2024 22:23:26 +0500
Subject: [PATCH 37/82] Use a TypedDict for the verbose cookie form.

---
 scrapy/downloadermiddlewares/cookies.py | 25 +++++++++----------------
 scrapy/http/request/__init__.py         | 20 ++++++++++++++++----
 scrapy/http/response/__init__.py        |  6 +++---
 scrapy/http/response/text.py            |  6 +++---
 4 files changed, 31 insertions(+), 26 deletions(-)

diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 6ada3b474..73c2c57fe 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -3,16 +3,7 @@ from __future__ import annotations
 import logging
 from collections import defaultdict
 from http.cookiejar import Cookie
-from typing import (
-    TYPE_CHECKING,
-    Any,
-    DefaultDict,
-    Dict,
-    Iterable,
-    Optional,
-    Sequence,
-    Union,
-)
+from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
 
 from tldextract import TLDExtract
 
@@ -21,6 +12,7 @@ from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
+from scrapy.http.request import VerboseCookie
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
@@ -128,7 +120,7 @@ class CookiesMiddleware:
                 msg = f"Received cookies from: {response}\n{cookies}"
                 logger.debug(msg, extra={"spider": spider})
 
-    def _format_cookie(self, cookie: Dict[str, Any], request: Request) -> Optional[str]:
+    def _format_cookie(self, cookie: VerboseCookie, request: Request) -> Optional[str]:
         """
         Given a dict consisting of cookie components, return its string representation.
         Decode from bytes if necessary.
@@ -142,18 +134,19 @@ class CookiesMiddleware:
                     logger.warning(msg)
                     return None
                 continue
-            if isinstance(cookie[key], (bool, float, int, str)):
-                decoded[key] = str(cookie[key])
+            # https://github.com/python/mypy/issues/7178, https://github.com/python/mypy/issues/9168
+            if isinstance(cookie[key], (bool, float, int, str)):  # type: ignore[literal-required]
+                decoded[key] = str(cookie[key])  # type: ignore[literal-required]
             else:
                 try:
-                    decoded[key] = cookie[key].decode("utf8")
+                    decoded[key] = cookie[key].decode("utf8")  # type: ignore[literal-required]
                 except UnicodeDecodeError:
                     logger.warning(
                         "Non UTF-8 encoded cookie found in request %s: %s",
                         request,
                         cookie,
                     )
-                    decoded[key] = cookie[key].decode("latin1", errors="replace")
+                    decoded[key] = cookie[key].decode("latin1", errors="replace")  # type: ignore[literal-required]
         for flag in ("secure",):
             value = cookie.get(flag, _UNSET)
             if value is _UNSET or not value:
@@ -174,7 +167,7 @@ class CookiesMiddleware:
         """
         if not request.cookies:
             return []
-        cookies: Iterable[Dict[str, Any]]
+        cookies: Iterable[VerboseCookie]
         if isinstance(request.cookies, dict):
             cookies = tuple({"name": k, "value": v} for k, v in request.cookies.items())
         else:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index dfb1dca89..96d0dc515 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -20,6 +20,7 @@ from typing import (
     NoReturn,
     Optional,
     Tuple,
+    TypedDict,
     Union,
     cast,
 )
@@ -34,8 +35,19 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    # typing.NotRequired and typing.Self require Python 3.11
+    from typing_extensions import NotRequired, Self
+
+
+class VerboseCookie(TypedDict):
+    name: str
+    value: str
+    domain: NotRequired[str]
+    path: NotRequired[str]
+    secure: NotRequired[bool]
+
+
+CookiesT = Union[Dict[str, str], List[VerboseCookie]]
 
 
 def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
@@ -97,7 +109,7 @@ class Request(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: str = "utf-8",
         priority: int = 0,
@@ -123,7 +135,7 @@ class Request(object_ref):
         self.callback: Optional[Callable] = callback
         self.errback: Optional[Callable] = errback
 
-        self.cookies: Union[Dict[str, str], List[Dict[str, str]]] = cookies or {}
+        self.cookies: CookiesT = cookies or {}
         self.headers: Headers = Headers(headers or {}, encoding=encoding)
         self.dont_filter: bool = dont_filter
 
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 14618e5e7..166c4de97 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -29,7 +29,7 @@ from twisted.internet.ssl import Certificate
 
 from scrapy.exceptions import NotSupported
 from scrapy.http.headers import Headers
-from scrapy.http.request import Request
+from scrapy.http.request import CookiesT, Request
 from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
@@ -181,7 +181,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
@@ -234,7 +234,7 @@ class Response(object_ref):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index a83279ac8..44c36b682 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -36,7 +36,7 @@ from w3lib.encoding import (
 )
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.http import Request
+from scrapy.http.request import CookiesT, Request
 from scrapy.http.response import Response
 from scrapy.link import Link
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
@@ -183,7 +183,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,
@@ -236,7 +236,7 @@ class TextResponse(Response):
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
-        cookies: Optional[Union[Dict[str, str], List[Dict[str, str]]]] = None,
+        cookies: Optional[CookiesT] = None,
         meta: Optional[Dict[str, Any]] = None,
         encoding: Optional[str] = None,
         priority: int = 0,

From 019f23e3b75a0a481a4fcc22dc93c867ce424b18 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 2 Jun 2024 18:42:01 +0500
Subject: [PATCH 38/82] Add parameters to some of typing.Callable.

---
 scrapy/core/downloader/handlers/__init__.py | 12 +++++++-----
 scrapy/core/engine.py                       | 12 +++++++++---
 scrapy/http/request/__init__.py             |  2 +-
 scrapy/utils/misc.py                        |  8 +++++---
 scrapy/utils/python.py                      |  6 +++---
 5 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index af5282553..5ec5ef6db 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -1,5 +1,7 @@
 """Download handlers for different schemes"""
 
+from __future__ import annotations
+
 import logging
 from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
 
@@ -19,16 +21,16 @@ logger = logging.getLogger(__name__)
 
 
 class DownloadHandlers:
-    def __init__(self, crawler: "Crawler"):
-        self._crawler: "Crawler" = crawler
-        self._schemes: Dict[str, Union[str, Callable]] = (
+    def __init__(self, crawler: Crawler):
+        self._crawler: Crawler = crawler
+        self._schemes: Dict[str, Union[str, Callable[..., Any]]] = (
             {}
         )  # stores acceptable schemes on instancing
         self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
-        handlers: Dict[str, Union[str, Callable]] = without_none_values(
+        handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values(
             cast(
-                Dict[str, Union[str, Callable]],
+                Dict[str, Union[str, Callable[..., Any]]],
                 crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
             )
         )
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4cb4454e3..b342ad7a3 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -86,7 +86,11 @@ class Slot:
 
 
 class ExecutionEngine:
-    def __init__(self, crawler: Crawler, spider_closed_callback: Callable) -> None:
+    def __init__(
+        self,
+        crawler: Crawler,
+        spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]],
+    ) -> None:
         self.crawler: Crawler = crawler
         self.settings: Settings = crawler.settings
         self.signals: SignalManager = crawler.signals
@@ -102,7 +106,9 @@ class ExecutionEngine:
         downloader_cls: Type[Downloader] = load_object(self.settings["DOWNLOADER"])
         self.downloader: Downloader = downloader_cls(crawler)
         self.scraper = Scraper(crawler)
-        self._spider_closed_callback: Callable = spider_closed_callback
+        self._spider_closed_callback: Callable[[Spider], Optional[Deferred[None]]] = (
+            spider_closed_callback
+        )
         self.start_time: Optional[float] = None
 
     def _get_scheduler_class(self, settings: BaseSettings) -> Type[BaseScheduler]:
@@ -427,7 +433,7 @@ class ExecutionEngine:
 
         dfd = self.slot.close()
 
-        def log_failure(msg: str) -> Callable:
+        def log_failure(msg: str) -> Callable[[Failure], None]:
             def errback(failure: Failure) -> None:
                 logger.error(
                     msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 96d0dc515..77149333c 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -266,7 +266,7 @@ class Request(object_ref):
         return d
 
 
-def _find_method(obj: Any, func: Callable) -> str:
+def _find_method(obj: Any, func: Callable[..., Any]) -> str:
     """Helper function for Request.to_dict"""
     # Only instance methods contain ``__func__``
     if obj and hasattr(func, "__func__"):
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index b678d1def..49f36de2d 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -56,7 +56,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     return [arg]
 
 
-def load_object(path: Union[str, Callable]) -> Any:
+def load_object(path: Union[str, Callable[..., Any]]) -> Any:
     """Load an object given its absolute object path, and return it.
 
     The object can be the import path of a class, function, variable or an
@@ -263,7 +263,7 @@ def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
-def is_generator_with_return_value(callable: Callable) -> bool:
+def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:
     """
     Returns True if a callable is a generator function which includes a
     'return' statement with a value different than None, False otherwise
@@ -300,7 +300,9 @@ def is_generator_with_return_value(callable: Callable) -> bool:
     return bool(_generator_callbacks_cache[callable])
 
 
-def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
+def warn_on_generator_with_return_value(
+    spider: Spider, callable: Callable[..., Any]
+) -> None:
     """
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index fc1eb4f69..37a84a350 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -217,7 +217,7 @@ def binary_is_text(data: bytes) -> bool:
     return all(c not in _BINARYCHARS for c in data)
 
 
-def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
+def get_func_args(func: Callable[..., Any], stripself: bool = False) -> List[str]:
     """Return the argument name list of a callable object"""
     if not callable(func):
         raise TypeError(f"func must be callable, got '{type(func).__name__}'")
@@ -247,7 +247,7 @@ def get_func_args(func: Callable, stripself: bool = False) -> List[str]:
     return args
 
 
-def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
+def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]:
     """Returns (args, kwargs) tuple for a function
     >>> import re
     >>> get_spec(re.match)
@@ -285,7 +285,7 @@ def get_spec(func: Callable) -> Tuple[List[str], Dict[str, Any]]:
 
 
 def equal_attributes(
-    obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable]]]
+    obj1: Any, obj2: Any, attributes: Optional[List[Union[str, Callable[[Any], Any]]]]
 ) -> bool:
     """Compare two objects attributes"""
     # not attributes given return False by default

From 492c3bce9dfc6cccdad8fc7002db4bec49cfcb35 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 3 Jun 2024 15:28:20 +0400
Subject: [PATCH 39/82] Don't run callbacks of requests from
 get_media_requests(). (#6386)

---
 scrapy/pipelines/media.py    | 10 ----------
 tests/test_pipeline_media.py |  7 -------
 2 files changed, 17 deletions(-)

diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 25e00b0ea..0e374265e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -24,10 +24,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def _DUMMY_CALLBACK(response):
-    return response
-
-
 class MediaPipeline(ABC):
     LOG_FAILED_RESULTS = True
 
@@ -89,10 +85,6 @@ class MediaPipeline(ABC):
 
     def _process_request(self, request, info, item):
         fp = self._fingerprinter.fingerprint(request)
-        if not request.callback or request.callback is NO_CALLBACK:
-            cb = _DUMMY_CALLBACK
-        else:
-            cb = request.callback
         eb = request.errback
         request.callback = NO_CALLBACK
         request.errback = None
@@ -100,14 +92,12 @@ class MediaPipeline(ABC):
         # Return cached result if request was already seen
         if fp in info.downloaded:
             d = defer_result(info.downloaded[fp])
-            d.addCallback(cb)
             if eb:
                 d.addErrback(eb)
             return d
 
         # Otherwise, wait for result
         wad = Deferred()
-        wad.addCallback(cb)
         if eb:
             wad.addErrback(eb)
         info.waiting[fp].append(wad)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 763453551..127775f43 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -211,10 +211,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
 class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     pipeline_class = MockedMediaPipeline
 
-    def _callback(self, result):
-        self.pipe._mockcalled.append("request_callback")
-        return result
-
     def _errback(self, result):
         self.pipe._mockcalled.append("request_errback")
         return result
@@ -225,7 +221,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         req = Request(
             "http://url1",
             meta={"response": rsp},
-            callback=self._callback,
             errback=self._errback,
         )
         item = {"requests": req}
@@ -237,7 +232,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
                 "get_media_requests",
                 "media_to_download",
                 "media_downloaded",
-                "request_callback",
                 "item_completed",
             ],
         )
@@ -249,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         req = Request(
             "http://url1",
             meta={"response": fail},
-            callback=self._callback,
             errback=self._errback,
         )
         item = {"requests": req}

From e56b425198bfe3e86f2c578e7bc1f2988c7d3ec9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 08:33:45 +0400
Subject: [PATCH 40/82] Full typing for scrapy/pipelines. (#6387)

---
 scrapy/pipelines/__init__.py |   3 +-
 scrapy/pipelines/files.py    | 274 +++++++++++++++++++++++++----------
 scrapy/pipelines/images.py   | 128 ++++++++++++----
 scrapy/pipelines/media.py    | 162 ++++++++++++++++-----
 4 files changed, 424 insertions(+), 143 deletions(-)

diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index f9544d329..0cfbc156f 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred
 
 from scrapy import Spider
 from scrapy.middleware import MiddlewareManager
+from scrapy.settings import Settings
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import deferred_f_from_coro_f
 
@@ -18,7 +19,7 @@ class ItemPipelineManager(MiddlewareManager):
     component_name = "item pipeline"
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings) -> List[Any]:
+    def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
         return build_component_list(settings.getwithbase("ITEM_PIPELINES"))
 
     def _add_middleware(self, pipe: Any) -> None:
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index c1ce0939c..85a8c77da 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -18,16 +18,35 @@ from ftplib import FTP
 from io import BytesIO
 from os import PathLike
 from pathlib import Path
-from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    DefaultDict,
+    Dict,
+    List,
+    NoReturn,
+    Optional,
+    Protocol,
+    Set,
+    Type,
+    TypedDict,
+    Union,
+    cast,
+)
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
 from twisted.internet import defer, threads
+from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
 
+from scrapy import Spider
 from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
@@ -40,10 +59,11 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+
 logger = logging.getLogger(__name__)
 
 
-def _to_string(path: Union[str, PathLike]) -> str:
+def _to_string(path: Union[str, PathLike[str]]) -> str:
     return str(path)  # convert a Path object to string
 
 
@@ -68,23 +88,54 @@ class FileException(Exception):
     """General media error exception"""
 
 
+class StatInfo(TypedDict, total=False):
+    checksum: str
+    last_modified: float
+
+
+class FilesStoreProtocol(Protocol):
+    def __init__(self, basedir: str): ...
+
+    def persist_file(
+        self,
+        path: str,
+        buf: BytesIO,
+        info: MediaPipeline.SpiderInfo,
+        meta: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Optional[Deferred[Any]]: ...
+
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Union[StatInfo, Deferred[StatInfo]]: ...
+
+
 class FSFilesStore:
-    def __init__(self, basedir: Union[str, PathLike]):
+    def __init__(self, basedir: Union[str, PathLike[str]]):
         basedir = _to_string(basedir)
         if "://" in basedir:
             basedir = basedir.split("://", 1)[1]
-        self.basedir = basedir
+        self.basedir: str = basedir
         self._mkdir(Path(self.basedir))
-        self.created_directories: DefaultDict[str, Set[str]] = defaultdict(set)
+        self.created_directories: DefaultDict[MediaPipeline.SpiderInfo, Set[str]] = (
+            defaultdict(set)
+        )
 
     def persist_file(
-        self, path: Union[str, PathLike], buf, info, meta=None, headers=None
-    ):
+        self,
+        path: Union[str, PathLike[str]],
+        buf: BytesIO,
+        info: MediaPipeline.SpiderInfo,
+        meta: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> None:
         absolute_path = self._get_filesystem_path(path)
         self._mkdir(absolute_path.parent, info)
         absolute_path.write_bytes(buf.getvalue())
 
-    def stat_file(self, path: Union[str, PathLike], info):
+    def stat_file(
+        self, path: Union[str, PathLike[str]], info: MediaPipeline.SpiderInfo
+    ) -> StatInfo:
         absolute_path = self._get_filesystem_path(path)
         try:
             last_modified = absolute_path.stat().st_mtime
@@ -96,12 +147,14 @@ class FSFilesStore:
 
         return {"last_modified": last_modified, "checksum": checksum}
 
-    def _get_filesystem_path(self, path: Union[str, PathLike]) -> Path:
+    def _get_filesystem_path(self, path: Union[str, PathLike[str]]) -> Path:
         path_comps = _to_string(path).split("/")
         return Path(self.basedir, *path_comps)
 
-    def _mkdir(self, dirname: Path, domain: Optional[str] = None):
-        seen = self.created_directories[domain] if domain else set()
+    def _mkdir(
+        self, dirname: Path, domain: Optional[MediaPipeline.SpiderInfo] = None
+    ) -> None:
+        seen: Set[str] = self.created_directories[domain] if domain else set()
         if str(dirname) not in seen:
             if not dirname.exists():
                 dirname.mkdir(parents=True)
@@ -122,7 +175,7 @@ class S3FilesStore:
         "Cache-Control": "max-age=172800",
     }
 
-    def __init__(self, uri):
+    def __init__(self, uri: str):
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
         import botocore.session
@@ -142,8 +195,10 @@ class S3FilesStore:
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
         self.bucket, self.prefix = uri[5:].split("/", 1)
 
-    def stat_file(self, path, info):
-        def _onsuccess(boto_key):
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+        def _onsuccess(boto_key: Dict[str, Any]) -> StatInfo:
             checksum = boto_key["ETag"].strip('"')
             last_modified = boto_key["LastModified"]
             modified_stamp = time.mktime(last_modified.timetuple())
@@ -151,13 +206,23 @@ class S3FilesStore:
 
         return self._get_boto_key(path).addCallback(_onsuccess)
 
-    def _get_boto_key(self, path):
+    def _get_boto_key(self, path: str) -> Deferred[Dict[str, Any]]:
         key_name = f"{self.prefix}{path}"
-        return threads.deferToThread(
-            self.s3_client.head_object, Bucket=self.bucket, Key=key_name
+        return cast(
+            "Deferred[Dict[str, Any]]",
+            threads.deferToThread(
+                self.s3_client.head_object, Bucket=self.bucket, Key=key_name  # type: ignore[attr-defined]
+            ),
         )
 
-    def persist_file(self, path, buf, info, meta=None, headers=None):
+    def persist_file(
+        self,
+        path: str,
+        buf: BytesIO,
+        info: MediaPipeline.SpiderInfo,
+        meta: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Deferred[Any]:
         """Upload file to S3 storage"""
         key_name = f"{self.prefix}{path}"
         buf.seek(0)
@@ -165,7 +230,7 @@ class S3FilesStore:
         if headers:
             extra.update(self._headers_to_botocore_kwargs(headers))
         return threads.deferToThread(
-            self.s3_client.put_object,
+            self.s3_client.put_object,  # type: ignore[attr-defined]
             Bucket=self.bucket,
             Key=key_name,
             Body=buf,
@@ -174,7 +239,7 @@ class S3FilesStore:
             **extra,
         )
 
-    def _headers_to_botocore_kwargs(self, headers):
+    def _headers_to_botocore_kwargs(self, headers: Dict[str, Any]) -> Dict[str, Any]:
         """Convert headers to botocore keyword arguments."""
         # This is required while we need to support both boto and botocore.
         mapping = CaseInsensitiveDict(
@@ -206,7 +271,7 @@ class S3FilesStore:
                 "X-Amz-Website-Redirect-Location": "WebsiteRedirectLocation",
             }
         )
-        extra = {}
+        extra: Dict[str, Any] = {}
         for key, value in headers.items():
             try:
                 kwarg = mapping[key]
@@ -226,13 +291,13 @@ class GCSFilesStore:
     # Overridden from settings.FILES_STORE_GCS_ACL in FilesPipeline.from_settings.
     POLICY = None
 
-    def __init__(self, uri):
+    def __init__(self, uri: str):
         from google.cloud import storage
 
         client = storage.Client(project=self.GCS_PROJECT_ID)
         bucket, prefix = uri[5:].split("/", 1)
         self.bucket = client.bucket(bucket)
-        self.prefix = prefix
+        self.prefix: str = prefix
         permissions = self.bucket.test_iam_permissions(
             ["storage.objects.get", "storage.objects.create"]
         )
@@ -248,8 +313,10 @@ class GCSFilesStore:
                 {"bucket": bucket},
             )
 
-    def stat_file(self, path, info):
-        def _onsuccess(blob):
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+        def _onsuccess(blob) -> StatInfo:
             if blob:
                 checksum = base64.b64decode(blob.md5_hash).hex()
                 last_modified = time.mktime(blob.updated.timetuple())
@@ -257,19 +324,29 @@ class GCSFilesStore:
             return {}
 
         blob_path = self._get_blob_path(path)
-        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
-            _onsuccess
+        return cast(
+            Deferred[StatInfo],
+            threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
+                _onsuccess
+            ),
         )
 
-    def _get_content_type(self, headers):
+    def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:
         if headers and "Content-Type" in headers:
             return headers["Content-Type"]
         return "application/octet-stream"
 
-    def _get_blob_path(self, path):
+    def _get_blob_path(self, path: str) -> str:
         return self.prefix + path
 
-    def persist_file(self, path, buf, info, meta=None, headers=None):
+    def persist_file(
+        self,
+        path: str,
+        buf: BytesIO,
+        info: MediaPipeline.SpiderInfo,
+        meta: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Deferred[Any]:
         blob_path = self._get_blob_path(path)
         blob = self.bucket.blob(blob_path)
         blob.cache_control = self.CACHE_CONTROL
@@ -283,22 +360,33 @@ class GCSFilesStore:
 
 
 class FTPFilesStore:
-    FTP_USERNAME = None
-    FTP_PASSWORD = None
-    USE_ACTIVE_MODE = None
+    FTP_USERNAME: Optional[str] = None
+    FTP_PASSWORD: Optional[str] = None
+    USE_ACTIVE_MODE: Optional[bool] = None
 
-    def __init__(self, uri):
+    def __init__(self, uri: str):
         if not uri.startswith("ftp://"):
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 'ftp'")
         u = urlparse(uri)
-        self.port = u.port
-        self.host = u.hostname
+        assert u.port
+        assert u.hostname
+        self.port: int = u.port
+        self.host: str = u.hostname
         self.port = int(u.port or 21)
-        self.username = u.username or self.FTP_USERNAME
-        self.password = u.password or self.FTP_PASSWORD
-        self.basedir = u.path.rstrip("/")
+        assert self.FTP_USERNAME
+        assert self.FTP_PASSWORD
+        self.username: str = u.username or self.FTP_USERNAME
+        self.password: str = u.password or self.FTP_PASSWORD
+        self.basedir: str = u.path.rstrip("/")
 
-    def persist_file(self, path, buf, info, meta=None, headers=None):
+    def persist_file(
+        self,
+        path: str,
+        buf: BytesIO,
+        info: MediaPipeline.SpiderInfo,
+        meta: Optional[Dict[str, Any]] = None,
+        headers: Optional[Dict[str, str]] = None,
+    ) -> Deferred[Any]:
         path = f"{self.basedir}/{path}"
         return threads.deferToThread(
             ftp_store_file,
@@ -311,8 +399,10 @@ class FTPFilesStore:
             use_active_mode=self.USE_ACTIVE_MODE,
         )
 
-    def stat_file(self, path, info):
-        def _stat_file(path):
+    def stat_file(
+        self, path: str, info: MediaPipeline.SpiderInfo
+    ) -> Deferred[StatInfo]:
+        def _stat_file(path: str) -> StatInfo:
             try:
                 ftp = FTP()
                 ftp.connect(self.host, self.port)
@@ -328,7 +418,7 @@ class FTPFilesStore:
             except Exception:
                 return {}
 
-        return threads.deferToThread(_stat_file, path)
+        return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))
 
 
 class FilesPipeline(MediaPipeline):
@@ -350,20 +440,23 @@ class FilesPipeline(MediaPipeline):
 
     """
 
-    MEDIA_NAME = "file"
-    EXPIRES = 90
-    STORE_SCHEMES = {
+    MEDIA_NAME: str = "file"
+    EXPIRES: int = 90
+    STORE_SCHEMES: Dict[str, Type[FilesStoreProtocol]] = {
         "": FSFilesStore,
         "file": FSFilesStore,
         "s3": S3FilesStore,
         "gs": GCSFilesStore,
         "ftp": FTPFilesStore,
     }
-    DEFAULT_FILES_URLS_FIELD = "file_urls"
-    DEFAULT_FILES_RESULT_FIELD = "files"
+    DEFAULT_FILES_URLS_FIELD: str = "file_urls"
+    DEFAULT_FILES_RESULT_FIELD: str = "files"
 
     def __init__(
-        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+        self,
+        store_uri: Union[str, PathLike[str]],
+        download_func: Optional[Callable[[Request, Spider], Response]] = None,
+        settings: Union[Settings, Dict[str, Any], None] = None,
     ):
         store_uri = _to_string(store_uri)
         if not store_uri:
@@ -372,26 +465,26 @@ class FilesPipeline(MediaPipeline):
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         cls_name = "FilesPipeline"
-        self.store = self._get_store(store_uri)
+        self.store: FilesStoreProtocol = self._get_store(store_uri)
         resolve = functools.partial(
             self._key_for_pipe, base_class_name=cls_name, settings=settings
         )
-        self.expires = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
+        self.expires: int = settings.getint(resolve("FILES_EXPIRES"), self.EXPIRES)
         if not hasattr(self, "FILES_URLS_FIELD"):
             self.FILES_URLS_FIELD = self.DEFAULT_FILES_URLS_FIELD
         if not hasattr(self, "FILES_RESULT_FIELD"):
             self.FILES_RESULT_FIELD = self.DEFAULT_FILES_RESULT_FIELD
-        self.files_urls_field = settings.get(
+        self.files_urls_field: str = settings.get(
             resolve("FILES_URLS_FIELD"), self.FILES_URLS_FIELD
         )
-        self.files_result_field = settings.get(
+        self.files_result_field: str = settings.get(
             resolve("FILES_RESULT_FIELD"), self.FILES_RESULT_FIELD
         )
 
         super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
-    def from_settings(cls, settings) -> Self:
+    def from_settings(cls, settings: Settings) -> Self:
         s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
         s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
         s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
@@ -418,7 +511,7 @@ class FilesPipeline(MediaPipeline):
         store_uri = settings["FILES_STORE"]
         return cls(store_uri, settings=settings)
 
-    def _get_store(self, uri: str):
+    def _get_store(self, uri: str) -> FilesStoreProtocol:
         if Path(uri).is_absolute():  # to support win32 paths like: C:\\some\dir
             scheme = "file"
         else:
@@ -426,19 +519,21 @@ class FilesPipeline(MediaPipeline):
         store_cls = self.STORE_SCHEMES[scheme]
         return store_cls(uri)
 
-    def media_to_download(self, request, info, *, item=None):
-        def _onsuccess(result):
+    def media_to_download(
+        self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
+    ) -> Deferred[Optional[FileInfo]]:
+        def _onsuccess(result: StatInfo) -> Optional[FileInfo]:
             if not result:
-                return  # returning None force download
+                return None  # returning None force download
 
             last_modified = result.get("last_modified", None)
             if not last_modified:
-                return  # returning None force download
+                return None  # returning None force download
 
             age_seconds = time.time() - last_modified
             age_days = age_seconds / 60 / 60 / 24
             if age_days > self.expires:
-                return  # returning None force download
+                return None  # returning None force download
 
             referer = referer_str(request)
             logger.debug(
@@ -458,19 +553,22 @@ class FilesPipeline(MediaPipeline):
             }
 
         path = self.file_path(request, info=info, item=item)
-        dfd = defer.maybeDeferred(self.store.stat_file, path, info)
-        dfd.addCallback(_onsuccess)
-        dfd.addErrback(lambda _: None)
-        dfd.addErrback(
+        # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
+        dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info)  # type: ignore[arg-type]
+        dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
+        dfd2.addErrback(lambda _: None)
+        dfd2.addErrback(
             lambda f: logger.error(
                 self.__class__.__name__ + ".store.stat_file",
                 exc_info=failure_to_exc_info(f),
                 extra={"spider": info.spider},
             )
         )
-        return dfd
+        return dfd2
 
-    def media_failed(self, failure, request, info):
+    def media_failed(
+        self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
+    ) -> NoReturn:
         if not isinstance(failure.value, IgnoreRequest):
             referer = referer_str(request)
             logger.warning(
@@ -487,7 +585,14 @@ class FilesPipeline(MediaPipeline):
 
         raise FileException
 
-    def media_downloaded(self, response, request, info, *, item=None):
+    def media_downloaded(
+        self,
+        response: Response,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> FileInfo:
         referer = referer_str(request)
 
         if response.status != 200:
@@ -546,16 +651,26 @@ class FilesPipeline(MediaPipeline):
             "status": status,
         }
 
-    def inc_stats(self, spider, status):
+    def inc_stats(self, spider: Spider, status: str) -> None:
+        assert spider.crawler.stats
         spider.crawler.stats.inc_value("file_count", spider=spider)
         spider.crawler.stats.inc_value(f"file_status_count/{status}", spider=spider)
 
     # Overridable Interface
-    def get_media_requests(self, item, info):
+    def get_media_requests(
+        self, item: Any, info: MediaPipeline.SpiderInfo
+    ) -> List[Request]:
         urls = ItemAdapter(item).get(self.files_urls_field, [])
         return [Request(u, callback=NO_CALLBACK) for u in urls]
 
-    def file_downloaded(self, response, request, info, *, item=None):
+    def file_downloaded(
+        self,
+        response: Response,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> str:
         path = self.file_path(request, response=response, info=info, item=item)
         buf = BytesIO(response.body)
         checksum = _md5sum(buf)
@@ -563,12 +678,21 @@ class FilesPipeline(MediaPipeline):
         self.store.persist_file(path, buf, info)
         return checksum
 
-    def item_completed(self, results, item, info):
+    def item_completed(
+        self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
+    ) -> Any:
         with suppress(KeyError):
             ItemAdapter(item)[self.files_result_field] = [x for ok, x in results if ok]
         return item
 
-    def file_path(self, request, response=None, info=None, *, item=None):
+    def file_path(
+        self,
+        request: Request,
+        response: Optional[Response] = None,
+        info: Optional[MediaPipeline.SpiderInfo] = None,
+        *,
+        item: Any = None,
+    ) -> str:
         media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         media_ext = Path(request.url).suffix
         # Handles empty and wild extensions by trying to guess the
@@ -577,5 +701,5 @@ class FilesPipeline(MediaPipeline):
             media_ext = ""
             media_type = mimetypes.guess_type(request.url)[0]
             if media_type:
-                media_ext = mimetypes.guess_extension(media_type)
+                media_ext = cast(str, mimetypes.guess_extension(media_type))
         return f"full/{media_guid}{media_ext}"
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index e7ef06fb3..27a57b17c 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -12,12 +12,25 @@ import warnings
 from contextlib import suppress
 from io import BytesIO
 from os import PathLike
-from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 from itemadapter import ItemAdapter
 
+from scrapy import Spider
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.files import (
     FileException,
@@ -27,20 +40,20 @@ from scrapy.pipelines.files import (
     S3FilesStore,
     _md5sum,
 )
-
-# TODO: from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.python import get_func_args, to_bytes
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
+    from PIL import Image
     from typing_extensions import Self
 
 
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         warnings.warn(
             "The NoimagesDrop class is deprecated",
             category=ScrapyDeprecationWarning,
@@ -56,19 +69,22 @@ class ImageException(FileException):
 class ImagesPipeline(FilesPipeline):
     """Abstract pipeline that implement the image thumbnail generation logic"""
 
-    MEDIA_NAME = "image"
+    MEDIA_NAME: str = "image"
 
     # Uppercase attributes kept for backward compatibility with code that subclasses
     # ImagesPipeline. They may be overridden by settings.
-    MIN_WIDTH = 0
-    MIN_HEIGHT = 0
-    EXPIRES = 90
+    MIN_WIDTH: int = 0
+    MIN_HEIGHT: int = 0
+    EXPIRES: int = 90
     THUMBS: Dict[str, Tuple[int, int]] = {}
     DEFAULT_IMAGES_URLS_FIELD = "image_urls"
     DEFAULT_IMAGES_RESULT_FIELD = "images"
 
     def __init__(
-        self, store_uri: Union[str, PathLike], download_func=None, settings=None
+        self,
+        store_uri: Union[str, PathLike[str]],
+        download_func: Optional[Callable[[Request, Spider], Response]] = None,
+        settings: Union[Settings, Dict[str, Any], None] = None,
     ):
         try:
             from PIL import Image
@@ -89,27 +105,33 @@ class ImagesPipeline(FilesPipeline):
             base_class_name="ImagesPipeline",
             settings=settings,
         )
-        self.expires = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
+        self.expires: int = settings.getint(resolve("IMAGES_EXPIRES"), self.EXPIRES)
 
         if not hasattr(self, "IMAGES_RESULT_FIELD"):
-            self.IMAGES_RESULT_FIELD = self.DEFAULT_IMAGES_RESULT_FIELD
+            self.IMAGES_RESULT_FIELD: str = self.DEFAULT_IMAGES_RESULT_FIELD
         if not hasattr(self, "IMAGES_URLS_FIELD"):
-            self.IMAGES_URLS_FIELD = self.DEFAULT_IMAGES_URLS_FIELD
+            self.IMAGES_URLS_FIELD: str = self.DEFAULT_IMAGES_URLS_FIELD
 
-        self.images_urls_field = settings.get(
+        self.images_urls_field: str = settings.get(
             resolve("IMAGES_URLS_FIELD"), self.IMAGES_URLS_FIELD
         )
-        self.images_result_field = settings.get(
+        self.images_result_field: str = settings.get(
             resolve("IMAGES_RESULT_FIELD"), self.IMAGES_RESULT_FIELD
         )
-        self.min_width = settings.getint(resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH)
-        self.min_height = settings.getint(resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT)
-        self.thumbs = settings.get(resolve("IMAGES_THUMBS"), self.THUMBS)
+        self.min_width: int = settings.getint(
+            resolve("IMAGES_MIN_WIDTH"), self.MIN_WIDTH
+        )
+        self.min_height: int = settings.getint(
+            resolve("IMAGES_MIN_HEIGHT"), self.MIN_HEIGHT
+        )
+        self.thumbs: Dict[str, Tuple[int, int]] = settings.get(
+            resolve("IMAGES_THUMBS"), self.THUMBS
+        )
 
-        self._deprecated_convert_image = None
+        self._deprecated_convert_image: Optional[bool] = None
 
     @classmethod
-    def from_settings(cls, settings) -> Self:
+    def from_settings(cls, settings: Settings) -> Self:
         s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
         s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
         s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
@@ -136,11 +158,25 @@ class ImagesPipeline(FilesPipeline):
         store_uri = settings["IMAGES_STORE"]
         return cls(store_uri, settings=settings)
 
-    def file_downloaded(self, response, request, info, *, item=None):
+    def file_downloaded(
+        self,
+        response: Response,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> str:
         return self.image_downloaded(response, request, info, item=item)
 
-    def image_downloaded(self, response, request, info, *, item=None):
-        checksum = None
+    def image_downloaded(
+        self,
+        response: Response,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> str:
+        checksum: Optional[str] = None
         for path, image, buf in self.get_images(response, request, info, item=item):
             if checksum is None:
                 buf.seek(0)
@@ -153,9 +189,17 @@ class ImagesPipeline(FilesPipeline):
                 meta={"width": width, "height": height},
                 headers={"Content-Type": "image/jpeg"},
             )
+        assert checksum is not None
         return checksum
 
-    def get_images(self, response, request, info, *, item=None):
+    def get_images(
+        self,
+        response: Response,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> Iterable[Tuple[str, Image.Image, BytesIO]]:
         path = self.file_path(request, response=response, info=info, item=item)
         orig_image = self._Image.open(BytesIO(response.body))
 
@@ -196,7 +240,12 @@ class ImagesPipeline(FilesPipeline):
                 thumb_image, thumb_buf = self.convert_image(image, size, buf)
             yield thumb_path, thumb_image, thumb_buf
 
-    def convert_image(self, image, size=None, response_body=None):
+    def convert_image(
+        self,
+        image: Image.Image,
+        size: Optional[Tuple[int, int]] = None,
+        response_body: Optional[BytesIO] = None,
+    ) -> Tuple[Image.Image, BytesIO]:
         if response_body is None:
             warnings.warn(
                 f"{self.__class__.__name__}.convert_image() method called in a deprecated way, "
@@ -225,7 +274,7 @@ class ImagesPipeline(FilesPipeline):
                 # when updating the minimum requirements for Pillow.
                 resampling_filter = self._Image.Resampling.LANCZOS
             except AttributeError:
-                resampling_filter = self._Image.ANTIALIAS
+                resampling_filter = self._Image.ANTIALIAS  # type: ignore[attr-defined]
             image.thumbnail(size, resampling_filter)
         elif response_body is not None and image.format == "JPEG":
             return image, response_body
@@ -234,19 +283,38 @@ class ImagesPipeline(FilesPipeline):
         image.save(buf, "JPEG")
         return image, buf
 
-    def get_media_requests(self, item, info):
+    def get_media_requests(
+        self, item: Any, info: MediaPipeline.SpiderInfo
+    ) -> List[Request]:
         urls = ItemAdapter(item).get(self.images_urls_field, [])
         return [Request(u, callback=NO_CALLBACK) for u in urls]
 
-    def item_completed(self, results, item, info):
+    def item_completed(
+        self, results: List[FileInfoOrError], item: Any, info: MediaPipeline.SpiderInfo
+    ) -> Any:
         with suppress(KeyError):
             ItemAdapter(item)[self.images_result_field] = [x for ok, x in results if ok]
         return item
 
-    def file_path(self, request, response=None, info=None, *, item=None):
+    def file_path(
+        self,
+        request: Request,
+        response: Optional[Response] = None,
+        info: Optional[MediaPipeline.SpiderInfo] = None,
+        *,
+        item: Any = None,
+    ) -> str:
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"full/{image_guid}.jpg"
 
-    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
+    def thumb_path(
+        self,
+        request: Request,
+        thumb_id: str,
+        response: Optional[Response] = None,
+        info: Optional[MediaPipeline.SpiderInfo] = None,
+        *,
+        item: Any = None,
+    ) -> str:
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 0e374265e..3e327105e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -4,54 +4,101 @@ import functools
 import logging
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import TYPE_CHECKING
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    DefaultDict,
+    Dict,
+    List,
+    Literal,
+    NoReturn,
+    Optional,
+    Set,
+    Tuple,
+    TypedDict,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
-from scrapy.http.request import NO_CALLBACK
+from scrapy import Spider
+from scrapy.crawler import Crawler
+from scrapy.http import Response
+from scrapy.http.request import NO_CALLBACK, Request
 from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import defer_result, mustbe_deferred
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
+from scrapy.utils.request import RequestFingerprinter
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+_T = TypeVar("_T")
+
+
+class FileInfo(TypedDict):
+    url: str
+    path: str
+    checksum: Optional[str]
+    status: str
+
+
+FileInfoOrError = Union[Tuple[Literal[True], FileInfo], Tuple[Literal[False], Failure]]
+
 
 logger = logging.getLogger(__name__)
 
 
 class MediaPipeline(ABC):
-    LOG_FAILED_RESULTS = True
+    crawler: Crawler
+    _fingerprinter: RequestFingerprinter
+
+    LOG_FAILED_RESULTS: bool = True
 
     class SpiderInfo:
-        def __init__(self, spider):
-            self.spider = spider
-            self.downloading = set()
-            self.downloaded = {}
-            self.waiting = defaultdict(list)
+        def __init__(self, spider: Spider):
+            self.spider: Spider = spider
+            self.downloading: Set[bytes] = set()
+            self.downloaded: Dict[bytes, Union[FileInfo, Failure]] = {}
+            self.waiting: DefaultDict[bytes, List[Deferred[FileInfo]]] = defaultdict(
+                list
+            )
 
-    def __init__(self, download_func=None, settings=None):
+    def __init__(
+        self,
+        download_func: Optional[Callable[[Request, Spider], Response]] = None,
+        settings: Union[Settings, Dict[str, Any], None] = None,
+    ):
         self.download_func = download_func
-        self._expects_item = {}
 
         if isinstance(settings, dict) or settings is None:
             settings = Settings(settings)
         resolve = functools.partial(
             self._key_for_pipe, base_class_name="MediaPipeline", settings=settings
         )
-        self.allow_redirects = settings.getbool(resolve("MEDIA_ALLOW_REDIRECTS"), False)
+        self.allow_redirects: bool = settings.getbool(
+            resolve("MEDIA_ALLOW_REDIRECTS"), False
+        )
         self._handle_statuses(self.allow_redirects)
 
-    def _handle_statuses(self, allow_redirects):
+    def _handle_statuses(self, allow_redirects: bool) -> None:
         self.handle_httpstatus_list = None
         if allow_redirects:
             self.handle_httpstatus_list = SequenceExclude(range(300, 400))
 
-    def _key_for_pipe(self, key, base_class_name=None, settings=None):
+    def _key_for_pipe(
+        self,
+        key: str,
+        base_class_name: Optional[str] = None,
+        settings: Optional[Settings] = None,
+    ) -> str:
         class_name = self.__class__.__name__
         formatted_key = f"{class_name.upper()}_{key}"
         if (
@@ -64,26 +111,34 @@ class MediaPipeline(ABC):
         return formatted_key
 
     @classmethod
-    def from_crawler(cls, crawler) -> Self:
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        pipe: Self
         try:
             pipe = cls.from_settings(crawler.settings)  # type: ignore[attr-defined]
         except AttributeError:
             pipe = cls()
         pipe.crawler = crawler
+        assert crawler.request_fingerprinter
         pipe._fingerprinter = crawler.request_fingerprinter
         return pipe
 
-    def open_spider(self, spider):
+    def open_spider(self, spider: Spider) -> None:
         self.spiderinfo = self.SpiderInfo(spider)
 
-    def process_item(self, item, spider):
+    def process_item(
+        self, item: Any, spider: Spider
+    ) -> Deferred[List[FileInfoOrError]]:
         info = self.spiderinfo
         requests = arg_to_iter(self.get_media_requests(item, info))
         dlist = [self._process_request(r, info, item) for r in requests]
-        dfd = DeferredList(dlist, consumeErrors=True)
+        dfd = cast(
+            "Deferred[List[FileInfoOrError]]", DeferredList(dlist, consumeErrors=True)
+        )
         return dfd.addCallback(self.item_completed, item, info)
 
-    def _process_request(self, request, info, item):
+    def _process_request(
+        self, request: Request, info: SpiderInfo, item: Any
+    ) -> Deferred[FileInfo]:
         fp = self._fingerprinter.fingerprint(request)
         eb = request.errback
         request.callback = NO_CALLBACK
@@ -97,7 +152,7 @@ class MediaPipeline(ABC):
             return d
 
         # Otherwise, wait for result
-        wad = Deferred()
+        wad: Deferred[FileInfo] = Deferred()
         if eb:
             wad.addErrback(eb)
         info.waiting[fp].append(wad)
@@ -108,36 +163,48 @@ class MediaPipeline(ABC):
 
         # Download request checking media_to_download hook output first
         info.downloading.add(fp)
-        dfd = mustbe_deferred(self.media_to_download, request, info, item=item)
-        dfd.addCallback(self._check_media_to_download, request, info, item=item)
-        dfd.addErrback(self._log_exception)
-        dfd.addBoth(self._cache_result_and_execute_waiters, fp, info)
-        return dfd.addBoth(lambda _: wad)  # it must return wad at last
+        dfd: Deferred[Optional[FileInfo]] = mustbe_deferred(
+            self.media_to_download, request, info, item=item
+        )
+        dfd2: Deferred[FileInfo] = dfd.addCallback(
+            self._check_media_to_download, request, info, item=item
+        )
+        dfd2.addErrback(self._log_exception)
+        dfd2.addBoth(self._cache_result_and_execute_waiters, fp, info)
+        return dfd2.addBoth(lambda _: wad)  # it must return wad at last
 
-    def _log_exception(self, result):
+    def _log_exception(self, result: Failure) -> Failure:
         logger.exception(result)
         return result
 
-    def _modify_media_request(self, request):
+    def _modify_media_request(self, request: Request) -> None:
         if self.handle_httpstatus_list:
             request.meta["handle_httpstatus_list"] = self.handle_httpstatus_list
         else:
             request.meta["handle_httpstatus_all"] = True
 
-    def _check_media_to_download(self, result, request, info, item):
+    def _check_media_to_download(
+        self, result: Optional[FileInfo], request: Request, info: SpiderInfo, item: Any
+    ) -> Union[FileInfo, Deferred[FileInfo]]:
         if result is not None:
             return result
+        dfd: Deferred[Response]
         if self.download_func:
             # this ugly code was left only to support tests. TODO: remove
             dfd = mustbe_deferred(self.download_func, request, info.spider)
         else:
             self._modify_media_request(request)
+            assert self.crawler.engine
             dfd = self.crawler.engine.download(request)
-        dfd.addCallback(self.media_downloaded, request, info, item=item)
-        dfd.addErrback(self.media_failed, request, info)
-        return dfd
+        dfd2: Deferred[FileInfo] = dfd.addCallback(
+            self.media_downloaded, request, info, item=item
+        )
+        dfd2.addErrback(self.media_failed, request, info)
+        return dfd2
 
-    def _cache_result_and_execute_waiters(self, result, fp, info):
+    def _cache_result_and_execute_waiters(
+        self, result: Union[FileInfo, Failure], fp: bytes, info: SpiderInfo
+    ) -> None:
         if isinstance(result, Failure):
             # minimize cached information for failure
             result.cleanFailure()
@@ -176,30 +243,44 @@ class MediaPipeline(ABC):
 
     # Overridable Interface
     @abstractmethod
-    def media_to_download(self, request, info, *, item=None):
+    def media_to_download(
+        self, request: Request, info: SpiderInfo, *, item: Any = None
+    ) -> Deferred[Optional[FileInfo]]:
         """Check request before starting download"""
         raise NotImplementedError()
 
     @abstractmethod
-    def get_media_requests(self, item, info):
+    def get_media_requests(self, item: Any, info: SpiderInfo) -> List[Request]:
         """Returns the media requests to download"""
         raise NotImplementedError()
 
     @abstractmethod
-    def media_downloaded(self, response, request, info, *, item=None):
+    def media_downloaded(
+        self,
+        response: Response,
+        request: Request,
+        info: SpiderInfo,
+        *,
+        item: Any = None,
+    ) -> FileInfo:
         """Handler for success downloads"""
         raise NotImplementedError()
 
     @abstractmethod
-    def media_failed(self, failure, request, info):
+    def media_failed(
+        self, failure: Failure, request: Request, info: SpiderInfo
+    ) -> NoReturn:
         """Handler for failed downloads"""
         raise NotImplementedError()
 
-    def item_completed(self, results, item, info):
+    def item_completed(
+        self, results: List[FileInfoOrError], item: Any, info: SpiderInfo
+    ) -> Any:
         """Called per item when all media requests has been processed"""
         if self.LOG_FAILED_RESULTS:
             for ok, value in results:
                 if not ok:
+                    assert isinstance(value, Failure)
                     logger.error(
                         "%(class)s found errors processing %(item)s",
                         {"class": self.__class__.__name__, "item": item},
@@ -209,6 +290,13 @@ class MediaPipeline(ABC):
         return item
 
     @abstractmethod
-    def file_path(self, request, response=None, info=None, *, item=None):
+    def file_path(
+        self,
+        request: Request,
+        response: Optional[Response] = None,
+        info: Optional[SpiderInfo] = None,
+        *,
+        item: Any = None,
+    ) -> str:
         """Returns the path where downloaded media should be stored"""
         raise NotImplementedError()

From 3f76853bd27d84f53ebaaa97cb819e8a29195a89 Mon Sep 17 00:00:00 2001
From: Suvan Banerjee <banerjeesuvan@gmail.com>
Date: Wed, 5 Jun 2024 10:04:46 +0530
Subject: [PATCH 41/82] Handle  AttributeError: 'NoneType' in contract parsing
 (#6388)

---
 scrapy/contracts/__init__.py |  3 ++-
 tests/test_contracts.py      | 28 ++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index b300b8457..27bc2fcba 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -120,7 +120,8 @@ class ContractsManager:
 
             if line.startswith("@"):
                 m = re.match(r"@(\w+)\s*(.*)", line)
-                assert m is not None
+                if m is None:
+                    continue
                 name, args = m.groups()
                 args = re.split(r"\s+", args)
 
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index 1459e0b5f..c9c12f0d8 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -182,6 +182,19 @@ class TestSpider(Spider):
         """
         pass
 
+    def invalid_regex(self, response):
+        """method with invalid regex
+        @ Scrapy is awsome
+        """
+        pass
+
+    def invalid_regex_with_valid_contract(self, response):
+        """method with invalid regex
+        @ scrapy is awsome
+        @url http://scrapy.org
+        """
+        pass
+
 
 class CustomContractSuccessSpider(Spider):
     name = "custom_contract_success_spider"
@@ -385,6 +398,21 @@ class ContractsManagerTest(unittest.TestCase):
         message = "ContractFail: Missing fields: name, url"
         assert message in self.results.failures[-1][-1]
 
+    def test_regex(self):
+        spider = TestSpider()
+        response = ResponseMock()
+
+        # invalid regex
+        request = self.conman.from_method(spider.invalid_regex, self.results)
+        self.should_succeed()
+
+        # invalid regex with valid contract
+        request = self.conman.from_method(
+            spider.invalid_regex_with_valid_contract, self.results
+        )
+        self.should_succeed()
+        request.callback(response)
+
     def test_custom_contracts(self):
         self.conman.from_spider(CustomContractSuccessSpider(), self.results)
         self.should_succeed()

From 2e214210f6707181a863dbceabf2d34e767396cb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 2 Jun 2024 01:48:37 +0500
Subject: [PATCH 42/82] Add parameters to iterable generics, replace generators
 with iterables.

---
 scrapy/commands/parse.py         |  8 ++--
 scrapy/core/engine.py            |  5 ++-
 scrapy/core/scraper.py           |  6 ++-
 scrapy/core/spidermw.py          | 70 ++++++++++++++++++++------------
 scrapy/http/response/__init__.py |  3 +-
 scrapy/http/response/text.py     |  3 +-
 scrapy/utils/iterators.py        | 28 ++++++-------
 scrapy/utils/misc.py             |  6 +--
 scrapy/utils/python.py           | 32 +++++++--------
 scrapy/utils/request.py          |  5 +--
 scrapy/utils/sitemap.py          |  4 +-
 scrapy/utils/spider.py           | 13 +++---
 tests/test_commands.py           |  4 +-
 13 files changed, 103 insertions(+), 84 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2453c0d39..f916a3e75 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -140,13 +140,13 @@ class Command(BaseRunSpiderCommand):
 
     @overload
     def iterate_spider_output(
-        self, result: Union[AsyncGenerator, CoroutineType]
-    ) -> Deferred: ...
+        self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
+    ) -> Deferred[_T]: ...
 
     @overload
-    def iterate_spider_output(self, result: _T) -> Iterable: ...
+    def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
 
-    def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
+    def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b342ad7a3..dededf99d 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -372,7 +372,10 @@ class ExecutionEngine:
 
     @inlineCallbacks
     def open_spider(
-        self, spider: Spider, start_requests: Iterable = (), close_if_idle: bool = True
+        self,
+        spider: Spider,
+        start_requests: Iterable[Request] = (),
+        close_if_idle: bool = True,
     ) -> Generator[Deferred, Any, None]:
         if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 566e6628b..3b7492838 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,6 +16,7 @@ from typing import (
     Set,
     Tuple,
     Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -47,6 +48,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
+_T = TypeVar("_T")
 QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
 
 
@@ -256,14 +258,14 @@ class Scraper:
 
     def handle_spider_output(
         self,
-        result: Union[Iterable, AsyncIterable],
+        result: Union[Iterable[_T], AsyncIterable[_T]],
         request: Request,
         response: Response,
         spider: Spider,
     ) -> Deferred:
         if not result:
             return defer_succeed(None)
-        it: Union[Iterable, AsyncIterable]
+        it: Union[Iterable[_T], AsyncIterable[_T]]
         if isinstance(result, AsyncIterable):
             it = aiter_errback(
                 result, self.handle_spider_error, request, response, spider
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 2cef2e1dd..cb1a93a68 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -9,7 +9,6 @@ from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import (
     Any,
-    AsyncGenerator,
     AsyncIterable,
     Callable,
     Generator,
@@ -17,6 +16,7 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
 )
@@ -42,6 +42,7 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain
 logger = logging.getLogger(__name__)
 
 
+_T = TypeVar("_T")
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
@@ -98,31 +99,39 @@ class SpiderMiddlewareManager(MiddlewareManager):
         self,
         response: Response,
         spider: Spider,
-        iterable: Union[Iterable, AsyncIterable],
+        iterable: Union[Iterable[_T], AsyncIterable[_T]],
         exception_processor_index: int,
-        recover_to: Union[MutableChain, MutableAsyncChain],
-    ) -> Union[Generator, AsyncGenerator]:
-        def process_sync(iterable: Iterable) -> Generator:
+        recover_to: Union[MutableChain[_T], MutableAsyncChain[_T]],
+    ) -> Union[Iterable[_T], AsyncIterable[_T]]:
+        def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
             try:
                 yield from iterable
             except Exception as ex:
-                exception_result = self._process_spider_exception(
-                    response, spider, Failure(ex), exception_processor_index
+                exception_result = cast(
+                    Union[Failure, MutableChain[_T]],
+                    self._process_spider_exception(
+                        response, spider, Failure(ex), exception_processor_index
+                    ),
                 )
                 if isinstance(exception_result, Failure):
                     raise
+                assert isinstance(recover_to, MutableChain)
                 recover_to.extend(exception_result)
 
-        async def process_async(iterable: AsyncIterable) -> AsyncGenerator:
+        async def process_async(iterable: AsyncIterable[_T]) -> AsyncIterable[_T]:
             try:
                 async for r in iterable:
                     yield r
             except Exception as ex:
-                exception_result = self._process_spider_exception(
-                    response, spider, Failure(ex), exception_processor_index
+                exception_result = cast(
+                    Union[Failure, MutableAsyncChain[_T]],
+                    self._process_spider_exception(
+                        response, spider, Failure(ex), exception_processor_index
+                    ),
                 )
                 if isinstance(exception_result, Failure):
                     raise
+                assert isinstance(recover_to, MutableAsyncChain)
                 recover_to.extend(exception_result)
 
         if isinstance(iterable, AsyncIterable):
@@ -135,7 +144,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
         _failure: Failure,
         start_index: int = 0,
-    ) -> Union[Failure, MutableChain]:
+    ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
         exception = _failure.value
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
@@ -151,14 +160,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                dfd: Deferred = self._process_spider_output(
-                    response, spider, result, method_index + 1
+                dfd: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
+                    self._process_spider_output(
+                        response, spider, result, method_index + 1
+                    )
                 )
                 # _process_spider_output() returns a Deferred only because of downgrading so this can be
                 # simplified when downgrading is removed.
                 if dfd.called:
                     # the result is available immediately if _process_spider_output didn't do downgrading
-                    return cast(MutableChain, dfd.result)
+                    return cast(
+                        Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
+                    )
                 # we forbid waiting here because otherwise we would need to return a deferred from
                 # _process_spider_exception too, which complicates the architecture
                 msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
@@ -181,12 +194,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
         self,
         response: Response,
         spider: Spider,
-        result: Union[Iterable, AsyncIterable],
+        result: Union[Iterable[_T], AsyncIterable[_T]],
         start_index: int = 0,
-    ) -> Generator[Deferred, Any, Union[MutableChain, MutableAsyncChain]]:
+    ) -> Generator[Deferred[Any], Any, Union[MutableChain[_T], MutableAsyncChain[_T]]]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        recovered: Union[MutableChain, MutableAsyncChain]
+        recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
         last_result_is_async = isinstance(result, AsyncIterable)
         if last_result_is_async:
             recovered = MutableAsyncChain()
@@ -237,7 +250,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
-                exception_result = self._process_spider_exception(
+                exception_result: Union[
+                    Failure, MutableChain[_T], MutableAsyncChain[_T]
+                ] = self._process_spider_exception(
                     response, spider, Failure(ex), method_index + 1
                 )
                 if isinstance(exception_result, Failure):
@@ -267,9 +282,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     async def _process_callback_output(
-        self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
-    ) -> Union[MutableChain, MutableAsyncChain]:
-        recovered: Union[MutableChain, MutableAsyncChain]
+        self,
+        response: Response,
+        spider: Spider,
+        result: Union[Iterable[_T], AsyncIterable[_T]],
+    ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
+        recovered: Union[MutableChain[_T], MutableAsyncChain[_T]]
         if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
@@ -293,14 +311,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
         spider: Spider,
     ) -> Deferred:
         async def process_callback_output(
-            result: Union[Iterable, AsyncIterable]
-        ) -> Union[MutableChain, MutableAsyncChain]:
+            result: Union[Iterable[_T], AsyncIterable[_T]]
+        ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
             return await self._process_callback_output(response, spider, result)
 
-        def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
+        def process_spider_exception(
+            _failure: Failure,
+        ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
             return self._process_spider_exception(response, spider, _failure)
 
-        dfd = mustbe_deferred(
+        dfd: Deferred = mustbe_deferred(
             self._process_spider_input, scrape_func, response, request, spider
         )
         dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 166c4de97..daf193f59 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -14,7 +14,6 @@ from typing import (
     AnyStr,
     Callable,
     Dict,
-    Generator,
     Iterable,
     List,
     Mapping,
@@ -242,7 +241,7 @@ class Response(object_ref):
         errback: Optional[Callable] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
-    ) -> Generator[Request, None, None]:
+    ) -> Iterable[Request]:
         """
         .. versionadded:: 2.0
 
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 44c36b682..df4d90829 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -15,7 +15,6 @@ from typing import (
     AnyStr,
     Callable,
     Dict,
-    Generator,
     Iterable,
     List,
     Mapping,
@@ -246,7 +245,7 @@ class TextResponse(Response):
         flags: Optional[List[str]] = None,
         css: Optional[str] = None,
         xpath: Optional[str] = None,
-    ) -> Generator[Request, None, None]:
+    ) -> Iterable[Request]:
         """
         A generator that produces :class:`~.Request` instances to follow all
         links in ``urls``. It accepts the same arguments as the :class:`~.Request`'s
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index cd6e9d04e..41a842386 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -6,8 +6,7 @@ from typing import (
     Any,
     Callable,
     Dict,
-    Generator,
-    Iterable,
+    Iterator,
     List,
     Literal,
     Optional,
@@ -22,14 +21,12 @@ from lxml import etree  # nosec
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch, to_unicode
+from scrapy.utils.python import re_rsearch
 
 logger = logging.getLogger(__name__)
 
 
-def xmliter(
-    obj: Union[Response, str, bytes], nodename: str
-) -> Generator[Selector, Any, None]:
+def xmliter(obj: Union[Response, str, bytes], nodename: str) -> Iterator[Selector]:
     """Return a iterator of Selector's over all nodes of a XML document,
        given the name of the node to iterate. Useful for parsing XML feeds.
 
@@ -90,7 +87,7 @@ def xmliter_lxml(
     nodename: str,
     namespace: Optional[str] = None,
     prefix: str = "x",
-) -> Generator[Selector, Any, None]:
+) -> Iterator[Selector]:
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
     iterable = etree.iterparse(
@@ -168,7 +165,7 @@ def csviter(
     headers: Optional[List[str]] = None,
     encoding: Optional[str] = None,
     quotechar: Optional[str] = None,
-) -> Generator[Dict[str, str], Any, None]:
+) -> Iterator[Dict[str, str]]:
     """Returns an iterator of dictionaries from the given csv object
 
     obj can be:
@@ -184,10 +181,13 @@ def csviter(
     quotechar is the character used to enclosure fields on the given obj.
     """
 
-    encoding = obj.encoding if isinstance(obj, TextResponse) else encoding or "utf-8"
-
-    def row_to_unicode(row_: Iterable) -> List[str]:
-        return [to_unicode(field, encoding) for field in row_]
+    if encoding is not None:
+        warn(
+            "The encoding argument of csviter() is ignored and will be removed"
+            " in a future Scrapy version.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
 
     lines = StringIO(_body_or_str(obj, unicode=True))
 
@@ -200,13 +200,11 @@ def csviter(
 
     if not headers:
         try:
-            row = next(csv_r)
+            headers = next(csv_r)
         except StopIteration:
             return
-        headers = row_to_unicode(row)
 
     for row in csv_r:
-        row = row_to_unicode(row)
         if len(row) != len(headers):
             logger.warning(
                 "ignoring row %(csvlnum)d (length: %(csvrow)d, "
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 49f36de2d..3d11c1035 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -20,8 +20,8 @@ from typing import (
     Any,
     Callable,
     Deque,
-    Generator,
     Iterable,
+    Iterator,
     List,
     Optional,
     Type,
@@ -227,7 +227,7 @@ def build_from_settings(
 
 
 @contextmanager
-def set_environ(**kwargs: str) -> Generator[None, Any, None]:
+def set_environ(**kwargs: str) -> Iterator[None]:
     """Temporarily set environment variables inside the context manager and
     fully restore previous environment afterwards
     """
@@ -244,7 +244,7 @@ def set_environ(**kwargs: str) -> Generator[None, Any, None]:
                 os.environ[k] = v
 
 
-def walk_callable(node: ast.AST) -> Generator[ast.AST, Any, None]:
+def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
     """Similar to ``ast.walk``, but walks only function body and skips nested
     functions defined within the node.
     """
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 37a84a350..059d8e04d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -15,12 +15,10 @@ from itertools import chain
 from typing import (
     TYPE_CHECKING,
     Any,
-    AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
     Callable,
     Dict,
-    Generator,
     Iterable,
     Iterator,
     List,
@@ -163,7 +161,7 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter() -> Generator[Tuple[str, int], Any, None]:
+    def _chunk_iter() -> Iterable[Tuple[str, int]]:
         offset = len(text)
         while True:
             offset -= chunk_size * 1024
@@ -351,43 +349,45 @@ else:
         gc.collect()
 
 
-class MutableChain(Iterable):
+class MutableChain(Iterable[_T]):
     """
     Thin wrapper around itertools.chain, allowing to add iterables "in-place"
     """
 
-    def __init__(self, *args: Iterable):
-        self.data = chain.from_iterable(args)
+    def __init__(self, *args: Iterable[_T]):
+        self.data: Iterator[_T] = chain.from_iterable(args)
 
-    def extend(self, *iterables: Iterable) -> None:
+    def extend(self, *iterables: Iterable[_T]) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
-    def __iter__(self) -> Iterator:
+    def __iter__(self) -> Iterator[_T]:
         return self
 
-    def __next__(self) -> Any:
+    def __next__(self) -> _T:
         return next(self.data)
 
 
-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+async def _async_chain(
+    *iterables: Union[Iterable[_T], AsyncIterable[_T]]
+) -> AsyncIterator[_T]:
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
 
 
-class MutableAsyncChain(AsyncIterable):
+class MutableAsyncChain(AsyncIterable[_T]):
     """
     Similar to MutableChain but for async iterables
     """
 
-    def __init__(self, *args: Union[Iterable, AsyncIterable]):
-        self.data = _async_chain(*args)
+    def __init__(self, *args: Union[Iterable[_T], AsyncIterable[_T]]):
+        self.data: AsyncIterator[_T] = _async_chain(*args)
 
-    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+    def extend(self, *iterables: Union[Iterable[_T], AsyncIterable[_T]]) -> None:
         self.data = _async_chain(self.data, _async_chain(*iterables))
 
-    def __aiter__(self) -> AsyncIterator:
+    def __aiter__(self) -> AsyncIterator[_T]:
         return self
 
-    async def __anext__(self) -> Any:
+    async def __anext__(self) -> _T:
         return await self.data.__anext__()
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 42a6537a8..45b8008f4 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -12,7 +12,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     Dict,
-    Generator,
     Iterable,
     List,
     Optional,
@@ -40,9 +39,7 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
-def _serialize_headers(
-    headers: Iterable[bytes], request: Request
-) -> Generator[bytes, Any, None]:
+def _serialize_headers(headers: Iterable[bytes], request: Request) -> Iterable[bytes]:
     for header in headers:
         if header in request.headers:
             yield header
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index cf429043d..7a91afe59 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
-from typing import Any, Dict, Generator, Iterator, Optional, Union
+from typing import Any, Dict, Iterable, Iterator, Optional, Union
 from urllib.parse import urljoin
 
 import lxml.etree  # nosec
@@ -42,7 +42,7 @@ class Sitemap:
 
 def sitemap_urls_from_robots(
     robots_text: str, base_url: Optional[str] = None
-) -> Generator[str, Any, None]:
+) -> Iterable[str]:
     """Return an iterator over all sitemap urls contained in the given
     robots.txt file
     """
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index cbbb01d85..b05135c04 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -7,7 +7,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncGenerator,
-    Generator,
     Iterable,
     Literal,
     Optional,
@@ -34,18 +33,20 @@ _T = TypeVar("_T")
 
 # https://stackoverflow.com/questions/60222982
 @overload
-def iterate_spider_output(result: AsyncGenerator) -> AsyncGenerator: ...  # type: ignore[overload-overlap]
+def iterate_spider_output(result: AsyncGenerator[_T, None]) -> AsyncGenerator[_T, None]: ...  # type: ignore[overload-overlap]
 
 
 @overload
-def iterate_spider_output(result: CoroutineType) -> Deferred: ...
+def iterate_spider_output(result: CoroutineType[Any, Any, _T]) -> Deferred[_T]: ...
 
 
 @overload
-def iterate_spider_output(result: _T) -> Iterable: ...
+def iterate_spider_output(result: _T) -> Iterable[Any]: ...
 
 
-def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferred]:
+def iterate_spider_output(
+    result: Any,
+) -> Union[Iterable[Any], AsyncGenerator[_T, None], Deferred[_T]]:
     if inspect.isasyncgen(result):
         return result
     if inspect.iscoroutine(result):
@@ -55,7 +56,7 @@ def iterate_spider_output(result: Any) -> Union[Iterable, AsyncGenerator, Deferr
     return arg_to_iter(deferred_from_coro(result))
 
 
-def iter_spider_classes(module: ModuleType) -> Generator[Type[Spider], Any, None]:
+def iter_spider_classes(module: ModuleType) -> Iterable[Type[Spider]]:
     """Return an iterator over all spider classes defined in the given module
     that can be instantiated (i.e. which have name)
     """
diff --git a/tests/test_commands.py b/tests/test_commands.py
index b9d468c66..857a56b73 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -13,7 +13,7 @@ from shutil import copytree, rmtree
 from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 from tempfile import TemporaryFile, mkdtemp
 from threading import Timer
-from typing import Dict, Generator, Optional, Union
+from typing import Dict, Iterator, Optional, Union
 from unittest import skipIf
 
 from pytest import mark
@@ -674,7 +674,7 @@ class BadSpider(scrapy.Spider):
         """
 
     @contextmanager
-    def _create_file(self, content, name=None) -> Generator[str, None, None]:
+    def _create_file(self, content, name=None) -> Iterator[str]:
         tmpdir = Path(self.mktemp())
         tmpdir.mkdir()
         if name:

From de146ad7cef9e3478290be021129979f69fc6d03 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:09:19 +0500
Subject: [PATCH 43/82] Bump typing deps.

---
 scrapy/extensions/httpcache.py | 5 ++---
 scrapy/http/headers.py         | 3 +--
 tox.ini                        | 9 ++++-----
 3 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 3f4af42b7..b7219bf07 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -370,12 +370,11 @@ class FilesystemCacheStorage:
         with self._open(rpath / "pickled_meta", "wb") as f:
             pickle.dump(metadata, f, protocol=4)
         with self._open(rpath / "response_headers", "wb") as f:
-            # headers_dict_to_raw() needs a better type hint
-            f.write(cast(bytes, headers_dict_to_raw(response.headers)))
+            f.write(headers_dict_to_raw(response.headers))
         with self._open(rpath / "response_body", "wb") as f:
             f.write(response.body)
         with self._open(rpath / "request_headers", "wb") as f:
-            f.write(cast(bytes, headers_dict_to_raw(request.headers)))
+            f.write(headers_dict_to_raw(request.headers))
         with self._open(rpath / "request_body", "wb") as f:
             f.write(request.body)
 
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 73aee7178..85b9229d3 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -118,8 +118,7 @@ class Headers(CaselessDict):
         ]
 
     def to_string(self) -> bytes:
-        # cast() can be removed if the headers_dict_to_raw() hint is improved
-        return cast(bytes, headers_dict_to_raw(self))
+        return headers_dict_to_raw(self)
 
     def to_unicode_dict(self) -> CaseInsensitiveDict:
         """Return headers as a CaseInsensitiveDict with str keys
diff --git a/tox.ini b/tox.ini
index 5a5e80496..023a86c5a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -47,18 +47,17 @@ install_command =
 basepython = python3
 deps =
     mypy==1.10.0
-    typing-extensions==4.11.0
+    typing-extensions==4.12.1
     types-lxml==2024.4.14
     types-Pygments==2.18.0.20240506
     types-pyOpenSSL==24.1.0.20240425
-    types-setuptools==69.5.0.20240518
+    types-setuptools==70.0.0.20240524
     botocore-stubs==1.34.94
-    boto3-stubs[s3]==1.34.108
+    boto3-stubs[s3]==1.34.119
     attrs >= 18.2.0
     Pillow >= 10.3.0
     pytest >= 8.2.0
-    # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
-    w3lib >= 2.1.2
+    w3lib >= 2.2.0
 commands =
     mypy {posargs: scrapy tests}
 

From 262c10d85bd34732b0c692bdc8d16375d83a178f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:11:34 +0500
Subject: [PATCH 44/82] Use typing.Coroutine instead of types.CoroutineType.

---
 scrapy/commands/parse.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index f916a3e75..ce6f4dc51 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -3,11 +3,11 @@ import functools
 import inspect
 import json
 import logging
-from types import CoroutineType
 from typing import (
     Any,
     AsyncGenerator,
     Callable,
+    Coroutine,
     Dict,
     Iterable,
     List,
@@ -140,7 +140,7 @@ class Command(BaseRunSpiderCommand):
 
     @overload
     def iterate_spider_output(
-        self, result: Union[AsyncGenerator[_T, None], CoroutineType[Any, Any, _T]]
+        self, result: Union[AsyncGenerator[_T, None], Coroutine[Any, Any, _T]]
     ) -> Deferred[_T]: ...
 
     @overload

From 480a11b68bee19162cc0da59e9bed42b29bc9cfe Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:48:16 +0500
Subject: [PATCH 45/82] Add mssing __future__ imports.

---
 scrapy/commands/parse.py | 2 ++
 scrapy/core/spidermw.py  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index ce6f4dc51..3320a1ee4 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import argparse
 import functools
 import inspect
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index cb1a93a68..58873f0d9 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,6 +4,8 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 
+from __future__ import annotations
+
 import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice

From feb0b8f7dcb78c3df012085f00b992a7fac81f7a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 5 Jun 2024 22:57:18 +0500
Subject: [PATCH 46/82] Add pyupgrade.

---
 .pre-commit-config.yaml                      |  6 ++
 scrapy/core/downloader/contextfactory.py     |  8 +--
 scrapy/downloadermiddlewares/retry.py        |  4 +-
 scrapy/extensions/debug.py                   |  2 +-
 scrapy/http/request/__init__.py              |  6 +-
 scrapy/http/request/rpc.py                   |  2 +-
 scrapy/settings/__init__.py                  |  6 +-
 scrapy/utils/benchserver.py                  |  4 +-
 scrapy/utils/request.py                      |  6 +-
 tests/mockserver.py                          |  6 +-
 tests/test_downloadermiddleware_cookies.py   |  4 +-
 tests/test_downloadermiddleware_redirect.py  |  2 +-
 tests/test_downloadermiddleware_robotstxt.py |  4 +-
 tests/test_feedexport.py                     | 58 ++++++++++----------
 tests/test_http_response.py                  |  4 +-
 tests/test_pipeline_crawl.py                 |  4 +-
 tests/test_pipeline_images.py                |  2 +-
 tests/test_responsetypes.py                  |  2 +-
 tests/test_robotstxt_interface.py            | 30 ++++------
 19 files changed, 74 insertions(+), 86 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index a911d4cfe..f76a04ca1 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -22,3 +22,9 @@ repos:
   - id: blacken-docs
     additional_dependencies:
     - black==24.2.0
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.15.2
+  hooks:
+  - id: pyupgrade
+    args: [--py38-plus, --keep-runtime-typing]
+    exclude: scrapy/__init__.py
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 0e77cd2fe..9f6edb630 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -107,7 +107,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
         return ctx
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         return ScrapyClientTLSOptions(
             hostname.decode("ascii"),
             self.getContext(),
@@ -134,7 +134,7 @@ class BrowserLikeContextFactory(ScrapyClientContextFactory):
     ``SSLv23_METHOD``) which allows TLS protocol negotiation.
     """
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         # trustRoot set to platformTrust() will use the platform's root CAs.
         #
         # This means that a website like https://www.cacert.org will be rejected
@@ -158,8 +158,8 @@ class AcceptableProtocolsContextFactory:
         self._wrapped_context_factory: Any = context_factory
         self._acceptable_protocols: List[bytes] = acceptable_protocols
 
-    def creatorForNetloc(self, hostname: bytes, port: int) -> "ClientTLSOptions":
-        options: "ClientTLSOptions" = self._wrapped_context_factory.creatorForNetloc(
+    def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
+        options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(
             hostname, port
         )
         _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 46587a898..0637f09d4 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -147,9 +147,7 @@ class RetryMiddleware(metaclass=BackwardsCompatibilityMetaclass):
         if not settings.getbool("RETRY_ENABLED"):
             raise NotConfigured
         self.max_retry_times = settings.getint("RETRY_TIMES")
-        self.retry_http_codes = set(
-            int(x) for x in settings.getlist("RETRY_HTTP_CODES")
-        )
+        self.retry_http_codes = {int(x) for x in settings.getlist("RETRY_HTTP_CODES")}
         self.priority_adjust = settings.getint("RETRY_PRIORITY_ADJUST")
 
         try:
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 26726b662..a0fc7b99f 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -55,7 +55,7 @@ class StackTraceDump:
         )
 
     def _thread_stacks(self) -> str:
-        id2name = dict((th.ident, th.name) for th in threading.enumerate())
+        id2name = {th.ident: th.name for th in threading.enumerate()}
         dumps = ""
         for id_, frame in sys._current_frames().items():
             name = id2name.get(id_, "")
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 77149333c..3da2e111d 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -189,10 +189,10 @@ class Request(object_ref):
     def __repr__(self) -> str:
         return f"<{self.method} {self.url}>"
 
-    def copy(self) -> "Request":
+    def copy(self) -> Request:
         return self.replace()
 
-    def replace(self, *args: Any, **kwargs: Any) -> "Request":
+    def replace(self, *args: Any, **kwargs: Any) -> Request:
         """Create a new Request with the same attributes except for those given new values"""
         for x in self.attributes:
             kwargs.setdefault(x, getattr(self, x))
@@ -237,7 +237,7 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
-    def to_dict(self, *, spider: Optional["scrapy.Spider"] = None) -> Dict[str, Any]:
+    def to_dict(self, *, spider: Optional[scrapy.Spider] = None) -> Dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
         Use :func:`~scrapy.utils.request.request_from_dict` to convert back into a :class:`~scrapy.Request` object.
diff --git a/scrapy/http/request/rpc.py b/scrapy/http/request/rpc.py
index e20e7c438..096ecd370 100644
--- a/scrapy/http/request/rpc.py
+++ b/scrapy/http/request/rpc.py
@@ -21,7 +21,7 @@ DUMPS_ARGS = get_func_args(xmlrpclib.dumps)
 class XmlRpcRequest(Request):
     def __init__(self, *args: Any, encoding: Optional[str] = None, **kwargs: Any):
         if "body" not in kwargs and "params" in kwargs:
-            kw = dict((k, kwargs.pop(k)) for k in DUMPS_ARGS if k in kwargs)
+            kw = {k: kwargs.pop(k) for k in DUMPS_ARGS if k in kwargs}
             kwargs["body"] = xmlrpclib.dumps(**kw)
 
         # spec defines that requests must use POST method
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 4448b6f4b..ea1db03f1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -275,7 +275,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         assert isinstance(value, (dict, list))
         return copy.deepcopy(value)
 
-    def getwithbase(self, name: _SettingsKeyT) -> "BaseSettings":
+    def getwithbase(self, name: _SettingsKeyT) -> BaseSettings:
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
 
@@ -438,7 +438,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         if self.frozen:
             raise TypeError("Trying to modify an immutable Settings object")
 
-    def copy(self) -> "Self":
+    def copy(self) -> Self:
         """
         Make a deep copy of current settings.
 
@@ -460,7 +460,7 @@ class BaseSettings(MutableMapping[_SettingsKeyT, Any]):
         """
         self.frozen = True
 
-    def frozencopy(self) -> "Self":
+    def frozencopy(self) -> Self:
         """
         Return an immutable copy of the current settings.
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index e9ea51aa1..550516141 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -22,9 +22,7 @@ class Root(Resource):
         for nl in nlist:
             args["n"] = nl
             argstr = urlencode(args, doseq=True)
-            request.write(
-                f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode("utf8")
-            )
+            request.write(f"<a href='/follow?{argstr}'>follow {nl}</a><br>".encode())
         request.write(b"</body></html>")
         return b""
 
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 42a6537a8..aa0b90ee8 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -49,9 +49,9 @@ def _serialize_headers(
             yield from request.headers.getlist(header)
 
 
-_fingerprint_cache: (
-    "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
-)
+_fingerprint_cache: WeakKeyDictionary[
+    Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]
+]
 _fingerprint_cache = WeakKeyDictionary()
 
 
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 647b0682e..233f6b934 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -189,10 +189,10 @@ class Raw(LeafResource):
 class Echo(LeafResource):
     def render_GET(self, request):
         output = {
-            "headers": dict(
-                (to_unicode(k), [to_unicode(v) for v in vs])
+            "headers": {
+                to_unicode(k): [to_unicode(v) for v in vs]
                 for k, vs in request.requestHeaders.getAllRawHeaders()
-            ),
+            },
             "body": to_unicode(request.content.read()),
         }
         return to_bytes(json.dumps(output))
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 5eccd396a..6e343d035 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -362,7 +362,7 @@ class CookiesMiddlewareTest(TestCase):
 
     def test_request_cookies_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request("http://example.org", cookies={"a": "á".encode("utf8")})
+        req1 = Request("http://example.org", cookies={"a": "á".encode()})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 
@@ -379,7 +379,7 @@ class CookiesMiddlewareTest(TestCase):
     @pytest.mark.xfail(reason="Cookie header is not currently being processed")
     def test_request_headers_cookie_encoding(self):
         # 1) UTF8-encoded bytes
-        req1 = Request("http://example.org", headers={"Cookie": "a=á".encode("utf8")})
+        req1 = Request("http://example.org", headers={"Cookie": "a=á".encode()})
         assert self.mw.process_request(req1, self.spider) is None
         self.assertCookieValEqual(req1.headers["Cookie"], b"a=\xc3\xa1")
 
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 4bfd34fe2..e37da9715 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -1125,7 +1125,7 @@ class RedirectMiddlewareTest(Base.Test):
 
     def test_utf8_location(self):
         req = Request("http://scrapytest.org/first")
-        utf8_location = "/ação".encode("utf-8")  # header using UTF-8 encoding
+        utf8_location = "/ação".encode()  # header using UTF-8 encoding
         resp = Response(
             "http://scrapytest.org/first",
             headers={"Location": utf8_location},
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 26898a6a1..e166cc000 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -40,9 +40,7 @@ Disallow: /wiki/K%C3%A4ytt%C3%A4j%C3%A4:
 Disallow: /wiki/Käyttäjä:
 User-Agent: UnicödeBöt
 Disallow: /some/randome/page.html
-""".encode(
-            "utf-8"
-        )
+""".encode()
         response = TextResponse("http://site.local/robots.txt", body=ROBOTS)
 
         def return_response(request):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index d7560b5ff..3771df8f1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1359,13 +1359,13 @@ class FeedExportTest(FeedExportTestBase):
         items = [dict({"foo": "Test\xd6"})]
 
         formats = {
-            "json": '[{"foo": "Test\\u00d6"}]'.encode("utf-8"),
-            "jsonlines": '{"foo": "Test\\u00d6"}\n'.encode("utf-8"),
+            "json": b'[{"foo": "Test\\u00d6"}]',
+            "jsonlines": b'{"foo": "Test\\u00d6"}\n',
             "xml": (
                 '<?xml version="1.0" encoding="utf-8"?>\n'
                 "<items><item><foo>Test\xd6</foo></item></items>"
-            ).encode("utf-8"),
-            "csv": "foo\r\nTest\xd6\r\n".encode("utf-8"),
+            ).encode(),
+            "csv": "foo\r\nTest\xd6\r\n".encode(),
         }
 
         for fmt, expected in formats.items():
@@ -1379,13 +1379,13 @@ class FeedExportTest(FeedExportTestBase):
             self.assertEqual(expected, data[fmt])
 
         formats = {
-            "json": '[{"foo": "Test\xd6"}]'.encode("latin-1"),
-            "jsonlines": '{"foo": "Test\xd6"}\n'.encode("latin-1"),
+            "json": b'[{"foo": "Test\xd6"}]',
+            "jsonlines": b'{"foo": "Test\xd6"}\n',
             "xml": (
-                '<?xml version="1.0" encoding="latin-1"?>\n'
-                "<items><item><foo>Test\xd6</foo></item></items>"
-            ).encode("latin-1"),
-            "csv": "foo\r\nTest\xd6\r\n".encode("latin-1"),
+                b'<?xml version="1.0" encoding="latin-1"?>\n'
+                b"<items><item><foo>Test\xd6</foo></item></items>"
+            ),
+            "csv": b"foo\r\nTest\xd6\r\n",
         }
 
         for fmt, expected in formats.items():
@@ -1404,12 +1404,12 @@ class FeedExportTest(FeedExportTestBase):
         items = [dict({"foo": "FOO", "bar": "BAR"})]
 
         formats = {
-            "json": '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
+            "json": b'[\n{"bar": "BAR"}\n]',
             "xml": (
-                '<?xml version="1.0" encoding="latin-1"?>\n'
-                "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
-            ).encode("latin-1"),
-            "csv": "bar,foo\r\nBAR,FOO\r\n".encode("utf-8"),
+                b'<?xml version="1.0" encoding="latin-1"?>\n'
+                b"<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+            ),
+            "csv": b"bar,foo\r\nBAR,FOO\r\n",
         }
 
         settings = {
@@ -1663,8 +1663,8 @@ class FeedExportTest(FeedExportTestBase):
     def test_extend_kwargs(self):
         items = [{"foo": "FOO", "bar": "BAR"}]
 
-        expected_with_title_csv = "foo,bar\r\nFOO,BAR\r\n".encode("utf-8")
-        expected_without_title_csv = "FOO,BAR\r\n".encode("utf-8")
+        expected_with_title_csv = b"foo,bar\r\nFOO,BAR\r\n"
+        expected_without_title_csv = b"FOO,BAR\r\n"
         test_cases = [
             # with title
             {
@@ -2519,22 +2519,22 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
         formats = {
             "json": [
-                '[\n{"bar": "BAR"}\n]'.encode("utf-8"),
-                '[\n{"bar": "BAR1"}\n]'.encode("utf-8"),
+                b'[\n{"bar": "BAR"}\n]',
+                b'[\n{"bar": "BAR1"}\n]',
             ],
             "xml": [
                 (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    "<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
-                ).encode("latin-1"),
+                    b'<?xml version="1.0" encoding="latin-1"?>\n'
+                    b"<items>\n  <item>\n    <foo>FOO</foo>\n  </item>\n</items>"
+                ),
                 (
-                    '<?xml version="1.0" encoding="latin-1"?>\n'
-                    "<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>"
-                ).encode("latin-1"),
+                    b'<?xml version="1.0" encoding="latin-1"?>\n'
+                    b"<items>\n  <item>\n    <foo>FOO1</foo>\n  </item>\n</items>"
+                ),
             ],
             "csv": [
-                "foo,bar\r\nFOO,BAR\r\n".encode("utf-8"),
-                "foo,bar\r\nFOO1,BAR1\r\n".encode("utf-8"),
+                b"foo,bar\r\nFOO,BAR\r\n",
+                b"foo,bar\r\nFOO1,BAR1\r\n",
             ],
         }
 
@@ -2577,8 +2577,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
         items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]
         formats = {
             "json": [
-                '[{"foo": "FOO"}]'.encode("utf-8"),
-                '[{"foo": "FOO1"}]'.encode("utf-8"),
+                b'[{"foo": "FOO"}]',
+                b'[{"foo": "FOO1"}]',
             ],
         }
         settings = {
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 80d46274b..b8a277295 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -728,9 +728,7 @@ class TextResponseTest(BaseResponseTest):
         resp1 = self.response_class(
             "http://example.com",
             encoding="utf8",
-            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(
-                "utf8"
-            ),
+            body='<html><body><a href="foo?привет">click me</a></body></html>'.encode(),
         )
         req = self._assert_followed_url(
             resp1.css("a")[0],
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 5a9a217ce..cd3442dd4 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -107,9 +107,7 @@ class FileDownloadCrawlTestCase(TestCase):
 
         # check that the images/files checksums are what we know they should be
         if self.expected_checksums is not None:
-            checksums = set(
-                i["checksum"] for item in items for i in item[self.media_key]
-            )
+            checksums = {i["checksum"] for item in items for i in item[self.media_key]}
             self.assertEqual(checksums, self.expected_checksums)
 
         # check that the image files where actually written to the media store
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 18a2454b3..7d7c78920 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -628,7 +628,7 @@ class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
 
 class NoimagesDropTestCase(unittest.TestCase):
     def test_deprecation_warning(self):
-        arg = str()
+        arg = ""
         with warnings.catch_warnings(record=True) as w:
             NoimagesDrop(arg)
             self.assertEqual(len(w), 1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2633cca5b..7be8150fc 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -29,7 +29,7 @@ class ResponseTypesTest(unittest.TestCase):
         mappings = [
             (b'attachment; filename="data.xml"', XmlResponse),
             (b"attachment; filename=data.xml", XmlResponse),
-            ("attachment;filename=data£.tar.gz".encode("utf-8"), Response),
+            ("attachment;filename=data£.tar.gz".encode(), Response),
             ("attachment;filename=dataµ.tar.gz".encode("latin-1"), Response),
             ("attachment;filename=data高.doc".encode("gbk"), Response),
             ("attachment;filename=دورهdata.html".encode("cp720"), HtmlResponse),
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 6ad30deed..28ad910a8 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -36,10 +36,10 @@ class BaseRobotParserTest:
 
     def test_allowed(self):
         robotstxt_robotstxt_body = (
-            "User-agent: * \n"
-            "Disallow: /disallowed \n"
-            "Allow: /allowed \n"
-            "Crawl-delay: 10".encode("utf-8")
+            b"User-agent: * \n"
+            b"Disallow: /disallowed \n"
+            b"Allow: /allowed \n"
+            b"Crawl-delay: 10"
         )
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
@@ -48,15 +48,13 @@ class BaseRobotParserTest:
         self.assertFalse(rp.allowed("https://www.site.local/disallowed", "*"))
 
     def test_allowed_wildcards(self):
-        robotstxt_robotstxt_body = """User-agent: first
+        robotstxt_robotstxt_body = b"""User-agent: first
                                 Disallow: /disallowed/*/end$
 
                                 User-agent: second
                                 Allow: /*allowed
                                 Disallow: /
-                                """.encode(
-            "utf-8"
-        )
+                                """
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -77,18 +75,14 @@ class BaseRobotParserTest:
         self.assertTrue(rp.allowed("https://www.site.local/is_allowed_too", "second"))
 
     def test_length_based_precedence(self):
-        robotstxt_robotstxt_body = (
-            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
-        )
+        robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
         self.assertTrue(rp.allowed("https://www.site.local/page", "*"))
 
     def test_order_based_precedence(self):
-        robotstxt_robotstxt_body = (
-            "User-agent: * \n" "Disallow: / \n" "Allow: /page".encode("utf-8")
-        )
+        robotstxt_robotstxt_body = b"User-agent: * \n" b"Disallow: / \n" b"Allow: /page"
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -123,9 +117,7 @@ class BaseRobotParserTest:
         Disallow: /wiki/Käyttäjä:
 
         User-Agent: UnicödeBöt
-        Disallow: /some/randome/page.html""".encode(
-            "utf-8"
-        )
+        Disallow: /some/randome/page.html""".encode()
         rp = self.parser_cls.from_crawler(
             crawler=None, robotstxt_body=robotstxt_robotstxt_body
         )
@@ -145,14 +137,14 @@ class BaseRobotParserTest:
 
 class DecodeRobotsTxtTest(unittest.TestCase):
     def test_native_string_conversion(self):
-        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        robotstxt_body = b"User-agent: *\nDisallow: /\n"
         decoded_content = decode_robotstxt(
             robotstxt_body, spider=None, to_native_str_type=True
         )
         self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
 
     def test_decode_utf8(self):
-        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        robotstxt_body = b"User-agent: *\nDisallow: /\n"
         decoded_content = decode_robotstxt(robotstxt_body, spider=None)
         self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
 

From 144ff6c756fa58da2bc1a85879aa6f89300030d1 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Wed, 5 Jun 2024 21:09:10 -0300
Subject: [PATCH 47/82] Document missing parts of response.json method

---
 docs/topics/dynamic-content.rst | 7 +++----
 docs/topics/selectors.rst       | 8 ++++++++
 scrapy/selector/unified.py      | 1 +
 3 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index a0f4b4411..a99f1e222 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -115,15 +115,14 @@ Handling different response formats
 Once you have a response with the desired data, how you extract the desired
 data from it depends on the type of response:
 
--   If the response is HTML or XML, use :ref:`selectors
+-   If the response is HTML, XML or JSON, use :ref:`selectors
     <topics-selectors>` as usual.
 
--   If the response is JSON, use :func:`json.loads` to load the desired data from
-    :attr:`response.text <scrapy.http.TextResponse.text>`:
+-   If the response is JSON, use :func:`response.json()` to load the desired data:
 
     .. code-block:: python
 
-        data = json.loads(response.text)
+        data = response.json()
 
     If the desired data is inside HTML or XML code embedded within JSON data,
     you can load that HTML or XML code into a
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index e32fc2b70..0aae41cc8 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -1060,6 +1060,12 @@ Selector objects
 
           For convenience, this method can be called as ``response.css()``
 
+  .. automethod:: jmespath
+
+      .. note::
+
+          For convenience, this method can be called as ``response.jmespath()``
+
   .. automethod:: get
 
      See also: :ref:`old-extraction-api`
@@ -1092,6 +1098,8 @@ SelectorList objects
 
    .. automethod:: css
 
+   .. automethod:: jmespath
+
    .. automethod:: getall
 
       See also: :ref:`old-extraction-api`
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index e852aadc7..bfddb87cb 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -59,6 +59,7 @@ class Selector(_ParselSelector, object_ref):
 
     * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
     * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
+    * ``"json"`` for :class:`~scrapy.http.TextResponse` type
     * ``"html"`` for anything else
 
     Otherwise, if ``type`` is set, the selector type will be forced and no

From 23b1214e901961057bf43a5fb2548b35dfe19b20 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 6 Jun 2024 21:44:07 +0500
Subject: [PATCH 48/82] Add a comment about pyupgrade and scrapy/__init__.py.

---
 .pre-commit-config.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f76a04ca1..505b3c57d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,4 +27,5 @@ repos:
   hooks:
   - id: pyupgrade
     args: [--py38-plus, --keep-runtime-typing]
+    # scrapy/__init__.py has a sys.version_info check we want to keep
     exclude: scrapy/__init__.py

From ed3a7acaf3169ed6b9f9ffbcffed35db63d840f7 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 7 Jun 2024 11:19:37 +0500
Subject: [PATCH 49/82] Remove the Python version check from
 scrapy/__init__.py.

---
 .pre-commit-config.yaml | 2 --
 scrapy/__init__.py      | 6 ------
 2 files changed, 8 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 505b3c57d..63da5544d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,5 +27,3 @@ repos:
   hooks:
   - id: pyupgrade
     args: [--py38-plus, --keep-runtime-typing]
-    # scrapy/__init__.py has a sys.version_info check we want to keep
-    exclude: scrapy/__init__.py
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index cc0e539c4..1c1a5c2cc 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -33,12 +33,6 @@ version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."
 twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
-# Check minimum required Python version
-if sys.version_info < (3, 8):
-    print(f"Scrapy {__version__} requires Python 3.8+")
-    sys.exit(1)
-
-
 # Ignore noisy twisted deprecation warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")
 

From ddc98fe91b454a0944a8558daa2000da08921b62 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 10 Jun 2024 13:16:26 +0500
Subject: [PATCH 50/82] Deprecate scrapy.utils.defer.process_chain_both().
 (#6397)

---
 scrapy/utils/defer.py     |  9 ++++++++-
 tests/test_utils_defer.py | 14 --------------
 2 files changed, 8 insertions(+), 15 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index abb7e1726..f60b7dde8 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -6,6 +6,7 @@ from __future__ import annotations
 
 import asyncio
 import inspect
+import warnings
 from asyncio import Future
 from functools import wraps
 from types import CoroutineType
@@ -35,7 +36,7 @@ from twisted.internet.task import Cooperator
 from twisted.python import failure
 from twisted.python.failure import Failure
 
-from scrapy.exceptions import IgnoreRequest
+from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
 from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed
 
 if TYPE_CHECKING:
@@ -281,6 +282,12 @@ def process_chain_both(
     **kw: _P.kwargs,
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
+    warnings.warn(
+        "process_chain_both() is deprecated and will be removed in a future"
+        " Scrapy version.",
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
     d: Deferred = Deferred()
     for cb, eb in zip(callbacks, errbacks):
         d.addCallback(cb, *a, **kw)
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index a7d54b565..ec0399865 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -14,7 +14,6 @@ from scrapy.utils.defer import (
     mustbe_deferred,
     parallel_async,
     process_chain,
-    process_chain_both,
     process_parallel,
 )
 
@@ -80,19 +79,6 @@ class DeferUtilsTest(unittest.TestCase):
             gotexc = True
         self.assertTrue(gotexc)
 
-    @defer.inlineCallbacks
-    def test_process_chain_both(self):
-        x = yield process_chain_both(
-            [cb_fail, cb2, cb3], [None, eb1, None], "res", "v1", "v2"
-        )
-        self.assertEqual(x, "(cb3 (eb1 TypeError v1 v2) v1 v2)")
-
-        fail = Failure(ZeroDivisionError())
-        x = yield process_chain_both(
-            [eb1, cb2, cb3], [eb1, None, None], fail, "v1", "v2"
-        )
-        self.assertEqual(x, "(cb3 (cb2 (eb1 ZeroDivisionError v1 v2) v1 v2) v1 v2)")
-
     @defer.inlineCallbacks
     def test_process_parallel(self):
         x = yield process_parallel([cb1, cb2, cb3], "res", "v1", "v2")

From 1282ddf8f77299edf613679c2ee0b606e96808ce Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 10 Jun 2024 13:27:50 +0500
Subject: [PATCH 51/82] Add parameters to most Deferred in scrapy/core. (#6395)

---
 scrapy/core/downloader/__init__.py          | 45 ++++++++++----
 scrapy/core/downloader/handlers/__init__.py | 38 +++++++++---
 scrapy/core/downloader/handlers/ftp.py      | 10 ++-
 scrapy/core/downloader/handlers/http10.py   |  3 +-
 scrapy/core/downloader/handlers/http11.py   | 69 ++++++++++++---------
 scrapy/core/downloader/handlers/http2.py    |  4 +-
 scrapy/core/downloader/handlers/s3.py       |  3 +-
 scrapy/core/downloader/middleware.py        | 23 ++++---
 scrapy/core/downloader/webclient.py         |  6 +-
 scrapy/core/engine.py                       | 55 +++++++++-------
 scrapy/core/scheduler.py                    |  8 +--
 scrapy/core/scraper.py                      | 61 +++++++++++-------
 scrapy/core/spidermw.py                     | 18 +++---
 scrapy/pipelines/__init__.py                |  4 +-
 scrapy/utils/defer.py                       | 20 +++---
 tests/test_downloadermiddleware.py          |  2 +-
 16 files changed, 237 insertions(+), 132 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 0ab3bdb77..41f729ed9 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,9 +1,22 @@
+from __future__ import annotations
+
 import random
 import warnings
 from collections import deque
 from datetime import datetime
 from time import time
-from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Deque,
+    Dict,
+    Optional,
+    Set,
+    Tuple,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from twisted.internet import task
 from twisted.internet.defer import Deferred
@@ -22,6 +35,8 @@ from scrapy.utils.httpobj import urlparse_cached
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
+_T = TypeVar("_T")
+
 
 class Slot:
     """Downloader slot"""
@@ -40,7 +55,7 @@ class Slot:
         self.throttle = throttle
 
         self.active: Set[Request] = set()
-        self.queue: Deque[Tuple[Request, Deferred]] = deque()
+        self.queue: Deque[Tuple[Request, Deferred[Response]]] = deque()
         self.transferring: Set[Request] = set()
         self.lastseen: float = 0
         self.latercall = None
@@ -93,7 +108,7 @@ def _get_concurrency_delay(
 class Downloader:
     DOWNLOAD_SLOT = "download_slot"
 
-    def __init__(self, crawler: "Crawler"):
+    def __init__(self, crawler: Crawler):
         self.settings: BaseSettings = crawler.settings
         self.signals: SignalManager = crawler.signals
         self.slots: Dict[str, Slot] = {}
@@ -114,13 +129,17 @@ class Downloader:
             "DOWNLOAD_SLOTS", {}
         )
 
-    def fetch(self, request: Request, spider: Spider) -> Deferred:
-        def _deactivate(response: Response) -> Response:
+    def fetch(
+        self, request: Request, spider: Spider
+    ) -> Deferred[Union[Response, Request]]:
+        def _deactivate(response: _T) -> _T:
             self.active.remove(request)
             return response
 
         self.active.add(request)
-        dfd = self.middleware.download(self._enqueue_request, request, spider)
+        dfd: Deferred[Union[Response, Request]] = self.middleware.download(
+            self._enqueue_request, request, spider
+        )
         return dfd.addBoth(_deactivate)
 
     def needs_backout(self) -> bool:
@@ -163,7 +182,7 @@ class Downloader:
         )
         return self.get_slot_key(request)
 
-    def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
+    def _enqueue_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         key, slot = self._get_slot(request, spider)
         request.meta[self.DOWNLOAD_SLOT] = key
 
@@ -175,7 +194,7 @@ class Downloader:
         self.signals.send_catch_log(
             signal=signals.request_reached_downloader, request=request, spider=spider
         )
-        deferred: Deferred = Deferred().addBoth(_deactivate)
+        deferred: Deferred[Response] = Deferred().addBoth(_deactivate)
         slot.queue.append((request, deferred))
         self._process_queue(spider, slot)
         return deferred
@@ -208,11 +227,15 @@ class Downloader:
                 self._process_queue(spider, slot)
                 break
 
-    def _download(self, slot: Slot, request: Request, spider: Spider) -> Deferred:
+    def _download(
+        self, slot: Slot, request: Request, spider: Spider
+    ) -> Deferred[Response]:
         # The order is very important for the following deferreds. Do not change!
 
         # 1. Create the download deferred
-        dfd = mustbe_deferred(self.handlers.download_request, request, spider)
+        dfd: Deferred[Response] = mustbe_deferred(
+            self.handlers.download_request, request, spider
+        )
 
         # 2. Notify response_downloaded listeners about the recent download
         # before querying queue for next request
@@ -233,7 +256,7 @@ class Downloader:
         # middleware itself)
         slot.transferring.add(request)
 
-        def finish_transferring(_: Any) -> Any:
+        def finish_transferring(_: _T) -> _T:
             slot.transferring.remove(request)
             self._process_queue(spider, slot)
             self.signals.send_catch_log(
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 5ec5ef6db..ebc4898b5 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -3,13 +3,25 @@
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Callable, Dict, Generator, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generator,
+    Optional,
+    Protocol,
+    Type,
+    Union,
+    cast,
+)
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
@@ -20,13 +32,21 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
+class DownloadHandlerProtocol(Protocol):
+    def download_request(
+        self, request: Request, spider: Spider
+    ) -> Deferred[Response]: ...
+
+
 class DownloadHandlers:
     def __init__(self, crawler: Crawler):
         self._crawler: Crawler = crawler
         self._schemes: Dict[str, Union[str, Callable[..., Any]]] = (
             {}
         )  # stores acceptable schemes on instancing
-        self._handlers: Dict[str, Any] = {}  # stores instanced handlers for schemes
+        self._handlers: Dict[str, DownloadHandlerProtocol] = (
+            {}
+        )  # stores instanced handlers for schemes
         self._notconfigured: Dict[str, str] = {}  # remembers failed handlers
         handlers: Dict[str, Union[str, Callable[..., Any]]] = without_none_values(
             cast(
@@ -40,7 +60,7 @@ class DownloadHandlers:
 
         crawler.signals.connect(self._close, signals.engine_stopped)
 
-    def _get_handler(self, scheme: str) -> Any:
+    def _get_handler(self, scheme: str) -> Optional[DownloadHandlerProtocol]:
         """Lazy-load the downloadhandler for a scheme
         only on the first request for that scheme.
         """
@@ -54,10 +74,12 @@ class DownloadHandlers:
 
         return self._load_handler(scheme)
 
-    def _load_handler(self, scheme: str, skip_lazy: bool = False) -> Any:
+    def _load_handler(
+        self, scheme: str, skip_lazy: bool = False
+    ) -> Optional[DownloadHandlerProtocol]:
         path = self._schemes[scheme]
         try:
-            dhcls = load_object(path)
+            dhcls: Type[DownloadHandlerProtocol] = load_object(path)
             if skip_lazy and getattr(dhcls, "lazy", True):
                 return None
             dh = build_from_crawler(
@@ -80,17 +102,17 @@ class DownloadHandlers:
             self._handlers[scheme] = dh
             return dh
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         scheme = urlparse_cached(request).scheme
         handler = self._get_handler(scheme)
         if not handler:
             raise NotSupported(
                 f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
             )
-        return cast(Deferred, handler.download_request(request, spider))
+        return handler.download_request(request, spider)
 
     @defer.inlineCallbacks
-    def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred, Any, None]:
+    def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]:
         for dh in self._handlers.values():
             if hasattr(dh, "close"):
                 yield dh.close()
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 77dcf3c38..724717ffd 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -91,7 +91,7 @@ class FTPDownloadHandler:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings)
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         from twisted.internet import reactor
 
         parsed_url = urlparse_cached(request)
@@ -103,10 +103,14 @@ class FTPDownloadHandler:
         creator = ClientCreator(
             reactor, FTPClient, user, password, passive=passive_mode
         )
-        dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
+        dfd: Deferred[FTPClient] = creator.connectTCP(
+            parsed_url.hostname, parsed_url.port or 21
+        )
         return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
 
-    def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:
+    def gotClient(
+        self, client: FTPClient, request: Request, filepath: str
+    ) -> Deferred[Response]:
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
         d = client.retrieveFile(filepath, protocol)
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index da9559525..3c4e48abb 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -9,6 +9,7 @@ from twisted.internet.defer import Deferred
 
 from scrapy import Request, Spider
 from scrapy.crawler import Crawler
+from scrapy.http import Response
 from scrapy.settings import BaseSettings
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
@@ -38,7 +39,7 @@ class HTTP10DownloadHandler:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         """Return a deferred for the HTTP download"""
         factory = self.HTTPClientFactory(request)
         self._connect(factory)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5e84be6ba..e2ad8f59a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,7 +8,7 @@ import re
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar, Union
 from urllib.parse import urldefrag, urlunparse
 
 from twisted.internet import ssl
@@ -38,12 +38,22 @@ from scrapy.settings import BaseSettings
 from scrapy.utils.python import to_bytes, to_unicode
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
-
+    # typing.NotRequired and typing.Self require Python 3.11
+    from typing_extensions import NotRequired, Self
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
+
+class _ResultT(TypedDict):
+    txresponse: TxResponse
+    body: bytes
+    flags: Optional[List[str]]
+    certificate: Optional[ssl.Certificate]
+    ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None]
+    failure: NotRequired[Optional[Failure]]
+
 
 class HTTP11DownloadHandler:
     lazy = False
@@ -71,7 +81,7 @@ class HTTP11DownloadHandler:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
@@ -83,10 +93,10 @@ class HTTP11DownloadHandler:
         )
         return agent.download_request(request)
 
-    def close(self) -> Deferred:
+    def close(self) -> Deferred[None]:
         from twisted.internet import reactor
 
-        d: Deferred = self._pool.closeCachedConnections()
+        d: Deferred[None] = self._pool.closeCachedConnections()
         # closeCachedConnections will hang on network or server issues, so
         # we'll manually timeout the deferred.
         #
@@ -97,7 +107,7 @@ class HTTP11DownloadHandler:
         # issue a callback after `_disconnect_timeout` seconds.
         delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
 
-        def cancel_delayed_call(result: Any) -> Any:
+        def cancel_delayed_call(result: _T) -> _T:
             if delayed_call.active():
                 delayed_call.cancel()
             return result
@@ -137,7 +147,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     ):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
         super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
-        self._tunnelReadyDeferred: Deferred = Deferred()
+        self._tunnelReadyDeferred: Deferred[Protocol] = Deferred()
         self._tunneledHost: str = host
         self._tunneledPort: int = port
         self._contextFactory: IPolicyForHTTPS = contextFactory
@@ -198,7 +208,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         """Propagates the errback to the appropriate deferred."""
         self._tunnelReadyDeferred.errback(reason)
 
-    def connect(self, protocolFactory: Factory) -> Deferred:
+    def connect(self, protocolFactory: Factory) -> Deferred[Protocol]:
         self._protocolFactory = protocolFactory
         connectDeferred = super().connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
@@ -271,7 +281,7 @@ class TunnelingAgent(Agent):
         headers: Optional[TxHeaders],
         bodyProducer: Optional[IBodyProducer],
         requestPath: bytes,
-    ) -> Deferred:
+    ) -> Deferred[TxResponse]:
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
@@ -310,7 +320,7 @@ class ScrapyProxyAgent(Agent):
         uri: bytes,
         headers: Optional[TxHeaders] = None,
         bodyProducer: Optional[IBodyProducer] = None,
-    ) -> Deferred:
+    ) -> Deferred[TxResponse]:
         """
         Issue a new request via the configured proxy.
         """
@@ -394,7 +404,7 @@ class ScrapyAgent:
             pool=self._pool,
         )
 
-    def download_request(self, request: Request) -> Deferred:
+    def download_request(self, request: Request) -> Deferred[Response]:
         from twisted.internet import reactor
 
         timeout = request.meta.get("download_timeout") or self._connectTimeout
@@ -411,22 +421,20 @@ class ScrapyAgent:
         else:
             bodyproducer = None
         start_time = time()
-        d: Deferred = agent.request(
+        d: Deferred[TxResponse] = agent.request(
             method, to_bytes(url, encoding="ascii"), headers, bodyproducer
         )
         # set download latency
         d.addCallback(self._cb_latency, request, start_time)
         # response body is ready to be consumed
-        d.addCallback(self._cb_bodyready, request)
-        d.addCallback(self._cb_bodydone, request, url)
+        d2: Deferred[_ResultT] = d.addCallback(self._cb_bodyready, request)
+        d3: Deferred[Response] = d2.addCallback(self._cb_bodydone, request, url)
         # check download timeout
-        self._timeout_cl = reactor.callLater(timeout, d.cancel)
-        d.addBoth(self._cb_timeout, request, url, timeout)
-        return d
+        self._timeout_cl = reactor.callLater(timeout, d3.cancel)
+        d3.addBoth(self._cb_timeout, request, url, timeout)
+        return d3
 
-    def _cb_timeout(
-        self, result: Any, request: Request, url: str, timeout: float
-    ) -> Any:
+    def _cb_timeout(self, result: _T, request: Request, url: str, timeout: float) -> _T:
         if self._timeout_cl.active():
             self._timeout_cl.cancel()
             return result
@@ -437,7 +445,7 @@ class ScrapyAgent:
 
         raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
 
-    def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:
+    def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
         request.meta["download_latency"] = time() - start_time
         return result
 
@@ -451,7 +459,7 @@ class ScrapyAgent:
 
     def _cb_bodyready(
         self, txresponse: TxResponse, request: Request
-    ) -> Union[Dict[str, Any], Deferred]:
+    ) -> Union[_ResultT, Deferred[_ResultT]]:
         headers_received_result = self._crawler.signals.send_catch_log(
             signal=signals.headers_received,
             headers=self._headers_from_twisted_response(txresponse),
@@ -520,7 +528,7 @@ class ScrapyAgent:
             # Abort connection immediately.
             txresponse._transport._producer.abortConnection()
 
-        d: Deferred = Deferred(_cancel)
+        d: Deferred[_ResultT] = Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
                 finished=d,
@@ -539,7 +547,7 @@ class ScrapyAgent:
         return d
 
     def _cb_bodydone(
-        self, result: Dict[str, Any], request: Request, url: str
+        self, result: _ResultT, request: Request, url: str
     ) -> Union[Response, Failure]:
         headers = self._headers_from_twisted_response(result["txresponse"])
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
@@ -559,8 +567,9 @@ class ScrapyAgent:
             protocol=protocol,
         )
         if result.get("failure"):
+            assert result["failure"]
             result["failure"].value.response = response
-            return cast(Failure, result["failure"])
+            return result["failure"]
         return response
 
 
@@ -570,7 +579,7 @@ class _RequestBodyProducer:
         self.body = body
         self.length = len(body)
 
-    def startProducing(self, consumer: IConsumer) -> Deferred:
+    def startProducing(self, consumer: IConsumer) -> Deferred[None]:
         consumer.write(self.body)
         return succeed(None)
 
@@ -584,7 +593,7 @@ class _RequestBodyProducer:
 class _ResponseReader(Protocol):
     def __init__(
         self,
-        finished: Deferred,
+        finished: Deferred[_ResultT],
         txresponse: TxResponse,
         request: Request,
         maxsize: int,
@@ -592,7 +601,7 @@ class _ResponseReader(Protocol):
         fail_on_dataloss: bool,
         crawler: Crawler,
     ):
-        self._finished: Deferred = finished
+        self._finished: Deferred[_ResultT] = finished
         self._txresponse: TxResponse = txresponse
         self._request: Request = request
         self._bodybuf: BytesIO = BytesIO()
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 16fc1e3ae..2ac4eca86 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -37,7 +37,7 @@ class H2DownloadHandler:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
@@ -98,7 +98,7 @@ class ScrapyH2Agent:
             pool=self._pool,
         )
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         from twisted.internet import reactor
 
         timeout = request.meta.get("download_timeout") or self._connect_timeout
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 1a3d36f45..0ad340721 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -8,6 +8,7 @@ from scrapy import Request, Spider
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
 from scrapy.settings import BaseSettings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
@@ -76,7 +77,7 @@ class S3DownloadHandler:
     def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
         return cls(crawler.settings, crawler=crawler, **kwargs)
 
-    def download_request(self, request: Request, spider: Spider) -> Deferred:
+    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
         p = urlparse_cached(request)
         scheme = "https" if request.meta.get("is_secure") else "http"
         bucket = p.hostname
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 52ebe4e22..2d8af114f 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -4,6 +4,8 @@ Downloader Middleware manager
 See documentation in docs/topics/downloader-middleware.rst
 """
 
+from __future__ import annotations
+
 from typing import Any, Callable, Generator, List, Union, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -34,10 +36,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             self.methods["process_exception"].appendleft(mw.process_exception)
 
     def download(
-        self, download_func: Callable, request: Request, spider: Spider
-    ) -> Deferred:
+        self,
+        download_func: Callable[[Request, Spider], Deferred[Response]],
+        request: Request,
+        spider: Spider,
+    ) -> Deferred[Union[Response, Request]]:
         @inlineCallbacks
-        def process_request(request: Request) -> Generator[Deferred, Any, Any]:
+        def process_request(
+            request: Request,
+        ) -> Generator[Deferred[Any], Any, Union[Response, Request]]:
             for method in self.methods["process_request"]:
                 method = cast(Callable, method)
                 response = yield deferred_from_coro(
@@ -52,12 +59,12 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                     )
                 if response:
                     return response
-            return (yield download_func(request=request, spider=spider))
+            return (yield download_func(request, spider))
 
         @inlineCallbacks
         def process_response(
             response: Union[Response, Request]
-        ) -> Generator[Deferred, Any, Union[Response, Request]]:
+        ) -> Generator[Deferred[Any], Any, Union[Response, Request]]:
             if response is None:
                 raise TypeError("Received None in process_response")
             elif isinstance(response, Request):
@@ -80,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         @inlineCallbacks
         def process_exception(
             failure: Failure,
-        ) -> Generator[Deferred, Any, Union[Failure, Response, Request]]:
+        ) -> Generator[Deferred[Any], Any, Union[Failure, Response, Request]]:
             exception = failure.value
             for method in self.methods["process_exception"]:
                 method = cast(Callable, method)
@@ -98,7 +105,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                     return response
             return failure
 
-        deferred = mustbe_deferred(process_request, request)
+        deferred: Deferred[Union[Response, Request]] = mustbe_deferred(
+            process_request, request
+        )
         deferred.addErrback(process_exception)
         deferred.addCallback(process_response)
         return deferred
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index bb1f73805..08a1d7c71 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -8,7 +8,7 @@ from twisted.internet.protocol import ClientFactory
 from twisted.web.http import HTTPClient
 
 from scrapy import Request
-from scrapy.http import Headers
+from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
@@ -145,7 +145,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         self.response_headers: Optional[Headers] = None
         self.timeout: float = request.meta.get("download_timeout") or timeout
         self.start_time: float = time()
-        self.deferred: defer.Deferred = defer.Deferred().addCallback(
+        self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
             self._build_response, request
         )
 
@@ -155,7 +155,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         # needed to add the callback _waitForDisconnect.
         # Specifically this avoids the AttributeError exception when
         # clientConnectionFailed method is called.
-        self._disconnectedDeferred: defer.Deferred = defer.Deferred()
+        self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred()
 
         self._set_connection_attributes(request)
 
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index dededf99d..4ffec78b9 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -19,6 +19,7 @@ from typing import (
     Optional,
     Set,
     Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -43,10 +44,13 @@ from scrapy.utils.reactor import CallLaterOnce
 
 if TYPE_CHECKING:
     from scrapy.core.scheduler import BaseScheduler
+    from scrapy.core.scraper import _HandleOutputDeferred
     from scrapy.crawler import Crawler
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
 
 class Slot:
     def __init__(
@@ -56,7 +60,7 @@ class Slot:
         nextcall: CallLaterOnce[None],
         scheduler: BaseScheduler,
     ) -> None:
-        self.closing: Optional[Deferred] = None
+        self.closing: Optional[Deferred[None]] = None
         self.inprogress: Set[Request] = set()
         self.start_requests: Optional[Iterator[Request]] = iter(start_requests)
         self.close_if_idle: bool = close_if_idle
@@ -71,7 +75,7 @@ class Slot:
         self.inprogress.remove(request)
         self._maybe_fire_closing()
 
-    def close(self) -> Deferred:
+    def close(self) -> Deferred[None]:
         self.closing = Deferred()
         self._maybe_fire_closing()
         return self.closing
@@ -123,20 +127,20 @@ class ExecutionEngine:
         return scheduler_cls
 
     @inlineCallbacks
-    def start(self) -> Generator[Deferred, Any, None]:
+    def start(self) -> Generator[Deferred[Any], Any, None]:
         if self.running:
             raise RuntimeError("Engine already running")
         self.start_time = time()
         yield self.signals.send_catch_log_deferred(signal=signals.engine_started)
         self.running = True
-        self._closewait: Deferred = Deferred()
+        self._closewait: Deferred[None] = Deferred()
         yield self._closewait
 
-    def stop(self) -> Deferred:
+    def stop(self) -> Deferred[None]:
         """Gracefully stop the execution engine"""
 
         @inlineCallbacks
-        def _finish_stopping_engine(_: Any) -> Generator[Deferred, Any, None]:
+        def _finish_stopping_engine(_: Any) -> Generator[Deferred[Any], Any, None]:
             yield self.signals.send_catch_log_deferred(signal=signals.engine_stopped)
             self._closewait.callback(None)
 
@@ -151,7 +155,7 @@ class ExecutionEngine:
         )
         return dfd.addBoth(_finish_stopping_engine)
 
-    def close(self) -> Deferred:
+    def close(self) -> Deferred[None]:
         """
         Gracefully close the execution engine.
         If it has already been started, stop it. In all cases, close the spider and the downloader.
@@ -214,7 +218,7 @@ class ExecutionEngine:
             or self.scraper.slot.needs_backout()
         )
 
-    def _next_request_from_scheduler(self) -> Optional[Deferred]:
+    def _next_request_from_scheduler(self) -> Optional[Deferred[None]]:
         assert self.slot is not None  # typing
         assert self.spider is not None  # typing
 
@@ -222,7 +226,7 @@ class ExecutionEngine:
         if request is None:
             return None
 
-        d = self._download(request)
+        d: Deferred[Union[Response, Request]] = self._download(request)
         d.addBoth(self._handle_downloader_output, request)
         d.addErrback(
             lambda f: logger.info(
@@ -236,8 +240,8 @@ class ExecutionEngine:
             assert self.slot
             self.slot.remove_request(request)
 
-        d.addBoth(_remove_request)
-        d.addErrback(
+        d2: Deferred[None] = d.addBoth(_remove_request)
+        d2.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
                 exc_info=failure_to_exc_info(f),
@@ -245,19 +249,19 @@ class ExecutionEngine:
             )
         )
         slot = self.slot
-        d.addBoth(lambda _: slot.nextcall.schedule())
-        d.addErrback(
+        d2.addBoth(lambda _: slot.nextcall.schedule())
+        d2.addErrback(
             lambda f: logger.info(
                 "Error while scheduling new request",
                 exc_info=failure_to_exc_info(f),
                 extra={"spider": self.spider},
             )
         )
-        return d
+        return d2
 
     def _handle_downloader_output(
         self, result: Union[Request, Response, Failure], request: Request
-    ) -> Optional[Deferred]:
+    ) -> Optional[_HandleOutputDeferred]:
         assert self.spider is not None  # typing
 
         if not isinstance(result, (Request, Response, Failure)):
@@ -319,20 +323,23 @@ class ExecutionEngine:
                 signals.request_dropped, request=request, spider=spider
             )
 
-    def download(self, request: Request) -> Deferred:
+    def download(self, request: Request) -> Deferred[Response]:
         """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
         if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
-        return self._download(request).addBoth(self._downloaded, request)
+        d: Deferred[Union[Response, Request]] = self._download(request)
+        # Deferred.addBoth() overloads don't seem to support a Union[_T, Deferred[_T]] return type
+        d2: Deferred[Response] = d.addBoth(self._downloaded, request)  # type: ignore[arg-type]
+        return d2
 
     def _downloaded(
         self, result: Union[Response, Request, Failure], request: Request
-    ) -> Union[Deferred, Response, Failure]:
+    ) -> Union[Deferred[Response], Response, Failure]:
         assert self.slot is not None  # typing
         self.slot.remove_request(request)
         return self.download(result) if isinstance(result, Request) else result
 
-    def _download(self, request: Request) -> Deferred:
+    def _download(self, request: Request) -> Deferred[Union[Response, Request]]:
         assert self.slot is not None  # typing
 
         self.slot.add_request(request)
@@ -359,13 +366,15 @@ class ExecutionEngine:
                 )
             return result
 
-        def _on_complete(_: Any) -> Any:
+        def _on_complete(_: _T) -> _T:
             assert self.slot is not None
             self.slot.nextcall.schedule()
             return _
 
         assert self.spider is not None
-        dwld = self.downloader.fetch(request, self.spider)
+        dwld: Deferred[Union[Response, Request]] = self.downloader.fetch(
+            request, self.spider
+        )
         dwld.addCallback(_on_success)
         dwld.addBoth(_on_complete)
         return dwld
@@ -376,7 +385,7 @@ class ExecutionEngine:
         spider: Spider,
         start_requests: Iterable[Request] = (),
         close_if_idle: bool = True,
-    ) -> Generator[Deferred, Any, None]:
+    ) -> Generator[Deferred[Any], Any, None]:
         if self.slot is not None:
             raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
         logger.info("Spider opened", extra={"spider": spider})
@@ -422,7 +431,7 @@ class ExecutionEngine:
             assert isinstance(ex, CloseSpider)  # typing
             self.close_spider(self.spider, reason=ex.reason)
 
-    def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred:
+    def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]:
         """Close (cancel) spider and clear all its outstanding requests"""
         if self.slot is None:
             raise RuntimeError("Engine slot not assigned")
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index e3b95e977..1e586c53a 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -71,7 +71,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
         """
         return cls()
 
-    def open(self, spider: Spider) -> Optional[Deferred]:
+    def open(self, spider: Spider) -> Optional[Deferred[None]]:
         """
         Called when the spider is opened by the engine. It receives the spider
         instance as argument and it's useful to execute initialization code.
@@ -81,7 +81,7 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
         """
         pass
 
-    def close(self, reason: str) -> Optional[Deferred]:
+    def close(self, reason: str) -> Optional[Deferred[None]]:
         """
         Called when the spider is closed by the engine. It receives the reason why the crawl
         finished as argument and it's useful to execute cleaning code.
@@ -216,7 +216,7 @@ class Scheduler(BaseScheduler):
     def has_pending_requests(self) -> bool:
         return len(self) > 0
 
-    def open(self, spider: Spider) -> Optional[Deferred]:
+    def open(self, spider: Spider) -> Optional[Deferred[None]]:
         """
         (1) initialize the memory queue
         (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
@@ -227,7 +227,7 @@ class Scheduler(BaseScheduler):
         self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None
         return self.df.open()
 
-    def close(self, reason: str) -> Optional[Deferred]:
+    def close(self, reason: str) -> Optional[Deferred[None]]:
         """
         (1) dump pending requests to disk if there is a disk queue
         (2) return the result of the dupefilter's ``close`` method
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 3b7492838..8a9e8f687 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -12,6 +12,7 @@ from typing import (
     Deque,
     Generator,
     Iterable,
+    Iterator,
     Optional,
     Set,
     Tuple,
@@ -33,6 +34,7 @@ from scrapy.logformatter import LogFormatter
 from scrapy.pipelines import ItemPipelineManager
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import (
+    DeferredListResultListT,
     aiter_errback,
     defer_fail,
     defer_succeed,
@@ -48,13 +50,18 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
-_T = TypeVar("_T")
-QueueTuple = Tuple[Union[Response, Failure], Request, Deferred]
-
-
 logger = logging.getLogger(__name__)
 
 
+_T = TypeVar("_T")
+_ParallelResult = DeferredListResultListT[Iterator[Any]]
+
+if TYPE_CHECKING:
+    # parameterized Deferreds require Twisted 21.7.0
+    _HandleOutputDeferred = Deferred[Union[_ParallelResult, None]]
+    QueueTuple = Tuple[Union[Response, Failure], Request, _HandleOutputDeferred]
+
+
 class Slot:
     """Scraper slot (one per running spider)"""
 
@@ -66,12 +73,12 @@ class Slot:
         self.active: Set[Request] = set()
         self.active_size: int = 0
         self.itemproc_size: int = 0
-        self.closing: Optional[Deferred] = None
+        self.closing: Optional[Deferred[Spider]] = None
 
     def add_response_request(
         self, result: Union[Response, Failure], request: Request
-    ) -> Deferred:
-        deferred: Deferred = Deferred()
+    ) -> _HandleOutputDeferred:
+        deferred: _HandleOutputDeferred = Deferred()
         self.queue.append((result, request, deferred))
         if isinstance(result, Response):
             self.active_size += max(len(result.body), self.MIN_RESPONSE_SIZE)
@@ -117,12 +124,12 @@ class Scraper:
         self.logformatter: LogFormatter = crawler.logformatter
 
     @inlineCallbacks
-    def open_spider(self, spider: Spider) -> Generator[Deferred, Any, None]:
+    def open_spider(self, spider: Spider) -> Generator[Deferred[Any], Any, None]:
         """Open the given spider for scraping and allocate resources for it"""
         self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
         yield self.itemproc.open_spider(spider)
 
-    def close_spider(self, spider: Spider) -> Deferred:
+    def close_spider(self, spider: Spider) -> Deferred[Spider]:
         """Close a spider being scraped and release its resources"""
         if self.slot is None:
             raise RuntimeError("Scraper slot not assigned")
@@ -142,12 +149,12 @@ class Scraper:
 
     def enqueue_scrape(
         self, result: Union[Response, Failure], request: Request, spider: Spider
-    ) -> Deferred:
+    ) -> _HandleOutputDeferred:
         if self.slot is None:
             raise RuntimeError("Scraper slot not assigned")
         dfd = self.slot.add_response_request(result, request)
 
-        def finish_scraping(_: Any) -> Any:
+        def finish_scraping(_: _T) -> _T:
             assert self.slot is not None
             self.slot.finish_response(result, request)
             self._check_if_closing(spider)
@@ -174,7 +181,7 @@ class Scraper:
 
     def _scrape(
         self, result: Union[Response, Failure], request: Request, spider: Spider
-    ) -> Deferred:
+    ) -> _HandleOutputDeferred:
         """
         Handle the downloaded response or failure through the spider callback/errback
         """
@@ -182,32 +189,35 @@ class Scraper:
             raise TypeError(
                 f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}"
             )
-        dfd = self._scrape2(
+        dfd: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = self._scrape2(
             result, request, spider
         )  # returns spider's processed output
         dfd.addErrback(self.handle_spider_error, request, result, spider)
-        dfd.addCallback(
+        dfd2: _HandleOutputDeferred = dfd.addCallback(
             self.handle_spider_output, request, cast(Response, result), spider
         )
-        return dfd
+        return dfd2
 
     def _scrape2(
         self, result: Union[Response, Failure], request: Request, spider: Spider
-    ) -> Deferred:
+    ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:
         """
         Handle the different cases of request's result been a Response or a Failure
         """
         if isinstance(result, Response):
-            return self.spidermw.scrape_response(
+            # Deferreds are invariant so Mutable*Chain isn't matched to *Iterable
+            return self.spidermw.scrape_response(  # type: ignore[return-value]
                 self.call_spider, result, request, spider
             )
         # else result is a Failure
         dfd = self.call_spider(result, request, spider)
-        return dfd.addErrback(self._log_download_errors, result, request, spider)
+        dfd.addErrback(self._log_download_errors, result, request, spider)
+        return dfd
 
     def call_spider(
         self, result: Union[Response, Failure], request: Request, spider: Spider
-    ) -> Deferred:
+    ) -> Deferred[Union[Iterable[Any], AsyncIterable[Any]]]:
+        dfd: Deferred[Any]
         if isinstance(result, Response):
             if getattr(result, "request", None) is None:
                 result.request = request
@@ -225,7 +235,10 @@ class Scraper:
             if request.errback:
                 warn_on_generator_with_return_value(spider, request.errback)
                 dfd.addErrback(request.errback)
-        return dfd.addCallback(iterate_spider_output)
+        dfd2: Deferred[Union[Iterable[Any], AsyncIterable[Any]]] = dfd.addCallback(
+            iterate_spider_output
+        )
+        return dfd2
 
     def handle_spider_error(
         self,
@@ -262,10 +275,11 @@ class Scraper:
         request: Request,
         response: Response,
         spider: Spider,
-    ) -> Deferred:
+    ) -> _HandleOutputDeferred:
         if not result:
             return defer_succeed(None)
         it: Union[Iterable[_T], AsyncIterable[_T]]
+        dfd: Deferred[_ParallelResult]
         if isinstance(result, AsyncIterable):
             it = aiter_errback(
                 result, self.handle_spider_error, request, response, spider
@@ -290,11 +304,12 @@ class Scraper:
                 response,
                 spider,
             )
-        return dfd
+        # returning Deferred[_ParallelResult] instead of Deferred[Union[_ParallelResult, None]]
+        return dfd  # type: ignore[return-value]
 
     def _process_spidermw_output(
         self, output: Any, request: Request, response: Response, spider: Spider
-    ) -> Optional[Deferred]:
+    ) -> Optional[Deferred[Any]]:
         """Process each Request/Item (given in the output parameter) returned
         from the given spider
         """
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 58873f0d9..e792f8ca7 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -45,7 +45,9 @@ logger = logging.getLogger(__name__)
 
 
 _T = TypeVar("_T")
-ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
+ScrapeFunc = Callable[
+    [Union[Response, Failure], Request, Spider], Union[Iterable[_T], AsyncIterable[_T]]
+]
 
 
 def _isiterable(o: Any) -> bool:
@@ -80,7 +82,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         response: Response,
         request: Request,
         spider: Spider,
-    ) -> Any:
+    ) -> Union[Iterable[_T], AsyncIterable[_T]]:
         for method in self.methods["process_spider_input"]:
             method = cast(Callable, method)
             try:
@@ -311,7 +313,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         response: Response,
         request: Request,
         spider: Spider,
-    ) -> Deferred:
+    ) -> Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]]:
         async def process_callback_output(
             result: Union[Iterable[_T], AsyncIterable[_T]]
         ) -> Union[MutableChain[_T], MutableAsyncChain[_T]]:
@@ -322,12 +324,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
         ) -> Union[Failure, MutableChain[_T], MutableAsyncChain[_T]]:
             return self._process_spider_exception(response, spider, _failure)
 
-        dfd: Deferred = mustbe_deferred(
+        dfd: Deferred[Union[Iterable[_T], AsyncIterable[_T]]] = mustbe_deferred(
             self._process_spider_input, scrape_func, response, request, spider
         )
-        dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
-        dfd.addErrback(process_spider_exception)
-        return dfd
+        dfd2: Deferred[Union[MutableChain[_T], MutableAsyncChain[_T]]] = (
+            dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
+        )
+        dfd2.addErrback(process_spider_exception)
+        return dfd2
 
     def process_start_requests(
         self, start_requests: Iterable[Request], spider: Spider
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 0cfbc156f..21d649e3c 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -4,6 +4,8 @@ Item pipeline
 See documentation in docs/item-pipeline.rst
 """
 
+from __future__ import annotations
+
 from typing import Any, List
 
 from twisted.internet.defer import Deferred
@@ -29,5 +31,5 @@ class ItemPipelineManager(MiddlewareManager):
                 deferred_f_from_coro_f(pipe.process_item)
             )
 
-    def process_item(self, item: Any, spider: Spider) -> Deferred:
+    def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
         return self._process_chain("process_item", item, spider)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index f60b7dde8..ddb68c86b 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -46,6 +46,12 @@ if TYPE_CHECKING:
     _P = ParamSpec("_P")
 
 _T = TypeVar("_T")
+_T2 = TypeVar("_T2")
+
+# copied from twisted.internet.defer
+_SelfResultT = TypeVar("_SelfResultT")
+_DeferredListResultItemT = Tuple[bool, _SelfResultT]
+DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]]
 
 
 def defer_fail(_failure: Failure) -> Deferred:
@@ -62,7 +68,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     return d
 
 
-def defer_succeed(result: Any) -> Deferred:
+def defer_succeed(result: _T) -> Deferred[_T]:
     """Same as twisted.internet.defer.succeed but delay calling callback until
     next reactor loop
 
@@ -128,10 +134,10 @@ def mustbe_deferred(
 def parallel(
     iterable: Iterable[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], Any],
+    callable: Callable[Concatenate[_T, _P], _T2],
     *args: _P.args,
     **named: _P.kwargs,
-) -> Deferred:
+) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
@@ -191,12 +197,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
     def __init__(
         self,
         aiterable: AsyncIterable[_T],
-        callable: Callable[Concatenate[_T, _P], Any],
+        callable: Callable[Concatenate[_T, _P], _T2],
         *callable_args: _P.args,
         **callable_kwargs: _P.kwargs,
     ):
         self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
-        self.callable: Callable[Concatenate[_T, _P], Any] = callable
+        self.callable: Callable[Concatenate[_T, _P], _T2] = callable
         self.callable_args: Tuple[Any, ...] = callable_args
         self.callable_kwargs: Dict[str, Any] = callable_kwargs
         self.finished: bool = False
@@ -249,10 +255,10 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
 def parallel_async(
     async_iterable: AsyncIterable[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], Any],
+    callable: Callable[Concatenate[_T, _P], _T2],
     *args: _P.args,
     **named: _P.kwargs,
-) -> Deferred:
+) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
     """Like parallel but for async iterators"""
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index 0155c62eb..dd3f8ceb9 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -36,7 +36,7 @@ class ManagerTestCase(TestCase):
         if not response:
             response = Response(request.url)
 
-        def download_func(**kwargs):
+        def download_func(request, spider):
             return response
 
         dfd = self.mwman.download(download_func, request, self.spider)

From 365c9e62ad9e99725eb1898cbd2806c63105cd58 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 17 Jun 2024 14:37:11 +0500
Subject: [PATCH 52/82] Removing empty example reference (#6402)

Co-authored-by: Michael Duane Mooring <mikeumus@gmail.com>
---
 docs/topics/link-extractors.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 1201c926d..f9744ed16 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -85,7 +85,7 @@ LxmlLinkExtractor
     :param restrict_xpaths: is an XPath (or list of XPath's) which defines
         regions inside the response where links should be extracted from.
         If given, only the text selected by those XPath will be scanned for
-        links. See examples below.
+        links.
     :type restrict_xpaths: str or list
 
     :param restrict_css: a CSS selector (or list of selectors) which defines

From a364560fadbbc0dd7cca78670bbd9d3c00d4d366 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 17 Jun 2024 14:38:10 +0500
Subject: [PATCH 53/82] Unpin markupsafe in extra-deps. (#6403)

---
 tox.ini | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tox.ini b/tox.ini
index 023a86c5a..d665fc5a5 100644
--- a/tox.ini
+++ b/tox.ini
@@ -147,9 +147,6 @@ deps =
     {[testenv]deps}
     boto3
     google-cloud-storage
-    # Twisted[http2] currently forces old mitmproxy because of h2 version
-    # restrictions in their deps, so we need to pin old markupsafe here too.
-    markupsafe < 2.1.0
     robotexclusionrulesparser
     Pillow
     Twisted[http2]

From d13219062500eae1a6d5330ceea3502590cd89cb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 19 Jun 2024 23:26:25 +0500
Subject: [PATCH 54/82] flake8-debugger

---
 .pre-commit-config.yaml    | 2 ++
 scrapy/extensions/debug.py | 2 +-
 scrapy/utils/console.py    | 4 ++--
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 63da5544d..eb3404b7f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -8,6 +8,8 @@ repos:
   rev: 7.0.0
   hooks:
   - id: flake8
+    additional_dependencies:
+    - flake8-debugger
 - repo: https://github.com/psf/black.git
   rev: 24.2.0
   hooks:
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index a0fc7b99f..b360ce48d 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -74,4 +74,4 @@ class Debugger:
 
     def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:
         assert frame
-        Pdb().set_trace(frame.f_back)
+        Pdb().set_trace(frame.f_back)  # noqa: T100
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index bf1803115..328219831 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -10,10 +10,10 @@ def _embed_ipython_shell(
 ) -> EmbedFuncT:
     """Start an IPython Shell"""
     try:
-        from IPython.terminal.embed import InteractiveShellEmbed
+        from IPython.terminal.embed import InteractiveShellEmbed  # noqa: T100
         from IPython.terminal.ipapp import load_default_config
     except ImportError:
-        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]
+        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]  # noqa: T100
             InteractiveShellEmbed,
         )
         from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]

From a617e04d2eb89b64f15df7a6a0326bfaf57f8dde Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 19 Jun 2024 23:28:58 +0500
Subject: [PATCH 55/82] flake8-string-format

---
 .flake8                 | 8 ++++++--
 .pre-commit-config.yaml | 1 +
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/.flake8 b/.flake8
index cf1a96476..b6048c9ee 100644
--- a/.flake8
+++ b/.flake8
@@ -1,8 +1,12 @@
 [flake8]
 
 max-line-length = 119
-ignore = E203, E501, E701, E704, W503
-
+ignore =
+    E203, E501, E701, E704, W503
+    # docstring does contain unindexed parameters
+    P102
+    # other string does contain unindexed parameters
+    P103
 exclude =
     docs/conf.py
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index eb3404b7f..47a3df53d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -10,6 +10,7 @@ repos:
   - id: flake8
     additional_dependencies:
     - flake8-debugger
+    - flake8-string-format
 - repo: https://github.com/psf/black.git
   rev: 24.2.0
   hooks:

From 1c70d3e60555084b4bec9dfd794adb93b24b2171 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 19 Jun 2024 23:36:36 +0500
Subject: [PATCH 56/82] flake8-comprehensions

---
 .pre-commit-config.yaml         |  1 +
 scrapy/extensions/feedexport.py |  2 +-
 tests/test_commands.py          |  2 +-
 tests/test_feedexport.py        | 10 +++++-----
 tests/test_loader.py            |  6 ++----
 tests/test_loader_deprecated.py |  4 ++--
 tests/test_scheduler.py         |  2 +-
 tests/test_spider.py            | 18 +++++++++---------
 8 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 47a3df53d..974d397c8 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,6 +9,7 @@ repos:
   hooks:
   - id: flake8
     additional_dependencies:
+    - flake8-comprehensions
     - flake8-debugger
     - flake8-string-format
 - repo: https://github.com/psf/black.git
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index de8a288f6..941bd4b26 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -104,7 +104,7 @@ class ItemFilter:
                 for item_class in feed_options.get("item_classes") or ()
             )
         else:
-            self.item_classes = tuple()
+            self.item_classes = ()
 
     def accepts(self, item: Any) -> bool:
         """
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 857a56b73..d829b1701 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -200,7 +200,7 @@ def get_permissions_dict(
 
     path_obj = Path(path)
 
-    renamings = renamings or tuple()
+    renamings = renamings or ()
     permissions_dict = {
         ".": get_permissions(path_obj),
     }
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3771df8f1..253987e15 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1356,7 +1356,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
-        items = [dict({"foo": "Test\xd6"})]
+        items = [{"foo": "Test\xd6"}]
 
         formats = {
             "json": b'[{"foo": "Test\\u00d6"}]',
@@ -1401,7 +1401,7 @@ class FeedExportTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
-        items = [dict({"foo": "FOO", "bar": "BAR"})]
+        items = [{"foo": "FOO", "bar": "BAR"}]
 
         formats = {
             "json": b'[\n{"bar": "BAR"}\n]',
@@ -2513,8 +2513,8 @@ class BatchDeliveriesTest(FeedExportTestBase):
     @defer.inlineCallbacks
     def test_export_multiple_configs(self):
         items = [
-            dict({"foo": "FOO", "bar": "BAR"}),
-            dict({"foo": "FOO1", "bar": "BAR1"}),
+            {"foo": "FOO", "bar": "BAR"},
+            {"foo": "FOO1", "bar": "BAR1"},
         ]
 
         formats = {
@@ -2574,7 +2574,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_batch_item_count_feeds_setting(self):
-        items = [dict({"foo": "FOO"}), dict({"foo": "FOO1"})]
+        items = [{"foo": "FOO"}, {"foo": "FOO1"}]
         formats = {
             "json": [
                 b'[{"foo": "FOO"}]',
diff --git a/tests/test_loader.py b/tests/test_loader.py
index b0b7f8723..8db929dcf 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -156,7 +156,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value("name"), ["foo"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(ItemAdapter(loaded_item).asdict(), dict({"name": ["foo"]}))
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo"]})
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
@@ -165,9 +165,7 @@ class InitializationTestMixin:
         self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(
-            ItemAdapter(loaded_item).asdict(), dict({"name": ["foo", "bar"]})
-        )
+        self.assertEqual(ItemAdapter(loaded_item).asdict(), {"name": ["foo", "bar"]})
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
diff --git a/tests/test_loader_deprecated.py b/tests/test_loader_deprecated.py
index 528efa142..0d245bec9 100644
--- a/tests/test_loader_deprecated.py
+++ b/tests/test_loader_deprecated.py
@@ -526,7 +526,7 @@ class InitializationFromDictTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["foo"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({"name": ["foo"]}))
+        self.assertEqual(loaded_item, {"name": ["foo"]})
 
     def test_get_output_value_list(self):
         """Getting output value must not remove value from item"""
@@ -535,7 +535,7 @@ class InitializationFromDictTest(unittest.TestCase):
         self.assertEqual(il.get_output_value("name"), ["foo", "bar"])
         loaded_item = il.load_item()
         self.assertIsInstance(loaded_item, self.item_class)
-        self.assertEqual(loaded_item, dict({"name": ["foo", "bar"]}))
+        self.assertEqual(loaded_item, {"name": ["foo", "bar"]})
 
     def test_values_single(self):
         """Values from initial item must be added to loader._values"""
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 02b50baa3..9b7bad4bf 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -284,7 +284,7 @@ class DownloaderAwareSchedulerTestMixin:
             downloader.decrement(slot)
 
         self.assertTrue(
-            _is_scheduling_fair(list(s for u, s in _URLS_WITH_SLOTS), dequeued_slots)
+            _is_scheduling_fair([s for u, s in _URLS_WITH_SLOTS], dequeued_slots)
         )
         self.assertEqual(sum(len(s.active) for s in downloader.slots.values()), 0)
 
diff --git a/tests/test_spider.py b/tests/test_spider.py
index d629d33af..18a863350 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -244,7 +244,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -270,7 +270,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -299,7 +299,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 2)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -324,7 +324,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -352,7 +352,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -383,7 +383,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -413,7 +413,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -445,7 +445,7 @@ class CrawlSpiderTest(SpiderTest):
         spider = _CrawlSpider()
         output = list(spider._requests_to_follow(response))
         self.assertEqual(len(output), 3)
-        self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
+        self.assertTrue(all(isinstance(r, Request) for r in output))
         self.assertEqual(
             [r.url for r in output],
             [
@@ -637,7 +637,7 @@ Sitemap: /sitemap-relative-url.xml
         class FilteredSitemapSpider(self.spider_class):
             def sitemap_filter(self, entries):
                 for entry in entries:
-                    alternate_links = entry.get("alternate", tuple())
+                    alternate_links = entry.get("alternate", ())
                     for link in alternate_links:
                         if "/deutsch/" in link:
                             entry["loc"] = link

From 1ef9c337cad36ac6c80eab86622f8ae9fc8d1075 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 19 Jun 2024 23:57:40 +0500
Subject: [PATCH 57/82] flake8-docstrings

---
 .flake8                      | 33 +++++++++++++++++++++++++++++++++
 .pre-commit-config.yaml      |  1 +
 tests/test_dupefilters.py    |  2 +-
 tests/test_linkextractors.py |  2 +-
 4 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/.flake8 b/.flake8
index b6048c9ee..222ba7179 100644
--- a/.flake8
+++ b/.flake8
@@ -2,11 +2,44 @@
 
 max-line-length = 119
 ignore =
+    # black disagrees with flake8 about these
     E203, E501, E701, E704, W503
     # docstring does contain unindexed parameters
     P102
     # other string does contain unindexed parameters
     P103
+    # Missing docstring in public module
+    D100
+    # Missing docstring in public class
+    D101
+    # Missing docstring in public method
+    D102
+    # Missing docstring in public function
+    D103
+    # Missing docstring in public package
+    D104
+    # Missing docstring in magic method
+    D105
+    # Missing docstring in public nested class
+    D106
+    # Missing docstring in __init__
+    D107
+    # One-line docstring should fit on one line with quotes
+    D200
+    # No blank lines allowed after function docstring
+    D202
+    # 1 blank line required between summary line and description
+    D205
+    # Multi-line docstring closing quotes should be on a separate line
+    D209
+    # First line should end with a period
+    D400
+    # First line should be in imperative mood; try rephrasing
+    D401
+    # First line should not be the function's "signature"
+    D402
+    # First word of the first line should be properly capitalized
+    D403
 exclude =
     docs/conf.py
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 974d397c8..6b60eff68 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -11,6 +11,7 @@ repos:
     additional_dependencies:
     - flake8-comprehensions
     - flake8-debugger
+    - flake8-docstrings
     - flake8-string-format
 - repo: https://github.com/psf/black.git
   rev: 24.2.0
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index aa0975555..f617fc027 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -146,7 +146,7 @@ class RFPDupeFilterTest(unittest.TestCase):
         case_insensitive_dupefilter.close("finished")
 
     def test_seenreq_newlines(self):
-        """Checks against adding duplicate \r to
+        r"""Checks against adding duplicate \r to
         line endings on Windows platforms."""
 
         r1 = Request("http://scrapytest.org/1")
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index d9c09a16a..b1043c111 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -186,7 +186,7 @@ class Base:
             )
 
         def test_nofollow(self):
-            '''Test the extractor's behaviour for links with rel="nofollow"'''
+            """Test the extractor's behaviour for links with rel='nofollow'"""
 
             html = b"""<html><head><title>Page title<title>
             <body>

From 3d8dbd5648406227c9b96736da62046b90c554e5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 20 Jun 2024 00:22:43 +0500
Subject: [PATCH 58/82] flake8-bugbear

---
 .flake8                           | 15 +++++++++++++++
 .pre-commit-config.yaml           |  1 +
 scrapy/pipelines/media.py         |  2 +-
 scrapy/utils/defer.py             |  2 +-
 scrapy/utils/python.py            |  2 +-
 scrapy/utils/signal.py            |  5 ++++-
 tests/test_cmdline/__init__.py    |  2 +-
 tests/test_command_version.py     |  4 ++--
 tests/test_commands.py            |  2 +-
 tests/test_downloader_handlers.py |  2 +-
 tests/test_engine.py              |  2 +-
 tests/test_request_dict.py        |  2 +-
 12 files changed, 30 insertions(+), 11 deletions(-)

diff --git a/.flake8 b/.flake8
index 222ba7179..57117d2cf 100644
--- a/.flake8
+++ b/.flake8
@@ -4,6 +4,21 @@ max-line-length = 119
 ignore =
     # black disagrees with flake8 about these
     E203, E501, E701, E704, W503
+    # Assigning to `os.environ` doesn't clear the environment.
+    B003
+    # Do not use mutable data structures for argument defaults.
+    B006
+    # Loop control variable not used within the loop body.
+    B007
+    # Do not perform function calls in argument defaults.
+    B008
+    # return/continue/break inside finally blocks cause exceptions to be
+    # silenced.
+    B012
+    # Star-arg unpacking after a keyword argument is strongly discouraged
+    B026
+    # No explicit stacklevel argument found.
+    B028
     # docstring does contain unindexed parameters
     P102
     # other string does contain unindexed parameters
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 6b60eff68..f70effc5d 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -9,6 +9,7 @@ repos:
   hooks:
   - id: flake8
     additional_dependencies:
+    - flake8-bugbear
     - flake8-comprehensions
     - flake8-debugger
     - flake8-docstrings
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 3e327105e..09e95cf5d 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -234,7 +234,7 @@ class MediaPipeline(ABC):
             # Exception Chaining (https://www.python.org/dev/peps/pep-3134/).
             context = getattr(result.value, "__context__", None)
             if isinstance(context, StopIteration):
-                setattr(result.value, "__context__", None)
+                result.value.__context__ = None
 
         info.downloading.remove(fp)
         info.downloaded[fp] = result  # cache result
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index ddb68c86b..877eb4388 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -407,7 +407,7 @@ def maybeDeferred_coro(
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)
-    except:  # noqa: E722
+    except:  # noqa: E722,B001
         return defer.fail(failure.Failure(captureVars=Deferred.debug))
 
     if isinstance(result, Deferred):
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 059d8e04d..f56950fdd 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -269,7 +269,7 @@ def get_spec(func: Callable[..., Any]) -> Tuple[List[str], Dict[str, Any]]:
 
     if inspect.isfunction(func) or inspect.ismethod(func):
         spec = inspect.getfullargspec(func)
-    elif hasattr(func, "__call__"):
+    elif hasattr(func, "__call__"):  # noqa: B004
         spec = inspect.getfullargspec(func.__call__)
     else:
         raise TypeError(f"{type(func)} is not callable")
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 89cfbd2ec..bb6d807ee 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -100,7 +100,10 @@ def send_catch_log_deferred(
         d.addErrback(logerror, receiver)
         # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
         d.addBoth(
-            lambda result: (receiver, result)  # pylint: disable=cell-var-from-loop
+            lambda result: (
+                receiver,  # pylint: disable=cell-var-from-loop  # noqa: B023
+                result,
+            )
         )
         dfds.append(d)
     d = DeferredList(dfds)
diff --git a/tests/test_cmdline/__init__.py b/tests/test_cmdline/__init__.py
index 25ded143c..4835e936b 100644
--- a/tests/test_cmdline/__init__.py
+++ b/tests/test_cmdline/__init__.py
@@ -20,7 +20,7 @@ class CmdlineTest(unittest.TestCase):
         self.env["SCRAPY_SETTINGS_MODULE"] = "tests.test_cmdline.settings"
 
     def _execute(self, *new_args, **kwargs):
-        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        encoding = sys.stdout.encoding or "utf-8"
         args = (sys.executable, "-m", "scrapy.cmdline") + new_args
         proc = Popen(args, stdout=PIPE, stderr=PIPE, env=self.env, **kwargs)
         comm = proc.communicate()[0].strip()
diff --git a/tests/test_command_version.py b/tests/test_command_version.py
index a52d0d13c..18c1c531c 100644
--- a/tests/test_command_version.py
+++ b/tests/test_command_version.py
@@ -12,7 +12,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_output(self):
-        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        encoding = sys.stdout.encoding or "utf-8"
         _, out, _ = yield self.execute([])
         self.assertEqual(
             out.strip().decode(encoding),
@@ -21,7 +21,7 @@ class VersionTest(ProcessTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_verbose_output(self):
-        encoding = getattr(sys.stdout, "encoding") or "utf-8"
+        encoding = sys.stdout.encoding or "utf-8"
         _, out, _ = yield self.execute(["-v"])
         headers = [
             line.partition(":")[0].strip()
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d829b1701..a23b7f4a9 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -101,7 +101,7 @@ class ProjectTest(unittest.TestCase):
         def kill_proc():
             p.kill()
             p.communicate()
-            assert False, "Command took too much time to complete"
+            raise AssertionError("Command took too much time to complete")
 
         timer = Timer(15, kill_proc)
         try:
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index d3fd63847..884491d01 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -892,7 +892,7 @@ class S3TestCase(unittest.TestCase):
         except Exception as e:
             self.assertIsInstance(e, (TypeError, NotConfigured))
         else:
-            assert False
+            raise AssertionError()
 
     def test_request_signing1(self):
         # gets an object from the johnsmith bucket.
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 33544e8db..86526420f 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -459,7 +459,7 @@ class EngineTest(unittest.TestCase):
         def kill_proc():
             p.kill()
             p.communicate()
-            assert False, "Command took too much time to complete"
+            raise AssertionError("Command took too much time to complete")
 
         timer = Timer(15, kill_proc)
         try:
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index 7312eb036..d3f416347 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -147,7 +147,7 @@ class RequestSerializationTest(unittest.TestCase):
 
         spider = MySpider()
         r = Request("http://www.example.com", callback=spider.parse)
-        setattr(spider, "parse", None)
+        spider.parse = None
         self.assertRaises(ValueError, r.to_dict, spider=spider)
 
     def test_callback_not_available(self):

From 13d3b1af470bbe7e82fda51017f0f72cb8eed9dd Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 20 Jun 2024 00:42:43 +0500
Subject: [PATCH 59/82] Split ignores into blocks.

---
 .flake8 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.flake8 b/.flake8
index 57117d2cf..be9d83eaf 100644
--- a/.flake8
+++ b/.flake8
@@ -4,6 +4,7 @@ max-line-length = 119
 ignore =
     # black disagrees with flake8 about these
     E203, E501, E701, E704, W503
+
     # Assigning to `os.environ` doesn't clear the environment.
     B003
     # Do not use mutable data structures for argument defaults.
@@ -19,10 +20,12 @@ ignore =
     B026
     # No explicit stacklevel argument found.
     B028
+
     # docstring does contain unindexed parameters
     P102
     # other string does contain unindexed parameters
     P103
+
     # Missing docstring in public module
     D100
     # Missing docstring in public class

From 326e323e11a7f5fc760250be6eae23d0159f6429 Mon Sep 17 00:00:00 2001
From: mlmsmith <mlmsmith@hotmail.co.uk>
Date: Fri, 21 Jun 2024 18:24:10 +0800
Subject: [PATCH 60/82] Apply grammar fixes (#6411)

---
 docs/intro/overview.rst | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 542760b4f..ef1294470 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -44,13 +44,13 @@ https://quotes.toscrape.com, following the pagination:
             if next_page is not None:
                 yield response.follow(next_page, self.parse)
 
-Put this in a text file, name it to something like ``quotes_spider.py``
+Put this in a text file, name it something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::
 
     scrapy runspider quotes_spider.py -o quotes.jsonl
 
 When this finishes you will have in the ``quotes.jsonl`` file a list of the
-quotes in JSON Lines format, containing text and author, looking like this::
+quotes in JSON Lines format, containing the text and author, which will look like this::
 
     {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
     {"author": "Steve Martin", "text": "\u201cA day without sunshine is like, you know, night.\u201d"}
@@ -72,11 +72,11 @@ using a CSS Selector, yield a Python dict with the extracted quote text and auth
 look for a link to the next page and schedule another request using the same
 ``parse`` method as callback.
 
-Here you notice one of the main advantages about Scrapy: requests are
+Here you will notice one of the main advantages of Scrapy: requests are
 :ref:`scheduled and processed asynchronously <topics-architecture>`.  This
 means that Scrapy doesn't need to wait for a request to be finished and
 processed, it can send another request or do other things in the meantime. This
-also means that other requests can keep going even if some request fails or an
+also means that other requests can keep going even if a request fails or an
 error happens while handling it.
 
 While this enables you to do very fast crawls (sending multiple concurrent

From d08f559600f0bb45b916be158a06e033753d45f5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 25 Jun 2024 13:20:59 +0500
Subject: [PATCH 61/82] Add flake8-type-checking. (#6413)

---
 .flake8                                       |  4 +++
 .pre-commit-config.yaml                       |  1 +
 scrapy/addons.py                              |  9 ++++--
 scrapy/cmdline.py                             |  3 +-
 scrapy/commands/__init__.py                   |  8 +++--
 scrapy/commands/bench.py                      |  8 +++--
 scrapy/commands/crawl.py                      |  8 +++--
 scrapy/commands/fetch.py                      |  8 +++--
 scrapy/commands/list.py                       |  8 +++--
 scrapy/commands/parse.py                      |  9 ++++--
 scrapy/commands/runspider.py                  | 12 ++++---
 scrapy/commands/shell.py                      |  8 +++--
 scrapy/contracts/__init__.py                  | 11 +++++--
 scrapy/core/downloader/__init__.py            |  5 +--
 scrapy/core/downloader/contextfactory.py      |  5 +--
 scrapy/core/downloader/handlers/__init__.py   |  6 ++--
 scrapy/core/downloader/handlers/datauri.py    |  8 +++--
 scrapy/core/downloader/handlers/file.py       |  9 ++++--
 scrapy/core/downloader/handlers/ftp.py        | 12 ++++---
 scrapy/core/downloader/handlers/http10.py     | 12 +++----
 scrapy/core/downloader/handlers/http11.py     | 11 ++++---
 scrapy/core/downloader/handlers/http2.py      | 16 +++++-----
 scrapy/core/downloader/handlers/s3.py         | 13 ++++----
 scrapy/core/downloader/middleware.py          | 11 ++++---
 scrapy/core/downloader/webclient.py           |  8 +++--
 scrapy/core/engine.py                         |  6 ++--
 scrapy/core/http2/agent.py                    | 18 +++++++----
 scrapy/core/http2/protocol.py                 | 21 ++++++++-----
 scrapy/core/http2/stream.py                   | 11 ++++---
 scrapy/core/scheduler.py                      | 16 +++++-----
 scrapy/core/spidermw.py                       |  6 +++-
 scrapy/downloadermiddlewares/ajaxcrawl.py     |  8 +++--
 scrapy/downloadermiddlewares/cookies.py       | 10 +++---
 .../downloadermiddlewares/defaultheaders.py   |  7 +++--
 .../downloadermiddlewares/downloadtimeout.py  |  5 +--
 scrapy/downloadermiddlewares/httpauth.py      |  5 +--
 scrapy/downloadermiddlewares/httpcache.py     | 13 ++++----
 .../downloadermiddlewares/httpcompression.py  |  6 ++--
 scrapy/downloadermiddlewares/httpproxy.py     |  7 +++--
 scrapy/downloadermiddlewares/offsite.py       |  6 ++--
 scrapy/downloadermiddlewares/redirect.py      |  8 +++--
 scrapy/downloadermiddlewares/retry.py         | 10 +++---
 scrapy/downloadermiddlewares/robotstxt.py     | 10 +++---
 scrapy/downloadermiddlewares/stats.py         |  9 +++---
 scrapy/downloadermiddlewares/useragent.py     |  5 +--
 scrapy/dupefilters.py                         | 10 +++---
 scrapy/extension.py                           |  8 +++--
 scrapy/extensions/closespider.py              | 10 +++---
 scrapy/extensions/corestats.py                |  5 +--
 scrapy/extensions/debug.py                    |  7 +++--
 scrapy/extensions/feedexport.py               | 17 +++++-----
 scrapy/extensions/httpcache.py                |  9 ++++--
 scrapy/extensions/logstats.py                 |  6 ++--
 scrapy/extensions/memdebug.py                 |  5 +--
 scrapy/extensions/memusage.py                 |  4 ++-
 scrapy/extensions/periodic_log.py             |  6 ++--
 scrapy/extensions/spiderstate.py              |  3 +-
 scrapy/extensions/statsmailer.py              |  9 +++---
 scrapy/extensions/telnet.py                   |  5 ++-
 scrapy/extensions/throttle.py                 |  8 +++--
 scrapy/http/cookies.py                        |  6 ++--
 scrapy/http/request/form.py                   |  3 +-
 scrapy/http/response/__init__.py              |  7 +++--
 scrapy/http/response/text.py                  |  3 +-
 scrapy/linkextractors/lxmlhtml.py             | 13 ++++++--
 scrapy/loader/__init__.py                     | 10 ++++--
 scrapy/logformatter.py                        |  5 +--
 scrapy/mail.py                                |  6 ++--
 scrapy/middleware.py                          |  8 ++---
 scrapy/pipelines/__init__.py                  | 12 ++++---
 scrapy/pipelines/files.py                     | 31 ++++++++++---------
 scrapy/pipelines/images.py                    | 11 ++++---
 scrapy/pipelines/media.py                     | 10 +++---
 scrapy/resolver.py                            |  3 +-
 scrapy/robotstxt.py                           |  3 +-
 scrapy/settings/__init__.py                   |  3 +-
 scrapy/signalmanager.py                       |  8 +++--
 scrapy/spiderloader.py                        |  8 +++--
 scrapy/spidermiddlewares/depth.py             |  8 +++--
 scrapy/spidermiddlewares/httperror.py         | 10 +++---
 scrapy/spidermiddlewares/offsite.py           |  6 ++--
 scrapy/spidermiddlewares/referer.py           |  6 ++--
 scrapy/spidermiddlewares/urllength.py         |  6 ++--
 scrapy/spiders/init.py                        |  8 +++--
 scrapy/squeues.py                             |  8 +++--
 scrapy/statscollectors.py                     | 10 +++---
 scrapy/utils/decorators.py                    |  8 ++---
 scrapy/utils/defer.py                         |  3 +-
 scrapy/utils/engine.py                        |  5 +--
 scrapy/utils/gz.py                            |  8 +++--
 scrapy/utils/httpobj.py                       | 10 ++++--
 scrapy/utils/job.py                           |  9 ++++--
 scrapy/utils/log.py                           |  3 +-
 scrapy/utils/misc.py                          |  4 ++-
 scrapy/utils/project.py                       |  5 +--
 scrapy/utils/reactor.py                       |  8 +++--
 scrapy/utils/spider.py                        | 10 +++---
 scrapy/utils/ssl.py                           |  8 +++--
 scrapy/utils/template.py                      |  8 +++--
 scrapy/utils/test.py                          | 19 ++++++++++--
 scrapy/utils/testproc.py                      |  6 ++--
 tests/mockserver.py                           |  8 +++--
 tests/test_feedexport.py                      |  8 +++--
 tests/test_http2_client_protocol.py           |  8 +++--
 104 files changed, 562 insertions(+), 300 deletions(-)

diff --git a/.flake8 b/.flake8
index be9d83eaf..c4814f13a 100644
--- a/.flake8
+++ b/.flake8
@@ -1,6 +1,7 @@
 [flake8]
 
 max-line-length = 119
+extend-select = TC, TC1
 ignore =
     # black disagrees with flake8 about these
     E203, E501, E701, E704, W503
@@ -58,6 +59,9 @@ ignore =
     D402
     # First word of the first line should be properly capitalized
     D403
+
+    # Annotation in typing.cast() should be a string literal
+    TC006
 exclude =
     docs/conf.py
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index f70effc5d..38526d720 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -14,6 +14,7 @@ repos:
     - flake8-debugger
     - flake8-docstrings
     - flake8-string-format
+    - flake8-type-checking
 - repo: https://github.com/psf/black.git
   rev: 24.2.0
   hooks:
diff --git a/scrapy/addons.py b/scrapy/addons.py
index 65d7a0310..f9ec58cea 100644
--- a/scrapy/addons.py
+++ b/scrapy/addons.py
@@ -1,13 +1,16 @@
+from __future__ import annotations
+
 import logging
 from typing import TYPE_CHECKING, Any, List
 
 from scrapy.exceptions import NotConfigured
-from scrapy.settings import Settings
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
+    from scrapy.settings import Settings
+
 
 logger = logging.getLogger(__name__)
 
@@ -15,8 +18,8 @@ logger = logging.getLogger(__name__)
 class AddonManager:
     """This class facilitates loading and storing :ref:`topics-addons`."""
 
-    def __init__(self, crawler: "Crawler") -> None:
-        self.crawler: "Crawler" = crawler
+    def __init__(self, crawler: Crawler) -> None:
+        self.crawler: Crawler = crawler
         self.addons: List[Any] = []
 
     def load_settings(self, settings: Settings) -> None:
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index da0e51386..e010b159a 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -12,7 +12,6 @@ import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import CrawlerProcess
 from scrapy.exceptions import UsageError
-from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
@@ -21,6 +20,8 @@ if TYPE_CHECKING:
     # typing.ParamSpec requires Python 3.10
     from typing_extensions import ParamSpec
 
+    from scrapy.settings import BaseSettings, Settings
+
     _P = ParamSpec("_P")
 
 
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 9fe803d3c..0322390e5 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -2,18 +2,22 @@
 Base class for Scrapy commands
 """
 
+from __future__ import annotations
+
 import argparse
 import builtins
 import os
 from pathlib import Path
-from typing import Any, Dict, Iterable, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional
 
 from twisted.python import failure
 
-from scrapy.crawler import Crawler, CrawlerProcess
 from scrapy.exceptions import UsageError
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler, CrawlerProcess
+
 
 class ScrapyCommand:
     requires_project: bool = False
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 0c4ebcd23..f91fec57e 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,16 +1,20 @@
+from __future__ import annotations
+
 import argparse
 import subprocess  # nosec
 import sys
 import time
-from typing import Any, Iterable, List
+from typing import TYPE_CHECKING, Any, Iterable, List
 from urllib.parse import urlencode
 
 import scrapy
-from scrapy import Request
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
 
+if TYPE_CHECKING:
+    from scrapy import Request
+
 
 class Command(ScrapyCommand):
     default_settings = {
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 6e023af81..fe1864372 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,11 +1,15 @@
-import argparse
-from typing import List, cast
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List, cast
 
 from twisted.python.failure import Failure
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 
+if TYPE_CHECKING:
+    import argparse
+
 
 class Command(BaseRunSpiderCommand):
     requires_project = True
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index 1acf2d26f..0bdc429da 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,6 +1,7 @@
+from __future__ import annotations
+
 import sys
-from argparse import ArgumentParser, Namespace
-from typing import Dict, List, Type
+from typing import TYPE_CHECKING, Dict, List, Type
 
 from w3lib.url import is_url
 
@@ -11,6 +12,9 @@ from scrapy.http import Request, Response
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
+if TYPE_CHECKING:
+    from argparse import ArgumentParser, Namespace
+
 
 class Command(ScrapyCommand):
     requires_project = False
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index dcc51a694..10330c92a 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,8 +1,12 @@
-import argparse
-from typing import List
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
 
 from scrapy.commands import ScrapyCommand
 
+if TYPE_CHECKING:
+    import argparse
+
 
 class Command(ScrapyCommand):
     requires_project = True
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 3320a1ee4..e6c5e2a47 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -6,6 +6,7 @@ import inspect
 import json
 import logging
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncGenerator,
     Callable,
@@ -22,13 +23,11 @@ from typing import (
 
 from itemadapter import ItemAdapter, is_item
 from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.python.failure import Failure
 from w3lib.url import is_url
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 from scrapy.http import Request, Response
-from scrapy.spiders import Spider
 from scrapy.utils import display
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import aiter_errback, deferred_from_coro
@@ -36,6 +35,12 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import spidercls_for_request
 
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
+    from scrapy.spiders import Spider
+
+
 logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 77850e7b5..87acf9a01 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,17 +1,21 @@
+from __future__ import annotations
+
 import argparse
 import sys
 from importlib import import_module
-from os import PathLike
 from pathlib import Path
-from types import ModuleType
-from typing import List, Union
+from typing import TYPE_CHECKING, List, Union
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.spider import iter_spider_classes
 
+if TYPE_CHECKING:
+    from os import PathLike
+    from types import ModuleType
 
-def _import_file(filepath: Union[str, PathLike]) -> ModuleType:
+
+def _import_file(filepath: Union[str, PathLike[str]]) -> ModuleType:
     abspath = Path(filepath).resolve()
     if abspath.suffix not in (".py", ".pyw"):
         raise ValueError(f"Not a Python source file: {abspath}")
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 668c95a7b..f03cf997a 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -4,9 +4,10 @@ Scrapy Shell
 See documentation in docs/topics/shell.rst
 """
 
-from argparse import ArgumentParser, Namespace
+from __future__ import annotations
+
 from threading import Thread
-from typing import Any, Dict, List, Type
+from typing import TYPE_CHECKING, Any, Dict, List, Type
 
 from scrapy import Spider
 from scrapy.commands import ScrapyCommand
@@ -15,6 +16,9 @@ from scrapy.shell import Shell
 from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 from scrapy.utils.url import guess_scheme
 
+if TYPE_CHECKING:
+    from argparse import ArgumentParser, Namespace
+
 
 class Command(ScrapyCommand):
     requires_project = False
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 27bc2fcba..440e0dc44 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 import re
 import sys
 from functools import wraps
 from inspect import getmembers
 from types import CoroutineType
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncGenerator,
     Callable,
@@ -16,13 +19,15 @@ from typing import (
 )
 from unittest import TestCase, TestResult
 
-from twisted.python.failure import Failure
-
-from scrapy import Spider
 from scrapy.http import Request, Response
 from scrapy.utils.python import get_spec
 from scrapy.utils.spider import iterate_spider_output
 
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
+    from scrapy import Spider
+
 
 class Contract:
     """Abstract class for contracts"""
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 41f729ed9..6786d7acf 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -25,15 +25,16 @@ from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.http import Response
 from scrapy.resolver import dnscache
-from scrapy.settings import BaseSettings
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.httpobj import urlparse_cached
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.settings import BaseSettings
+
 
 _T = TypeVar("_T")
 
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 9f6edb630..2b388a9f5 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -21,8 +21,6 @@ from scrapy.core.downloader.tls import (
     ScrapyClientTLSOptions,
     openssl_methods,
 )
-from scrapy.crawler import Crawler
-from scrapy.settings import BaseSettings
 from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
@@ -31,6 +29,9 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
 
 @implementer(IPolicyForHTTPS)
 class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index ebc4898b5..70d356b83 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -17,17 +17,19 @@ from typing import (
 )
 
 from twisted.internet import defer
-from twisted.internet.defer import Deferred
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured, NotSupported
-from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.py
index a7ae56a85..bf6879521 100644
--- a/scrapy/core/downloader/handlers/datauri.py
+++ b/scrapy/core/downloader/handlers/datauri.py
@@ -1,12 +1,16 @@
-from typing import Any, Dict
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Dict
 
 from w3lib.url import parse_data_uri
 
-from scrapy import Request, Spider
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+
 
 class DataURIDownloadHandler:
     lazy = False
diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 17dd7483b..d55c516f0 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -1,12 +1,17 @@
+from __future__ import annotations
+
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from w3lib.url import file_uri_to_path
 
-from scrapy import Request, Spider
-from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.http import Response
+
 
 class FileDownloadHandler:
     lazy = False
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 724717ffd..69c2d88e1 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -35,23 +35,25 @@ from io import BytesIO
 from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional
 from urllib.parse import unquote
 
-from twisted.internet.defer import Deferred
 from twisted.internet.protocol import ClientCreator, Protocol
 from twisted.protocols.ftp import CommandFailed, FTPClient
-from twisted.python.failure import Failure
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+    from twisted.python.failure import Failure
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
 
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename: Optional[str] = None):
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 3c4e48abb..98f62efcf 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -5,21 +5,21 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Type
 
-from twisted.internet.defer import Deferred
-
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
     from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
     from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.settings import BaseSettings
 
 
 class HTTP10DownloadHandler:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e2ad8f59a..c06d90f01 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -12,11 +12,9 @@ from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypedDict, TypeVar
 from urllib.parse import urldefrag, urlunparse
 
 from twisted.internet import ssl
-from twisted.internet.base import ReactorBase
 from twisted.internet.defer import CancelledError, Deferred, succeed
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
-from twisted.internet.interfaces import IConsumer
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
 from twisted.web.client import URI, Agent, HTTPConnectionPool
@@ -30,17 +28,22 @@ from zope.interface import implementer
 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
-from scrapy.crawler import Crawler
 from scrapy.exceptions import StopDownload
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
 from scrapy.utils.python import to_bytes, to_unicode
 
 if TYPE_CHECKING:
+    from twisted.internet.base import ReactorBase
+    from twisted.internet.interfaces import IConsumer
+
     # typing.NotRequired and typing.Self require Python 3.11
     from typing_extensions import NotRequired, Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index 2ac4eca86..4722c612d 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -4,25 +4,27 @@ from time import time
 from typing import TYPE_CHECKING, Optional
 from urllib.parse import urldefrag
 
-from twisted.internet.base import DelayedCall
-from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
-from twisted.web.iweb import IPolicyForHTTPS
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
-from scrapy.crawler import Crawler
-from scrapy.http import Request, Response
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
+    from twisted.internet.base import DelayedCall
+    from twisted.internet.defer import Deferred
+    from twisted.web.iweb import IPolicyForHTTPS
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Request, Response
+    from scrapy.settings import Settings
+    from scrapy.spiders import Spider
+
 
 class H2DownloadHandler:
     def __init__(self, settings: Settings, crawler: Crawler):
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 0ad340721..edf370193 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,22 +2,23 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Optional, Type
 
-from twisted.internet.defer import Deferred
-
-from scrapy import Request, Spider
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.settings import BaseSettings
+
 
 class S3DownloadHandler:
     def __init__(
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 2d8af114f..0bdb756c8 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -6,19 +6,22 @@ See documentation in docs/topics/downloader-middleware.rst
 
 from __future__ import annotations
 
-from typing import Any, Callable, Generator, List, Union, cast
+from typing import TYPE_CHECKING, Any, Callable, Generator, List, Union, cast
 
 from twisted.internet.defer import Deferred, inlineCallbacks
-from twisted.python.failure import Failure
 
-from scrapy import Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.settings import BaseSettings
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
 
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
+    from scrapy import Spider
+    from scrapy.settings import BaseSettings
+
 
 class DownloaderMiddlewareManager(MiddlewareManager):
     component_name = "downloader middleware"
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 08a1d7c71..99502f0d2 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,18 +1,22 @@
+from __future__ import annotations
+
 import re
 from time import time
-from typing import Optional, Tuple
+from typing import TYPE_CHECKING, Optional, Tuple
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
 from twisted.internet import defer
 from twisted.internet.protocol import ClientFactory
 from twisted.web.http import HTTPClient
 
-from scrapy import Request
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    from scrapy import Request
+
 
 def _parsed_url_args(parsed: ParseResult) -> Tuple[bytes, bytes, bytes, int, bytes]:
     # Assume parsed is urlparse-d from Request.url,
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 4ffec78b9..5318cbd64 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -34,9 +34,8 @@ from scrapy.core.scraper import Scraper
 from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
 from scrapy.http import Request, Response
 from scrapy.logformatter import LogFormatter
-from scrapy.settings import BaseSettings, Settings
+from scrapy.settings import Settings
 from scrapy.signalmanager import SignalManager
-from scrapy.spiders import Spider
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import global_object_name
@@ -46,6 +45,9 @@ if TYPE_CHECKING:
     from scrapy.core.scheduler import BaseScheduler
     from scrapy.core.scraper import _HandleOutputDeferred
     from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+    from scrapy.spiders import Spider
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 999764a6e..d291a5b8a 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -1,10 +1,10 @@
+from __future__ import annotations
+
 from collections import deque
-from typing import Deque, Dict, List, Optional, Tuple
+from typing import TYPE_CHECKING, Deque, Dict, List, Optional, Tuple
 
 from twisted.internet import defer
-from twisted.internet.base import ReactorBase
 from twisted.internet.defer import Deferred
-from twisted.internet.endpoints import HostnameEndpoint
 from twisted.python.failure import Failure
 from twisted.web.client import (
     URI,
@@ -16,9 +16,15 @@ from twisted.web.error import SchemeNotSupported
 
 from scrapy.core.downloader.contextfactory import AcceptableProtocolsContextFactory
 from scrapy.core.http2.protocol import H2ClientFactory, H2ClientProtocol
-from scrapy.http.request import Request
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
+
+if TYPE_CHECKING:
+    from twisted.internet.base import ReactorBase
+    from twisted.internet.endpoints import HostnameEndpoint
+
+    from scrapy.http.request import Request
+    from scrapy.settings import Settings
+    from scrapy.spiders import Spider
+
 
 ConnectionKeyT = Tuple[bytes, bytes, int]
 
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index f2f1cb0b8..a6809102b 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import ipaddress
 import itertools
 import logging
 from collections import deque
-from ipaddress import IPv4Address, IPv6Address
-from typing import Any, Deque, Dict, List, Optional, Union
+from typing import TYPE_CHECKING, Any, Deque, Dict, List, Optional, Union
 
 from h2.config import H2Configuration
 from h2.connection import H2Connection
@@ -20,7 +21,6 @@ from h2.events import (
     WindowUpdated,
 )
 from h2.exceptions import FrameTooLargeError, H2Error
-from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.internet.interfaces import (
     IAddress,
@@ -30,14 +30,21 @@ from twisted.internet.interfaces import (
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.internet.ssl import Certificate
 from twisted.protocols.policies import TimeoutMixin
-from twisted.python.failure import Failure
-from twisted.web.client import URI
 from zope.interface import implementer
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
 from scrapy.http import Request
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
+
+if TYPE_CHECKING:
+    from ipaddress import IPv4Address, IPv6Address
+
+    from twisted.internet.defer import Deferred
+    from twisted.python.failure import Failure
+    from twisted.web.client import URI
+
+    from scrapy.settings import Settings
+    from scrapy.spiders import Spider
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 7c70e86db..a02fbb328 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 from enum import Enum
 from io import BytesIO
@@ -5,19 +7,20 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
 
 from h2.errors import ErrorCodes
 from h2.exceptions import H2Error, ProtocolError, StreamClosedError
-from hpack import HeaderTuple
 from twisted.internet.defer import CancelledError, Deferred
 from twisted.internet.error import ConnectionClosed
 from twisted.python.failure import Failure
 from twisted.web.client import ResponseFailed
 
-from scrapy.http import Request
 from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 
 if TYPE_CHECKING:
+    from hpack import HeaderTuple
+
     from scrapy.core.http2.protocol import H2ClientProtocol
+    from scrapy.http import Request
 
 
 logger = logging.getLogger(__name__)
@@ -87,7 +90,7 @@ class Stream:
         self,
         stream_id: int,
         request: Request,
-        protocol: "H2ClientProtocol",
+        protocol: H2ClientProtocol,
         download_maxsize: int = 0,
         download_warnsize: int = 0,
     ) -> None:
@@ -99,7 +102,7 @@ class Stream:
         """
         self.stream_id: int = stream_id
         self._request: Request = request
-        self._protocol: "H2ClientProtocol" = protocol
+        self._protocol: H2ClientProtocol = protocol
 
         self._download_maxsize = self._request.meta.get(
             "download_maxsize", download_maxsize
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 1e586c53a..d4286c874 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -6,14 +6,10 @@ from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, List, Optional, Type, cast
 
-from twisted.internet.defer import Deferred
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from twisted.internet.defer import Deferred  # noqa: TC002
 
-from scrapy.crawler import Crawler
-from scrapy.dupefilters import BaseDupeFilter
-from scrapy.http.request import Request
-from scrapy.pqueues import ScrapyPriorityQueue
-from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
+from scrapy.spiders import Spider  # noqa: TC001
 from scrapy.utils.job import job_dir
 from scrapy.utils.misc import build_from_crawler, load_object
 
@@ -24,6 +20,12 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.dupefilters import BaseDupeFilter
+    from scrapy.http.request import Request
+    from scrapy.pqueues import ScrapyPriorityQueue
+    from scrapy.statscollectors import StatsCollector
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index e792f8ca7..37a666605 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -10,6 +10,7 @@ import logging
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncIterable,
     Callable,
@@ -30,7 +31,6 @@ from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.settings import BaseSettings
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import (
@@ -41,6 +41,10 @@ from scrapy.utils.defer import (
 )
 from scrapy.utils.python import MutableAsyncChain, MutableChain
 
+if TYPE_CHECKING:
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
index 0e757e4be..5fc7f31a3 100644
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ b/scrapy/downloadermiddlewares/ajaxcrawl.py
@@ -6,16 +6,18 @@ from typing import TYPE_CHECKING, Union
 
 from w3lib import html
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.http import HtmlResponse, Response
-from scrapy.settings import BaseSettings
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 73c2c57fe..23140d263 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -2,24 +2,26 @@ from __future__ import annotations
 
 import logging
 from collections import defaultdict
-from http.cookiejar import Cookie
 from typing import TYPE_CHECKING, Any, DefaultDict, Iterable, Optional, Sequence, Union
 
 from tldextract import TLDExtract
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
-from scrapy.http.request import VerboseCookie
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
+    from http.cookiejar import Cookie
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http.request import VerboseCookie
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/downloadermiddlewares/defaultheaders.py b/scrapy/downloadermiddlewares/defaultheaders.py
index 58fd415b9..49b9fdc05 100644
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@@ -8,15 +8,16 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Iterable, Tuple, Union
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
 from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 class DefaultHeadersMiddleware:
     def __init__(self, headers: Iterable[Tuple[str, str]]):
diff --git a/scrapy/downloadermiddlewares/downloadtimeout.py b/scrapy/downloadermiddlewares/downloadtimeout.py
index fd7c03a38..ee7a24825 100644
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@@ -9,13 +9,14 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Union
 
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 class DownloadTimeoutMiddleware:
     def __init__(self, timeout: float = 180):
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.py
index 63490a37a..39165e155 100644
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@@ -11,14 +11,15 @@ from typing import TYPE_CHECKING, Union
 from w3lib.http import basic_auth_header
 
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
 from scrapy.utils.url import url_is_from_any_domain
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 class HttpAuthMiddleware:
     """Set Basic HTTP Authorization header
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index 971473403..8377a3c1d 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -16,19 +16,20 @@ from twisted.internet.error import (
 from twisted.web.client import ResponseFailed
 
 from scrapy import signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
-from scrapy.http.request import Request
-from scrapy.http.response import Response
-from scrapy.settings import Settings
-from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.misc import load_object
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http.request import Request
+    from scrapy.http.response import Response
+    from scrapy.settings import Settings
+    from scrapy.spiders import Spider
+    from scrapy.statscollectors import StatsCollector
+
 
 class HttpCacheMiddleware:
     DOWNLOAD_EXCEPTIONS = (
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index f3647e05f..6b0a56f7f 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -6,11 +6,9 @@ from logging import getLogger
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils._compression import (
     _DecompressionMaxSizeExceeded,
     _inflate,
@@ -24,6 +22,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = getLogger(__name__)
 
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 5b56ad449..a7af83f7d 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -9,10 +9,7 @@ from urllib.request import (  # type: ignore[attr-defined]
     proxy_bypass,
 )
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
@@ -20,6 +17,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 class HttpProxyMiddleware:
     def __init__(self, auth_encoding: Optional[str] = "latin-1"):
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index bd8dbe329..6f67e3975 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -6,15 +6,17 @@ import warnings
 from typing import TYPE_CHECKING, Set
 
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.httpobj import urlparse_cached
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 371e2fd3b..53081237c 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -6,11 +6,8 @@ from urllib.parse import urljoin
 
 from w3lib.url import safe_url_string
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import HtmlResponse, Response
-from scrapy.settings import BaseSettings
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.response import get_meta_refresh
 
@@ -18,6 +15,11 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index 0637f09d4..8d7b7293c 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -16,12 +16,8 @@ import warnings
 from logging import Logger, getLogger
 from typing import TYPE_CHECKING, Any, Optional, Tuple, Type, Union
 
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Response
-from scrapy.http.request import Request
 from scrapy.settings import BaseSettings, Settings
-from scrapy.spiders import Spider
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
@@ -30,6 +26,12 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.http.request import Request
+    from scrapy.spiders import Spider
+
+
 retry_logger = getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 6a0ecb7bf..70393576b 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -10,22 +10,24 @@ import logging
 from typing import TYPE_CHECKING, Any, Dict, Optional, Union
 
 from twisted.internet.defer import Deferred, maybeDeferred
-from twisted.python.failure import Failure
 
-from scrapy import Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.robotstxt import RobotParser
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import load_object
 
 if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.crawler import Crawler
+    from scrapy.robotstxt import RobotParser
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.py
index 444702757..0faae7b5a 100644
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@@ -4,11 +4,7 @@ from typing import TYPE_CHECKING, Dict, List, Tuple, Union
 
 from twisted.web import http
 
-from scrapy import Request, Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.python import global_object_name, to_bytes
 from scrapy.utils.request import request_httprepr
 
@@ -16,6 +12,11 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.statscollectors import StatsCollector
+
 
 def get_header_size(
     headers: Dict[str, Union[List[Union[str, bytes]], Tuple[Union[str, bytes], ...]]]
diff --git a/scrapy/downloadermiddlewares/useragent.py b/scrapy/downloadermiddlewares/useragent.py
index 92f1ec897..109f1a4d9 100644
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@@ -5,13 +5,14 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Union
 
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.http import Response
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
 
 class UserAgentMiddleware:
     """This middleware allows spiders to override the user_agent"""
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index dd2420e98..ffaf783a7 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -4,11 +4,6 @@ import logging
 from pathlib import Path
 from typing import TYPE_CHECKING, Optional, Set
 
-from twisted.internet.defer import Deferred
-
-from scrapy.http.request import Request
-from scrapy.settings import BaseSettings
-from scrapy.spiders import Spider
 from scrapy.utils.job import job_dir
 from scrapy.utils.request import (
     RequestFingerprinter,
@@ -17,10 +12,15 @@ from scrapy.utils.request import (
 )
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import Request
+    from scrapy.settings import BaseSettings
+    from scrapy.spiders import Spider
 
 
 class BaseDupeFilter:
diff --git a/scrapy/extension.py b/scrapy/extension.py
index 8221b675e..8c81ab356 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -4,12 +4,16 @@ The Extension Manager
 See documentation in docs/topics/extensions.rst
 """
 
-from typing import Any, List
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List
 
 from scrapy.middleware import MiddlewareManager
-from scrapy.settings import Settings
 from scrapy.utils.conf import build_component_list
 
+if TYPE_CHECKING:
+    from scrapy.settings import Settings
+
 
 class ExtensionManager(MiddlewareManager):
     component_name = "extension"
diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 812b3553c..4627e7f98 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -10,17 +10,19 @@ import logging
 from collections import defaultdict
 from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
-from twisted.python.failure import Failure
-
 from scrapy import Request, Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
 
 if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index f3ac19623..6ef2d0382 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -8,13 +8,14 @@ from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, Optional
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
-from scrapy.statscollectors import StatsCollector
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
 
 class CoreStats:
     def __init__(self, stats: StatsCollector):
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index b360ce48d..c54871e02 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -12,17 +12,20 @@ import sys
 import threading
 import traceback
 from pdb import Pdb
-from types import FrameType
 from typing import TYPE_CHECKING, Optional
 
-from scrapy.crawler import Crawler
 from scrapy.utils.engine import format_engine_status
 from scrapy.utils.trackref import format_live_refs
 
 if TYPE_CHECKING:
+    from types import FrameType
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 941bd4b26..43c2d2815 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -31,18 +31,15 @@ from typing import (
 )
 from urllib.parse import unquote, urlparse
 
-from twisted.internet import threads
 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
-from twisted.python.failure import Failure
+from twisted.internet.threads import deferToThread
 from w3lib.url import file_uri_to_path
 from zope.interface import Interface, implementer
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.exporters import BaseItemExporter
 from scrapy.extensions.postprocessing import PostProcessingManager
-from scrapy.settings import BaseSettings, Settings
+from scrapy.settings import Settings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -54,11 +51,14 @@ from scrapy.utils.python import without_none_values
 
 if TYPE_CHECKING:
     from _typeshed import OpenBinaryMode
+    from twisted.python.failure import Failure
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
-logger = logging.getLogger(__name__)
+    from scrapy.crawler import Crawler
+    from scrapy.exporters import BaseItemExporter
+    from scrapy.settings import BaseSettings
 
 try:
     import boto3  # noqa: F401
@@ -67,6 +67,9 @@ try:
 except ImportError:
     IS_BOTO3_AVAILABLE = False
 
+
+logger = logging.getLogger(__name__)
+
 UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]
 
 _StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
@@ -160,7 +163,7 @@ class BlockingFeedStorage:
         return NamedTemporaryFile(prefix="feed-", dir=path)
 
     def store(self, file: IO[bytes]) -> Optional[Deferred]:
-        return threads.deferToThread(self._store_in_thread, file)
+        return deferToThread(self._store_in_thread, file)
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
         raise NotImplementedError
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index b7219bf07..448d5f1ab 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import gzip
 import logging
 import os
@@ -13,10 +15,7 @@ from weakref import WeakKeyDictionary
 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
 
 from scrapy.http import Headers, Response
-from scrapy.http.request import Request
 from scrapy.responsetypes import responsetypes
-from scrapy.settings import BaseSettings
-from scrapy.spiders import Spider
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
@@ -26,6 +25,10 @@ if TYPE_CHECKING:
     # typing.Concatenate requires Python 3.10
     from typing_extensions import Concatenate
 
+    from scrapy.http.request import Request
+    from scrapy.settings import BaseSettings
+    from scrapy.spiders import Spider
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 2388afa75..c4f43482d 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -6,14 +6,16 @@ from typing import TYPE_CHECKING, Optional, Tuple, Union
 from twisted.internet import task
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index f304e1bf2..3cbbb64e5 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -10,15 +10,16 @@ import gc
 from typing import TYPE_CHECKING
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.trackref import live_refs
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
 
 class MemoryDebugger:
     def __init__(self, stats: StatsCollector):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de06b24d..25f63ecc6 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -16,7 +16,6 @@ from typing import TYPE_CHECKING, List
 from twisted.internet import task
 
 from scrapy import signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.utils.engine import get_engine_status
@@ -25,6 +24,9 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 9567f948a..80c0a3b26 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -8,15 +8,17 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 from twisted.internet import task
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index c6eb20277..567efd7a1 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -5,7 +5,6 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Optional
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
@@ -13,6 +12,8 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+
 
 class SpiderState:
     """Store and load spider state during a scraping job"""
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 20b8f910c..e43de6f5c 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,18 +8,19 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, List, Optional
 
-from twisted.internet.defer import Deferred
-
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
-from scrapy.statscollectors import StatsCollector
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
 
 class StatsMailer:
     def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index 00c69434c..c4e01b3d9 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -26,7 +26,6 @@ except (ImportError, SyntaxError):
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy import signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.decorators import defers
 from scrapy.utils.engine import print_engine_status
@@ -36,6 +35,10 @@ from scrapy.utils.trackref import print_live_refs
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
+
+    from scrapy.crawler import Crawler
+
+
 logger = logging.getLogger(__name__)
 
 # signal to update telnet variables
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index 217e61a81..6ce9ce63a 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -4,15 +4,17 @@ import logging
 from typing import TYPE_CHECKING, Optional, Tuple
 
 from scrapy import Request, Spider, signals
-from scrapy.core.downloader import Slot
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.core.downloader import Slot
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 8af89c74f..cc88a9420 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -17,8 +17,6 @@ from typing import (
     cast,
 )
 
-from scrapy import Request
-from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
@@ -26,6 +24,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request
+    from scrapy.http import Response
+
+
 # Defined in the http.cookiejar module, but undocumented:
 # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index ea98ed795..a8c242e8b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -28,13 +28,14 @@ from lxml.html import TextareaElement  # nosec
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
-from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import is_listlike, to_bytes
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.http.response.text import TextResponse
+
 
 FormdataVType = Union[str, Iterable[str]]
 FormdataKVType = Tuple[str, FormdataVType]
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 684439097..ff3581abb 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,7 +7,6 @@ See documentation in docs/topics/request-response.rst
 
 from __future__ import annotations
 
-from ipaddress import IPv4Address, IPv6Address
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -26,8 +25,6 @@ from typing import (
 )
 from urllib.parse import urljoin
 
-from twisted.internet.ssl import Certificate
-
 from scrapy.exceptions import NotSupported
 from scrapy.http.headers import Headers
 from scrapy.http.request import CookiesT, Request
@@ -35,6 +32,10 @@ from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
 if TYPE_CHECKING:
+    from ipaddress import IPv4Address, IPv6Address
+
+    from twisted.internet.ssl import Certificate
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index df4d90829..0635f744f 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -35,15 +35,16 @@ from w3lib.encoding import (
 )
 from w3lib.html import strip_html5_whitespace
 
-from scrapy.http.request import CookiesT, Request
 from scrapy.http.response import Response
 from scrapy.link import Link
 from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
 if TYPE_CHECKING:
+    from scrapy.http.request import CookiesT, Request
     from scrapy.selector import Selector, SelectorList
 
+
 _NONE = object()
 
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 33a10cd6c..d27a132b3 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -2,10 +2,13 @@
 Link extractor based on lxml.html
 """
 
+from __future__ import annotations
+
 import logging
 import operator
 from functools import partial
 from typing import (
+    TYPE_CHECKING,
     Any,
     Callable,
     Iterable,
@@ -20,13 +23,10 @@ from typing import (
 from urllib.parse import urljoin, urlparse
 
 from lxml import etree  # nosec
-from lxml.html import HtmlElement  # nosec
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
-from scrapy import Selector
-from scrapy.http import TextResponse
 from scrapy.link import Link
 from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
@@ -34,6 +34,13 @@ from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
 from scrapy.utils.url import url_has_any_extension, url_is_from_any_domain
 
+if TYPE_CHECKING:
+    from lxml.html import HtmlElement  # nosec
+
+    from scrapy import Selector
+    from scrapy.http import TextResponse
+
+
 logger = logging.getLogger(__name__)
 
 # from lxml/src/lxml/html/__init__.py
diff --git a/scrapy/loader/__init__.py b/scrapy/loader/__init__.py
index db0b4820f..9644cc093 100644
--- a/scrapy/loader/__init__.py
+++ b/scrapy/loader/__init__.py
@@ -4,14 +4,18 @@ Item Loader
 See documentation in docs/topics/loaders.rst
 """
 
-from typing import Any, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Optional
 
 import itemloaders
 
-from scrapy.http import TextResponse
 from scrapy.item import Item
 from scrapy.selector import Selector
 
+if TYPE_CHECKING:
+    from scrapy.http import TextResponse
+
 
 class ItemLoader(itemloaders.ItemLoader):
     """
@@ -91,7 +95,7 @@ class ItemLoader(itemloaders.ItemLoader):
         selector: Optional[Selector] = None,
         response: Optional[TextResponse] = None,
         parent: Optional[itemloaders.ItemLoader] = None,
-        **context: Any
+        **context: Any,
     ):
         if selector is None and response is not None:
             try:
diff --git a/scrapy/logformatter.py b/scrapy/logformatter.py
index 42a03b560..601209fb0 100644
--- a/scrapy/logformatter.py
+++ b/scrapy/logformatter.py
@@ -6,8 +6,9 @@ from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, TypedDict, Union
 
 from twisted.python.failure import Failure
 
-from scrapy import Request, Spider
-from scrapy.http import Response
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from scrapy import Request, Spider  # noqa: TC001
+from scrapy.http import Response  # noqa: TC001
 from scrapy.utils.request import referer_str
 
 if TYPE_CHECKING:
diff --git a/scrapy/mail.py b/scrapy/mail.py
index f4ce2800c..3ea20e831 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -30,20 +30,22 @@ from typing import (
 from twisted import version as twisted_version
 from twisted.internet import ssl
 from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
 from twisted.python.versions import Version
 
-from scrapy.settings import BaseSettings
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
     # imports twisted.internet.reactor
     from twisted.mail.smtp import ESMTPSenderFactory
+    from twisted.python.failure import Failure
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index f60c726f9..ea5488ba1 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -17,19 +17,19 @@ from typing import (
     cast,
 )
 
-from twisted.internet.defer import Deferred
-
-from scrapy import Spider
 from scrapy.exceptions import NotConfigured
-from scrapy.settings import Settings
 from scrapy.utils.defer import process_chain, process_parallel
 from scrapy.utils.misc import build_from_crawler, build_from_settings, load_object
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
     from scrapy.crawler import Crawler
+    from scrapy.settings import Settings
 
 
 logger = logging.getLogger(__name__)
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 21d649e3c..480a5a58c 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -6,16 +6,18 @@ See documentation in docs/item-pipeline.rst
 
 from __future__ import annotations
 
-from typing import Any, List
+from typing import TYPE_CHECKING, Any, List
 
-from twisted.internet.defer import Deferred
-
-from scrapy import Spider
 from scrapy.middleware import MiddlewareManager
-from scrapy.settings import Settings
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import deferred_f_from_coro_f
 
+if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
+    from scrapy import Spider
+    from scrapy.settings import Settings
+
 
 class ItemPipelineManager(MiddlewareManager):
     component_name = "item pipeline"
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 85a8c77da..1a13aeaf2 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -16,7 +16,6 @@ from collections import defaultdict
 from contextlib import suppress
 from ftplib import FTP
 from io import BytesIO
-from os import PathLike
 from pathlib import Path
 from typing import (
     IO,
@@ -38,11 +37,9 @@ from typing import (
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
-from twisted.internet import defer, threads
-from twisted.internet.defer import Deferred
-from twisted.python.failure import Failure
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.threads import deferToThread
 
-from scrapy import Spider
 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
@@ -56,9 +53,15 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 
 if TYPE_CHECKING:
+    from os import PathLike
+
+    from twisted.python.failure import Failure
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+
 
 logger = logging.getLogger(__name__)
 
@@ -210,7 +213,7 @@ class S3FilesStore:
         key_name = f"{self.prefix}{path}"
         return cast(
             "Deferred[Dict[str, Any]]",
-            threads.deferToThread(
+            deferToThread(
                 self.s3_client.head_object, Bucket=self.bucket, Key=key_name  # type: ignore[attr-defined]
             ),
         )
@@ -229,7 +232,7 @@ class S3FilesStore:
         extra = self._headers_to_botocore_kwargs(self.HEADERS)
         if headers:
             extra.update(self._headers_to_botocore_kwargs(headers))
-        return threads.deferToThread(
+        return deferToThread(
             self.s3_client.put_object,  # type: ignore[attr-defined]
             Bucket=self.bucket,
             Key=key_name,
@@ -326,9 +329,7 @@ class GCSFilesStore:
         blob_path = self._get_blob_path(path)
         return cast(
             Deferred[StatInfo],
-            threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(
-                _onsuccess
-            ),
+            deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess),
         )
 
     def _get_content_type(self, headers: Optional[Dict[str, str]]) -> str:
@@ -351,7 +352,7 @@ class GCSFilesStore:
         blob = self.bucket.blob(blob_path)
         blob.cache_control = self.CACHE_CONTROL
         blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
-        return threads.deferToThread(
+        return deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),
             content_type=self._get_content_type(headers),
@@ -388,7 +389,7 @@ class FTPFilesStore:
         headers: Optional[Dict[str, str]] = None,
     ) -> Deferred[Any]:
         path = f"{self.basedir}/{path}"
-        return threads.deferToThread(
+        return deferToThread(
             ftp_store_file,
             path=path,
             file=buf,
@@ -418,7 +419,7 @@ class FTPFilesStore:
             except Exception:
                 return {}
 
-        return cast("Deferred[StatInfo]", threads.deferToThread(_stat_file, path))
+        return cast("Deferred[StatInfo]", deferToThread(_stat_file, path))
 
 
 class FilesPipeline(MediaPipeline):
@@ -553,8 +554,8 @@ class FilesPipeline(MediaPipeline):
             }
 
         path = self.file_path(request, info=info, item=item)
-        # defer.maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
-        dfd: Deferred[StatInfo] = defer.maybeDeferred(self.store.stat_file, path, info)  # type: ignore[arg-type]
+        # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
+        dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info)  # type: ignore[arg-type]
         dfd2: Deferred[Optional[FileInfo]] = dfd.addCallback(_onsuccess)
         dfd2.addErrback(lambda _: None)
         dfd2.addErrback(
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 27a57b17c..166f81314 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -11,7 +11,6 @@ import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
-from os import PathLike
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -28,7 +27,6 @@ from typing import (
 
 from itemadapter import ItemAdapter
 
-from scrapy import Spider
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
@@ -40,15 +38,20 @@ from scrapy.pipelines.files import (
     S3FilesStore,
     _md5sum,
 )
-from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.python import get_func_args, to_bytes
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
+    from os import PathLike
+
     from PIL import Image
+
+    # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.pipelines.media import FileInfoOrError, MediaPipeline
+
 
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 09e95cf5d..ea36a9e8a 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -25,21 +25,23 @@ from typing import (
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
 
-from scrapy import Spider
-from scrapy.crawler import Crawler
-from scrapy.http import Response
 from scrapy.http.request import NO_CALLBACK, Request
 from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import defer_result, mustbe_deferred
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.request import RequestFingerprinter
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.utils.request import RequestFingerprinter
+
+
 _T = TypeVar("_T")
 
 
diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index ba7cd716b..d5eedf9b1 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type
 
 from twisted.internet import defer
 from twisted.internet.base import ReactorBase, ThreadedResolver
-from twisted.internet.defer import Deferred
 from twisted.internet.interfaces import (
     IAddress,
     IHostnameResolver,
@@ -17,6 +16,8 @@ from zope.interface.declarations import implementer, provider
 from scrapy.utils.datatypes import LocalCache
 
 if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index a33f73306..0d282dc37 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -6,7 +6,6 @@ from abc import ABCMeta, abstractmethod
 from typing import TYPE_CHECKING, Optional, Union
 from warnings import warn
 
-from scrapy import Spider
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
@@ -14,8 +13,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
     from scrapy.crawler import Crawler
 
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index ea1db03f1..6703c569f 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -4,7 +4,6 @@ import copy
 import json
 from importlib import import_module
 from pprint import pformat
-from types import ModuleType
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -27,6 +26,8 @@ from scrapy.settings import default_settings
 _SettingsKeyT = Union[bool, float, int, str, None]
 
 if TYPE_CHECKING:
+    from types import ModuleType
+
     # https://github.com/python/typing/issues/445#issuecomment-1131458824
     from _typeshed import SupportsItems
 
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index f6df191d8..3d37b8235 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -1,10 +1,14 @@
-from typing import Any, List, Tuple
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Tuple
 
 from pydispatch import dispatcher
-from twisted.internet.defer import Deferred
 
 from scrapy.utils import signal as _signal
 
+if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
 
 class SignalManager:
     def __init__(self, sender: Any = dispatcher.Anonymous):
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index d855c962c..b8fe65668 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -3,21 +3,23 @@ from __future__ import annotations
 import traceback
 import warnings
 from collections import defaultdict
-from types import ModuleType
 from typing import TYPE_CHECKING, DefaultDict, Dict, List, Tuple, Type
 
 from zope.interface import implementer
 
-from scrapy import Request, Spider
 from scrapy.interfaces import ISpiderLoader
-from scrapy.settings import BaseSettings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.spider import iter_spider_classes
 
 if TYPE_CHECKING:
+    from types import ModuleType
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request, Spider
+    from scrapy.settings import BaseSettings
+
 
 @implementer(ISpiderLoader)
 class SpiderLoader:
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 1e96654e2..c5b7f0749 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -9,15 +9,17 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable
 
-from scrapy import Spider
-from scrapy.crawler import Crawler
 from scrapy.http import Request, Response
-from scrapy.statscollectors import StatsCollector
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/spidermiddlewares/httperror.py b/scrapy/spidermiddlewares/httperror.py
index 35c869a75..ea1686c25 100644
--- a/scrapy/spidermiddlewares/httperror.py
+++ b/scrapy/spidermiddlewares/httperror.py
@@ -9,16 +9,18 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING, Any, Iterable, List, Optional
 
-from scrapy import Spider
-from scrapy.crawler import Crawler
 from scrapy.exceptions import IgnoreRequest
-from scrapy.http import Response
-from scrapy.settings import BaseSettings
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.crawler import Crawler
+    from scrapy.http import Response
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 50c93ac9f..379c5d0a3 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -12,10 +12,8 @@ import warnings
 from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable, Set
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
-from scrapy.statscollectors import StatsCollector
 from scrapy.utils.httpobj import urlparse_cached
 
 warnings.warn(
@@ -28,6 +26,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.statscollectors import StatsCollector
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 8af0bdf5b..d35cf8f71 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -23,10 +23,8 @@ from urllib.parse import urlparse
 from w3lib.url import safe_url_string
 
 from scrapy import Spider, signals
-from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
@@ -35,6 +33,10 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
+
+
 LOCAL_SCHEMES: Tuple[str, ...] = (
     "about",
     "blob",
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index e2aa554a7..34df54ca7 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -9,15 +9,17 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING, Any, AsyncIterable, Iterable
 
-from scrapy import Spider
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
-from scrapy.settings import BaseSettings
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Spider
+    from scrapy.settings import BaseSettings
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index a0898a0cf..ce0f1bbaa 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,10 +1,14 @@
-from typing import Any, Iterable, Optional, cast
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Iterable, Optional, cast
 
 from scrapy import Request
-from scrapy.http import Response
 from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
+if TYPE_CHECKING:
+    from scrapy.http import Response
+
 
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 6f80ee388..d3e7896c5 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -6,20 +6,22 @@ from __future__ import annotations
 
 import marshal
 import pickle  # nosec
-from os import PathLike
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
 
 from queuelib import queue
 
-from scrapy import Request
-from scrapy.crawler import Crawler
 from scrapy.utils.request import request_from_dict
 
 if TYPE_CHECKING:
+    from os import PathLike
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy import Request
+    from scrapy.crawler import Crawler
+
 
 def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
     class DirectoriesCreated(queue_class):  # type: ignore[valid-type,misc]
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index ab571a3ab..88e72f366 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -2,15 +2,17 @@
 Scrapy extension for collecting scraping stats
 """
 
+from __future__ import annotations
+
 import logging
 import pprint
 from typing import TYPE_CHECKING, Any, Dict, Optional
 
-from scrapy import Spider
-
 if TYPE_CHECKING:
+    from scrapy import Spider
     from scrapy.crawler import Crawler
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -18,7 +20,7 @@ StatsT = Dict[str, Any]
 
 
 class StatsCollector:
-    def __init__(self, crawler: "Crawler"):
+    def __init__(self, crawler: Crawler):
         self._dump: bool = crawler.settings.getbool("STATS_DUMP")
         self._stats: StatsT = {}
 
@@ -67,7 +69,7 @@ class StatsCollector:
 
 
 class MemoryStatsCollector(StatsCollector):
-    def __init__(self, crawler: "Crawler"):
+    def __init__(self, crawler: Crawler):
         super().__init__(crawler)
         self.spider_stats: Dict[str, StatsT] = {}
 
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 7e82dd519..2240f0b58 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -4,8 +4,8 @@ import warnings
 from functools import wraps
 from typing import TYPE_CHECKING, Any, Callable, TypeVar
 
-from twisted.internet import defer, threads
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.threads import deferToThread
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -48,7 +48,7 @@ def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
 
     @wraps(func)
     def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
-        return defer.maybeDeferred(func, *a, **kw)
+        return maybeDeferred(func, *a, **kw)
 
     return wrapped
 
@@ -60,6 +60,6 @@ def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
 
     @wraps(func)
     def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
-        return threads.deferToThread(func, *a, **kw)
+        return deferToThread(func, *a, **kw)
 
     return wrapped
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 877eb4388..1d578e8a3 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -34,12 +34,13 @@ from twisted.internet import defer
 from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
 from twisted.internet.task import Cooperator
 from twisted.python import failure
-from twisted.python.failure import Failure
 
 from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning
 from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed
 
 if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
     # typing.Concatenate and typing.ParamSpec require Python 3.10
     from typing_extensions import Concatenate, ParamSpec
 
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index fdcf484d4..770ee0b1b 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -4,9 +4,10 @@ from __future__ import annotations
 
 # used in global tests code
 from time import time  # noqa: F401
-from typing import Any, List, Tuple
+from typing import TYPE_CHECKING, Any, List, Tuple
 
-from scrapy.core.engine import ExecutionEngine
+if TYPE_CHECKING:
+    from scrapy.core.engine import ExecutionEngine
 
 
 def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:
diff --git a/scrapy/utils/gz.py b/scrapy/utils/gz.py
index 2e487d88b..85324361c 100644
--- a/scrapy/utils/gz.py
+++ b/scrapy/utils/gz.py
@@ -1,11 +1,15 @@
+from __future__ import annotations
+
 import struct
 from gzip import GzipFile
 from io import BytesIO
-
-from scrapy.http import Response
+from typing import TYPE_CHECKING
 
 from ._compression import _CHUNK_SIZE, _DecompressionMaxSizeExceeded
 
+if TYPE_CHECKING:
+    from scrapy.http import Response
+
 
 def gunzip(data: bytes, *, max_size: int = 0) -> bytes:
     """Gunzip the given data and return as much data as possible.
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index d502e8910..3cf9585ec 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -1,12 +1,16 @@
 """Helper functions for scrapy.http objects (Request, Response)"""
 
-from typing import Union
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Union
 from urllib.parse import ParseResult, urlparse
 from weakref import WeakKeyDictionary
 
-from scrapy.http import Request, Response
+if TYPE_CHECKING:
+    from scrapy.http import Request, Response
 
-_urlparse_cache: "WeakKeyDictionary[Union[Request, Response], ParseResult]" = (
+
+_urlparse_cache: WeakKeyDictionary[Union[Request, Response], ParseResult] = (
     WeakKeyDictionary()
 )
 
diff --git a/scrapy/utils/job.py b/scrapy/utils/job.py
index e230e4235..488c7994b 100644
--- a/scrapy/utils/job.py
+++ b/scrapy/utils/job.py
@@ -1,7 +1,10 @@
-from pathlib import Path
-from typing import Optional
+from __future__ import annotations
 
-from scrapy.settings import BaseSettings
+from pathlib import Path
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    from scrapy.settings import BaseSettings
 
 
 def job_dir(settings: BaseSettings) -> Optional[str]:
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index cbfd170ed..439b065a9 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -21,12 +21,13 @@ from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
 
 import scrapy
-from scrapy.logformatter import LogFormatterResult
 from scrapy.settings import Settings, _SettingsKeyT
 from scrapy.utils.versions import scrapy_components_versions
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
+    from scrapy.logformatter import LogFormatterResult
+
 
 logger = logging.getLogger(__name__)
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 3d11c1035..3c787e50f 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,6 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from types import ModuleType
 from typing import (
     IO,
     TYPE_CHECKING,
@@ -35,10 +34,13 @@ from scrapy.item import Item
 from scrapy.utils.datatypes import LocalWeakReferencedCache
 
 if TYPE_CHECKING:
+    from types import ModuleType
+
     from scrapy import Spider
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
 
+
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
 T = TypeVar("T")
 
diff --git a/scrapy/utils/project.py b/scrapy/utils/project.py
index de3c8eaf9..efb6af299 100644
--- a/scrapy/utils/project.py
+++ b/scrapy/utils/project.py
@@ -1,7 +1,8 @@
+from __future__ import annotations
+
 import os
 import warnings
 from importlib import import_module
-from os import PathLike
 from pathlib import Path
 from typing import Union
 
@@ -46,7 +47,7 @@ def project_data_dir(project: str = "default") -> str:
     return str(d)
 
 
-def data_path(path: Union[str, PathLike], createdir: bool = False) -> str:
+def data_path(path: Union[str, os.PathLike[str]], createdir: bool = False) -> str:
     """
     Return the given path joined with the .scrapy data directory.
     If given an absolute path, return it unmodified.
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 5af6d22eb..a627db601 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import asyncio
 import sys
-from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
 from typing import (
     TYPE_CHECKING,
@@ -20,13 +19,16 @@ from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
 from twisted.internet.base import DelayedCall
-from twisted.internet.protocol import ServerFactory
-from twisted.internet.tcp import Port
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
 
 if TYPE_CHECKING:
+    from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
+
+    from twisted.internet.protocol import ServerFactory
+    from twisted.internet.tcp import Port
+
     # typing.ParamSpec requires Python 3.10
     from typing_extensions import ParamSpec
 
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index b05135c04..ce754fad3 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import inspect
 import logging
-from types import CoroutineType, ModuleType
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -16,16 +15,19 @@ from typing import (
     overload,
 )
 
-from twisted.internet.defer import Deferred
-
-from scrapy import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
 
 if TYPE_CHECKING:
+    from types import CoroutineType, ModuleType
+
+    from twisted.internet.defer import Deferred
+
+    from scrapy import Request
     from scrapy.spiderloader import SpiderLoader
 
+
 logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index d520ef809..95611ebd9 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -1,12 +1,16 @@
-from typing import Any, Optional
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, Optional
 
 import OpenSSL._util as pyOpenSSLutil
 import OpenSSL.SSL
 import OpenSSL.version
-from OpenSSL.crypto import X509Name
 
 from scrapy.utils.python import to_unicode
 
+if TYPE_CHECKING:
+    from OpenSSL.crypto import X509Name
+
 
 def ffi_buf_to_string(buf: Any) -> str:
     return to_unicode(pyOpenSSLutil.ffi.string(buf))
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 6b22f3bfa..08f3f2dc9 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -1,10 +1,14 @@
 """Helper functions for working with templates"""
 
+from __future__ import annotations
+
 import re
 import string
-from os import PathLike
 from pathlib import Path
-from typing import Any, Union
+from typing import TYPE_CHECKING, Any, Union
+
+if TYPE_CHECKING:
+    from os import PathLike
 
 
 def render_templatefile(path: Union[str, PathLike], **kwargs: Any) -> None:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 268d8d4be..fe2bfa042 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -2,21 +2,36 @@
 This module contains some assorted functions used in tests
 """
 
+from __future__ import annotations
+
 import asyncio
 import os
 from importlib import import_module
 from pathlib import Path
 from posixpath import split
-from typing import Any, Awaitable, Dict, List, Optional, Tuple, Type, TypeVar
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Awaitable,
+    Dict,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
 from unittest import TestCase, mock
 
-from twisted.internet.defer import Deferred
 from twisted.trial.unittest import SkipTest
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
 from scrapy.utils.boto import is_botocore_available
 
+if TYPE_CHECKING:
+    from twisted.internet.defer import Deferred
+
+
 _T = TypeVar("_T")
 
 
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 3bdffcaa7..8882bfc5f 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -2,12 +2,14 @@ from __future__ import annotations
 
 import os
 import sys
-from typing import Iterable, List, Optional, Tuple, cast
+from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, cast
 
 from twisted.internet.defer import Deferred
 from twisted.internet.error import ProcessTerminated
 from twisted.internet.protocol import ProcessProtocol
-from twisted.python.failure import Failure
+
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
 
 
 class ProcessTest:
diff --git a/tests/mockserver.py b/tests/mockserver.py
index 233f6b934..6ec46aa3d 100644
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import argparse
 import json
 import os
@@ -7,12 +9,11 @@ from pathlib import Path
 from shutil import rmtree
 from subprocess import PIPE, Popen
 from tempfile import mkdtemp
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 from urllib.parse import urlencode
 
 from OpenSSL import SSL
 from twisted.internet import defer, reactor, ssl
-from twisted.internet.protocol import ServerFactory
 from twisted.internet.task import deferLater
 from twisted.names import dns, error
 from twisted.names.server import DNSServerFactory
@@ -23,6 +24,9 @@ from twisted.web.util import redirectTo
 
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    from twisted.internet.protocol import ServerFactory
+
 
 def getarg(request, name, default=None, type=None):
     if name in request.args:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 253987e15..ea3ed3b05 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import bz2
 import csv
 import gzip
@@ -14,10 +16,9 @@ from collections import defaultdict
 from contextlib import ExitStack
 from io import BytesIO
 from logging import getLogger
-from os import PathLike
 from pathlib import Path
 from string import ascii_letters, digits
-from typing import Union
+from typing import TYPE_CHECKING, Union
 from unittest import mock
 from urllib.parse import quote, urljoin
 from urllib.request import pathname2url
@@ -53,6 +54,9 @@ from scrapy.utils.test import get_crawler, mock_google_cloud_storage, skip_if_no
 from tests.mockserver import MockFTPServer, MockServer
 from tests.spiders import ItemSpider
 
+if TYPE_CHECKING:
+    from os import PathLike
+
 
 def path_to_url(path):
     return urljoin("file:", pathname2url(str(path)))
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 995c02a1a..7ea3fe8c9 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import random
 import re
@@ -6,7 +8,7 @@ import string
 from ipaddress import IPv4Address
 from pathlib import Path
 from tempfile import mkdtemp
-from typing import Dict
+from typing import TYPE_CHECKING, Dict
 from unittest import mock, skipIf
 from urllib.parse import urlencode
 
@@ -20,7 +22,6 @@ from twisted.internet.defer import (
 from twisted.internet.endpoints import SSL4ClientEndpoint, SSL4ServerEndpoint
 from twisted.internet.error import TimeoutError
 from twisted.internet.ssl import Certificate, PrivateCertificate, optionsForClientTLS
-from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase
 from twisted.web.client import URI, ResponseFailed
 from twisted.web.http import H2_ENABLED
@@ -33,6 +34,9 @@ from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from tests.mockserver import LeafResource, Status, ssl_context_factory
 
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
 
 def generate_random_string(size):
     return "".join(random.choices(string.ascii_uppercase + string.digits, k=size))

From e47110f9a5a16f0628e53e16b9cb5f6a4f9721d3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 26 Jun 2024 13:01:43 +0500
Subject: [PATCH 62/82] Add parameteres to most Deferred instances. (#6414)

---
 scrapy/commands/parse.py                  |  6 +-
 scrapy/core/downloader/handlers/http10.py |  3 +-
 scrapy/core/http2/agent.py                | 26 +++++----
 scrapy/core/http2/protocol.py             | 18 ++++--
 scrapy/core/http2/stream.py               |  6 +-
 scrapy/core/scraper.py                    |  6 +-
 scrapy/core/spidermw.py                   |  7 ++-
 scrapy/crawler.py                         | 33 +++++++----
 scrapy/downloadermiddlewares/robotstxt.py | 26 ++++++---
 scrapy/dupefilters.py                     |  4 +-
 scrapy/extensions/feedexport.py           | 16 ++---
 scrapy/extensions/statsmailer.py          |  2 +-
 scrapy/mail.py                            | 12 ++--
 scrapy/middleware.py                      | 27 ++++++---
 scrapy/shell.py                           | 10 +++-
 scrapy/signalmanager.py                   |  4 +-
 scrapy/spiders/__init__.py                | 10 ++--
 scrapy/utils/defer.py                     | 71 ++++++++++++-----------
 scrapy/utils/signal.py                    | 24 ++++----
 scrapy/utils/test.py                      |  3 +-
 scrapy/utils/testproc.py                  |  4 +-
 21 files changed, 190 insertions(+), 128 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index e6c5e2a47..1265aa38e 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -153,7 +153,7 @@ class Command(BaseRunSpiderCommand):
     @overload
     def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
 
-    def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred]:
+    def iterate_spider_output(self, result: Any) -> Union[Iterable[Any], Deferred[Any]]:
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
@@ -233,7 +233,7 @@ class Command(BaseRunSpiderCommand):
         response: Response,
         callback: Callable,
         cb_kwargs: Optional[Dict[str, Any]] = None,
-    ) -> Deferred:
+    ) -> Deferred[Any]:
         cb_kwargs = cb_kwargs or {}
         d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
         return d
@@ -345,7 +345,7 @@ class Command(BaseRunSpiderCommand):
     def prepare_request(
         self, spider: Spider, request: Request, opts: argparse.Namespace
     ) -> Request:
-        def callback(response: Response, **cb_kwargs: Any) -> Deferred:
+        def callback(response: Response, **cb_kwargs: Any) -> Deferred[List[Any]]:
             # memorize first request
             if not self.first_response:
                 self.first_response = response
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 98f62efcf..8d7b0635c 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -10,6 +10,7 @@ from scrapy.utils.python import to_unicode
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
+    from twisted.internet.interfaces import IConnector
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
@@ -45,7 +46,7 @@ class HTTP10DownloadHandler:
         self._connect(factory)
         return factory.deferred
 
-    def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
+    def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:
         from twisted.internet import reactor
 
         host, port = to_unicode(factory.host), factory.port
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index d291a5b8a..640fb7129 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
     from twisted.internet.base import ReactorBase
     from twisted.internet.endpoints import HostnameEndpoint
 
-    from scrapy.http.request import Request
+    from scrapy.http import Request, Response
     from scrapy.settings import Settings
     from scrapy.spiders import Spider
 
@@ -39,16 +39,18 @@ class H2ConnectionPool:
         self._connections: Dict[ConnectionKeyT, H2ClientProtocol] = {}
 
         # Save all requests that arrive before the connection is established
-        self._pending_requests: Dict[ConnectionKeyT, Deque[Deferred]] = {}
+        self._pending_requests: Dict[
+            ConnectionKeyT, Deque[Deferred[H2ClientProtocol]]
+        ] = {}
 
     def get_connection(
         self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
-    ) -> Deferred:
+    ) -> Deferred[H2ClientProtocol]:
         if key in self._pending_requests:
             # Received a request while connecting to remote
             # Create a deferred which will fire with the H2ClientProtocol
             # instance
-            d: Deferred = Deferred()
+            d: Deferred[H2ClientProtocol] = Deferred()
             self._pending_requests[key].append(d)
             return d
 
@@ -63,17 +65,17 @@ class H2ConnectionPool:
 
     def _new_connection(
         self, key: ConnectionKeyT, uri: URI, endpoint: HostnameEndpoint
-    ) -> Deferred:
+    ) -> Deferred[H2ClientProtocol]:
         self._pending_requests[key] = deque()
 
-        conn_lost_deferred: Deferred = Deferred()
+        conn_lost_deferred: Deferred[List[BaseException]] = Deferred()
         conn_lost_deferred.addCallback(self._remove_connection, key)
 
         factory = H2ClientFactory(uri, self.settings, conn_lost_deferred)
         conn_d = endpoint.connect(factory)
         conn_d.addCallback(self.put_connection, key)
 
-        d: Deferred = Deferred()
+        d: Deferred[H2ClientProtocol] = Deferred()
         self._pending_requests[key].append(d)
         return d
 
@@ -141,7 +143,7 @@ class H2Agent:
         """
         return uri.scheme, uri.host, uri.port
 
-    def request(self, request: Request, spider: Spider) -> Deferred:
+    def request(self, request: Request, spider: Spider) -> Deferred[Response]:
         uri = URI.fromBytes(bytes(request.url, encoding="utf-8"))
         try:
             endpoint = self.get_endpoint(uri)
@@ -149,9 +151,11 @@ class H2Agent:
             return defer.fail(Failure())
 
         key = self.get_key(uri)
-        d = self._pool.get_connection(key, uri, endpoint)
-        d.addCallback(lambda conn: conn.request(request, spider))
-        return d
+        d: Deferred[H2ClientProtocol] = self._pool.get_connection(key, uri, endpoint)
+        d2: Deferred[Response] = d.addCallback(
+            lambda conn: conn.request(request, spider)
+        )
+        return d2
 
 
 class ScrapyProxyH2Agent(H2Agent):
diff --git a/scrapy/core/http2/protocol.py b/scrapy/core/http2/protocol.py
index a6809102b..8aebbaab4 100644
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@@ -33,7 +33,7 @@ from twisted.protocols.policies import TimeoutMixin
 from zope.interface import implementer
 
 from scrapy.core.http2.stream import Stream, StreamCloseReason
-from scrapy.http import Request
+from scrapy.http import Request, Response
 
 if TYPE_CHECKING:
     from ipaddress import IPv4Address, IPv6Address
@@ -88,7 +88,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
     IDLE_TIMEOUT = 240
 
     def __init__(
-        self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
+        self,
+        uri: URI,
+        settings: Settings,
+        conn_lost_deferred: Deferred[List[BaseException]],
     ) -> None:
         """
         Arguments:
@@ -99,7 +102,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
             conn_lost_deferred -- Deferred fires with the reason: Failure to notify
                 that connection was lost
         """
-        self._conn_lost_deferred = conn_lost_deferred
+        self._conn_lost_deferred: Deferred[List[BaseException]] = conn_lost_deferred
 
         config = H2Configuration(client_side=True, header_encoding="utf-8")
         self.conn = H2Connection(config=config)
@@ -215,14 +218,14 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
         data = self.conn.data_to_send()
         self.transport.write(data)
 
-    def request(self, request: Request, spider: Spider) -> Deferred:
+    def request(self, request: Request, spider: Spider) -> Deferred[Response]:
         if not isinstance(request, Request):
             raise TypeError(
                 f"Expected scrapy.http.Request, received {request.__class__.__qualname__}"
             )
 
         stream = self._new_stream(request, spider)
-        d = stream.get_response()
+        d: Deferred[Response] = stream.get_response()
 
         # Add the stream to the request pool
         self._pending_request_stream_pool.append(stream)
@@ -436,7 +439,10 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
 @implementer(IProtocolNegotiationFactory)
 class H2ClientFactory(Factory):
     def __init__(
-        self, uri: URI, settings: Settings, conn_lost_deferred: Deferred
+        self,
+        uri: URI,
+        settings: Settings,
+        conn_lost_deferred: Deferred[List[BaseException]],
     ) -> None:
         self.uri = uri
         self.settings = settings
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index a02fbb328..d8b5cc8eb 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -20,7 +20,7 @@ if TYPE_CHECKING:
     from hpack import HeaderTuple
 
     from scrapy.core.http2.protocol import H2ClientProtocol
-    from scrapy.http import Request
+    from scrapy.http import Request, Response
 
 
 logger = logging.getLogger(__name__)
@@ -154,7 +154,7 @@ class Stream:
             else:
                 self.close(StreamCloseReason.CANCELLED)
 
-        self._deferred_response: Deferred = Deferred(_cancel)
+        self._deferred_response: Deferred[Response] = Deferred(_cancel)
 
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"
@@ -180,7 +180,7 @@ class Stream:
             and not self.metadata["reached_warnsize"]
         )
 
-    def get_response(self) -> Deferred:
+    def get_response(self) -> Deferred[Response]:
         """Simply return a Deferred which fires when response
         from the asynchronous request is available
         """
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8a9e8f687..a7d65e1e3 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -13,6 +13,7 @@ from typing import (
     Generator,
     Iterable,
     Iterator,
+    List,
     Optional,
     Set,
     Tuple,
@@ -34,7 +35,6 @@ from scrapy.logformatter import LogFormatter
 from scrapy.pipelines import ItemPipelineManager
 from scrapy.signalmanager import SignalManager
 from scrapy.utils.defer import (
-    DeferredListResultListT,
     aiter_errback,
     defer_fail,
     defer_succeed,
@@ -54,7 +54,7 @@ logger = logging.getLogger(__name__)
 
 
 _T = TypeVar("_T")
-_ParallelResult = DeferredListResultListT[Iterator[Any]]
+_ParallelResult = List[Tuple[bool, Iterator[Any]]]
 
 if TYPE_CHECKING:
     # parameterized Deferreds require Twisted 21.7.0
@@ -374,7 +374,7 @@ class Scraper:
 
     def _itemproc_finished(
         self, output: Any, item: Any, response: Response, spider: Spider
-    ) -> Deferred:
+    ) -> Deferred[Any]:
         """ItemProcessor finished for the given ``item`` and returned ``output``"""
         assert self.slot is not None  # typing
         self.slot.itemproc_size -= 1
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 37a666605..c9feac29c 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -302,7 +302,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
         result = await maybe_deferred_to_future(
-            self._process_spider_output(response, spider, result)
+            cast(
+                "Deferred[Union[Iterable[_T], AsyncIterable[_T]]]",
+                self._process_spider_output(response, spider, result),
+            )
         )
         if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
@@ -339,7 +342,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     def process_start_requests(
         self, start_requests: Iterable[Request], spider: Spider
-    ) -> Deferred:
+    ) -> Deferred[Iterable[Request]]:
         return self._process_chain("process_start_requests", start_requests, spider)
 
     # This method is only needed until _async compatibility methods are removed.
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 4fe5987a7..877ea5928 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -4,7 +4,18 @@ import logging
 import pprint
 import signal
 import warnings
-from typing import TYPE_CHECKING, Any, Dict, Generator, Optional, Set, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Generator,
+    Optional,
+    Set,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+)
 
 from twisted.internet.defer import (
     Deferred,
@@ -54,6 +65,8 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
 
 class Crawler:
     def __init__(
@@ -140,7 +153,7 @@ class Crawler:
         )
 
     @inlineCallbacks
-    def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred, Any, None]:
+    def crawl(self, *args: Any, **kwargs: Any) -> Generator[Deferred[Any], Any, None]:
         if self.crawling:
             raise RuntimeError("Crawling already taking place")
         if self._started:
@@ -172,7 +185,7 @@ class Crawler:
         return ExecutionEngine(self, lambda _: self.stop())
 
     @inlineCallbacks
-    def stop(self) -> Generator[Deferred, Any, None]:
+    def stop(self) -> Generator[Deferred[Any], Any, None]:
         """Starts a graceful stop of the crawler and returns a deferred that is
         fired when the crawler is stopped."""
         if self.crawling:
@@ -256,7 +269,7 @@ class CrawlerRunner:
         self.settings = settings
         self.spider_loader = self._get_spider_loader(settings)
         self._crawlers: Set[Crawler] = set()
-        self._active: Set[Deferred] = set()
+        self._active: Set[Deferred[None]] = set()
         self.bootstrap_failed = False
 
     def crawl(
@@ -264,7 +277,7 @@ class CrawlerRunner:
         crawler_or_spidercls: Union[Type[Spider], str, Crawler],
         *args: Any,
         **kwargs: Any,
-    ) -> Deferred:
+    ) -> Deferred[None]:
         """
         Run a crawler with the provided arguments.
 
@@ -294,12 +307,12 @@ class CrawlerRunner:
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
-    def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred:
+    def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> Deferred[None]:
         self.crawlers.add(crawler)
         d = crawler.crawl(*args, **kwargs)
         self._active.add(d)
 
-        def _done(result: Any) -> Any:
+        def _done(result: _T) -> _T:
             self.crawlers.discard(crawler)
             self._active.discard(d)
             self.bootstrap_failed |= not getattr(crawler, "spider", None)
@@ -335,7 +348,7 @@ class CrawlerRunner:
         # temporary cast until self.spider_loader is typed
         return Crawler(cast(Type[Spider], spidercls), self.settings)
 
-    def stop(self) -> Deferred:
+    def stop(self) -> Deferred[Any]:
         """
         Stops simultaneously all the crawling jobs taking place.
 
@@ -344,7 +357,7 @@ class CrawlerRunner:
         return DeferredList([c.stop() for c in list(self.crawlers)])
 
     @inlineCallbacks
-    def join(self) -> Generator[Deferred, Any, None]:
+    def join(self) -> Generator[Deferred[Any], Any, None]:
         """
         join()
 
@@ -460,7 +473,7 @@ class CrawlerProcess(CrawlerRunner):
             )
         reactor.run(installSignalHandlers=install_signal_handlers)  # blocking call
 
-    def _graceful_stop_reactor(self) -> Deferred:
+    def _graceful_stop_reactor(self) -> Deferred[Any]:
         d = self.stop()
         d.addBoth(self._stop_reactor)
         return d
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 70393576b..73757162f 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Optional, Union
+from typing import TYPE_CHECKING, Dict, Optional, TypeVar, Union
 
 from twisted.internet.defer import Deferred, maybeDeferred
 
@@ -31,6 +31,8 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
 
 class RobotsTxtMiddleware:
     DOWNLOAD_PRIORITY: int = 1000
@@ -43,7 +45,9 @@ class RobotsTxtMiddleware:
             "ROBOTSTXT_USER_AGENT", None
         )
         self.crawler: Crawler = crawler
-        self._parsers: Dict[str, Union[RobotParser, Deferred, None]] = {}
+        self._parsers: Dict[
+            str, Union[RobotParser, Deferred[Optional[RobotParser]], None]
+        ] = {}
         self._parserimpl: RobotParser = load_object(
             crawler.settings.get("ROBOTSTXT_PARSER")
         )
@@ -55,14 +59,18 @@ class RobotsTxtMiddleware:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def process_request(self, request: Request, spider: Spider) -> Optional[Deferred]:
+    def process_request(
+        self, request: Request, spider: Spider
+    ) -> Optional[Deferred[None]]:
         if request.meta.get("dont_obey_robotstxt"):
             return None
         if request.url.startswith("data:") or request.url.startswith("file:"):
             return None
-        d: Deferred = maybeDeferred(self.robot_parser, request, spider)
-        d.addCallback(self.process_request_2, request, spider)
-        return d
+        d: Deferred[Optional[RobotParser]] = maybeDeferred(
+            self.robot_parser, request, spider  # type: ignore[arg-type]
+        )
+        d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
+        return d2
 
     def process_request_2(
         self, rp: Optional[RobotParser], request: Request, spider: Spider
@@ -86,7 +94,7 @@ class RobotsTxtMiddleware:
 
     def robot_parser(
         self, request: Request, spider: Spider
-    ) -> Union[RobotParser, Deferred, None]:
+    ) -> Union[RobotParser, Deferred[Optional[RobotParser]], None]:
         url = urlparse_cached(request)
         netloc = url.netloc
 
@@ -109,9 +117,9 @@ class RobotsTxtMiddleware:
 
         parser = self._parsers[netloc]
         if isinstance(parser, Deferred):
-            d: Deferred = Deferred()
+            d: Deferred[Optional[RobotParser]] = Deferred()
 
-            def cb(result: Any) -> Any:
+            def cb(result: Optional[RobotParser]) -> Optional[RobotParser]:
                 d.callback(result)
                 return result
 
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index ffaf783a7..40ea48510 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -31,10 +31,10 @@ class BaseDupeFilter:
     def request_seen(self, request: Request) -> bool:
         return False
 
-    def open(self) -> Optional[Deferred]:
+    def open(self) -> Optional[Deferred[None]]:
         pass
 
-    def close(self, reason: str) -> Optional[Deferred]:
+    def close(self, reason: str) -> Optional[Deferred[None]]:
         pass
 
     def log(self, request: Request, spider: Spider) -> None:
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 43c2d2815..0d7f5bfd4 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -149,7 +149,7 @@ class FeedStorageProtocol(Protocol):
         """Open the storage for the given spider. It must return a file-like
         object that will be used for the exporters"""
 
-    def store(self, file: IO[bytes]) -> Optional[Deferred]:
+    def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
         """Store the given file stream"""
 
 
@@ -162,7 +162,7 @@ class BlockingFeedStorage:
 
         return NamedTemporaryFile(prefix="feed-", dir=path)
 
-    def store(self, file: IO[bytes]) -> Optional[Deferred]:
+    def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
         return deferToThread(self._store_in_thread, file)
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
@@ -192,7 +192,7 @@ class StdoutFeedStorage:
     def open(self, spider: Spider) -> IO[bytes]:
         return self._stdout
 
-    def store(self, file: IO[bytes]) -> Optional[Deferred]:
+    def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
         pass
 
 
@@ -211,7 +211,7 @@ class FileFeedStorage:
             dirname.mkdir(parents=True)
         return Path(self.path).open(self.write_mode)
 
-    def store(self, file: IO[bytes]) -> Optional[Deferred]:
+    def store(self, file: IO[bytes]) -> Optional[Deferred[None]]:
         file.close()
         return None
 
@@ -483,7 +483,7 @@ _FeedSlot = create_deprecated_class(
 
 
 class FeedExporter:
-    _pending_deferreds: List[Deferred] = []
+    _pending_deferreds: List[Deferred[None]] = []
 
     @classmethod
     def from_crawler(cls, crawler: Crawler) -> Self:
@@ -570,7 +570,7 @@ class FeedExporter:
             self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed)
         )
 
-    def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]:
+    def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred[None]]:
         def get_file(slot_: FeedSlot) -> IO[bytes]:
             assert slot_.file
             if isinstance(slot_.file, PostProcessingManager):
@@ -590,7 +590,7 @@ class FeedExporter:
             return None
 
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
-        d: Deferred = maybeDeferred(slot.storage.store, get_file(slot))
+        d: Deferred[None] = maybeDeferred(slot.storage.store, get_file(slot))  # type: ignore[arg-type]
 
         d.addCallback(
             self._handle_store_success, logmsg, spider, type(slot.storage).__name__
@@ -621,7 +621,7 @@ class FeedExporter:
         self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
 
     def _handle_store_success(
-        self, f: Failure, logmsg: str, spider: Spider, slot_type: str
+        self, result: Any, logmsg: str, spider: Spider, slot_type: str
     ) -> None:
         logger.info("Stored %s", logmsg, extra={"spider": spider})
         assert self.crawler.stats
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index e43de6f5c..cad607514 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -39,7 +39,7 @@ class StatsMailer:
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_closed(self, spider: Spider) -> Optional[Deferred]:
+    def spider_closed(self, spider: Spider) -> Optional[Deferred[None]]:
         spider_stats = self.stats.get_stats(spider)
         body = "Global stats\n\n"
         body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 3ea20e831..c020732f9 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -103,7 +103,7 @@ class MailSender:
         mimetype: str = "text/plain",
         charset: Optional[str] = None,
         _callback: Optional[Callable[..., None]] = None,
-    ) -> Optional[Deferred]:
+    ) -> Optional[Deferred[None]]:
         from twisted.internet import reactor
 
         msg: MIMEBase
@@ -155,7 +155,9 @@ class MailSender:
             )
             return None
 
-        dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
+        dfd: Deferred[Any] = self._sendmail(
+            rcpts, msg.as_string().encode(charset or "utf-8")
+        )
         dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs))
         dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs))
         reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
@@ -198,11 +200,11 @@ class MailSender:
         )
         return failure
 
-    def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred:
+    def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred[Any]:
         from twisted.internet import reactor
 
         msg_io = BytesIO(msg)
-        d: Deferred = Deferred()
+        d: Deferred[Any] = Deferred()
 
         factory = self._create_sender_factory(to_addrs, msg_io, d)
 
@@ -216,7 +218,7 @@ class MailSender:
         return d
 
     def _create_sender_factory(
-        self, to_addrs: List[str], msg: IO[bytes], d: Deferred
+        self, to_addrs: List[str], msg: IO[bytes], d: Deferred[Any]
     ) -> ESMTPSenderFactory:
         from twisted.mail.smtp import ESMTPSenderFactory
 
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index ea5488ba1..2296db90e 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -13,6 +13,7 @@ from typing import (
     List,
     Optional,
     Tuple,
+    TypeVar,
     Union,
     cast,
 )
@@ -24,16 +25,22 @@ from scrapy.utils.misc import build_from_crawler, build_from_settings, load_obje
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
 
+    # typing.Concatenate and typing.ParamSpec require Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    from typing_extensions import Concatenate, ParamSpec, Self
 
     from scrapy import Spider
     from scrapy.crawler import Crawler
     from scrapy.settings import Settings
 
+    _P = ParamSpec("_P")
+
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+_T2 = TypeVar("_T2")
+
 
 class MiddlewareManager:
     """Base class for implementing middleware managers"""
@@ -98,16 +105,22 @@ class MiddlewareManager:
         if hasattr(mw, "close_spider"):
             self.methods["close_spider"].appendleft(mw.close_spider)
 
-    def _process_parallel(self, methodname: str, obj: Any, *args: Any) -> Deferred:
-        methods = cast(Iterable[Callable], self.methods[methodname])
+    def _process_parallel(
+        self, methodname: str, obj: _T, *args: Any
+    ) -> Deferred[List[_T2]]:
+        methods = cast(
+            "Iterable[Callable[Concatenate[_T, _P], _T2]]", self.methods[methodname]
+        )
         return process_parallel(methods, obj, *args)
 
-    def _process_chain(self, methodname: str, obj: Any, *args: Any) -> Deferred:
-        methods = cast(Iterable[Callable], self.methods[methodname])
+    def _process_chain(self, methodname: str, obj: _T, *args: Any) -> Deferred[_T]:
+        methods = cast(
+            "Iterable[Callable[Concatenate[_T, _P], _T]]", self.methods[methodname]
+        )
         return process_chain(methods, obj, *args)
 
-    def open_spider(self, spider: Spider) -> Deferred:
+    def open_spider(self, spider: Spider) -> Deferred[List[None]]:
         return self._process_parallel("open_spider", spider)
 
-    def close_spider(self, spider: Spider) -> Deferred:
+    def close_spider(self, spider: Spider) -> Deferred[List[None]]:
         return self._process_parallel("close_spider", spider)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 2c22d3d8f..b7e46274f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -4,6 +4,8 @@ See documentation in docs/topics/shell.rst
 
 """
 
+from __future__ import annotations
+
 import os
 import signal
 from typing import Any, Callable, Dict, Optional, Tuple, Union
@@ -92,7 +94,9 @@ class Shell:
                 self.vars, shells=shells, banner=self.vars.pop("banner", "")
             )
 
-    def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred:
+    def _schedule(
+        self, request: Request, spider: Optional[Spider]
+    ) -> defer.Deferred[Any]:
         if is_asyncio_reactor_installed():
             # set the asyncio event loop for the current thread
             event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
@@ -209,7 +213,7 @@ def inspect_response(response: Response, spider: Spider) -> None:
     signal.signal(signal.SIGINT, sigint_handler)
 
 
-def _request_deferred(request: Request) -> defer.Deferred:
+def _request_deferred(request: Request) -> defer.Deferred[Any]:
     """Wrap a request inside a Deferred.
 
     This function is harmful, do not use it until you know what you are doing.
@@ -228,7 +232,7 @@ def _request_deferred(request: Request) -> defer.Deferred:
         request.errback = request_errback
         return result
 
-    d: defer.Deferred = defer.Deferred()
+    d: defer.Deferred[Any] = defer.Deferred()
     d.addBoth(_restore_callbacks)
     if request.callback:
         d.addCallback(request.callback)
diff --git a/scrapy/signalmanager.py b/scrapy/signalmanager.py
index 3d37b8235..b2c6dea5d 100644
--- a/scrapy/signalmanager.py
+++ b/scrapy/signalmanager.py
@@ -50,7 +50,9 @@ class SignalManager:
         kwargs.setdefault("sender", self.sender)
         return _signal.send_catch_log(signal, **kwargs)
 
-    def send_catch_log_deferred(self, signal: Any, **kwargs: Any) -> Deferred:
+    def send_catch_log_deferred(
+        self, signal: Any, **kwargs: Any
+    ) -> Deferred[List[Tuple[Any, Any]]]:
         """
         Like :meth:`send_catch_log` but supports returning
         :class:`~twisted.internet.defer.Deferred` objects from signal handlers.
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 7b43f04f2..f0b0c0988 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -7,9 +7,7 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import logging
-from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union, cast
-
-from twisted.internet.defer import Deferred
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, cast
 
 from scrapy import signals
 from scrapy.http import Request, Response
@@ -19,6 +17,8 @@ from scrapy.utils.url import url_is_from_spider
 if TYPE_CHECKING:
     from collections.abc import Callable
 
+    from twisted.internet.defer import Deferred
+
     # typing.Concatenate requires Python 3.10
     # typing.Self requires Python 3.11
     from typing_extensions import Concatenate, Self
@@ -105,10 +105,10 @@ class Spider(object_ref):
         return url_is_from_spider(request.url, cls)
 
     @staticmethod
-    def close(spider: Spider, reason: str) -> Union[Deferred, None]:
+    def close(spider: Spider, reason: str) -> Optional[Deferred[None]]:
         closed = getattr(spider, "closed", None)
         if callable(closed):
-            return cast(Union[Deferred, None], closed(reason))
+            return cast("Optional[Deferred[None]]", closed(reason))
         return None
 
     def __repr__(self) -> str:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 1d578e8a3..c5763a06c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -49,13 +49,8 @@ if TYPE_CHECKING:
 _T = TypeVar("_T")
 _T2 = TypeVar("_T2")
 
-# copied from twisted.internet.defer
-_SelfResultT = TypeVar("_SelfResultT")
-_DeferredListResultItemT = Tuple[bool, _SelfResultT]
-DeferredListResultListT = List[_DeferredListResultItemT[_SelfResultT]]
 
-
-def defer_fail(_failure: Failure) -> Deferred:
+def defer_fail(_failure: Failure) -> Deferred[Any]:
     """Same as twisted.internet.defer.fail but delay calling errback until
     next reactor loop
 
@@ -64,7 +59,7 @@ def defer_fail(_failure: Failure) -> Deferred:
     """
     from twisted.internet import reactor
 
-    d: Deferred = Deferred()
+    d: Deferred[Any] = Deferred()
     reactor.callLater(0.1, d.errback, _failure)
     return d
 
@@ -78,12 +73,12 @@ def defer_succeed(result: _T) -> Deferred[_T]:
     """
     from twisted.internet import reactor
 
-    d: Deferred = Deferred()
+    d: Deferred[_T] = Deferred()
     reactor.callLater(0.1, d.callback, result)
     return d
 
 
-def defer_result(result: Any) -> Deferred:
+def defer_result(result: Any) -> Deferred[Any]:
     if isinstance(result, Deferred):
         return result
     if isinstance(result, failure.Failure):
@@ -138,14 +133,14 @@ def parallel(
     callable: Callable[Concatenate[_T, _P], _T2],
     *args: _P.args,
     **named: _P.kwargs,
-) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
+) -> Deferred[List[Tuple[bool, Iterator[_T2]]]]:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
 
     Taken from: https://jcalderone.livejournal.com/24285.html
     """
     coop = Cooperator()
-    work = (callable(elem, *args, **named) for elem in iterable)
+    work: Iterator[_T2] = (callable(elem, *args, **named) for elem in iterable)
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
@@ -198,16 +193,16 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
     def __init__(
         self,
         aiterable: AsyncIterable[_T],
-        callable: Callable[Concatenate[_T, _P], _T2],
+        callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]],
         *callable_args: _P.args,
         **callable_kwargs: _P.kwargs,
     ):
         self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
-        self.callable: Callable[Concatenate[_T, _P], _T2] = callable
+        self.callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]] = callable
         self.callable_args: Tuple[Any, ...] = callable_args
         self.callable_kwargs: Dict[str, Any] = callable_kwargs
         self.finished: bool = False
-        self.waiting_deferreds: List[Deferred] = []
+        self.waiting_deferreds: List[Deferred[Any]] = []
         self.anext_deferred: Optional[Deferred[_T]] = None
 
     def _callback(self, result: _T) -> None:
@@ -241,12 +236,12 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
         self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
-    def __next__(self) -> Deferred:
+    def __next__(self) -> Deferred[Any]:
         # This puts a new Deferred into self.waiting_deferreds and returns it.
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d: Deferred = Deferred()
+        d: Deferred[Any] = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
@@ -256,25 +251,29 @@ class _AsyncCooperatorAdapter(Iterator[Deferred]):
 def parallel_async(
     async_iterable: AsyncIterable[_T],
     count: int,
-    callable: Callable[Concatenate[_T, _P], _T2],
+    callable: Callable[Concatenate[_T, _P], Optional[Deferred[Any]]],
     *args: _P.args,
     **named: _P.kwargs,
-) -> Deferred[DeferredListResultListT[Iterator[_T2]]]:
-    """Like parallel but for async iterators"""
+) -> Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]]:
+    """Like ``parallel`` but for async iterators"""
     coop = Cooperator()
-    work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
-    dl: Deferred = DeferredList([coop.coiterate(work) for _ in range(count)])
+    work: Iterator[Deferred[Any]] = _AsyncCooperatorAdapter(
+        async_iterable, callable, *args, **named
+    )
+    dl: Deferred[List[Tuple[bool, Iterator[Deferred[Any]]]]] = DeferredList(
+        [coop.coiterate(work) for _ in range(count)]
+    )
     return dl
 
 
 def process_chain(
-    callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
-    input: Any,
+    callbacks: Iterable[Callable[Concatenate[_T, _P], _T]],
+    input: _T,
     *a: _P.args,
     **kw: _P.kwargs,
-) -> Deferred:
+) -> Deferred[_T]:
     """Return a Deferred built by chaining the given callbacks"""
-    d: Deferred = Deferred()
+    d: Deferred[_T] = Deferred()
     for x in callbacks:
         d.addCallback(x, *a, **kw)
     d.callback(input)
@@ -307,19 +306,21 @@ def process_chain_both(
 
 
 def process_parallel(
-    callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
-    input: Any,
+    callbacks: Iterable[Callable[Concatenate[_T, _P], _T2]],
+    input: _T,
     *a: _P.args,
     **kw: _P.kwargs,
-) -> Deferred:
+) -> Deferred[List[_T2]]:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
-    d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
-    d.addCallback(lambda r: [x[1] for x in r])
-    d.addErrback(lambda f: f.value.subFailure)
-    return d
+    d: Deferred[List[Tuple[bool, _T2]]] = DeferredList(
+        dfds, fireOnOneErrback=True, consumeErrors=True
+    )
+    d2: Deferred[List[_T2]] = d.addCallback(lambda r: [x[1] for x in r])
+    d2.addErrback(lambda f: f.value.subFailure)
+    return d2
 
 
 def iter_errback(
@@ -404,7 +405,7 @@ def deferred_f_from_coro_f(
 
 def maybeDeferred_coro(
     f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs
-) -> Deferred:
+) -> Deferred[Any]:
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)
@@ -420,7 +421,7 @@ def maybeDeferred_coro(
     return defer.succeed(result)
 
 
-def deferred_to_future(d: Deferred) -> Future:
+def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
     """
     .. versionadded:: 2.6.0
 
@@ -442,7 +443,7 @@ def deferred_to_future(d: Deferred) -> Future:
     return d.asFuture(_get_asyncio_event_loop())
 
 
-def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
+def maybe_deferred_to_future(d: Deferred[_T]) -> Union[Deferred[_T], Future[_T]]:
     """
     .. versionadded:: 2.6.0
 
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index bb6d807ee..4310c1d56 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -1,5 +1,7 @@
 """Helper functions for working with signals"""
 
+from __future__ import annotations
+
 import collections.abc
 import logging
 from typing import Any as TypingAny
@@ -27,7 +29,7 @@ def send_catch_log(
     signal: TypingAny = Any,
     sender: TypingAny = Anonymous,
     *arguments: TypingAny,
-    **named: TypingAny
+    **named: TypingAny,
 ) -> List[Tuple[TypingAny, TypingAny]]:
     """Like pydispatcher.robust.sendRobust but it also logs errors and returns
     Failures instead of exceptions.
@@ -73,8 +75,8 @@ def send_catch_log_deferred(
     signal: TypingAny = Any,
     sender: TypingAny = Anonymous,
     *arguments: TypingAny,
-    **named: TypingAny
-) -> Deferred:
+    **named: TypingAny,
+) -> Deferred[List[Tuple[TypingAny, TypingAny]]]:
     """Like send_catch_log but supports returning deferreds on signal handlers.
     Returns a deferred that gets fired once all signal handlers deferreds were
     fired.
@@ -92,23 +94,25 @@ def send_catch_log_deferred(
 
     dont_log = named.pop("dont_log", None)
     spider = named.get("spider", None)
-    dfds = []
+    dfds: List[Deferred[Tuple[TypingAny, TypingAny]]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
-        d = maybeDeferred_coro(
+        d: Deferred[TypingAny] = maybeDeferred_coro(
             robustApply, receiver, signal=signal, sender=sender, *arguments, **named
         )
         d.addErrback(logerror, receiver)
         # TODO https://pylint.readthedocs.io/en/latest/user_guide/messages/warning/cell-var-from-loop.html
-        d.addBoth(
+        d2: Deferred[Tuple[TypingAny, TypingAny]] = d.addBoth(
             lambda result: (
                 receiver,  # pylint: disable=cell-var-from-loop  # noqa: B023
                 result,
             )
         )
-        dfds.append(d)
-    d = DeferredList(dfds)
-    d.addCallback(lambda out: [x[1] for x in out])
-    return d
+        dfds.append(d2)
+    dl = DeferredList(dfds)
+    d3: Deferred[List[Tuple[TypingAny, TypingAny]]] = dl.addCallback(
+        lambda out: [x[1] for x in out]
+    )
+    return d3
 
 
 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index fe2bfa042..30f235592 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -30,6 +30,7 @@ from scrapy.utils.boto import is_botocore_available
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred
+    from twisted.web.client import Response as TxResponse
 
 
 _T = TypeVar("_T")
@@ -159,7 +160,7 @@ def mock_google_cloud_storage() -> Tuple[Any, Any, Any]:
     return (client_mock, bucket_mock, blob_mock)
 
 
-def get_web_client_agent_req(url: str) -> Deferred:
+def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:
     from twisted.internet import reactor
     from twisted.web.client import Agent  # imports twisted.internet.reactor
 
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
index 8882bfc5f..bb269a9f5 100644
--- a/scrapy/utils/testproc.py
+++ b/scrapy/utils/testproc.py
@@ -22,7 +22,7 @@ class ProcessTest:
         args: Iterable[str],
         check_code: bool = True,
         settings: Optional[str] = None,
-    ) -> Deferred:
+    ) -> Deferred[TestProcessProtocol]:
         from twisted.internet import reactor
 
         env = os.environ.copy()
@@ -49,7 +49,7 @@ class ProcessTest:
 
 class TestProcessProtocol(ProcessProtocol):
     def __init__(self) -> None:
-        self.deferred: Deferred = Deferred()
+        self.deferred: Deferred[TestProcessProtocol] = Deferred()
         self.out: bytes = b""
         self.err: bytes = b""
         self.exitcode: Optional[int] = None

From 96d6519b25a3d7b02e8efa1180f6f59e5244f977 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 26 Jun 2024 17:43:59 +0500
Subject: [PATCH 63/82] Bump twine in twinecheck. (#6416)

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index d665fc5a5..c325064d9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,7 +88,7 @@ commands =
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==4.0.2
+    twine==5.0.0
     build==1.0.3
 commands =
     python -m build --sdist

From 41e15e93e7459673e93ff2591462b47b7ae01566 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 26 Jun 2024 17:44:12 +0500
Subject: [PATCH 64/82] Remove an obsolete import. (#6415)

---
 scrapy/crawler.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 877ea5928..ecb0a8150 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -23,13 +23,6 @@ from twisted.internet.defer import (
     inlineCallbacks,
     maybeDeferred,
 )
-
-try:
-    # zope >= 5.0 only supports MultipleInvalid
-    from zope.interface.exceptions import MultipleInvalid
-except ImportError:
-    MultipleInvalid = None
-
 from zope.interface.verify import verifyClass
 
 from scrapy import Spider, signals

From 558b1d11d2f1e3063aba59d444fdb93d42a9ddb9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 1 Jul 2024 12:30:49 +0500
Subject: [PATCH 65/82] Use CallbackT for Request.callback. (#6422)

---
 scrapy/commands/parse.py         | 16 ++++++++--------
 scrapy/contracts/__init__.py     |  7 ++++---
 scrapy/http/request/__init__.py  | 20 ++++++++++++++------
 scrapy/http/response/__init__.py | 12 +++++++-----
 scrapy/http/response/text.py     | 12 +++++++-----
 scrapy/spiders/__init__.py       |  8 ++------
 scrapy/spiders/crawl.py          | 15 +++++++++------
 scrapy/spiders/sitemap.py        |  8 ++++----
 8 files changed, 55 insertions(+), 43 deletions(-)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 1265aa38e..fbd200d88 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -9,7 +9,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AsyncGenerator,
-    Callable,
     Coroutine,
     Dict,
     Iterable,
@@ -38,6 +37,7 @@ from scrapy.utils.spider import spidercls_for_request
 if TYPE_CHECKING:
     from twisted.python.failure import Failure
 
+    from scrapy.http.request import CallbackT
     from scrapy.spiders import Spider
 
 
@@ -218,8 +218,8 @@ class Command(BaseRunSpiderCommand):
         opts: argparse.Namespace,
         depth: int,
         spider: Spider,
-        callback: Callable,
-    ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
+        callback: CallbackT,
+    ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT]:
         items, requests = [], []
         for x in spider_output:
             if is_item(x):
@@ -231,7 +231,7 @@ class Command(BaseRunSpiderCommand):
     def run_callback(
         self,
         response: Response,
-        callback: Callable,
+        callback: CallbackT,
         cb_kwargs: Optional[Dict[str, Any]] = None,
     ) -> Deferred[Any]:
         cb_kwargs = cb_kwargs or {}
@@ -240,7 +240,7 @@ class Command(BaseRunSpiderCommand):
 
     def get_callback_from_rules(
         self, spider: Spider, response: Response
-    ) -> Union[Callable, str, None]:
+    ) -> Union[CallbackT, str, None]:
         if getattr(spider, "rules", None):
             for rule in spider.rules:  # type: ignore[attr-defined]
                 if rule.link_extractor.matches(response.url):
@@ -286,7 +286,7 @@ class Command(BaseRunSpiderCommand):
     def scraped_data(
         self,
         args: Tuple[
-            List[Any], List[Request], argparse.Namespace, int, Spider, Callable
+            List[Any], List[Request], argparse.Namespace, int, Spider, CallbackT
         ],
     ) -> List[Any]:
         items, requests, opts, depth, spider, callback = args
@@ -313,8 +313,8 @@ class Command(BaseRunSpiderCommand):
         spider: Spider,
         opts: argparse.Namespace,
         response: Optional[Response] = None,
-    ) -> Callable:
-        cb: Union[str, Callable, None] = None
+    ) -> CallbackT:
+        cb: Union[str, CallbackT, None] = None
         if response:
             cb = response.meta["_callback"]
         if not cb:
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 440e0dc44..a7e129948 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -16,6 +16,7 @@ from typing import (
     Optional,
     Tuple,
     Type,
+    cast,
 )
 from unittest import TestCase, TestResult
 
@@ -62,7 +63,7 @@ class Contract:
                     if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                         raise TypeError("Contracts don't support async callbacks")
                     return list(  # pylint: disable=return-in-finally
-                        iterate_spider_output(cb_result)
+                        cast(Iterable[Any], iterate_spider_output(cb_result))
                     )
 
             request.callback = wrapper
@@ -79,7 +80,7 @@ class Contract:
                 cb_result = cb(response, **cb_kwargs)
                 if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                     raise TypeError("Contracts don't support async callbacks")
-                output = list(iterate_spider_output(cb_result))
+                output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
                 try:
                     results.startTest(self.testcase_post)
                     self.post_process(output)
@@ -195,7 +196,7 @@ class ContractsManager:
         def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
             try:
                 output = cb(response, **cb_kwargs)
-                output = list(iterate_spider_output(output))
+                output = list(cast(Iterable[Any], iterate_spider_output(output)))
             except Exception:
                 case = _create_testcase(method, "callback")
                 results.addError(case, sys.exc_info())
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 04589dd37..9381a6cb3 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -12,7 +12,6 @@ from typing import (
     TYPE_CHECKING,
     Any,
     AnyStr,
-    Callable,
     Dict,
     Iterable,
     List,
@@ -37,8 +36,17 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from twisted.python.failure import Failure
+
+    # typing.Concatenate requires Python 3.10
     # typing.NotRequired and typing.Self require Python 3.11
-    from typing_extensions import NotRequired, Self
+    from typing_extensions import Concatenate, NotRequired, Self
+
+    from scrapy.http import Response
+
+    CallbackT = Callable[Concatenate[Response, ...], Any]
 
 
 class VerboseCookie(TypedDict):
@@ -110,7 +118,7 @@ class Request(object_ref):
     def __init__(
         self,
         url: str,
-        callback: Optional[Callable] = None,
+        callback: Optional[CallbackT] = None,
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
@@ -119,7 +127,7 @@ class Request(object_ref):
         encoding: str = "utf-8",
         priority: int = 0,
         dont_filter: bool = False,
-        errback: Optional[Callable] = None,
+        errback: Optional[Callable[[Failure], Any]] = None,
         flags: Optional[List[str]] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
     ) -> None:
@@ -137,8 +145,8 @@ class Request(object_ref):
             )
         if not (callable(errback) or errback is None):
             raise TypeError(f"errback must be a callable, got {type(errback).__name__}")
-        self.callback: Optional[Callable] = callback
-        self.errback: Optional[Callable] = errback
+        self.callback: Optional[CallbackT] = callback
+        self.errback: Optional[Callable[[Failure], Any]] = errback
 
         self.cookies: CookiesT = cookies or {}
         self.headers: Headers = Headers(headers or {}, encoding=encoding)
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index ff3581abb..92e4852b6 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -27,7 +27,7 @@ from urllib.parse import urljoin
 
 from scrapy.exceptions import NotSupported
 from scrapy.http.headers import Headers
-from scrapy.http.request import CookiesT, Request
+from scrapy.http.request import Request
 from scrapy.link import Link
 from scrapy.utils.trackref import object_ref
 
@@ -35,10 +35,12 @@ if TYPE_CHECKING:
     from ipaddress import IPv4Address, IPv6Address
 
     from twisted.internet.ssl import Certificate
+    from twisted.python.failure import Failure
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.http.request import CallbackT, CookiesT
     from scrapy.selector import SelectorList
 
 
@@ -196,7 +198,7 @@ class Response(object_ref):
     def follow(
         self,
         url: Union[str, Link],
-        callback: Optional[Callable] = None,
+        callback: Optional[CallbackT] = None,
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
@@ -205,7 +207,7 @@ class Response(object_ref):
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
         dont_filter: bool = False,
-        errback: Optional[Callable] = None,
+        errback: Optional[Callable[[Failure], Any]] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
     ) -> Request:
@@ -249,7 +251,7 @@ class Response(object_ref):
     def follow_all(
         self,
         urls: Iterable[Union[str, Link]],
-        callback: Optional[Callable] = None,
+        callback: Optional[CallbackT] = None,
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
@@ -258,7 +260,7 @@ class Response(object_ref):
         encoding: Optional[str] = "utf-8",
         priority: int = 0,
         dont_filter: bool = False,
-        errback: Optional[Callable] = None,
+        errback: Optional[Callable[[Failure], Any]] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
     ) -> Iterable[Request]:
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 0635f744f..588695002 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -41,7 +41,9 @@ from scrapy.utils.python import memoizemethod_noargs, to_unicode
 from scrapy.utils.response import get_base_url
 
 if TYPE_CHECKING:
-    from scrapy.http.request import CookiesT, Request
+    from twisted.python.failure import Failure
+
+    from scrapy.http.request import CallbackT, CookiesT, Request
     from scrapy.selector import Selector, SelectorList
 
 
@@ -179,7 +181,7 @@ class TextResponse(Response):
     def follow(
         self,
         url: Union[str, Link, parsel.Selector],
-        callback: Optional[Callable] = None,
+        callback: Optional[CallbackT] = None,
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
@@ -188,7 +190,7 @@ class TextResponse(Response):
         encoding: Optional[str] = None,
         priority: int = 0,
         dont_filter: bool = False,
-        errback: Optional[Callable] = None,
+        errback: Optional[Callable[[Failure], Any]] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
     ) -> Request:
@@ -232,7 +234,7 @@ class TextResponse(Response):
     def follow_all(
         self,
         urls: Union[Iterable[Union[str, Link]], parsel.SelectorList, None] = None,
-        callback: Optional[Callable] = None,
+        callback: Optional[CallbackT] = None,
         method: str = "GET",
         headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
         body: Optional[Union[bytes, str]] = None,
@@ -241,7 +243,7 @@ class TextResponse(Response):
         encoding: Optional[str] = None,
         priority: int = 0,
         dont_filter: bool = False,
-        errback: Optional[Callable] = None,
+        errback: Optional[Callable[[Failure], Any]] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         flags: Optional[List[str]] = None,
         css: Optional[str] = None,
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index f0b0c0988..d977acd26 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -15,20 +15,16 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 
 if TYPE_CHECKING:
-    from collections.abc import Callable
-
     from twisted.internet.defer import Deferred
 
-    # typing.Concatenate requires Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Concatenate, Self
+    from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import CallbackT
     from scrapy.settings import BaseSettings, _SettingsKeyT
     from scrapy.utils.log import SpiderLoggerAdapter
 
-    CallbackT = Callable[Concatenate[Response, ...], Any]
-
 
 class Spider(object_ref):
     """Base class for scrapy spiders. All spiders must inherit from this
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 48c830d2a..2639f14b2 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -39,6 +39,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import CallbackT
 
 
 _T = TypeVar("_T")
@@ -73,7 +74,7 @@ class Rule:
     def __init__(
         self,
         link_extractor: Optional[LinkExtractor] = None,
-        callback: Union[Callable, str, None] = None,
+        callback: Union[CallbackT, str, None] = None,
         cb_kwargs: Optional[Dict[str, Any]] = None,
         follow: Optional[bool] = None,
         process_links: Union[ProcessLinksT, str, None] = None,
@@ -81,7 +82,7 @@ class Rule:
         errback: Union[Callable[[Failure], Any], str, None] = None,
     ):
         self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor
-        self.callback: Union[Callable, str, None] = callback
+        self.callback: Union[CallbackT, str, None] = callback
         self.errback: Union[Callable[[Failure], Any], str, None] = errback
         self.cb_kwargs: Dict[str, Any] = cb_kwargs or {}
         self.process_links: Union[ProcessLinksT, str] = process_links or _identity
@@ -92,7 +93,7 @@ class Rule:
 
     def _compile(self, spider: Spider) -> None:
         # this replaces method names with methods and we can't express this in type hints
-        self.callback = _get_method(self.callback, spider)
+        self.callback = cast("CallbackT", _get_method(self.callback, spider))
         self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
         self.process_links = cast(
             ProcessLinksT, _get_method(self.process_links, spider)
@@ -122,7 +123,9 @@ class CrawlSpider(Spider):
     def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
         return []
 
-    def process_results(self, response: Response, results: Any) -> Any:
+    def process_results(
+        self, response: Response, results: Iterable[Any]
+    ) -> Iterable[Any]:
         return results
 
     def _build_request(self, rule_index: int, link: Link) -> Request:
@@ -152,7 +155,7 @@ class CrawlSpider(Spider):
         rule = self._rules[cast(int, response.meta["rule"])]
         return self._parse_response(
             response,
-            cast(Callable, rule.callback),
+            cast("CallbackT", rule.callback),
             {**rule.cb_kwargs, **cb_kwargs},
             rule.follow,
         )
@@ -166,7 +169,7 @@ class CrawlSpider(Spider):
     async def _parse_response(
         self,
         response: Response,
-        callback: Optional[Callable],
+        callback: Optional[CallbackT],
         cb_kwargs: Dict[str, Any],
         follow: bool = True,
     ) -> AsyncIterable[Any]:
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d082fbfdb..1542ef79c 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -5,7 +5,6 @@ import re
 from typing import (
     TYPE_CHECKING,
     Any,
-    Callable,
     Dict,
     Iterable,
     List,
@@ -27,6 +26,7 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from scrapy.crawler import Crawler
+    from scrapy.http.request import CallbackT
 
 logger = logging.getLogger(__name__)
 
@@ -34,7 +34,7 @@ logger = logging.getLogger(__name__)
 class SitemapSpider(Spider):
     sitemap_urls: Sequence[str] = ()
     sitemap_rules: Sequence[
-        Tuple[Union[re.Pattern[str], str], Union[str, Callable]]
+        Tuple[Union[re.Pattern[str], str], Union[str, CallbackT]]
     ] = [("", "parse")]
     sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""]
     sitemap_alternate_links: bool = False
@@ -54,10 +54,10 @@ class SitemapSpider(Spider):
 
     def __init__(self, *a: Any, **kw: Any):
         super().__init__(*a, **kw)
-        self._cbs: List[Tuple[re.Pattern[str], Callable]] = []
+        self._cbs: List[Tuple[re.Pattern[str], CallbackT]] = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
-                c = cast(Callable, getattr(self, c))
+                c = cast("CallbackT", getattr(self, c))
             self._cbs.append((regex(r), c))
         self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
 

From d8ecd28c5557e27f42e00bd1223b457468ea2ea7 Mon Sep 17 00:00:00 2001
From: mlmsmith <mlmsmith@hotmail.co.uk>
Date: Thu, 4 Jul 2024 18:16:26 +0800
Subject: [PATCH 66/82] Documentation improvements (#6429)

---
 docs/intro/install.rst  | 13 ++++---------
 docs/intro/overview.rst | 12 ++++++------
 2 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index c90c1d2bf..e6c9a683b 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -37,7 +37,7 @@ Note that sometimes this may require solving compilation issues for some Scrapy
 dependencies depending on your operating system, so be sure to check the
 :ref:`intro-install-platform-notes`.
 
-For more detailed and platform specifics instructions, as well as
+For more detailed and platform-specific instructions, as well as
 troubleshooting information, read on.
 
 
@@ -101,7 +101,7 @@ Windows
 -------
 
 Though it's possible to install Scrapy on Windows using pip, we recommend you
-to install `Anaconda`_ or `Miniconda`_ and use the package from the
+install `Anaconda`_ or `Miniconda`_ and use the package from the
 `conda-forge`_ channel, which will avoid most installation issues.
 
 Once you've installed `Anaconda`_ or `Miniconda`_, install Scrapy with::
@@ -141,7 +141,7 @@ But it should support older versions of Ubuntu too, like Ubuntu 14.04,
 albeit with potential issues with TLS connections.
 
 **Don't** use the ``python-scrapy`` package provided by Ubuntu, they are
-typically too old and slow to catch up with latest Scrapy.
+typically too old and slow to catch up with the latest Scrapy release.
 
 
 To install Scrapy on Ubuntu (or Ubuntu-based) systems, you need to install
@@ -170,7 +170,7 @@ macOS
 
 Building Scrapy's dependencies requires the presence of a C compiler and
 development headers. On macOS this is typically provided by Apple’s Xcode
-development tools. To install the Xcode command line tools open a terminal
+development tools. To install the Xcode command-line tools, open a terminal
 window and run::
 
     xcode-select --install
@@ -200,11 +200,6 @@ solutions:
 
       brew install python
 
-  * Latest versions of python have ``pip`` bundled with them so you won't need
-    to install it separately. If this is not the case, upgrade python::
-
-      brew update; brew upgrade python
-
 *   *(Optional)* :ref:`Install Scrapy inside a Python virtual environment
     <intro-using-virtualenv>`.
 
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index ef1294470..cd17b1968 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -65,7 +65,7 @@ When you ran the command ``scrapy runspider quotes_spider.py``, Scrapy looked fo
 Spider definition inside it and ran it through its crawler engine.
 
 The crawl started by making requests to the URLs defined in the ``start_urls``
-attribute (in this case, only the URL for quotes in *humor* category)
+attribute (in this case, only the URL for quotes in the *humor* category)
 and called the default callback method ``parse``, passing the response object as
 an argument. In the ``parse`` callback, we loop through the quote elements
 using a CSS Selector, yield a Python dict with the extracted quote text and author,
@@ -83,9 +83,9 @@ While this enables you to do very fast crawls (sending multiple concurrent
 requests at the same time, in a fault-tolerant way) Scrapy also gives you
 control over the politeness of the crawl through :ref:`a few settings
 <topics-settings-ref>`. You can do things like setting a download delay between
-each request, limiting amount of concurrent requests per domain or per IP, and
+each request, limiting the amount of concurrent requests per domain or per IP, and
 even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
-to figure out these automatically.
+to figure these settings out automatically.
 
 .. note::
 
@@ -106,10 +106,10 @@ scraping easy and efficient, such as:
 
 * Built-in support for :ref:`selecting and extracting <topics-selectors>` data
   from HTML/XML sources using extended CSS selectors and XPath expressions,
-  with helper methods to extract using regular expressions.
+  with helper methods for extraction using regular expressions.
 
 * An :ref:`interactive shell console <topics-shell>` (IPython aware) for trying
-  out the CSS and XPath expressions to scrape data, very useful when writing or
+  out the CSS and XPath expressions to scrape data, which is very useful when writing or
   debugging your spiders.
 
 * Built-in support for :ref:`generating feed exports <topics-feed-exports>` in
@@ -124,7 +124,7 @@ scraping easy and efficient, such as:
   well-defined API (middlewares, :ref:`extensions <topics-extensions>`, and
   :ref:`pipelines <topics-item-pipeline>`).
 
-* Wide range of built-in extensions and middlewares for handling:
+* A wide range of built-in extensions and middlewares for handling:
 
   - cookies and session handling
   - HTTP features like compression, authentication, caching

From ceedb026f8c8ccb049187baa14202f98b2a3a60c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 9 Jul 2024 11:34:58 +0500
Subject: [PATCH 67/82] Remove top-level imports that install the reactor from
 scrapy.extensions.telnet. (#6432)

---
 scrapy/extensions/telnet.py | 22 +++++++---------------
 tests/test_crawler.py       |  4 ----
 tests/test_utils_log.py     |  4 ----
 3 files changed, 7 insertions(+), 23 deletions(-)

diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index c4e01b3d9..c64a0b417 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -10,21 +10,11 @@ import binascii
 import logging
 import os
 import pprint
-import traceback
 from typing import TYPE_CHECKING, Any, Dict, List
 
 from twisted.internet import protocol
 from twisted.internet.tcp import Port
 
-try:
-    from twisted.conch import manhole, telnet
-    from twisted.conch.insults import insults
-
-    TWISTED_CONCH_AVAILABLE = True
-except (ImportError, SyntaxError):
-    _TWISTED_CONCH_TRACEBACK = traceback.format_exc()
-    TWISTED_CONCH_AVAILABLE = False
-
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.decorators import defers
@@ -33,6 +23,8 @@ from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.trackref import print_live_refs
 
 if TYPE_CHECKING:
+    from twisted.conch import telnet
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
@@ -50,11 +42,7 @@ class TelnetConsole(protocol.ServerFactory):
     def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
             raise NotConfigured
-        if not TWISTED_CONCH_AVAILABLE:
-            raise NotConfigured(
-                "TELNETCONSOLE_ENABLED setting is True but required twisted "
-                "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
-            )
+
         self.crawler: Crawler = crawler
         self.noisy: bool = False
         self.portrange: List[int] = [
@@ -88,6 +76,10 @@ class TelnetConsole(protocol.ServerFactory):
         self.port.stopListening()
 
     def protocol(self) -> telnet.TelnetTransport:  # type: ignore[override]
+        # these import twisted.internet.reactor
+        from twisted.conch import manhole, telnet
+        from twisted.conch.insults import insults
+
         class Portal:
             """An implementation of IPortal"""
 
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 791ea1faa..c87e65758 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -21,7 +21,6 @@ import scrapy
 from scrapy import Spider
 from scrapy.crawler import Crawler, CrawlerProcess, CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.extensions import telnet
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
@@ -482,7 +481,6 @@ class CrawlerLoggingTestCase(unittest.TestCase):
                 "LOG_FILE": str(log_file),
                 # settings to avoid extra warnings
                 "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
-                "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
             }
 
         configure_logging()
@@ -516,8 +514,6 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             custom_settings = {
                 "LOG_FILE": str(log_file),
                 "LOG_FILE_APPEND": False,
-                # disable telnet if not available to avoid an extra warning
-                "TELNETCONSOLE_ENABLED": telnet.TWISTED_CONCH_AVAILABLE,
             }
 
         configure_logging()
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index a8d080822..0f75bdb5c 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -11,7 +11,6 @@ import pytest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
-from scrapy.extensions import telnet
 from scrapy.utils.log import (
     LogCounterHandler,
     SpiderLoggerAdapter,
@@ -70,9 +69,6 @@ class TopLevelFormatterTest(unittest.TestCase):
 class LogCounterHandlerTest(unittest.TestCase):
     def setUp(self):
         settings = {"LOG_LEVEL": "WARNING"}
-        if not telnet.TWISTED_CONCH_AVAILABLE:
-            # disable it to avoid the extra warning
-            settings["TELNETCONSOLE_ENABLED"] = False
         self.logger = logging.getLogger("test")
         self.logger.setLevel(logging.NOTSET)
         self.logger.propagate = False

From 0b8604bb5d8bffbbd5c78783022965fa2606c131 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Tue, 9 Jul 2024 15:52:49 -0300
Subject: [PATCH 68/82] add CLOSESPIDER_PAGECOUNT_NO_ITEM to CloseSpider
 extension

---
 scrapy/extensions/closespider.py | 25 +++++++++++++++++++++++++
 tests/keys/mitmproxy-dhparam.pem | 14 ++++++++++++++
 tests/test_closespider.py        | 13 +++++++++++++
 3 files changed, 52 insertions(+)
 create mode 100644 tests/keys/mitmproxy-dhparam.pem

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 4627e7f98..6ebf98e65 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured
+from scrapy.signalmanager import dispatcher
 
 if TYPE_CHECKING:
     from twisted.python.failure import Failure
@@ -36,6 +37,9 @@ class CloseSpider:
             "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
             "errorcount": crawler.settings.getint("CLOSESPIDER_ERRORCOUNT"),
             "timeout_no_item": crawler.settings.getint("CLOSESPIDER_TIMEOUT_NO_ITEM"),
+            "pagecount_no_item": crawler.settings.getint(
+                "CLOSESPIDER_PAGECOUNT_NO_ITEM"
+            ),
         }
 
         if not any(self.close_on.values()):
@@ -60,6 +64,19 @@ class CloseSpider:
             crawler.signals.connect(
                 self.item_scraped_no_item, signal=signals.item_scraped
             )
+        if self.close_on.get("pagecount_no_item"):
+            if self.page_count not in dispatcher.getReceivers(
+                signal=signals.response_received
+            ):
+                crawler.signals.connect(
+                    self.page_count, signal=signals.response_received
+                )
+
+            if self.item_scraped not in dispatcher.getReceivers(
+                signal=signals.item_scraped
+            ):
+                crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
+
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
@@ -74,9 +91,16 @@ class CloseSpider:
 
     def page_count(self, response: Response, request: Request, spider: Spider) -> None:
         self.counter["pagecount"] += 1
+        self.counter["pagecount_since_last_item"] += 1
         if self.counter["pagecount"] == self.close_on["pagecount"]:
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_pagecount")
+        if self.close_on["pagecount_no_item"] and (
+            self.counter["pagecount_since_last_item"]
+            >= self.close_on["pagecount_no_item"]
+        ):
+            assert self.crawler.engine
+            self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")
 
     def spider_opened(self, spider: Spider) -> None:
         from twisted.internet import reactor
@@ -91,6 +115,7 @@ class CloseSpider:
 
     def item_scraped(self, item: Any, spider: Spider) -> None:
         self.counter["itemcount"] += 1
+        self.counter["pagecount_since_last_item"] = 0
         if self.counter["itemcount"] == self.close_on["itemcount"]:
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_itemcount")
diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem
new file mode 100644
index 000000000..c10121fbf
--- /dev/null
+++ b/tests/keys/mitmproxy-dhparam.pem
@@ -0,0 +1,14 @@
+
+-----BEGIN DH PARAMETERS-----
+MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3
+O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv
+j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ
+Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB
+chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC
+ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq
+o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX
+IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv
+A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8
+6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I
+rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
+-----END DH PARAMETERS-----
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 38ede70e4..caaa9f183 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -34,6 +34,19 @@ class TestCloseSpider(TestCase):
         pagecount = crawler.stats.get_value("response_received_count")
         self.assertTrue(pagecount >= close_on)
 
+    @defer.inlineCallbacks
+    def test_closespider_pagecount_no_item(self):
+        close_on = 5
+        crawler = get_crawler(
+            FollowAllSpider,
+            {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on},
+        )
+        yield crawler.crawl(mockserver=self.mockserver)
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_pagecount_no_item")
+        pagecount = crawler.stats.get_value("response_received_count")
+        self.assertTrue(pagecount >= close_on)
+
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):
         close_on = 5

From a44818afeacc25cc5e05705bf8ae5804e0545c89 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Tue, 9 Jul 2024 16:07:55 -0300
Subject: [PATCH 69/82] restore mitmproxy-dhparam

---
 tests/keys/mitmproxy-dhparam.pem | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 tests/keys/mitmproxy-dhparam.pem

diff --git a/tests/keys/mitmproxy-dhparam.pem b/tests/keys/mitmproxy-dhparam.pem
deleted file mode 100644
index c10121fbf..000000000
--- a/tests/keys/mitmproxy-dhparam.pem
+++ /dev/null
@@ -1,14 +0,0 @@
-
------BEGIN DH PARAMETERS-----
-MIICCAKCAgEAyT6LzpwVFS3gryIo29J5icvgxCnCebcdSe/NHMkD8dKJf8suFCg3
-O2+dguLakSVif/t6dhImxInJk230HmfC8q93hdcg/j8rLGJYDKu3ik6H//BAHKIv
-j5O9yjU3rXCfmVJQic2Nne39sg3CreAepEts2TvYHhVv3TEAzEqCtOuTjgDv0ntJ
-Gwpj+BJBRQGG9NvprX1YGJ7WOFBP/hWU7d6tgvE6Xa7T/u9QIKpYHMIkcN/l3ZFB
-chZEqVlyrcngtSXCROTPcDOQ6Q8QzhaBJS+Z6rcsd7X+haiQqvoFcmaJ08Ks6LQC
-ZIL2EtYJw8V8z7C0igVEBIADZBI6OTbuuhDwRw//zU1uq52Oc48CIZlGxTYG/Evq
-o9EWAXUYVzWkDSTeBH1r4z/qLPE2cnhtMxbFxuvK53jGB0emy2y1Ei6IhKshJ5qX
-IB/aE7SSHyQ3MDHHkCmQJCsOd4Mo26YX61NZ+n501XjqpCBQ2+DfZCBh8Va2wDyv
-A2Ryg9SUz8j0AXViRNMJgJrr446yro/FuJZwnQcO3WQnXeqSBnURqKjmqkeFP+d8
-6mk2tqJaY507lRNqtGlLnj7f5RNoBFJDCLBNurVgfvq9TCVWKDIFD4vZRjCrnl6I
-rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
------END DH PARAMETERS-----

From d6352f9f66f655f11332fe6c52ed71ebb2e55bf4 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Wed, 10 Jul 2024 11:03:01 -0300
Subject: [PATCH 70/82] refactor changes on closespider.py and improve test

---
 scrapy/extensions/closespider.py | 18 +++---------------
 tests/spiders.py                 | 17 +++++++++++++++++
 tests/test_closespider.py        | 24 +++++++++++++++++++-----
 3 files changed, 39 insertions(+), 20 deletions(-)

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 6ebf98e65..cef5527b7 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -12,7 +12,6 @@ from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured
-from scrapy.signalmanager import dispatcher
 
 if TYPE_CHECKING:
     from twisted.python.failure import Failure
@@ -49,11 +48,11 @@ class CloseSpider:
 
         if self.close_on.get("errorcount"):
             crawler.signals.connect(self.error_count, signal=signals.spider_error)
-        if self.close_on.get("pagecount"):
+        if self.close_on.get("pagecount") or self.close_on.get("pagecount_no_item"):
             crawler.signals.connect(self.page_count, signal=signals.response_received)
         if self.close_on.get("timeout"):
             crawler.signals.connect(self.spider_opened, signal=signals.spider_opened)
-        if self.close_on.get("itemcount"):
+        if self.close_on.get("itemcount") or self.close_on.get("pagecount_no_item"):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
         if self.close_on.get("timeout_no_item"):
             self.timeout_no_item: int = self.close_on["timeout_no_item"]
@@ -64,18 +63,6 @@ class CloseSpider:
             crawler.signals.connect(
                 self.item_scraped_no_item, signal=signals.item_scraped
             )
-        if self.close_on.get("pagecount_no_item"):
-            if self.page_count not in dispatcher.getReceivers(
-                signal=signals.response_received
-            ):
-                crawler.signals.connect(
-                    self.page_count, signal=signals.response_received
-                )
-
-            if self.item_scraped not in dispatcher.getReceivers(
-                signal=signals.item_scraped
-            ):
-                crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
 
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
@@ -95,6 +82,7 @@ class CloseSpider:
         if self.counter["pagecount"] == self.close_on["pagecount"]:
             assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_pagecount")
+            return
         if self.close_on["pagecount_no_item"] and (
             self.counter["pagecount_since_last_item"]
             >= self.close_on["pagecount_no_item"]
diff --git a/tests/spiders.py b/tests/spiders.py
index ea419afbd..2bcec5624 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -283,6 +283,23 @@ class ItemSpider(FollowAllSpider):
             yield {}
 
 
+class MaxItemsSpider(ItemSpider):
+    def __init__(self, max_items=10, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.max_items = max_items
+        self.items_scraped = 0
+
+    def parse(self, response):
+        for item_or_req in super().parse(response):
+            if isinstance(item_or_req, Request):
+                yield item_or_req
+            else:
+                if self.items_scraped >= self.max_items:
+                    continue
+                self.items_scraped += 1
+                yield item_or_req
+
+
 class DefaultError(Exception):
     pass
 
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index caaa9f183..0046b4e29 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -3,7 +3,13 @@ from twisted.trial.unittest import TestCase
 
 from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
-from tests.spiders import ErrorSpider, FollowAllSpider, ItemSpider, SlowSpider
+from tests.spiders import (
+    ErrorSpider,
+    FollowAllSpider,
+    ItemSpider,
+    MaxItemsSpider,
+    SlowSpider,
+)
 
 
 class TestCloseSpider(TestCase):
@@ -37,15 +43,23 @@ class TestCloseSpider(TestCase):
     @defer.inlineCallbacks
     def test_closespider_pagecount_no_item(self):
         close_on = 5
+        close_on_pagecount = 20
+        max_items = 5
         crawler = get_crawler(
-            FollowAllSpider,
-            {"CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on},
+            MaxItemsSpider,
+            {
+                "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
+                "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
+            },
         )
-        yield crawler.crawl(mockserver=self.mockserver)
+        yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)
         reason = crawler.spider.meta["close_reason"]
         self.assertEqual(reason, "closespider_pagecount_no_item")
         pagecount = crawler.stats.get_value("response_received_count")
-        self.assertTrue(pagecount >= close_on)
+        itemcount = crawler.stats.get_value("item_scraped_count")
+        self.assertEqual(itemcount, max_items)
+        self.assertLess(pagecount, close_on_pagecount)
+        self.assertTrue((pagecount - itemcount) >= close_on)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):

From 59782d73088e46618d1c042e74ce5197e880536a Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Wed, 10 Jul 2024 11:08:22 -0300
Subject: [PATCH 71/82] update docs

---
 docs/topics/extensions.rst | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index f7b2f3799..a503fd746 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -317,6 +317,18 @@ crawls more than that, the spider will be closed with the reason
 ``closespider_pagecount``. If zero (or non set), spiders won't be closed by
 number of crawled responses.
 
+.. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM
+
+CLOSESPIDER_PAGECOUNT_NO_ITEM
+"""""""""""""""""""""
+
+Default: ``0``
+
+An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider
+crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason
+``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by
+number of crawled responses with no items.
+
 .. setting:: CLOSESPIDER_ERRORCOUNT
 
 CLOSESPIDER_ERRORCOUNT

From 8646d2ec7bc44ef96f5df015e03ff37ceb5554c0 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Wed, 10 Jul 2024 11:44:44 -0300
Subject: [PATCH 72/82] fix docs detail

---
 docs/topics/extensions.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index a503fd746..29bcaa0f2 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -320,7 +320,7 @@ number of crawled responses.
 .. setting:: CLOSESPIDER_PAGECOUNT_NO_ITEM
 
 CLOSESPIDER_PAGECOUNT_NO_ITEM
-"""""""""""""""""""""
+"""""""""""""""""""""""""""""
 
 Default: ``0``
 

From 129dbfa0bf1ad464ab6b50f3dee0da39853de6a1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Jul 2024 12:20:36 +0500
Subject: [PATCH 73/82] Bump tool versions.

---
 .github/workflows/checks.yml        |  4 ++--
 .github/workflows/publish.yml       |  4 ++--
 .github/workflows/tests-macos.yml   |  2 +-
 .github/workflows/tests-ubuntu.yml  |  2 +-
 .github/workflows/tests-windows.yml |  2 +-
 .pre-commit-config.yaml             | 12 ++++++------
 tox.ini                             | 14 +++++++-------
 7 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index ed1629b67..1841bda1c 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -32,7 +32,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
@@ -46,4 +46,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
-    - uses: pre-commit/action@v3.0.0
+    - uses: pre-commit/action@v3.0.1
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index affaa32a5..03e94f761 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -13,13 +13,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.12
       - run: | 
           pip install --upgrade build twine
           python -m build
       - name: Publish to PyPI
-        uses: pypa/gh-action-pypi-publish@v1.6.4
+        uses: pypa/gh-action-pypi-publish@v1.9.0
         with:
           password: ${{ secrets.PYPI_TOKEN }}
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index a297f494c..8ebe7f1db 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index f50a4d104..763de9eff 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -62,7 +62,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 757d62285..80d09e7a0 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -35,7 +35,7 @@ jobs:
     - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 38526d720..addad838f 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,11 +1,11 @@
 repos:
 - repo: https://github.com/PyCQA/bandit
-  rev: 1.7.7
+  rev: 1.7.9
   hooks:
   - id: bandit
     args: [-r, -c, .bandit.yml]
 - repo: https://github.com/PyCQA/flake8
-  rev: 7.0.0
+  rev: 7.1.0
   hooks:
   - id: flake8
     additional_dependencies:
@@ -16,7 +16,7 @@ repos:
     - flake8-string-format
     - flake8-type-checking
 - repo: https://github.com/psf/black.git
-  rev: 24.2.0
+  rev: 24.4.2
   hooks:
   - id: black
 - repo: https://github.com/pycqa/isort
@@ -24,13 +24,13 @@ repos:
   hooks:
   - id: isort
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: 1.16.0
+  rev: 1.18.0
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==24.2.0
+    - black==24.4.2
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.15.2
+  rev: v3.16.0
   hooks:
   - id: pyupgrade
     args: [--py38-plus, --keep-runtime-typing]
diff --git a/tox.ini b/tox.ini
index c325064d9..29d240031 100644
--- a/tox.ini
+++ b/tox.ini
@@ -46,14 +46,14 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.10.0
-    typing-extensions==4.12.1
+    mypy==1.10.1
+    typing-extensions==4.12.2
     types-lxml==2024.4.14
     types-Pygments==2.18.0.20240506
     types-pyOpenSSL==24.1.0.20240425
-    types-setuptools==70.0.0.20240524
-    botocore-stubs==1.34.94
-    boto3-stubs[s3]==1.34.119
+    types-setuptools==70.3.0.20240710
+    botocore-stubs==1.34.143
+    boto3-stubs[s3]==1.34.143
     attrs >= 18.2.0
     Pillow >= 10.3.0
     pytest >= 8.2.0
@@ -88,8 +88,8 @@ commands =
 [testenv:twinecheck]
 basepython = python3
 deps =
-    twine==5.0.0
-    build==1.0.3
+    twine==5.1.1
+    build==1.2.1
 commands =
     python -m build --sdist
     twine check dist/*

From 435686830cbe86d14aa09c9259157695596be07f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Jul 2024 12:25:13 +0500
Subject: [PATCH 74/82] Bump the Python version for RTD.

---
 .github/workflows/checks.yml | 2 +-
 .readthedocs.yml             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 1841bda1c..2be6a9502 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -21,7 +21,7 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing-tests
-        - python-version: "3.11"  # Keep in sync with .readthedocs.yml
+        - python-version: "3.12"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
         - python-version: "3.12"
diff --git a/.readthedocs.yml b/.readthedocs.yml
index e71d34f3a..0c544df7e 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -9,7 +9,7 @@ build:
   tools:
     # For available versions, see:
     # https://docs.readthedocs.io/en/stable/config-file/v2.html#build-tools-python
-    python: "3.11"  # Keep in sync with .github/workflows/checks.yml
+    python: "3.12"  # Keep in sync with .github/workflows/checks.yml
 
 python:
   install:

From 3c9c1a31bcdcced96e87e299689aaa7be8f5bdee Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 11 Jul 2024 12:30:12 +0500
Subject: [PATCH 75/82] Bump pylint.

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 29d240031..e3dd96425 100644
--- a/tox.ini
+++ b/tox.ini
@@ -81,7 +81,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==3.1.0
+    pylint==3.2.5
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 

From a40d5281cfb8fdaf7d7edce80d3addbddef897a6 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Thu, 11 Jul 2024 11:14:30 -0300
Subject: [PATCH 76/82] improve test_closespider_pagecount_no_item and
 MaxItemsSpider

---
 tests/spiders.py          | 13 +++++--------
 tests/test_closespider.py | 10 ++++------
 2 files changed, 9 insertions(+), 14 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index 2bcec5624..d1998ca69 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -283,21 +283,18 @@ class ItemSpider(FollowAllSpider):
             yield {}
 
 
-class MaxItemsSpider(ItemSpider):
+class MaxItemsKeepCrawlingSpider(FollowAllSpider):
     def __init__(self, max_items=10, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.max_items = max_items
         self.items_scraped = 0
 
     def parse(self, response):
-        for item_or_req in super().parse(response):
-            if isinstance(item_or_req, Request):
-                yield item_or_req
-            else:
-                if self.items_scraped >= self.max_items:
-                    continue
+        for request in super().parse(response):
+            yield request
+            if self.items_scraped < self.max_items:
+                yield Item()
                 self.items_scraped += 1
-                yield item_or_req
 
 
 class DefaultError(Exception):
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 0046b4e29..50b483a74 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -7,7 +7,7 @@ from tests.spiders import (
     ErrorSpider,
     FollowAllSpider,
     ItemSpider,
-    MaxItemsSpider,
+    MaxItemsKeepCrawlingSpider,
     SlowSpider,
 )
 
@@ -43,13 +43,11 @@ class TestCloseSpider(TestCase):
     @defer.inlineCallbacks
     def test_closespider_pagecount_no_item(self):
         close_on = 5
-        close_on_pagecount = 20
         max_items = 5
         crawler = get_crawler(
-            MaxItemsSpider,
+            MaxItemsKeepCrawlingSpider,
             {
                 "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
-                "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
             },
         )
         yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)
@@ -58,8 +56,8 @@ class TestCloseSpider(TestCase):
         pagecount = crawler.stats.get_value("response_received_count")
         itemcount = crawler.stats.get_value("item_scraped_count")
         self.assertEqual(itemcount, max_items)
-        self.assertLess(pagecount, close_on_pagecount)
-        self.assertTrue((pagecount - itemcount) >= close_on)
+        self.assertLessEqual(pagecount, close_on + itemcount)
+        self.assertGreater(pagecount, itemcount)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):

From 5f0fad16f5d86134bcf72964f6e453541031eb06 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Thu, 11 Jul 2024 13:26:22 -0300
Subject: [PATCH 77/82] improve test_closespider_pagecount_no_item and
 corresponding test spider

---
 tests/spiders.py          | 12 ++++++++----
 tests/test_closespider.py | 11 ++++++-----
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/tests/spiders.py b/tests/spiders.py
index d1998ca69..743811893 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -283,15 +283,19 @@ class ItemSpider(FollowAllSpider):
             yield {}
 
 
-class MaxItemsKeepCrawlingSpider(FollowAllSpider):
-    def __init__(self, max_items=10, *args, **kwargs):
+class MaxItemsAndRequestsSpider(FollowAllSpider):
+    def __init__(self, max_items=10, max_requests=10, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self.max_items = max_items
-        self.items_scraped = 0
+        self.max_requests = max_requests
 
     def parse(self, response):
+        self.items_scraped = 0
+        self.pages_crawled = 1  # account for the start url
         for request in super().parse(response):
-            yield request
+            if self.pages_crawled < self.max_requests:
+                yield request
+                self.pages_crawled += 1
             if self.items_scraped < self.max_items:
                 yield Item()
                 self.items_scraped += 1
diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 50b483a74..9810d10fb 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -7,7 +7,7 @@ from tests.spiders import (
     ErrorSpider,
     FollowAllSpider,
     ItemSpider,
-    MaxItemsKeepCrawlingSpider,
+    MaxItemsAndRequestsSpider,
     SlowSpider,
 )
 
@@ -44,20 +44,21 @@ class TestCloseSpider(TestCase):
     def test_closespider_pagecount_no_item(self):
         close_on = 5
         max_items = 5
+        max_requests = close_on + max_items
         crawler = get_crawler(
-            MaxItemsKeepCrawlingSpider,
+            MaxItemsAndRequestsSpider,
             {
                 "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on,
             },
         )
-        yield crawler.crawl(max_items=max_items, mockserver=self.mockserver)
+        yield crawler.crawl(
+            max_items=max_items, max_requests=max_requests, mockserver=self.mockserver
+        )
         reason = crawler.spider.meta["close_reason"]
         self.assertEqual(reason, "closespider_pagecount_no_item")
         pagecount = crawler.stats.get_value("response_received_count")
         itemcount = crawler.stats.get_value("item_scraped_count")
-        self.assertEqual(itemcount, max_items)
         self.assertLessEqual(pagecount, close_on + itemcount)
-        self.assertGreater(pagecount, itemcount)
 
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):

From 9cdbcb4f63922f09194fab7d211ba297319b5135 Mon Sep 17 00:00:00 2001
From: guillermo-bondonno <guillermo.bondonno@zyte.com>
Date: Thu, 11 Jul 2024 14:02:24 -0300
Subject: [PATCH 78/82] add test_closespider_pagecount_no_item_with_pagecount

---
 tests/test_closespider.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tests/test_closespider.py b/tests/test_closespider.py
index 9810d10fb..9a837350f 100644
--- a/tests/test_closespider.py
+++ b/tests/test_closespider.py
@@ -60,6 +60,23 @@ class TestCloseSpider(TestCase):
         itemcount = crawler.stats.get_value("item_scraped_count")
         self.assertLessEqual(pagecount, close_on + itemcount)
 
+    @defer.inlineCallbacks
+    def test_closespider_pagecount_no_item_with_pagecount(self):
+        close_on_pagecount_no_item = 5
+        close_on_pagecount = 20
+        crawler = get_crawler(
+            FollowAllSpider,
+            {
+                "CLOSESPIDER_PAGECOUNT_NO_ITEM": close_on_pagecount_no_item,
+                "CLOSESPIDER_PAGECOUNT": close_on_pagecount,
+            },
+        )
+        yield crawler.crawl(mockserver=self.mockserver)
+        reason = crawler.spider.meta["close_reason"]
+        self.assertEqual(reason, "closespider_pagecount_no_item")
+        pagecount = crawler.stats.get_value("response_received_count")
+        self.assertLess(pagecount, close_on_pagecount)
+
     @defer.inlineCallbacks
     def test_closespider_errorcount(self):
         close_on = 5

From 026d6065287e882c244d9b90e0c4fa5e873e29fe Mon Sep 17 00:00:00 2001
From: guillermo-bondonno
 <95530227+guillermo-bondonno@users.noreply.github.com>
Date: Fri, 12 Jul 2024 08:09:03 -0300
Subject: [PATCH 79/82] clean closespider_pagecount_no_item docs section

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/extensions.rst | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 29bcaa0f2..7b34a19d5 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -324,10 +324,11 @@ CLOSESPIDER_PAGECOUNT_NO_ITEM
 
 Default: ``0``
 
-An integer which specifies the maximum number of consecutive responses to crawl without items scraped. If the spider
-crawls more consecutive responses than that and no items are scraped in the meantime, the spider will be closed with the reason
-``closespider_pagecount_no_item``. If zero (or non set), spiders won't be closed by
-number of crawled responses with no items.
+An integer which specifies the maximum number of consecutive responses to crawl
+without items scraped. If the spider crawls more consecutive responses than that
+and no items are scraped in the meantime, the spider will be closed with the
+reason ``closespider_pagecount_no_item``. If zero (or not set), spiders won't be
+closed by number of crawled responses with no items.
 
 .. setting:: CLOSESPIDER_ERRORCOUNT
 

From e376c0b31a01cedd8a8c5c1ccd423d72ae1fb169 Mon Sep 17 00:00:00 2001
From: mlmsmith <mlmsmith@hotmail.co.uk>
Date: Wed, 24 Jul 2024 12:40:01 +0800
Subject: [PATCH 80/82] Tutorial edits (#6440)

---
 docs/intro/tutorial.rst | 53 ++++++++++++++++++++---------------------
 1 file changed, 26 insertions(+), 27 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 8ea98f29b..ee6a1184c 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -18,11 +18,11 @@ This tutorial will walk you through these tasks:
 4. Changing spider to recursively follow links
 5. Using spider arguments
 
-Scrapy is written in Python_. If you're new to the language you might want to
-start by getting an idea of what the language is like, to get the most out of
-Scrapy.
+Scrapy is written in Python_. The more you learn about Python, the more you
+can get out of Scrapy.
 
-If you're already familiar with other languages, and want to learn Python quickly, the `Python Tutorial`_ is a good resource.
+If you're already familiar with other languages and want to learn Python quickly, the
+`Python Tutorial`_ is a good resource.
 
 If you're new to programming and want to start with Python, the following books
 may be useful to you:
@@ -76,10 +76,9 @@ This will create a ``tutorial`` directory with the following contents::
 Our first Spider
 ================
 
-Spiders are classes that you define and that Scrapy uses to scrape information
-from a website (or a group of websites). They must subclass
-:class:`~scrapy.Spider` and define the initial requests to make,
-optionally how to follow links in the pages, and how to parse the downloaded
+Spiders are classes that you define and that Scrapy uses to scrape information from a website
+(or a group of websites). They must subclass :class:`~scrapy.Spider` and define the initial
+requests to be made, and optionally, how to follow links in pages and parse the downloaded
 page content to extract data.
 
 This is the code for our first Spider. Save it in a file named
@@ -138,7 +137,7 @@ To put our spider to work, go to the project's top level directory and run::
 
    scrapy crawl quotes
 
-This command runs the spider with name ``quotes`` that we've just added, that
+This command runs the spider named ``quotes`` that we've just added, that
 will send some requests for the ``quotes.toscrape.com`` domain. You will get an output
 similar to this::
 
@@ -169,7 +168,7 @@ Scrapy schedules the :class:`scrapy.Request <scrapy.Request>` objects
 returned by the ``start_requests`` method of the Spider. Upon receiving a
 response for each one, it instantiates :class:`~scrapy.http.Response` objects
 and calls the callback method associated with the request (in this case, the
-``parse`` method) passing the response as argument.
+``parse`` method) passing the response as an argument.
 
 
 A shortcut to the start_requests method
@@ -217,7 +216,7 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
 
 .. note::
 
-   Remember to always enclose urls in quotes when running Scrapy shell from
+   Remember to always enclose urls in quotes when running Scrapy shell from the
    command-line, otherwise urls containing arguments (i.e. ``&`` character)
    will not work.
 
@@ -257,7 +256,7 @@ object:
 The result of running ``response.css('title')`` is a list-like object called
 :class:`~scrapy.selector.SelectorList`, which represents a list of
 :class:`~scrapy.Selector` objects that wrap around XML/HTML elements
-and allow you to run further queries to fine-grain the selection or extract the
+and allow you to run further queries to refine the selection or extract the
 data.
 
 To extract the text from the title above, you can do:
@@ -354,12 +353,12 @@ Besides `CSS`_, Scrapy selectors also support using `XPath`_ expressions:
 
 XPath expressions are very powerful, and are the foundation of Scrapy
 Selectors. In fact, CSS selectors are converted to XPath under-the-hood. You
-can see that if you read closely the text representation of the selector
-objects in the shell.
+can see that if you read the text representation of the selector
+objects in the shell closely.
 
 While perhaps not as popular as CSS selectors, XPath expressions offer more
 power because besides navigating the structure, it can also look at the
-content. Using XPath, you're able to select things like: *select the link
+content. Using XPath, you're able to select things like: *the link
 that contains the text "Next Page"*. This makes XPath very fitting to the task
 of scraping, and we encourage you to learn XPath even if you already know how to
 construct CSS selectors, it will make scraping much easier.
@@ -422,7 +421,7 @@ variable, so that we can run our CSS selectors directly on a particular quote:
 
     >>> quote = response.css("div.quote")[0]
 
-Now, let's extract ``text``, ``author`` and the ``tags`` from that quote
+Now, let's extract the ``text``, ``author`` and ``tags`` from that quote
 using the ``quote`` object we just created:
 
 .. code-block:: pycon
@@ -448,7 +447,7 @@ to get all of them:
   from sys import version_info
 
 Having figured out how to extract each bit, we can now iterate over all the
-quotes elements and put them together into a Python dictionary:
+quote elements and put them together into a Python dictionary:
 
 .. code-block:: pycon
 
@@ -465,8 +464,8 @@ quotes elements and put them together into a Python dictionary:
 Extracting data in our spider
 -----------------------------
 
-Let's get back to our spider. Until now, it doesn't extract any data in
-particular, just saves the whole HTML page to a local file. Let's integrate the
+Let's get back to our spider. Until now, it hasn't extracted any data in
+particular, just saving the whole HTML page to a local file. Let's integrate the
 extraction logic above into our spider.
 
 A Scrapy spider typically generates many dictionaries containing the data
@@ -529,8 +528,8 @@ using a different serialization format, such as `JSON Lines`_::
 
     scrapy crawl quotes -o quotes.jsonl
 
-The `JSON Lines`_ format is useful because it's stream-like, you can easily
-append new records to it. It doesn't have the same problem of JSON when you run
+The `JSON Lines`_ format is useful because it's stream-like, so you can easily
+append new records to it. It doesn't have the same problem as JSON when you run
 twice. Also, as each record is a separate line, you can process big files
 without having to fit everything in memory, there are tools like `JQ`_ to help
 do that at the command-line.
@@ -555,7 +554,7 @@ from https://quotes.toscrape.com, you want quotes from all the pages in the webs
 Now that you know how to extract data from pages, let's see how to follow links
 from them.
 
-First thing is to extract the link to the page we want to follow.  Examining
+The first thing to do is extract the link to the page we want to follow.  Examining
 our page, we can see there is a link to the next page with the following
 markup:
 
@@ -589,7 +588,7 @@ There is also an ``attrib`` property available
     >>> response.css("li.next a").attrib["href"]
     '/page/2/'
 
-Let's see now our spider modified to recursively follow the link to the next
+Now let's see our spider, modified to recursively follow the link to the next
 page, extracting data from it:
 
 .. code-block:: python
@@ -756,8 +755,8 @@ Another interesting thing this spider demonstrates is that, even if there are
 many quotes from the same author, we don't need to worry about visiting the
 same author page multiple times. By default, Scrapy filters out duplicated
 requests to URLs already visited, avoiding the problem of hitting servers too
-much because of a programming mistake. This can be configured by the setting
-:setting:`DUPEFILTER_CLASS`.
+much because of a programming mistake. This can be configured in the
+:setting:`DUPEFILTER_CLASS` setting.
 
 Hopefully by now you have a good understanding of how to use the mechanism
 of following links and callbacks with Scrapy.
@@ -824,12 +823,12 @@ Next steps
 ==========
 
 This tutorial covered only the basics of Scrapy, but there's a lot of other
-features not mentioned here. Check the :ref:`topics-whatelse` section in
+features not mentioned here. Check the :ref:`topics-whatelse` section in the
 :ref:`intro-overview` chapter for a quick overview of the most important ones.
 
 You can continue from the section :ref:`section-basics` to know more about the
 command-line tool, spiders, selectors and other things the tutorial hasn't covered like
-modeling the scraped data. If you prefer to play with an example project, check
+modeling the scraped data. If you'd prefer to play with an example project, check
 the :ref:`intro-examples` section.
 
 .. _JSON: https://en.wikipedia.org/wiki/JSON

From 03a15ced4f0a4284c75a917fdfb07c44b21f9ff2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 31 Jul 2024 11:37:19 +0200
Subject: [PATCH 81/82] Do not suggest logging dropped items twice (#6448)

Co-authored-by: Kevin Lloyd Bernal <kevinoxy@gmail.com>
---
 docs/topics/item-pipeline.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index a5f6e07b8..58c922e0d 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -99,7 +99,7 @@ contain a price:
                     adapter["price"] = adapter["price"] * self.vat_factor
                 return item
             else:
-                raise DropItem(f"Missing price in {item}")
+                raise DropItem("Missing price")
 
 
 Write items to a JSON lines file
@@ -254,7 +254,7 @@ returns multiples items with the same id:
         def process_item(self, item, spider):
             adapter = ItemAdapter(item)
             if adapter["id"] in self.ids_seen:
-                raise DropItem(f"Duplicate item found: {item!r}")
+                raise DropItem(f"Item ID already seen: {adapter['id']}")
             else:
                 self.ids_seen.add(adapter["id"])
                 return item

From b9ef1326a51140f70325609501265300fdac5e9b Mon Sep 17 00:00:00 2001
From: mlmsmith <mlmsmith@hotmail.co.uk>
Date: Thu, 1 Aug 2024 15:29:11 +0800
Subject: [PATCH 82/82] Proofread the commands documentation (#6449)

---
 docs/intro/tutorial.rst  |  4 ++--
 docs/topics/commands.rst | 18 +++++++++---------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index ee6a1184c..dd1efd3b3 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -216,8 +216,8 @@ using the :ref:`Scrapy shell <topics-shell>`. Run::
 
 .. note::
 
-   Remember to always enclose urls in quotes when running Scrapy shell from the
-   command-line, otherwise urls containing arguments (i.e. ``&`` character)
+   Remember to always enclose URLs in quotes when running Scrapy shell from the
+   command line, otherwise URLs containing arguments (i.e. ``&`` character)
    will not work.
 
    On Windows, use double quotes instead::
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 1d37895c2..6eb4af9bd 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -6,7 +6,7 @@
 Command line tool
 =================
 
-Scrapy is controlled through the ``scrapy`` command-line tool, to be referred
+Scrapy is controlled through the ``scrapy`` command-line tool, to be referred to
 here as the "Scrapy tool" to differentiate it from the sub-commands, which we
 just call "commands" or "Scrapy commands".
 
@@ -185,8 +185,8 @@ And you can see all available commands with::
 
 There are two kinds of commands, those that only work from inside a Scrapy
 project (Project-specific commands) and those that also work without an active
-Scrapy project (Global commands), though they may behave slightly different
-when running from inside a project (as they would use the project overridden
+Scrapy project (Global commands), though they may behave slightly differently
+when run from inside a project (as they would use the project overridden
 settings).
 
 Global commands:
@@ -236,7 +236,7 @@ genspider
 .. versionadded:: 2.6.0
    The ability to pass a URL instead of a domain.
 
-Create a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
+Creates a new spider in the current folder or in the current project's ``spiders`` folder, if called from inside a project. The ``<name>`` parameter is set as the spider's ``name``, while ``<domain or URL>`` is used to generate the ``allowed_domains`` and ``start_urls`` spider's attributes.
 
 Usage example::
 
@@ -253,7 +253,7 @@ Usage example::
     $ scrapy genspider -t crawl scrapyorg scrapy.org
     Created spider 'scrapyorg' using template 'crawl'
 
-This is just a convenience shortcut command for creating spiders based on
+This is just a convenient shortcut command for creating spiders based on
 pre-defined templates, but certainly not the only way to create spiders. You
 can just create the spider source code files yourself, instead of using this
 command.
@@ -274,9 +274,9 @@ Supported options:
 
 * ``-a NAME=VALUE``: set a spider argument (may be repeated)
 
-* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout), to define format set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
+* ``--output FILE`` or ``-o FILE``: append scraped items to the end of FILE (use - for stdout). To define the output format, set a colon at the end of the output URI (i.e. ``-o FILE:FORMAT``)
 
-* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file, to define format set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
+* ``--overwrite-output FILE`` or ``-O FILE``: dump scraped items into FILE, overwriting any existing file. To define the output format, set a colon at the end of the output URI (i.e. ``-O FILE:FORMAT``)
 
 * ``--output-format FORMAT`` or ``-t FORMAT``: deprecated way to define format to use for dumping items, does not work in combination with ``-O``
 
@@ -353,7 +353,7 @@ edit
 Edit the given spider using the editor defined in the ``EDITOR`` environment
 variable or (if unset) the :setting:`EDITOR` setting.
 
-This command is provided only as a convenience shortcut for the most common
+This command is provided only as a convenient shortcut for the most common
 case, the developer is of course free to choose any tool or IDE to write and
 debug spiders.
 
@@ -372,7 +372,7 @@ fetch
 Downloads the given URL using the Scrapy downloader and writes the contents to
 standard output.
 
-The interesting thing about this command is that it fetches the page how the
+The interesting thing about this command is that it fetches the page the way the
 spider would download it. For example, if the spider has a ``USER_AGENT``
 attribute which overrides the User Agent, it will use that one.