From bccb4cf18ba38c8bf09d61d19e0ffabaf15554b1 Mon Sep 17 00:00:00 2001
From: Jalil SA <61639983+jxlil@users.noreply.github.com>
Date: Wed, 14 Feb 2024 12:29:29 -0600
Subject: [PATCH 01/74] fix: LxmlLinkExtractor unique_list missing key

---
 scrapy/linkextractors/lxmlhtml.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 23cbd0116..98781ba7f 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -248,5 +248,5 @@ class LxmlLinkExtractor:
             links = self._extract_links(doc, response.url, response.encoding, base_url)
             all_links.extend(self._process_links(links))
         if self.link_extractor.unique:
-            return unique_list(all_links)
+            return unique_list(all_links, key=self.link_extractor.link_key)
         return all_links

From 5e51417a485f296354e9639f85fb0b51a4a3e533 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 16 Feb 2024 20:10:52 +0100
Subject: [PATCH 02/74] Add tests, fix canonicalize passing

---
 scrapy/linkextractors/lxmlhtml.py |   2 +-
 tests/test_linkextractors.py      | 112 ++++++++++++++++++++++++++++++
 2 files changed, 113 insertions(+), 1 deletion(-)

diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 98781ba7f..7abdaaec4 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -153,7 +153,7 @@ class LxmlLinkExtractor:
             unique=unique,
             process=process_value,
             strip=strip,
-            canonicalized=canonicalize,
+            canonicalized=not canonicalize,
         )
         self.allow_res = [
             x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 18e9608c1..f23b8988e 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -745,6 +745,118 @@ class Base:
             lx = self.extractor_cls()
             self.assertIsInstance(pickle.loads(pickle.dumps(lx)), self.extractor_cls)
 
+        def test_link_extractor_aggregation(self):
+            """When a parameter like restrict_css is used, the underlying
+            implementation calls its internal link extractor once per selector
+            matching the specified restrictions, and then aggregates the
+            extracted links.
+
+            Test that aggregation respects the unique and canonicalize
+            parameters.
+            """
+            # unique=True (default), canonicalize=False (default)
+            lx = self.extractor_cls(restrict_css=("div",))
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/b?b=2&a=1", text="b2"),
+                ],
+            )
+
+            # unique=True (default), canonicalize=True
+            lx = self.extractor_cls(restrict_css=("div",), canonicalize=True)
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                ],
+            )
+
+            # unique=False, canonicalize=False (default)
+            lx = self.extractor_cls(restrict_css=("div",), unique=False)
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/a", text="a2"),
+                    Link(url="https://example.com/b?b=2&a=1", text="b2"),
+                ],
+            )
+
+            # unique=False, canonicalize=True
+            lx = self.extractor_cls(
+                restrict_css=("div",), unique=False, canonicalize=True
+            )
+            response = HtmlResponse(
+                "https://example.com",
+                body=b"""
+                    <div>
+                        <a href="/a">a1</a>
+                        <a href="/b?a=1&b=2">b1</a>
+                    </div>
+                    <div>
+                        <a href="/a">a2</a>
+                        <a href="/b?b=2&a=1">b2</a>
+                    </div>
+                """,
+            )
+            actual = lx.extract_links(response)
+            self.assertEqual(
+                actual,
+                [
+                    Link(url="https://example.com/a", text="a1"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b1"),
+                    Link(url="https://example.com/a", text="a2"),
+                    Link(url="https://example.com/b?a=1&b=2", text="b2"),
+                ],
+            )
+
 
 class LxmlLinkExtractorTestCase(Base.LinkExtractorTestCase):
     extractor_cls = LxmlLinkExtractor

From f096f17fa4ac1307fa1c81ae082bb52e9f86653a Mon Sep 17 00:00:00 2001
From: Elias Ram <eliasram@kth.se>
Date: Tue, 20 Feb 2024 20:32:02 +0100
Subject: [PATCH 03/74] test #6 added tests for check command

---
 tests/test_command_check.py | 55 +++++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 592494aba..d503628b8 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -1,3 +1,8 @@
+import sys
+from io import StringIO
+from unittest.mock import Mock, PropertyMock, patch
+
+from scrapy.commands.check import Command
 from tests.test_commands import CommandTest
 
 
@@ -94,3 +99,53 @@ class CheckSpider(scrapy.Spider):
             raise Exception('SCRAPY_CHECK not set')
         """
         self._test_contract(parse_def=parse_def)
+
+    @patch("scrapy.commands.check.ContractsManager")
+    def test_run_with_opts_list_prints_spider(self, cm_cls_mock):
+        output = StringIO()
+        sys.stdout = output
+        cmd = Command()
+        cmd.settings = Mock(getwithbase=Mock(return_value={}))
+        cm_cls_mock.return_value = cm_mock = Mock()
+        spider_loader_mock = Mock()
+        cmd.crawler_process = Mock(spider_loader=spider_loader_mock)
+        spider_name = "FakeSpider"
+        spider_cls_mock = Mock()
+        type(spider_cls_mock).name = PropertyMock(return_value=spider_name)
+        spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[
+            x
+        ]
+        tested_methods = ["fakeMethod1", "fakeMethod2"]
+        cm_mock.tested_methods_from_spidercls.side_effect = lambda x: {
+            spider_cls_mock: tested_methods
+        }[x]
+
+        cmd.run([spider_name], Mock(list=True))
+
+        self.assertEqual(
+            "FakeSpider\n  * fakeMethod1\n  * fakeMethod2\n", output.getvalue()
+        )
+        sys.stdout = sys.__stdout__
+
+    @patch("scrapy.commands.check.ContractsManager")
+    def test_run_without_opts_list_does_not_crawl_spider_with_no_tested_methods(
+        self, cm_cls_mock
+    ):
+        cmd = Command()
+        cmd.settings = Mock(getwithbase=Mock(return_value={}))
+        cm_cls_mock.return_value = cm_mock = Mock()
+        spider_loader_mock = Mock()
+        cmd.crawler_process = Mock(spider_loader=spider_loader_mock)
+        spider_name = "FakeSpider"
+        spider_cls_mock = Mock()
+        spider_loader_mock.load.side_effect = lambda x: {spider_name: spider_cls_mock}[
+            x
+        ]
+        tested_methods = []
+        cm_mock.tested_methods_from_spidercls.side_effect = lambda x: {
+            spider_cls_mock: tested_methods
+        }[x]
+
+        cmd.run([spider_name], Mock(list=False))
+
+        cmd.crawler_process.crawl.assert_not_called()

From c513e7d6e58efc99c5dfc8ad135dc23df528c5fd Mon Sep 17 00:00:00 2001
From: Can Kupeli <cankupeli101@hotmail.com>
Date: Wed, 21 Feb 2024 23:52:35 +0100
Subject: [PATCH 04/74] implemented tests for branch coverage for function
 printSummary

---
 tests/test_command_check.py | 52 +++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 592494aba..70098a4e7 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -1,3 +1,6 @@
+from unittest.mock import Mock, call, patch
+
+from scrapy.commands.check import TextTestResult
 from tests.test_commands import CommandTest
 
 
@@ -94,3 +97,52 @@ class CheckSpider(scrapy.Spider):
             raise Exception('SCRAPY_CHECK not set')
         """
         self._test_contract(parse_def=parse_def)
+
+    def test_printSummary_with_unsuccessful_test_result_without_errors_and_without_failures(
+        self,
+    ):
+        result = TextTestResult(Mock(), descriptions=False, verbosity=1)
+        start_time = 1.0
+        stop_time = 2.0
+        result.testsRun = 5
+        result.failures = []
+        result.errors = []
+        result.unexpectedSuccesses = ["a", "b"]
+        with patch.object(result.stream, "write") as mock_write:
+            result.printSummary(start_time, stop_time)
+            mock_write.assert_has_calls([call("FAILED"), call("\n")])
+
+    def test_printSummary_with_unsuccessful_test_result_with_only_failures(self):
+        result = TextTestResult(Mock(), descriptions=False, verbosity=1)
+        start_time = 1.0
+        stop_time = 2.0
+        result.testsRun = 5
+        result.failures = [(self, "failure")]
+        result.errors = []
+        with patch.object(result.stream, "writeln") as mock_write:
+            result.printSummary(start_time, stop_time)
+            mock_write.assert_called_with(" (failures=1)")
+
+    def test_printSummary_with_unsuccessful_test_result_with_only_errors(self):
+        result = TextTestResult(Mock(), descriptions=False, verbosity=1)
+        start_time = 1.0
+        stop_time = 2.0
+        result.testsRun = 5
+        result.failures = []
+        result.errors = [(self, "error")]
+        with patch.object(result.stream, "writeln") as mock_write:
+            result.printSummary(start_time, stop_time)
+            mock_write.assert_called_with(" (errors=1)")
+
+    def test_printSummary_with_unsuccessful_test_result_with_both_failures_and_errors(
+        self,
+    ):
+        result = TextTestResult(Mock(), descriptions=False, verbosity=1)
+        start_time = 1.0
+        stop_time = 2.0
+        result.testsRun = 5
+        result.failures = [(self, "failure")]
+        result.errors = [(self, "error")]
+        with patch.object(result.stream, "writeln") as mock_write:
+            result.printSummary(start_time, stop_time)
+            mock_write.assert_called_with(" (failures=1, errors=1)")

From 2169810414a700fcbfe33eafe1e85e46e7f62413 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Thu, 29 Feb 2024 06:41:14 -0300
Subject: [PATCH 05/74] fix: Proxy tests don't use custom certificate authority

---
 tests/test_proxy_connect.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
index 46d42e9f6..93f006c76 100644
--- a/tests/test_proxy_connect.py
+++ b/tests/test_proxy_connect.py
@@ -27,7 +27,7 @@ from mitmproxy.tools.main import mitmdump
 sys.argv[0] = "mitmdump"
 sys.exit(mitmdump())
         """
-        cert_path = Path(__file__).parent.resolve() / "keys" / "mitmproxy-ca.pem"
+        cert_path = Path(__file__).parent.resolve() / "keys"
         self.proc = Popen(
             [
                 sys.executable,
@@ -40,8 +40,8 @@ sys.exit(mitmdump())
                 "0",
                 "--proxyauth",
                 f"{self.auth_user}:{self.auth_pass}",
-                "--certs",
-                str(cert_path),
+                "--set",
+                f"confdir={cert_path}",
                 "--ssl-insecure",
             ],
             stdout=PIPE,

From 2bfd9a2257c79ae56955e95b46f2bc7b23e1eabd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 29 Feb 2024 11:11:42 +0100
Subject: [PATCH 06/74] bandit: allow-list false positives

---
 .bandit.yml                         | 11 +----------
 scrapy/commands/bench.py            |  6 ++++--
 scrapy/commands/edit.py             |  2 +-
 scrapy/commands/genspider.py        |  2 +-
 scrapy/core/downloader/__init__.py  |  2 +-
 scrapy/exporters.py                 |  4 ++--
 scrapy/extensions/httpcache.py      |  6 +++---
 scrapy/extensions/spiderstate.py    |  4 ++--
 scrapy/settings/default_settings.py |  2 +-
 scrapy/shell.py                     |  2 +-
 scrapy/squeues.py                   |  2 +-
 scrapy/utils/benchserver.py         |  2 +-
 scrapy/utils/engine.py              |  2 +-
 13 files changed, 20 insertions(+), 27 deletions(-)

diff --git a/.bandit.yml b/.bandit.yml
index 6e8331c0f..4fcd75c57 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,19 +1,10 @@
 skips:
-- B101  # assert_used
-- B105  # hardcoded_password_string
-- B301  # pickle
-- B307  # eval
-- B311  # random
+- B101  # assert_used, needed for mypy
 - B320  # xml_bad_etree
 - B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
 - B324  # hashlib "Use of weak SHA1 hash for security"
 - B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B403  # import_pickle
-- B404  # import_subprocess
-- B406  # import_xml_sax
 - B410  # import_lxml
 - B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
 - B503  # ssl_with_bad_defaults
-- B603  # subprocess_without_shell_equals_true
-- B605  # start_process_with_a_shell
 exclude_dirs: ['tests']
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index e1ccdc451..aaf5a439f 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,4 +1,4 @@
-import subprocess
+import subprocess  # nosec
 import sys
 import time
 from urllib.parse import urlencode
@@ -29,7 +29,9 @@ class _BenchServer:
         from scrapy.utils.test import get_testenv
 
         pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
-        self.proc = subprocess.Popen(pargs, stdout=subprocess.PIPE, env=get_testenv())
+        self.proc = subprocess.Popen(
+            pargs, stdout=subprocess.PIPE, env=get_testenv()
+        )  # nosec
         self.proc.stdout.readline()
 
     def __exit__(self, exc_type, exc_value, traceback):
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index 03a8ed5c7..e85d2c9ec 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -37,4 +37,4 @@ class Command(ScrapyCommand):
 
         sfile = sys.modules[spidercls.__module__].__file__
         sfile = sfile.replace(".pyc", ".py")
-        self.exitcode = os.system(f'{editor} "{sfile}"')
+        self.exitcode = os.system(f'{editor} "{sfile}"')  # nosec
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 68cbe8ff6..567ebcdc0 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -113,7 +113,7 @@ class Command(ScrapyCommand):
         if template_file:
             self._genspider(module, name, url, opts.template, template_file)
             if opts.edit:
-                self.exitcode = os.system(f'scrapy edit "{name}"')
+                self.exitcode = os.system(f'scrapy edit "{name}"')  # nosec
 
     def _genspider(self, module, name, url, template_name, template_file):
         """Generate the spider module, based on the given template"""
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index c84525160..666282856 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -40,7 +40,7 @@ class Slot:
 
     def download_delay(self) -> float:
         if self.randomize_delay:
-            return random.uniform(0.5 * self.delay, 1.5 * self.delay)
+            return random.uniform(0.5 * self.delay, 1.5 * self.delay)  # nosec
         return self.delay
 
     def close(self) -> None:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index f85f1dad8..79fd4e56f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -5,10 +5,10 @@ Item Exporters are used to export/serialize items into different formats.
 import csv
 import io
 import marshal
-import pickle
+import pickle  # nosec
 import pprint
 from collections.abc import Mapping
-from xml.sax.saxutils import XMLGenerator
+from xml.sax.saxutils import XMLGenerator  # nosec
 
 from itemadapter import ItemAdapter, is_item
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 7e4f047a8..335728502 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,6 +1,6 @@
 import gzip
 import logging
-import pickle
+import pickle  # nosec
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
@@ -274,7 +274,7 @@ class DbmCacheStorage:
         if 0 < self.expiration_secs < time() - float(ts):
             return  # expired
 
-        return pickle.loads(db[f"{key}_data"])
+        return pickle.loads(db[f"{key}_data"])  # nosec
 
 
 class FilesystemCacheStorage:
@@ -352,7 +352,7 @@ class FilesystemCacheStorage:
         if 0 < self.expiration_secs < time() - mtime:
             return  # expired
         with self._open(metapath, "rb") as f:
-            return pickle.load(f)
+            return pickle.load(f)  # nosec
 
 
 def parse_cachecontrol(header):
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 929a3be70..43359401b 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,4 +1,4 @@
-import pickle
+import pickle  # nosec
 from pathlib import Path
 
 from scrapy import signals
@@ -31,7 +31,7 @@ class SpiderState:
     def spider_opened(self, spider):
         if self.jobdir and Path(self.statefn).exists():
             with Path(self.statefn).open("rb") as f:
-                spider.state = pickle.load(f)
+                spider.state = pickle.load(f)  # nosec
         else:
             spider.state = {}
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 49ab1b5ef..2b3d95a0e 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -177,7 +177,7 @@ FILES_STORE_S3_ACL = "private"
 FILES_STORE_GCS_ACL = ""
 
 FTP_USER = "anonymous"
-FTP_PASSWORD = "guest"
+FTP_PASSWORD = "guest"  # nosec
 FTP_PASSIVE_MODE = True
 
 GCS_PROJECT_ID = None
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 05909977a..63ea33892 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -50,7 +50,7 @@ class Shell:
         else:
             self.populate_vars()
         if self.code:
-            print(eval(self.code, globals(), self.vars))
+            print(eval(self.code, globals(), self.vars))  # nosec
         else:
             """
             Detect interactive shell setting in scrapy.cfg
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index f665ad88c..e20f60f06 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -3,7 +3,7 @@ Scheduler queues
 """
 
 import marshal
-import pickle
+import pickle  # nosec
 from os import PathLike
 from pathlib import Path
 from typing import Union
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 38884a9f0..f6f704d4b 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -14,7 +14,7 @@ class Root(Resource):
     def render(self, request):
         total = _getarg(request, b"total", 100, int)
         show = _getarg(request, b"show", 10, int)
-        nlist = [random.randint(1, total) for _ in range(show)]
+        nlist = [random.randint(1, total) for _ in range(show)]  # nosec
         request.write(b"<html><head></head><body>")
         args = request.args.copy()
         for nl in nlist:
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index a5f2a8c6e..0b2722663 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -30,7 +30,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
     checks: List[Tuple[str, Any]] = []
     for test in tests:
         try:
-            checks += [(test, eval(test))]
+            checks += [(test, eval(test))]  # nosec
         except Exception as e:
             checks += [(test, f"{type(e).__name__} (exception)")]
 

From 31cbbb57584fe2a7c42d30acf2aa4707039457b5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 29 Feb 2024 11:31:39 +0100
Subject: [PATCH 07/74] bandit: ignore md5 usage for download slot names

---
 scrapy/pqueues.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index b62d2fe58..593667f1f 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -20,7 +20,7 @@ def _path_safe(text):
     pathable_slot = "".join([c if c.isalnum() or c in "-._" else "_" for c in text])
     # as we replace some letters we can get collision for different slots
     # add we add unique part
-    unique_slot = hashlib.md5(text.encode("utf8")).hexdigest()
+    unique_slot = hashlib.md5(text.encode("utf8")).hexdigest()  # nosec
     return "-".join([pathable_slot, unique_slot])
 
 

From 1311e7db05204fe2cae7d1c5caf8b0ffe9371cd0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 29 Feb 2024 16:31:06 +0500
Subject: [PATCH 08/74] Regenerate the expired mitmproxy-ca.pem.

---
 tests/keys/mitmproxy-ca.pem | 93 ++++++++++++++++++-------------------
 1 file changed, 45 insertions(+), 48 deletions(-)

diff --git a/tests/keys/mitmproxy-ca.pem b/tests/keys/mitmproxy-ca.pem
index cdef75f99..61a690cc8 100644
--- a/tests/keys/mitmproxy-ca.pem
+++ b/tests/keys/mitmproxy-ca.pem
@@ -1,50 +1,47 @@
------BEGIN PRIVATE KEY-----
-MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCYp6U4G9YWITYB
-/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7ZDiT
-NUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMgz1BJ
-u6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniIggUH
-JrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE6HFB
-eIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/0zls
-90iyQ3E/AgMBAAECggEBAJA1dyAdM85uC04vKVNUJM1GDp0xS+0syBReJaKRI3nJ
-epoCj+RqxGag1pdaYLI0G84NTPqECz9LOyLdqpPgEfKRIxWlf9oWmSnfnXskArd8
-VfVcWYl6tEPv1TToTZIBmCbYLBFVbLxG/GrbK6uokdhUsqbdXwEKok2IEaSTRlDn
-v8BVXte00d9VEKKpmI6EY3f45uPQPHuJNcitP2HGW1mT/C6XoZR6wj+VvoRgUGQT
-I7PuktbYpQlLV+oX0uZz9frPGhjydUq0Jti5v3QAJEb+7D0cKrkZW+7fYDx4YkRU
-oDiuWEyO2kfpff52Qxs+xUXMiAyw6/8+TamKoAi1TIECgYEAyAzoztW6W4CjL2au
-/hN5VmbAvuBxq1m1G5KgXM1myX9V2CgH6OKwzJQNSCEfKMNOjqxB99T7C3tMCjgG
-gmbUzylTeciQFF+crrl2Rn/6qZS9dCo1hagb3K5eXMhLXoP425Y4sypNPPqULhPn
-YrUDFNAf89rRLqP1KMPLZ+uO7EECgYEAw1lWPxGV+X85iQxYN9xoX85htfJSBXTf
-dLirQ4bkykOxSA6ZzFuhDO/G373Q1rze4tmEO790uOCeaiXGgeWC1A+2PMO957i5
-9FqhDIkmerfdIttdEUMM9rQwuTcLnixGZkT5GHDzjtNinaIVB+pv7twRAESqN9dC
-QXh7IF7g/X8CgYBMhQOX+hCqZ24D95cAAJrs/ajEWj2geVPZFCDa3oZulJJVeBpu
-bieKWScra9/rS6mE0Ub6cTEFl0fisMNspcDI7NnNP3Y9FMVt3+rp1JIgw5AkGvEW
-CtN9egUGIGcT5A8Qj0lo3slkhcSgS2S6UNq431MZh51z5askyJ/JREULAQKBgFrR
-OatwfYzUfOcd+hVePpfr1rlDwqYOw6P8BoMKP2tZNR4Oy6maH7Fn98kk8eYjQGuu
-PC+avqUEqCEpFrRlAwGbnFl7ltoXozvatmyhhmYe/Iur+ASCa5B2DQDOenQ6mTAK
-eNPIDzMjSwGFzMk1UHx3it/ZDFmRlZfibzuJYIf5AoGBAIaPHk4qadK/XpcD4Wwx
-BOsDEIz27DGWdwWfd5r3EcV4zX/wNzH0G1Z8eydNjUqKzufMZgFwpcTu0Evesl1/
-B8kC8sLHxQoG5SvBu4dBxMwKIU9O9uFnX5SUYZUDpCtUYyZ+GtGom41Jwg5ENrwy
-HzPh2taMnCA0h1fNLFFBkw88
------END PRIVATE KEY-----
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAuq7ujTpHoSwQn9/hFJT837jU/T7xLuyXjkAEfL7uVDuPWSdF
+AJy+QJsuL6INMKMlxLUb1RRzxQgAmtYN1dIEbTPplffCNbfYm3FXg0mZlxg1UBg9
+rE1bPwuz/B+M76S35EIiKQlpaCFErLQi5oyhw9FIBvYLZxxgfeDfDPiXQlBvtHix
+n9TFqNoLNZkAX+auh2Wj2SSjM74pBQWsuVZLkF5CAwYuMQkpEplCV/QHNX3ZeNdQ
+YNFvpA7CxENa3sTZyHpeTmWoOdJXgJFveAWL4ZhSvkSd0HDuPPJp4JckETNESa9B
+qOwFfj36SM+5dRiCwiuzwAQ+oaFjisMXuRbVFQIDAQABAoIBABqFaJmCupNgnboA
+xcq1QdmMuiGCNCRs8zj/ykNoopYv4fUR+aEVI4gtI5obxRDwVJjF+/7BCZNnyCI3
+H78NN5jGA7zM9nfINwsaRor9xUasZ0KKNxTH5pslz/uVBeIzvfY9GPpIfoOPGmEI
+tF6Zgw+9JyTqBoOvCdxIOpfupxqB8TQ0z4UbFUuBiEkGuJ+o8C1rX8Wze0JUl0qG
+BOwhQtaCn/yrm+dTXZ5XaelJY5mcwgFy22Jiynmm6TbLhyZlACd2Q/MGak7o1TJL
+QgvvGMlcVrK7MZ3TJN+wzwWfwAAjXnT3Xvd5pD5yunZJoNe8YyFOCMlh0swNG5Zt
+0tGeX78CgYEA+m6gYGKTNWFnqlqMZRfGTqiqVZeVQKftcLdA1dkscnffRP9bvKOW
+9TbgzoGHiyZnjZBDFTUuy67El8RXIMsxYy3GYuGRdUSLS63Fr2af4pBQIYvW6OmG
+UZlcAP6ZAhUzn409XGlXaac3F30hFeKdC62+V5ZMnfPlVhHRCoKaaqsCgYEAvtV4
+FuU5sFKyhKPPV3rzaNZtL0swvtBIuODH1oAWhPNySQvCu+45W0EOOAPPpsYP4wGX
+G+otOSp4RLdlVXNhkh1rpJzeK77KZ5ZY+ShkuHD+uL/iRARwl/gh2Ve1aqUrm1LE
+9ldchmQGvLalN9HalzeW//xHA3X9SF4Vo16Dvz8CgYABeZlUOABp9hLoO/RLvCIc
+4H1wV543bUXGvi2RlN/gJLiZ7W8a41PGSfZ1AOpNdYJyoQDkJRYLeRILWsqwlMHL
+tb9PYci7ihXP8kwRxmb2rKbsK6iuYoG6BU83akh4bKuLKwfLfYtYQfXfG4uQV29Z
+XEKcvXPiEkethBlZGH/UVQKBgCa9Pvum3OcmYob6mgSwOOl3XgLTyLlzns+pEehB
+aFDk+rZJZOaxnYMg2boVS/oXCvKSSBKqnzOTo4aPlEqceZonzspD7fYDbSNKKhWq
+VYf7qDno+g3EuPagsH5mh7V2gjutub4oTegaNiPpD/Ec8Lrx1f1xQRk6wogGUW4w
+qZ4RAoGBAPg1LezV8mlesF5mhj+KubYP4l1Zf9geAeQprjDbFsA0BEAS2KsWgmwR
+Ye1fmek7jDjCPLQ4Amq030mLJuQGEM3cZPqjKX2sBZ8fQcgw7pWJWMvKMTBA4Aah
+zQx1KXwHJANMWq/0QSFDq/LGJ2OYMlV2F0tH3P5Kp7ZASTyc78ux
+-----END RSA PRIVATE KEY-----
 -----BEGIN CERTIFICATE-----
-MIIDoTCCAomgAwIBAgIGDodLQx9+MA0GCSqGSIb3DQEBCwUAMCgxEjAQBgNVBAMM
-CW1pdG1wcm94eTESMBAGA1UECgwJbWl0bXByb3h5MB4XDTIwMDgxMjE3MDMyNloX
-DTIzMDgxNDE3MDMyNlowKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAlt
-aXRtcHJveHkwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCYp6U4G9YW
-ITYB/JlZ+Hd08c/9a157WVl03hbR2DSK8FnK+D8cp2dGzuTfC08w8M/yvVYPcbb7
-ZDiTNUsVwboFvmr/6mN6M9uQioCRStrP6Rkm2Wuagyj+GjqLwogTJlPiPwEPhlMg
-z1BJu6jQQSgiMsxKWMkVz3pCYERUMRX0DEgYST9rjYUAwD4rPv8XXtLLSPs0VniI
-ggUHJrngDUrtoK5Wuf098NJPIwW8uE2ev+DXH2Iuwn2fNKt5lSYypJdUZjyamwuE
-6HFBeIBAIIKijMz/8UV1+H8Q0OcU2Sva2FglHREQtA/S5FlpcuTZt/77Vnxv75y/
-0zls90iyQ3E/AgMBAAGjgdAwgc0wDwYDVR0TAQH/BAUwAwEB/zARBglghkgBhvhC
-AQEEBAMCAgQweAYDVR0lBHEwbwYIKwYBBQUHAwEGCCsGAQUFBwMCBggrBgEFBQcD
-BAYIKwYBBQUHAwgGCisGAQQBgjcCARUGCisGAQQBgjcCARYGCisGAQQBgjcKAwEG
-CisGAQQBgjcKAwMGCisGAQQBgjcKAwQGCWCGSAGG+EIEATAOBgNVHQ8BAf8EBAMC
-AQYwHQYDVR0OBBYEFBCsLPpFz3l9rOOfGmfs+VRc3jhJMA0GCSqGSIb3DQEBCwUA
-A4IBAQADTpA15na6U5qqDCe0rr39fkS1/dY804Xnz7g/L3AsxPE1KOMijuJa8sKd
-kKwba1173FwMupfK39zY8jUxL8Qprdi92RO6CpoFUsL/icpA///lYhzUSqt32qwe
-gRNW3mtYBimOk6KH1NOfQnJolWpJh+g1OEsitQKEeKwIn5Hz+8/yS5tbwLgdnMlY
-1/it1H70JSdE7nfJueqN4cFfBsm6XaHZzacJJmN7WP88fd+zztnSQsBFbLlnjnqj
-envCDIwCrMywKNMqEBMwmBEGSAF47fVNYj6KzDAtMvBdDkYaHWpBf4tnFfk6v0wj
-wiKjdLjCmJgjGAQjRw5VYJ8JI0XO
+MIIDNTCCAh2gAwIBAgIUcGDiCmOuhfxMGFS/otcGGFkOSAEwDQYJKoZIhvcNAQEL
+BQAwKDESMBAGA1UEAwwJbWl0bXByb3h5MRIwEAYDVQQKDAltaXRtcHJveHkwHhcN
+MjQwMjI3MTMwNTQ4WhcNMzQwMjI2MTMwNTQ4WjAoMRIwEAYDVQQDDAltaXRtcHJv
+eHkxEjAQBgNVBAoMCW1pdG1wcm94eTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALqu7o06R6EsEJ/f4RSU/N+41P0+8S7sl45ABHy+7lQ7j1knRQCcvkCb
+Li+iDTCjJcS1G9UUc8UIAJrWDdXSBG0z6ZX3wjW32JtxV4NJmZcYNVAYPaxNWz8L
+s/wfjO+kt+RCIikJaWghRKy0IuaMocPRSAb2C2ccYH3g3wz4l0JQb7R4sZ/Uxaja
+CzWZAF/mrodlo9kkozO+KQUFrLlWS5BeQgMGLjEJKRKZQlf0BzV92XjXUGDRb6QO
+wsRDWt7E2ch6Xk5lqDnSV4CRb3gFi+GYUr5EndBw7jzyaeCXJBEzREmvQajsBX49
++kjPuXUYgsIrs8AEPqGhY4rDF7kW1RUCAwEAAaNXMFUwDwYDVR0TAQH/BAUwAwEB
+/zATBgNVHSUEDDAKBggrBgEFBQcDATAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE
+FOjFT0G7itqsrCij2InhRSfB0sEkMA0GCSqGSIb3DQEBCwUAA4IBAQCVMa5/xlH4
+GUbrWNMdxr9LL7Dh+vK0wYCfAsc/kO2zCq8iVt/MaqVLel/bKcQhvE5RZHvyep13
+x7378OfCqqHkDDDNroWIvij84ZtMUaM53tF13G/ZGOlNsoLNynWs9IVVvqGKsH7o
+/buJ1RNArI/0irF0UD7qrMmo1p6SYanZhqdh2PphNy9NS3FsfrfnuWvf+/TRp9Ts
+L8058B0p/LIL0OB5trYFircC3iKSOuRl0ERD2ufgSqsSVEYm1mc6UIxv+d1iFD+Q
+8CRUF88icQXrec1TCbhh0CfdDxz+FYSTnW0DR0L75coa/CBmRxAjnrkLoXRr3Y1d
+sTjU4zDdBcBw
 -----END CERTIFICATE-----

From 4cd94aa668c60f92b1d9f4e5cf27752e1fe9c9cd Mon Sep 17 00:00:00 2001
From: "Yuri H. Galvao" <yuri@galvao.ca>
Date: Fri, 1 Mar 2024 04:07:38 -0600
Subject: [PATCH 09/74] Restore brotlipy support (#6261)

---
 scrapy/utils/_compression.py | 32 +++++++++++++++++++++++++++++++-
 tox.ini                      |  1 +
 2 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 5610595d3..14531df3f 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -1,10 +1,40 @@
 import zlib
 from io import BytesIO
+from warnings import warn
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
     import brotli
 except ImportError:
     pass
+else:
+    try:
+        brotli.Decompressor.process
+    except AttributeError:
+
+        warn(
+            (
+                "You have brotlipy installed, and Scrapy will use it, but "
+                "Scrapy support for brotlipy is deprecated and will stop "
+                "working in a future version of Scrapy. brotlipy itself is "
+                "deprecated, it has been superseded by brotlicffi (not "
+                "currently supported by Scrapy). Please, uninstall brotlipy "
+                "and install brotli instead. brotlipy has the same import "
+                "name as brotli, so keeping both installed is strongly "
+                "discouraged."
+            ),
+            ScrapyDeprecationWarning,
+        )
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.decompress(data)
+
+    else:
+
+        def _brotli_decompress(decompressor, data):
+            return decompressor.process(data)
+
 
 try:
     import zstandard
@@ -61,7 +91,7 @@ def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
     decompressed_size = 0
     while output_chunk:
         input_chunk = input_stream.read(_CHUNK_SIZE)
-        output_chunk = decompressor.process(input_chunk)
+        output_chunk = _brotli_decompress(decompressor, input_chunk)
         decompressed_size += len(output_chunk)
         if max_size and decompressed_size > max_size:
             raise _DecompressionMaxSizeExceeded(
diff --git a/tox.ini b/tox.ini
index 4ed9b3bd7..237aa489c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -135,6 +135,7 @@ deps =
     google-cloud-storage==1.29.0
     Pillow==7.1.0
     robotexclusionrulesparser==1.6.2
+    brotlipy
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}

From aa1bf6907964f0281264052cabc28197c5d28107 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Mar 2024 12:48:00 +0100
Subject: [PATCH 10/74] Mark hashlib usages as not intended for security
 (#6264)

---
 .bandit.yml                |  1 -
 scrapy/pipelines/files.py  | 28 ++++++++++++++++++++++------
 scrapy/pipelines/images.py |  9 ++++-----
 scrapy/utils/misc.py       | 10 +++++++++-
 scrapy/utils/request.py    |  2 +-
 5 files changed, 36 insertions(+), 14 deletions(-)

diff --git a/.bandit.yml b/.bandit.yml
index 4fcd75c57..db2fbb84c 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -2,7 +2,6 @@ skips:
 - B101  # assert_used, needed for mypy
 - B320  # xml_bad_etree
 - B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B324  # hashlib "Use of weak SHA1 hash for security"
 - B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
 - B410  # import_lxml
 - B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 1d7625299..d04218089 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -16,7 +16,7 @@ from ftplib import FTP
 from io import BytesIO
 from os import PathLike
 from pathlib import Path
-from typing import DefaultDict, Optional, Set, Union
+from typing import IO, DefaultDict, Optional, Set, Union
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -31,7 +31,6 @@ from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.ftp import ftp_store_file
 from scrapy.utils.log import failure_to_exc_info
-from scrapy.utils.misc import md5sum
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 
@@ -42,6 +41,23 @@ def _to_string(path: Union[str, PathLike]) -> str:
     return str(path)  # convert a Path object to string
 
 
+def _md5sum(file: IO) -> str:
+    """Calculate the md5 checksum of a file-like object without reading its
+    whole content in memory.
+
+    >>> from io import BytesIO
+    >>> _md5sum(BytesIO(b'file content to hash'))
+    '784406af91dd5a54fbb9c84c2236595a'
+    """
+    m = hashlib.md5()  # nosec
+    while True:
+        d = file.read(8096)
+        if not d:
+            break
+        m.update(d)
+    return m.hexdigest()
+
+
 class FileException(Exception):
     """General media error exception"""
 
@@ -70,7 +86,7 @@ class FSFilesStore:
             return {}
 
         with absolute_path.open("rb") as f:
-            checksum = md5sum(f)
+            checksum = _md5sum(f)
 
         return {"last_modified": last_modified, "checksum": checksum}
 
@@ -299,7 +315,7 @@ class FTPFilesStore:
                     ftp.set_pasv(False)
                 file_path = f"{self.basedir}/{path}"
                 last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
-                m = hashlib.md5()
+                m = hashlib.md5()  # nosec
                 ftp.retrbinary(f"RETR {file_path}", m.update)
                 return {"last_modified": last_modified, "checksum": m.hexdigest()}
             # The file doesn't exist
@@ -531,7 +547,7 @@ class FilesPipeline(MediaPipeline):
     def file_downloaded(self, response, request, info, *, item=None):
         path = self.file_path(request, response=response, info=info, item=item)
         buf = BytesIO(response.body)
-        checksum = md5sum(buf)
+        checksum = _md5sum(buf)
         buf.seek(0)
         self.store.persist_file(path, buf, info)
         return checksum
@@ -542,7 +558,7 @@ class FilesPipeline(MediaPipeline):
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
-        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         media_ext = Path(request.url).suffix
         # Handles empty and wild extensions by trying to guess the
         # mime type then extension or default to empty string otherwise
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 8169583f8..137aa7a9a 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -17,11 +17,10 @@ from itemadapter import ItemAdapter
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FileException, FilesPipeline
+from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
 
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
-from scrapy.utils.misc import md5sum
 from scrapy.utils.python import get_func_args, to_bytes
 
 
@@ -128,7 +127,7 @@ class ImagesPipeline(FilesPipeline):
         for path, image, buf in self.get_images(response, request, info, item=item):
             if checksum is None:
                 buf.seek(0)
-                checksum = md5sum(buf)
+                checksum = _md5sum(buf)
             width, height = image.size
             self.store.persist_file(
                 path,
@@ -228,9 +227,9 @@ class ImagesPipeline(FilesPipeline):
         return item
 
     def file_path(self, request, response=None, info=None, *, item=None):
-        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"full/{image_guid}.jpg"
 
     def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
-        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
+        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()  # nosec
         return f"thumbs/{thumb_id}/{thumb_guid}.jpg"
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 7b43760a8..7f83d06fb 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -113,7 +113,15 @@ def md5sum(file: IO) -> str:
     >>> md5sum(BytesIO(b'file content to hash'))
     '784406af91dd5a54fbb9c84c2236595a'
     """
-    m = hashlib.md5()
+    warnings.warn(
+        (
+            "The scrapy.utils.misc.md5sum function is deprecated, and will be "
+            "removed in a future version of Scrapy."
+        ),
+        ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+    m = hashlib.md5()  # nosec
     while True:
         d = file.read(8096)
         if not d:
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index e99d1eeb5..1f07d58eb 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -111,7 +111,7 @@ def fingerprint(
             "headers": headers,
         }
         fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
-        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
+        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()  # nosec
     return cache[cache_key]
 
 

From bf149356fc6e519e92fb55150a60b40b14e45ae8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Fri, 1 Mar 2024 16:02:03 +0100
Subject: [PATCH 11/74] Bandit: allow-list lxml usages (#6265)

---
 .bandit.yml                       |  2 --
 scrapy/http/request/form.py       | 17 ++++++-----------
 scrapy/linkextractors/lxmlhtml.py |  2 +-
 scrapy/selector/unified.py        |  3 ++-
 scrapy/utils/_compression.py      |  1 -
 scrapy/utils/iterators.py         |  5 +++--
 scrapy/utils/sitemap.py           |  4 ++--
 scrapy/utils/versions.py          |  2 +-
 8 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/.bandit.yml b/.bandit.yml
index db2fbb84c..b7f1817e0 100644
--- a/.bandit.yml
+++ b/.bandit.yml
@@ -1,9 +1,7 @@
 skips:
 - B101  # assert_used, needed for mypy
-- B320  # xml_bad_etree
 - B321  # ftplib, https://github.com/scrapy/scrapy/issues/4180
 - B402  # import_ftplib, https://github.com/scrapy/scrapy/issues/4180
-- B410  # import_lxml
 - B411  # import_xmlrpclib, https://github.com/PyCQA/bandit/issues/1082
 - B503  # ssl_with_bad_defaults
 exclude_dirs: ['tests']
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 0f80a0ab7..3206d79cd 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -10,21 +10,16 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any, Iterable, List, Optional, Tuple, Union, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
 
-from lxml.html import (
-    FormElement,
-    HTMLParser,
-    InputElement,
-    MultipleSelectOptions,
-    SelectElement,
-    TextareaElement,
-)
-from parsel.selector import create_root_node
+from lxml.html import FormElement  # nosec
+from lxml.html import InputElement  # nosec
+from lxml.html import MultipleSelectOptions  # nosec
+from lxml.html import SelectElement  # nosec
+from lxml.html import TextareaElement  # nosec
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http.request import Request
 from scrapy.http.response.text import TextResponse
 from scrapy.utils.python import is_listlike, to_bytes
-from scrapy.utils.response import get_base_url
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
@@ -120,7 +115,7 @@ def _get_form(
     formxpath: Optional[str],
 ) -> FormElement:
     """Find the wanted form element within the given response."""
-    root = create_root_node(response.text, HTMLParser, base_url=get_base_url(response))
+    root = response.selector.root
     forms = root.xpath("//form")
     if not forms:
         raise ValueError(f"No <form> element found in {response}")
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index d76db20ba..55bc0fc43 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -7,7 +7,7 @@ import operator
 from functools import partial
 from urllib.parse import urljoin, urlparse
 
-from lxml import etree
+from lxml import etree  # nosec
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 75d5e9fbd..aa9581fcd 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -8,6 +8,7 @@ from parsel import Selector as _ParselSelector
 
 from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.utils.python import to_bytes
+from scrapy.utils.response import get_base_url
 from scrapy.utils.trackref import object_ref
 
 __all__ = ["Selector", "SelectorList"]
@@ -88,7 +89,7 @@ class Selector(_ParselSelector, object_ref):
 
         if response is not None:
             text = response.text
-            kwargs.setdefault("base_url", response.url)
+            kwargs.setdefault("base_url", get_base_url(response))
 
         self.response = response
 
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 14531df3f..7c40d0a02 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -12,7 +12,6 @@ else:
     try:
         brotli.Decompressor.process
     except AttributeError:
-
         warn(
             (
                 "You have brotlipy installed, and Scrapy will use it, but "
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 93a2ba7a1..49493e9c6 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -18,7 +18,7 @@ from typing import (
 )
 from warnings import warn
 
-from lxml import etree
+from lxml import etree  # nosec
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
@@ -26,7 +26,7 @@ from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
 if TYPE_CHECKING:
-    from lxml._types import SupportsReadClose
+    from lxml._types import SupportsReadClose  # nosec
 
 logger = logging.getLogger(__name__)
 
@@ -101,6 +101,7 @@ def xmliter_lxml(
         cast("SupportsReadClose[bytes]", reader),
         encoding=reader.encoding,
         events=("end", "start-ns"),
+        resolve_entities=False,
         huge_tree=True,
     )
     selxpath = "//" + (f"{prefix}:{nodename}" if namespace else nodename)
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 8bf941eb2..7dcee3a2f 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -8,7 +8,7 @@ SitemapSpider, its API is subject to change without notice.
 from typing import Any, Dict, Generator, Iterator, Optional
 from urllib.parse import urljoin
 
-import lxml.etree
+import lxml.etree  # nosec
 
 
 class Sitemap:
@@ -19,7 +19,7 @@ class Sitemap:
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )
-        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)
+        self._root = lxml.etree.fromstring(xmltext, parser=xmlp)  # nosec
         rt = self._root.tag
         self.type = self._root.tag.split("}", 1)[1] if "}" in rt else rt
 
diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 9b637bdb0..42e5e9be4 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -4,7 +4,7 @@ from typing import List, Tuple
 
 import cryptography
 import cssselect
-import lxml.etree
+import lxml.etree  # nosec
 import parsel
 import twisted
 import w3lib

From cab1016bb6f719b15043f65502c63fbaa191df36 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 20:24:17 -0300
Subject: [PATCH 12/74] Add brotlicffi support

---
 scrapy/utils/_compression.py                  |  2 ++
 ...st_downloadermiddleware_httpcompression.py | 35 +++++++++++++++----
 tox.ini                                       |  1 +
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 7c40d0a02..7896f4c01 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -6,6 +6,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
     import brotli
+except ImportError:
+    import brotlicffi as brotli
 except ImportError:
     pass
 else:
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index ae5569d0a..7c36f748e 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -130,7 +130,10 @@ class HttpCompressionTest(TestCase):
 
     def test_process_response_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         response = self._getresponse("br")
@@ -448,7 +451,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_setting("br")
@@ -486,7 +492,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_spider_attr("br")
@@ -522,7 +531,10 @@ class HttpCompressionTest(TestCase):
 
     def test_compression_bomb_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_compression_bomb_request_meta("br")
@@ -568,7 +580,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_setting_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_setting("br")
@@ -616,7 +631,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_spider_attr_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_spider_attr("br")
@@ -662,7 +680,10 @@ class HttpCompressionTest(TestCase):
 
     def test_download_warnsize_request_meta_br(self):
         try:
-            import brotli  # noqa: F401
+            try:
+                import brotli  # noqa: F401
+            except ImportError:
+                import brotlicffi  # noqa: F401
         except ImportError:
             raise SkipTest("no brotli")
         self._test_download_warnsize_request_meta("br")
diff --git a/tox.ini b/tox.ini
index 237aa489c..6b804b78c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -162,6 +162,7 @@ basepython = {[testenv:pypy3]basepython}
 deps =
     {[pinned]deps}
     PyPyDispatcher==2.1.0
+    brotlicffi
 commands =
     pytest --durations=10 scrapy tests
 install_command = {[pinned]install_command}

From 3421823dce94a693ee86915110d899d8da6f3e9f Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 20:26:23 -0300
Subject: [PATCH 13/74] Nested try-except block

---
 scrapy/utils/_compression.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 7896f4c01..477573588 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -5,9 +5,10 @@ from warnings import warn
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
-    import brotli
-except ImportError:
-    import brotlicffi as brotli
+    try:
+        import brotli
+    except ImportError:
+        import brotlicffi as brotli
 except ImportError:
     pass
 else:

From a52429ae08ec15d70f7f3e2079d32933bb639d6b Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 21:12:25 -0300
Subject: [PATCH 14/74] Update disclaimer

---
 scrapy/utils/_compression.py | 4 ++--
 tox.ini                      | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 477573588..4b3fd342d 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -20,8 +20,8 @@ else:
                 "You have brotlipy installed, and Scrapy will use it, but "
                 "Scrapy support for brotlipy is deprecated and will stop "
                 "working in a future version of Scrapy. brotlipy itself is "
-                "deprecated, it has been superseded by brotlicffi (not "
-                "currently supported by Scrapy). Please, uninstall brotlipy "
+                "deprecated, it has been superseded by brotlicffi "
+                "Please, uninstall brotlipy "
                 "and install brotli instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "
                 "discouraged."
diff --git a/tox.ini b/tox.ini
index 6b804b78c..9cf3c92ad 100644
--- a/tox.ini
+++ b/tox.ini
@@ -124,7 +124,7 @@ deps =
     robotexclusionrulesparser
     Pillow
     Twisted[http2]
-    brotli
+    brotli; implementation_name != 'pypy'
     zstandard
 
 [testenv:extra-deps-pinned]

From 16864ea602ebc3d1a764aaf6c101a5f20ff57bee Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 21:18:24 -0300
Subject: [PATCH 15/74] Remove PyPy condition

---
 tox.ini | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tox.ini b/tox.ini
index 9cf3c92ad..6b804b78c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -124,7 +124,7 @@ deps =
     robotexclusionrulesparser
     Pillow
     Twisted[http2]
-    brotli; implementation_name != 'pypy'
+    brotli
     zstandard
 
 [testenv:extra-deps-pinned]

From 532cd2eabd8b280e64a1087c49b8f5eb5f05530f Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 21:30:20 -0300
Subject: [PATCH 16/74] Use brotlicffi for PyPy

---
 tests/requirements.txt | 3 +--
 tox.ini                | 1 -
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/requirements.txt b/tests/requirements.txt
index 5b75674f5..ca5f6ddbd 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -11,8 +11,7 @@ uvloop; platform_system != "Windows"
 
 bpython  # optional for shell wrapper tests
 brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
-# 1.1.0 is broken on PyPy: https://github.com/google/brotli/issues/1072
-brotli==1.0.9; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
+brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
 zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
 ipython
 pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index 6b804b78c..237aa489c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -162,7 +162,6 @@ basepython = {[testenv:pypy3]basepython}
 deps =
     {[pinned]deps}
     PyPyDispatcher==2.1.0
-    brotlicffi
 commands =
     pytest --durations=10 scrapy tests
 install_command = {[pinned]install_command}

From 7f1fbdba3cc6f118cbf11285ed26e488f854aed1 Mon Sep 17 00:00:00 2001
From: Laerte Pereira <laertefbk@gmail.com>
Date: Tue, 5 Mar 2024 22:11:11 -0300
Subject: [PATCH 17/74] Check brotlicffi for ACCEPTED_ENCODINGS

---
 scrapy/downloadermiddlewares/httpcompression.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index aa3abe853..0e5e215ac 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -29,7 +29,10 @@ logger = getLogger(__name__)
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
 
 try:
-    import brotli  # noqa: F401
+    try:
+        import brotli  # noqa: F401
+    except ImportError:
+        import brotlicffi  # noqa: F401
 except ImportError:
     pass
 else:

From 7be919138d84ec00feb80a78e13721dff998c10c Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 6 Mar 2024 05:49:31 -0300
Subject: [PATCH 18/74] Update scrapy/utils/_compression.py

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 scrapy/utils/_compression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 4b3fd342d..349fd9ac0 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -20,7 +20,7 @@ else:
                 "You have brotlipy installed, and Scrapy will use it, but "
                 "Scrapy support for brotlipy is deprecated and will stop "
                 "working in a future version of Scrapy. brotlipy itself is "
-                "deprecated, it has been superseded by brotlicffi "
+                "deprecated, it has been superseded by brotlicffi. "
                 "Please, uninstall brotlipy "
                 "and install brotli instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "

From 99f7165c63a8a2dba72090f65ba1093d476d669a Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 6 Mar 2024 09:02:01 -0300
Subject: [PATCH 19/74] Update scrapy/utils/_compression.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/utils/_compression.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 349fd9ac0..84c255c28 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -22,7 +22,7 @@ else:
                 "working in a future version of Scrapy. brotlipy itself is "
                 "deprecated, it has been superseded by brotlicffi. "
                 "Please, uninstall brotlipy "
-                "and install brotli instead. brotlipy has the same import "
+                "and install brotli or brotlicffi instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "
                 "discouraged."
             ),

From 6ecc9e0a34be6317d1b35a3ca1fc13cb98129732 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 6 Mar 2024 17:21:08 +0500
Subject: [PATCH 20/74] Add typing for scrapy/commands (#6268)

---
 scrapy/commands/__init__.py     |  48 +++++++-----
 scrapy/commands/bench.py        |  18 +++--
 scrapy/commands/check.py        |  18 +++--
 scrapy/commands/crawl.py        |  14 +++-
 scrapy/commands/edit.py         |  14 ++--
 scrapy/commands/fetch.py        |  23 +++---
 scrapy/commands/genspider.py    |  31 +++++---
 scrapy/commands/list.py         |   8 +-
 scrapy/commands/parse.py        | 128 +++++++++++++++++++++++---------
 scrapy/commands/runspider.py    |  12 +--
 scrapy/commands/settings.py     |  13 ++--
 scrapy/commands/shell.py        |  19 ++---
 scrapy/commands/startproject.py |  22 +++---
 scrapy/commands/version.py      |  13 ++--
 scrapy/commands/view.py         |  15 +++-
 scrapy/shell.py                 |  70 +++++++++++------
 scrapy/utils/console.py         |  48 ++++++++----
 scrapy/utils/response.py        |  26 +++----
 18 files changed, 355 insertions(+), 185 deletions(-)

diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 27993710e..9fe803d3c 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -3,61 +3,62 @@ Base class for Scrapy commands
 """
 
 import argparse
+import builtins
 import os
 from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
 
 from twisted.python import failure
 
-from scrapy.crawler import CrawlerProcess
+from scrapy.crawler import Crawler, CrawlerProcess
 from scrapy.exceptions import UsageError
 from scrapy.utils.conf import arglist_to_dict, feed_process_params_from_cli
 
 
 class ScrapyCommand:
-    requires_project = False
+    requires_project: bool = False
     crawler_process: Optional[CrawlerProcess] = None
 
     # default settings to be used for this command instead of global defaults
     default_settings: Dict[str, Any] = {}
 
-    exitcode = 0
+    exitcode: int = 0
 
     def __init__(self) -> None:
         self.settings: Any = None  # set in scrapy.cmdline
 
-    def set_crawler(self, crawler):
+    def set_crawler(self, crawler: Crawler) -> None:
         if hasattr(self, "_crawler"):
             raise RuntimeError("crawler already set")
-        self._crawler = crawler
+        self._crawler: Crawler = crawler
 
-    def syntax(self):
+    def syntax(self) -> str:
         """
         Command syntax (preferably one-line). Do not include command name.
         """
         return ""
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         """
         A short description of the command
         """
         return ""
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         """A long description of the command. Return short description when not
         available. It cannot contain newlines since contents will be formatted
         by optparser which removes newlines and wraps text.
         """
         return self.short_desc()
 
-    def help(self):
+    def help(self) -> str:
         """An extensive help for the command. It will be shown when using the
         "help" command. It can contain newlines since no post-formatting will
         be applied to its contents.
         """
         return self.long_desc()
 
-    def add_options(self, parser):
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
         """
         Populate option parse with options available for this command
         """
@@ -92,7 +93,7 @@ class ScrapyCommand:
         )
         group.add_argument("--pdb", action="store_true", help="enable pdb on failure")
 
-    def process_options(self, args, opts):
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
         try:
             self.settings.setdict(arglist_to_dict(opts.set), priority="cmdline")
         except ValueError:
@@ -129,8 +130,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
     Common class used to share functionality between the crawl, parse and runspider commands
     """
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-a",
             dest="spargs",
@@ -162,8 +163,8 @@ class BaseRunSpiderCommand(ScrapyCommand):
             help="format to use for dumping items",
         )
 
-    def process_options(self, args, opts):
-        ScrapyCommand.process_options(self, args, opts)
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
+        super().process_options(args, opts)
         try:
             opts.spargs = arglist_to_dict(opts.spargs)
         except ValueError:
@@ -183,7 +184,13 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
     Help Formatter for scrapy command line help messages.
     """
 
-    def __init__(self, prog, indent_increment=2, max_help_position=24, width=None):
+    def __init__(
+        self,
+        prog: str,
+        indent_increment: int = 2,
+        max_help_position: int = 24,
+        width: Optional[int] = None,
+    ):
         super().__init__(
             prog,
             indent_increment=indent_increment,
@@ -191,11 +198,12 @@ class ScrapyHelpFormatter(argparse.HelpFormatter):
             width=width,
         )
 
-    def _join_parts(self, part_strings):
-        parts = self.format_part_strings(part_strings)
+    def _join_parts(self, part_strings: Iterable[str]) -> str:
+        # scrapy.commands.list shadows builtins.list
+        parts = self.format_part_strings(builtins.list(part_strings))
         return super()._join_parts(parts)
 
-    def format_part_strings(self, part_strings):
+    def format_part_strings(self, part_strings: List[str]) -> List[str]:
         """
         Underline and title case command line help message headers.
         """
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index aaf5a439f..2e6bb5d86 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -1,10 +1,14 @@
+import argparse
 import subprocess  # nosec
 import sys
 import time
+from typing import Any, Iterable, List
 from urllib.parse import urlencode
 
 import scrapy
+from scrapy import Request
 from scrapy.commands import ScrapyCommand
+from scrapy.http import Response
 from scrapy.linkextractors import LinkExtractor
 
 
@@ -15,26 +19,28 @@ class Command(ScrapyCommand):
         "CLOSESPIDER_TIMEOUT": 10,
     }
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run quick benchmark test"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         with _BenchServer():
+            assert self.crawler_process
             self.crawler_process.crawl(_BenchSpider, total=100000)
             self.crawler_process.start()
 
 
 class _BenchServer:
-    def __enter__(self):
+    def __enter__(self) -> None:
         from scrapy.utils.test import get_testenv
 
         pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
         self.proc = subprocess.Popen(
             pargs, stdout=subprocess.PIPE, env=get_testenv()
         )  # nosec
+        assert self.proc.stdout
         self.proc.stdout.readline()
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
         self.proc.kill()
         self.proc.wait()
         time.sleep(0.2)
@@ -49,11 +55,11 @@ class _BenchSpider(scrapy.Spider):
     baseurl = "http://localhost:8998"
     link_extractor = LinkExtractor()
 
-    def start_requests(self):
+    def start_requests(self) -> Iterable[Request]:
         qargs = {"total": self.total, "show": self.show}
         url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
-    def parse(self, response):
+    def parse(self, response: Response) -> Any:  # type: ignore[override]
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/commands/check.py b/scrapy/commands/check.py
index de54ca4d3..22c8abf7a 100644
--- a/scrapy/commands/check.py
+++ b/scrapy/commands/check.py
@@ -1,5 +1,7 @@
+import argparse
 import time
 from collections import defaultdict
+from typing import List
 from unittest import TextTestResult as _TextTestResult
 from unittest import TextTestRunner
 
@@ -10,9 +12,10 @@ from scrapy.utils.misc import load_object, set_environ
 
 
 class TextTestResult(_TextTestResult):
-    def printSummary(self, start, stop):
+    def printSummary(self, start: float, stop: float) -> None:
         write = self.stream.write
-        writeln = self.stream.writeln
+        # _WritelnDecorator isn't implemented in typeshed yet
+        writeln = self.stream.writeln  # type: ignore[attr-defined]
 
         run = self.testsRun
         plural = "s" if run != 1 else ""
@@ -42,14 +45,14 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Check spider contracts"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-l",
             "--list",
@@ -66,7 +69,7 @@ class Command(ScrapyCommand):
             help="print contract tests for all spiders",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         # load contracts
         contracts = build_component_list(self.settings.getwithbase("SPIDER_CONTRACTS"))
         conman = ContractsManager(load_object(c) for c in contracts)
@@ -76,6 +79,7 @@ class Command(ScrapyCommand):
         # contract requests
         contract_reqs = defaultdict(list)
 
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
 
         with set_environ(SCRAPY_CHECK="true"):
diff --git a/scrapy/commands/crawl.py b/scrapy/commands/crawl.py
index 2f0f1c7b9..6e023af81 100644
--- a/scrapy/commands/crawl.py
+++ b/scrapy/commands/crawl.py
@@ -1,3 +1,8 @@
+import argparse
+from typing import List, cast
+
+from twisted.python.failure import Failure
+
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
 
@@ -5,13 +10,13 @@ from scrapy.exceptions import UsageError
 class Command(BaseRunSpiderCommand):
     requires_project = True
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run a spider"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) < 1:
             raise UsageError()
         elif len(args) > 1:
@@ -20,10 +25,11 @@ class Command(BaseRunSpiderCommand):
             )
         spname = args[0]
 
+        assert self.crawler_process
         crawl_defer = self.crawler_process.crawl(spname, **opts.spargs)
 
         if getattr(crawl_defer, "result", None) is not None and issubclass(
-            crawl_defer.result.type, Exception
+            cast(Failure, crawl_defer.result).type, Exception
         ):
             self.exitcode = 1
         else:
diff --git a/scrapy/commands/edit.py b/scrapy/commands/edit.py
index e85d2c9ec..04012bee8 100644
--- a/scrapy/commands/edit.py
+++ b/scrapy/commands/edit.py
@@ -1,5 +1,7 @@
+import argparse
 import os
 import sys
+from typing import List
 
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
@@ -9,32 +11,34 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "<spider>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Edit spider"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Edit a spider using the editor defined in the EDITOR environment"
             " variable or else the EDITOR setting"
         )
 
-    def _err(self, msg):
+    def _err(self, msg: str) -> None:
         sys.stderr.write(msg + os.linesep)
         self.exitcode = 1
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) != 1:
             raise UsageError()
 
         editor = self.settings["EDITOR"]
+        assert self.crawler_process
         try:
             spidercls = self.crawler_process.spider_loader.load(args[0])
         except KeyError:
             return self._err(f"Spider not found: {args[0]}")
 
         sfile = sys.modules[spidercls.__module__].__file__
+        assert sfile
         sfile = sfile.replace(".pyc", ".py")
         self.exitcode = os.system(f'{editor} "{sfile}"')  # nosec
diff --git a/scrapy/commands/fetch.py b/scrapy/commands/fetch.py
index cdb7ad4ae..1acf2d26f 100644
--- a/scrapy/commands/fetch.py
+++ b/scrapy/commands/fetch.py
@@ -1,13 +1,13 @@
 import sys
-from argparse import Namespace
-from typing import List, Type
+from argparse import ArgumentParser, Namespace
+from typing import Dict, List, Type
 
 from w3lib.url import is_url
 
 from scrapy import Spider
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
-from scrapy.http import Request
+from scrapy.http import Request, Response
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 
@@ -15,20 +15,20 @@ from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 class Command(ScrapyCommand):
     requires_project = False
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <url>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Fetch a URL using the Scrapy downloader"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Fetch a URL using the Scrapy downloader and print its content"
             " to stdout. You may want to use --nolog to disable logging"
         )
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument("--spider", dest="spider", help="use this spider")
         parser.add_argument(
             "--headers",
@@ -44,20 +44,21 @@ class Command(ScrapyCommand):
             help="do not handle HTTP 3xx status codes and print response as-is",
         )
 
-    def _print_headers(self, headers, prefix):
+    def _print_headers(self, headers: Dict[bytes, List[bytes]], prefix: bytes) -> None:
         for key, values in headers.items():
             for value in values:
                 self._print_bytes(prefix + b" " + key + b": " + value)
 
-    def _print_response(self, response, opts):
+    def _print_response(self, response: Response, opts: Namespace) -> None:
         if opts.headers:
+            assert response.request
             self._print_headers(response.request.headers, b">")
             print(">")
             self._print_headers(response.headers, b"<")
         else:
             self._print_bytes(response.body)
 
-    def _print_bytes(self, bytes_):
+    def _print_bytes(self, bytes_: bytes) -> None:
         sys.stdout.buffer.write(bytes_ + b"\n")
 
     def run(self, args: List[str], opts: Namespace) -> None:
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 567ebcdc0..2649fb23d 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -1,9 +1,10 @@
+import argparse
 import os
 import shutil
 import string
 from importlib import import_module
 from pathlib import Path
-from typing import Optional, cast
+from typing import List, Optional, Union, cast
 from urllib.parse import urlparse
 
 import scrapy
@@ -12,7 +13,7 @@ from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
 
-def sanitize_module_name(module_name):
+def sanitize_module_name(module_name: str) -> str:
     """Sanitize the given module name, by replacing dashes and points
     with underscores and prefixing it with a letter if it doesn't start
     with one
@@ -23,7 +24,7 @@ def sanitize_module_name(module_name):
     return module_name
 
 
-def extract_domain(url):
+def extract_domain(url: str) -> str:
     """Extract domain name from URL string"""
     o = urlparse(url)
     if o.scheme == "" and o.netloc == "":
@@ -31,7 +32,7 @@ def extract_domain(url):
     return o.netloc
 
 
-def verify_url_scheme(url):
+def verify_url_scheme(url: str) -> str:
     """Check url for scheme and insert https if none found."""
     parsed = urlparse(url)
     if parsed.scheme == "" and parsed.netloc == "":
@@ -43,14 +44,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <name> <domain>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Generate new spider using pre-defined templates"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-l",
             "--list",
@@ -86,7 +87,7 @@ class Command(ScrapyCommand):
             help="If the spider already exists, overwrite it with the template",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if opts.list:
             self._list_templates()
             return
@@ -115,7 +116,14 @@ class Command(ScrapyCommand):
             if opts.edit:
                 self.exitcode = os.system(f'scrapy edit "{name}"')  # nosec
 
-    def _genspider(self, module, name, url, template_name, template_file):
+    def _genspider(
+        self,
+        module: str,
+        name: str,
+        url: str,
+        template_name: str,
+        template_file: Union[str, os.PathLike],
+    ) -> None:
         """Generate the spider module, based on the given template"""
         capitalized_module = "".join(s.capitalize() for s in module.split("_"))
         domain = extract_domain(url)
@@ -130,6 +138,7 @@ class Command(ScrapyCommand):
         }
         if self.settings.get("NEWSPIDER_MODULE"):
             spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
+            assert spiders_module.__file__
             spiders_dir = Path(spiders_module.__file__).parent.resolve()
         else:
             spiders_module = None
@@ -152,7 +161,7 @@ class Command(ScrapyCommand):
         print('Use "scrapy genspider --list" to see all available templates.')
         return None
 
-    def _list_templates(self):
+    def _list_templates(self) -> None:
         print("Available templates:")
         for file in sorted(Path(self.templates_dir).iterdir()):
             if file.suffix == ".tmpl":
diff --git a/scrapy/commands/list.py b/scrapy/commands/list.py
index 2f5032360..dcc51a694 100644
--- a/scrapy/commands/list.py
+++ b/scrapy/commands/list.py
@@ -1,3 +1,6 @@
+import argparse
+from typing import List
+
 from scrapy.commands import ScrapyCommand
 
 
@@ -5,9 +8,10 @@ class Command(ScrapyCommand):
     requires_project = True
     default_settings = {"LOG_ENABLED": False}
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "List available spiders"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         for s in sorted(self.crawler_process.spider_loader.list()):
             print(s)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index c9f8586d3..2453c0d39 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -1,16 +1,32 @@
+import argparse
 import functools
 import inspect
 import json
 import logging
-from typing import Dict
+from types import CoroutineType
+from typing import (
+    Any,
+    AsyncGenerator,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Tuple,
+    TypeVar,
+    Union,
+    overload,
+)
 
 from itemadapter import ItemAdapter, is_item
-from twisted.internet.defer import maybeDeferred
+from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.python.failure import Failure
 from w3lib.url import is_url
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
-from scrapy.http import Request
+from scrapy.http import Request, Response
+from scrapy.spiders import Spider
 from scrapy.utils import display
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import aiter_errback, deferred_from_coro
@@ -20,24 +36,26 @@ from scrapy.utils.spider import spidercls_for_request
 
 logger = logging.getLogger(__name__)
 
+_T = TypeVar("_T")
+
 
 class Command(BaseRunSpiderCommand):
     requires_project = True
 
     spider = None
-    items: Dict[int, list] = {}
-    requests: Dict[int, list] = {}
+    items: Dict[int, List[Any]] = {}
+    requests: Dict[int, List[Request]] = {}
 
     first_response = None
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <url>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Parse URL (using its spider) and print the results"
 
-    def add_options(self, parser):
-        BaseRunSpiderCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--spider",
             dest="spider",
@@ -106,7 +124,7 @@ class Command(BaseRunSpiderCommand):
         )
 
     @property
-    def max_level(self):
+    def max_level(self) -> int:
         max_items, max_requests = 0, 0
         if self.items:
             max_items = max(self.items)
@@ -114,13 +132,21 @@ class Command(BaseRunSpiderCommand):
             max_requests = max(self.requests)
         return max(max_items, max_requests)
 
-    def handle_exception(self, _failure):
+    def handle_exception(self, _failure: Failure) -> None:
         logger.error(
             "An error is caught while iterating the async iterable",
             exc_info=failure_to_exc_info(_failure),
         )
 
-    def iterate_spider_output(self, result):
+    @overload
+    def iterate_spider_output(
+        self, result: Union[AsyncGenerator, CoroutineType]
+    ) -> Deferred: ...
+
+    @overload
+    def iterate_spider_output(self, result: _T) -> Iterable: ...
+
+    def iterate_spider_output(self, result: Any) -> Union[Iterable, Deferred]:
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
@@ -133,15 +159,15 @@ class Command(BaseRunSpiderCommand):
             return d
         return arg_to_iter(deferred_from_coro(result))
 
-    def add_items(self, lvl, new_items):
+    def add_items(self, lvl: int, new_items: List[Any]) -> None:
         old_items = self.items.get(lvl, [])
         self.items[lvl] = old_items + new_items
 
-    def add_requests(self, lvl, new_reqs):
+    def add_requests(self, lvl: int, new_reqs: List[Request]) -> None:
         old_reqs = self.requests.get(lvl, [])
         self.requests[lvl] = old_reqs + new_reqs
 
-    def print_items(self, lvl=None, colour=True):
+    def print_items(self, lvl: Optional[int] = None, colour: bool = True) -> None:
         if lvl is None:
             items = [item for lst in self.items.values() for item in lst]
         else:
@@ -150,7 +176,7 @@ class Command(BaseRunSpiderCommand):
         print("# Scraped Items ", "-" * 60)
         display.pprint([ItemAdapter(x).asdict() for x in items], colorize=colour)
 
-    def print_requests(self, lvl=None, colour=True):
+    def print_requests(self, lvl: Optional[int] = None, colour: bool = True) -> None:
         if lvl is None:
             if self.requests:
                 requests = self.requests[max(self.requests)]
@@ -162,7 +188,7 @@ class Command(BaseRunSpiderCommand):
         print("# Requests ", "-" * 65)
         display.pprint(requests, colorize=colour)
 
-    def print_results(self, opts):
+    def print_results(self, opts: argparse.Namespace) -> None:
         colour = not opts.nocolour
 
         if opts.verbose:
@@ -179,7 +205,14 @@ class Command(BaseRunSpiderCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
+    def _get_items_and_requests(
+        self,
+        spider_output: Iterable[Any],
+        opts: argparse.Namespace,
+        depth: int,
+        spider: Spider,
+        callback: Callable,
+    ) -> Tuple[List[Any], List[Request], argparse.Namespace, int, Spider, Callable]:
         items, requests = [], []
         for x in spider_output:
             if is_item(x):
@@ -188,14 +221,21 @@ class Command(BaseRunSpiderCommand):
                 requests.append(x)
         return items, requests, opts, depth, spider, callback
 
-    def run_callback(self, response, callback, cb_kwargs=None):
+    def run_callback(
+        self,
+        response: Response,
+        callback: Callable,
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+    ) -> Deferred:
         cb_kwargs = cb_kwargs or {}
         d = maybeDeferred(self.iterate_spider_output, callback(response, **cb_kwargs))
         return d
 
-    def get_callback_from_rules(self, spider, response):
+    def get_callback_from_rules(
+        self, spider: Spider, response: Response
+    ) -> Union[Callable, str, None]:
         if getattr(spider, "rules", None):
-            for rule in spider.rules:
+            for rule in spider.rules:  # type: ignore[attr-defined]
                 if rule.link_extractor.matches(response.url):
                     return rule.callback or "parse"
         else:
@@ -204,8 +244,10 @@ class Command(BaseRunSpiderCommand):
                 "please specify a callback to use for parsing",
                 {"spider": spider.name},
             )
+        return None
 
-    def set_spidercls(self, url, opts):
+    def set_spidercls(self, url: str, opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         spider_loader = self.crawler_process.spider_loader
         if opts.spider:
             try:
@@ -219,13 +261,14 @@ class Command(BaseRunSpiderCommand):
             if not self.spidercls:
                 logger.error("Unable to find spider for: %(url)s", {"url": url})
 
-        def _start_requests(spider):
+        def _start_requests(spider: Spider) -> Iterable[Request]:
             yield self.prepare_request(spider, Request(url), opts)
 
         if self.spidercls:
             self.spidercls.start_requests = _start_requests
 
-    def start_parsing(self, url, opts):
+    def start_parsing(self, url: str, opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
         self.pcrawler = list(self.crawler_process.crawlers)[0]
         self.crawler_process.start()
@@ -233,7 +276,12 @@ class Command(BaseRunSpiderCommand):
         if not self.first_response:
             logger.error("No response downloaded for: %(url)s", {"url": url})
 
-    def scraped_data(self, args):
+    def scraped_data(
+        self,
+        args: Tuple[
+            List[Any], List[Request], argparse.Namespace, int, Spider, Callable
+        ],
+    ) -> List[Any]:
         items, requests, opts, depth, spider, callback = args
         if opts.pipelines:
             itemproc = self.pcrawler.engine.scraper.itemproc
@@ -252,8 +300,14 @@ class Command(BaseRunSpiderCommand):
 
         return scraped_data
 
-    def _get_callback(self, *, spider, opts, response=None):
-        cb = None
+    def _get_callback(
+        self,
+        *,
+        spider: Spider,
+        opts: argparse.Namespace,
+        response: Optional[Response] = None,
+    ) -> Callable:
+        cb: Union[str, Callable, None] = None
         if response:
             cb = response.meta["_callback"]
         if not cb:
@@ -270,6 +324,7 @@ class Command(BaseRunSpiderCommand):
                 cb = "parse"
 
         if not callable(cb):
+            assert cb is not None
             cb_method = getattr(spider, cb, None)
             if callable(cb_method):
                 cb = cb_method
@@ -277,10 +332,13 @@ class Command(BaseRunSpiderCommand):
                 raise ValueError(
                     f"Cannot find callback {cb!r} in spider: {spider.name}"
                 )
+        assert callable(cb)
         return cb
 
-    def prepare_request(self, spider, request, opts):
-        def callback(response, **cb_kwargs):
+    def prepare_request(
+        self, spider: Spider, request: Request, opts: argparse.Namespace
+    ) -> Request:
+        def callback(response: Response, **cb_kwargs: Any) -> Deferred:
             # memorize first request
             if not self.first_response:
                 self.first_response = response
@@ -288,7 +346,7 @@ class Command(BaseRunSpiderCommand):
             cb = self._get_callback(spider=spider, opts=opts, response=response)
 
             # parse items and requests
-            depth = response.meta["_depth"]
+            depth: int = response.meta["_depth"]
 
             d = self.run_callback(response, cb, cb_kwargs)
             d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
@@ -311,13 +369,13 @@ class Command(BaseRunSpiderCommand):
         request.callback = callback
         return request
 
-    def process_options(self, args, opts):
-        BaseRunSpiderCommand.process_options(self, args, opts)
+    def process_options(self, args: List[str], opts: argparse.Namespace) -> None:
+        super().process_options(args, opts)
 
         self.process_request_meta(opts)
         self.process_request_cb_kwargs(opts)
 
-    def process_request_meta(self, opts):
+    def process_request_meta(self, opts: argparse.Namespace) -> None:
         if opts.meta:
             try:
                 opts.meta = json.loads(opts.meta)
@@ -328,7 +386,7 @@ class Command(BaseRunSpiderCommand):
                     print_help=False,
                 )
 
-    def process_request_cb_kwargs(self, opts):
+    def process_request_cb_kwargs(self, opts: argparse.Namespace) -> None:
         if opts.cbkwargs:
             try:
                 opts.cbkwargs = json.loads(opts.cbkwargs)
@@ -339,7 +397,7 @@ class Command(BaseRunSpiderCommand):
                     print_help=False,
                 )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         # parse arguments
         if not len(args) == 1 or not is_url(args[0]):
             raise UsageError()
diff --git a/scrapy/commands/runspider.py b/scrapy/commands/runspider.py
index 58ed89a81..77850e7b5 100644
--- a/scrapy/commands/runspider.py
+++ b/scrapy/commands/runspider.py
@@ -1,9 +1,10 @@
+import argparse
 import sys
 from importlib import import_module
 from os import PathLike
 from pathlib import Path
 from types import ModuleType
-from typing import Union
+from typing import List, Union
 
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.exceptions import UsageError
@@ -27,16 +28,16 @@ class Command(BaseRunSpiderCommand):
     requires_project = False
     default_settings = {"SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options] <spider_file>"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Run a self-contained spider (without creating a project)"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return "Run the spider defined in the given file"
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) != 1:
             raise UsageError()
         filename = Path(args[0])
@@ -51,6 +52,7 @@ class Command(BaseRunSpiderCommand):
             raise UsageError(f"No spider found in file: {filename}\n")
         spidercls = spclasses.pop()
 
+        assert self.crawler_process
         self.crawler_process.crawl(spidercls, **opts.spargs)
         self.crawler_process.start()
 
diff --git a/scrapy/commands/settings.py b/scrapy/commands/settings.py
index 318187204..dbda73b44 100644
--- a/scrapy/commands/settings.py
+++ b/scrapy/commands/settings.py
@@ -1,4 +1,6 @@
+import argparse
 import json
+from typing import List
 
 from scrapy.commands import ScrapyCommand
 from scrapy.settings import BaseSettings
@@ -8,14 +10,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[options]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Get settings values"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--get", dest="get", metavar="SETTING", help="print raw setting value"
         )
@@ -44,7 +46,8 @@ class Command(ScrapyCommand):
             help="print setting value, interpreted as a list",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
+        assert self.crawler_process
         settings = self.crawler_process.settings
         if opts.get:
             s = settings.get(opts.get)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index f72a23c6a..668c95a7b 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -4,9 +4,9 @@ Scrapy Shell
 See documentation in docs/topics/shell.rst
 """
 
-from argparse import Namespace
+from argparse import ArgumentParser, Namespace
 from threading import Thread
-from typing import List, Type
+from typing import Any, Dict, List, Type
 
 from scrapy import Spider
 from scrapy.commands import ScrapyCommand
@@ -24,20 +24,20 @@ class Command(ScrapyCommand):
         "DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
     }
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[url|file]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Interactive scraping console"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Interactive console for scraping the given url or file. "
             "Use ./file.html syntax or full path for local file."
         )
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "-c",
             dest="code",
@@ -52,7 +52,7 @@ class Command(ScrapyCommand):
             help="do not handle HTTP 3xx status codes and print response as-is",
         )
 
-    def update_vars(self, vars):
+    def update_vars(self, vars: Dict[str, Any]) -> None:
         """You can use this function to update the Scrapy objects that will be
         available in the shell
         """
@@ -88,7 +88,8 @@ class Command(ScrapyCommand):
         shell = Shell(crawler, update_vars=self.update_vars, code=opts.code)
         shell.start(url=url, redirect=not opts.no_redirect)
 
-    def _start_crawler_thread(self):
+    def _start_crawler_thread(self) -> None:
+        assert self.crawler_process
         t = Thread(
             target=self.crawler_process.start,
             kwargs={"stop_after_crawl": False, "install_signal_handlers": False},
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
index fde609c6f..58c1aa28f 100644
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,3 +1,4 @@
+import argparse
 import os
 import re
 import string
@@ -5,13 +6,14 @@ from importlib.util import find_spec
 from pathlib import Path
 from shutil import copy2, copystat, ignore_patterns, move
 from stat import S_IWUSR as OWNER_WRITE_PERMISSION
+from typing import List, Tuple, Union
 
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.exceptions import UsageError
 from scrapy.utils.template import render_templatefile, string_camelcase
 
-TEMPLATES_TO_RENDER = (
+TEMPLATES_TO_RENDER: Tuple[Tuple[str, ...], ...] = (
     ("scrapy.cfg",),
     ("${project_name}", "settings.py.tmpl"),
     ("${project_name}", "items.py.tmpl"),
@@ -22,7 +24,7 @@ TEMPLATES_TO_RENDER = (
 IGNORE = ignore_patterns("*.pyc", "__pycache__", ".svn")
 
 
-def _make_writable(path):
+def _make_writable(path: Union[str, os.PathLike]) -> None:
     current_permissions = os.stat(path).st_mode
     os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
@@ -31,14 +33,14 @@ class Command(ScrapyCommand):
     requires_project = False
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "<project_name> [project_dir]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Create new project"
 
-    def _is_valid_name(self, project_name):
-        def _module_exists(module_name):
+    def _is_valid_name(self, project_name: str) -> bool:
+        def _module_exists(module_name: str) -> bool:
             spec = find_spec(module_name)
             return spec is not None and spec.loader is not None
 
@@ -53,7 +55,7 @@ class Command(ScrapyCommand):
             return True
         return False
 
-    def _copytree(self, src: Path, dst: Path):
+    def _copytree(self, src: Path, dst: Path) -> None:
         """
         Since the original function always creates the directory, to resolve
         the issue a new function had to be created. It's a simple copy and
@@ -84,7 +86,7 @@ class Command(ScrapyCommand):
         copystat(src, dst)
         _make_writable(dst)
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if len(args) not in (1, 2):
             raise UsageError()
 
@@ -105,7 +107,9 @@ class Command(ScrapyCommand):
             return
 
         self._copytree(Path(self.templates_dir), project_dir.resolve())
-        move(project_dir / "module", project_dir / project_name)
+        # On 3.8 shutil.move doesn't fully support Path args, but it supports our use case
+        # See https://bugs.python.org/issue32689
+        move(project_dir / "module", project_dir / project_name)  # type: ignore[arg-type]
         for paths in TEMPLATES_TO_RENDER:
             tplfile = Path(
                 project_dir,
diff --git a/scrapy/commands/version.py b/scrapy/commands/version.py
index 47582866b..f057e8544 100644
--- a/scrapy/commands/version.py
+++ b/scrapy/commands/version.py
@@ -1,3 +1,6 @@
+import argparse
+from typing import List
+
 import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.utils.versions import scrapy_components_versions
@@ -6,14 +9,14 @@ from scrapy.utils.versions import scrapy_components_versions
 class Command(ScrapyCommand):
     default_settings = {"LOG_ENABLED": False, "SPIDER_LOADER_WARN_ONLY": True}
 
-    def syntax(self):
+    def syntax(self) -> str:
         return "[-v]"
 
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Print Scrapy version"
 
-    def add_options(self, parser):
-        ScrapyCommand.add_options(self, parser)
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
+        super().add_options(parser)
         parser.add_argument(
             "--verbose",
             "-v",
@@ -22,7 +25,7 @@ class Command(ScrapyCommand):
             help="also display twisted/python/platform info (useful for bug reports)",
         )
 
-    def run(self, args, opts):
+    def run(self, args: List[str], opts: argparse.Namespace) -> None:
         if opts.verbose:
             versions = scrapy_components_versions()
             width = max(len(n) for (n, _) in versions)
diff --git a/scrapy/commands/view.py b/scrapy/commands/view.py
index ebdfa10a8..21679e3aa 100644
--- a/scrapy/commands/view.py
+++ b/scrapy/commands/view.py
@@ -1,21 +1,28 @@
 import argparse
+import logging
 
 from scrapy.commands import fetch
+from scrapy.http import Response, TextResponse
 from scrapy.utils.response import open_in_browser
 
+logger = logging.getLogger(__name__)
+
 
 class Command(fetch.Command):
-    def short_desc(self):
+    def short_desc(self) -> str:
         return "Open URL in browser, as seen by Scrapy"
 
-    def long_desc(self):
+    def long_desc(self) -> str:
         return (
             "Fetch a URL using the Scrapy downloader and show its contents in a browser"
         )
 
-    def add_options(self, parser):
+    def add_options(self, parser: argparse.ArgumentParser) -> None:
         super().add_options(parser)
         parser.add_argument("--headers", help=argparse.SUPPRESS)
 
-    def _print_response(self, response, opts):
+    def _print_response(self, response: Response, opts: argparse.Namespace) -> None:
+        if not isinstance(response, TextResponse):
+            logger.error("Cannot view a non-text response.")
+            return
         open_in_browser(response)
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 63ea33892..fac42e8a2 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -6,6 +6,7 @@ See documentation in docs/topics/shell.rst
 
 import os
 import signal
+from typing import Any, Callable, Dict, Optional, Tuple, Union
 
 from itemadapter import is_item
 from twisted.internet import defer, threads
@@ -26,18 +27,32 @@ from scrapy.utils.response import open_in_browser
 
 
 class Shell:
-    relevant_classes = (Crawler, Spider, Request, Response, Settings)
+    relevant_classes: Tuple[type, ...] = (Crawler, Spider, Request, Response, Settings)
 
-    def __init__(self, crawler, update_vars=None, code=None):
-        self.crawler = crawler
-        self.update_vars = update_vars or (lambda x: None)
-        self.item_class = load_object(crawler.settings["DEFAULT_ITEM_CLASS"])
-        self.spider = None
-        self.inthread = not threadable.isInIOThread()
-        self.code = code
-        self.vars = {}
+    def __init__(
+        self,
+        crawler: Crawler,
+        update_vars: Optional[Callable[[Dict[str, Any]], None]] = None,
+        code: Optional[str] = None,
+    ):
+        self.crawler: Crawler = crawler
+        self.update_vars: Callable[[Dict[str, Any]], None] = update_vars or (
+            lambda x: None
+        )
+        self.item_class: type = load_object(crawler.settings["DEFAULT_ITEM_CLASS"])
+        self.spider: Optional[Spider] = None
+        self.inthread: bool = not threadable.isInIOThread()
+        self.code: Optional[str] = code
+        self.vars: Dict[str, Any] = {}
 
-    def start(self, url=None, request=None, response=None, spider=None, redirect=True):
+    def start(
+        self,
+        url: Optional[str] = None,
+        request: Optional[Request] = None,
+        response: Optional[Response] = None,
+        spider: Optional[Spider] = None,
+        redirect: bool = True,
+    ) -> None:
         # disable accidental Ctrl-C key press from shutting down the engine
         signal.signal(signal.SIGINT, signal.SIG_IGN)
         if url:
@@ -77,7 +92,7 @@ class Shell:
                 self.vars, shells=shells, banner=self.vars.pop("banner", "")
             )
 
-    def _schedule(self, request, spider):
+    def _schedule(self, request: Request, spider: Optional[Spider]) -> defer.Deferred:
         if is_asyncio_reactor_installed():
             # set the asyncio event loop for the current thread
             event_loop_path = self.crawler.settings["ASYNCIO_EVENT_LOOP"]
@@ -85,10 +100,11 @@ class Shell:
         spider = self._open_spider(request, spider)
         d = _request_deferred(request)
         d.addCallback(lambda x: (x, spider))
+        assert self.crawler.engine
         self.crawler.engine.crawl(request)
         return d
 
-    def _open_spider(self, request, spider):
+    def _open_spider(self, request: Request, spider: Optional[Spider]) -> Spider:
         if self.spider:
             return self.spider
 
@@ -96,11 +112,18 @@ class Shell:
             spider = self.crawler.spider or self.crawler._create_spider()
 
         self.crawler.spider = spider
+        assert self.crawler.engine
         self.crawler.engine.open_spider(spider, close_if_idle=False)
         self.spider = spider
         return spider
 
-    def fetch(self, request_or_url, spider=None, redirect=True, **kwargs):
+    def fetch(
+        self,
+        request_or_url: Union[Request, str],
+        spider: Optional[Spider] = None,
+        redirect: bool = True,
+        **kwargs: Any,
+    ) -> None:
         from twisted.internet import reactor
 
         if isinstance(request_or_url, Request):
@@ -123,7 +146,12 @@ class Shell:
             pass
         self.populate_vars(response, request, spider)
 
-    def populate_vars(self, response=None, request=None, spider=None):
+    def populate_vars(
+        self,
+        response: Optional[Response] = None,
+        request: Optional[Request] = None,
+        spider: Optional[Spider] = None,
+    ) -> None:
         import scrapy
 
         self.vars["scrapy"] = scrapy
@@ -141,10 +169,10 @@ class Shell:
         if not self.code:
             self.vars["banner"] = self.get_help()
 
-    def print_help(self):
+    def print_help(self) -> None:
         print(self.get_help())
 
-    def get_help(self):
+    def get_help(self) -> str:
         b = []
         b.append("Available Scrapy objects:")
         b.append(
@@ -168,11 +196,11 @@ class Shell:
 
         return "\n".join(f"[s] {line}" for line in b)
 
-    def _is_relevant(self, value):
+    def _is_relevant(self, value: Any) -> bool:
         return isinstance(value, self.relevant_classes) or is_item(value)
 
 
-def inspect_response(response, spider):
+def inspect_response(response: Response, spider: Spider) -> None:
     """Open a shell to inspect the given response"""
     # Shell.start removes the SIGINT handler, so save it and re-add it after
     # the shell has closed
@@ -181,7 +209,7 @@ def inspect_response(response, spider):
     signal.signal(signal.SIGINT, sigint_handler)
 
 
-def _request_deferred(request):
+def _request_deferred(request: Request) -> defer.Deferred:
     """Wrap a request inside a Deferred.
 
     This function is harmful, do not use it until you know what you are doing.
@@ -195,12 +223,12 @@ def _request_deferred(request):
     request_callback = request.callback
     request_errback = request.errback
 
-    def _restore_callbacks(result):
+    def _restore_callbacks(result: Any) -> Any:
         request.callback = request_callback
         request.errback = request_errback
         return result
 
-    d = defer.Deferred()
+    d: defer.Deferred = defer.Deferred()
     d.addBoth(_restore_callbacks)
     if request.callback:
         d.addCallbacks(request.callback, request.errback)
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 100f040bb..bf1803115 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,17 +1,27 @@
 from functools import wraps
+from typing import Any, Callable, Dict, Iterable, Optional
+
+EmbedFuncT = Callable[..., None]
+KnownShellsT = Dict[str, Callable[..., EmbedFuncT]]
 
 
-def _embed_ipython_shell(namespace={}, banner=""):
+def _embed_ipython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start an IPython Shell"""
     try:
         from IPython.terminal.embed import InteractiveShellEmbed
         from IPython.terminal.ipapp import load_default_config
     except ImportError:
-        from IPython.frontend.terminal.embed import InteractiveShellEmbed
-        from IPython.frontend.terminal.ipapp import load_default_config
+        from IPython.frontend.terminal.embed import (  # type: ignore[no-redef]
+            InteractiveShellEmbed,
+        )
+        from IPython.frontend.terminal.ipapp import (  # type: ignore[no-redef]
+            load_default_config,
+        )
 
     @wraps(_embed_ipython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         config = load_default_config()
         # Always use .instance() to ensure _instance propagation to all parents
         # this is needed for <TAB> completion works well for new imports
@@ -26,30 +36,36 @@ def _embed_ipython_shell(namespace={}, banner=""):
     return wrapper
 
 
-def _embed_bpython_shell(namespace={}, banner=""):
+def _embed_bpython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a bpython shell"""
     import bpython
 
     @wraps(_embed_bpython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         bpython.embed(locals_=namespace, banner=banner)
 
     return wrapper
 
 
-def _embed_ptpython_shell(namespace={}, banner=""):
+def _embed_ptpython_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a ptpython shell"""
     import ptpython.repl
 
     @wraps(_embed_ptpython_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         print(banner)
         ptpython.repl.embed(locals=namespace)
 
     return wrapper
 
 
-def _embed_standard_shell(namespace={}, banner=""):
+def _embed_standard_shell(
+    namespace: Dict[str, Any] = {}, banner: str = ""
+) -> EmbedFuncT:
     """Start a standard python shell"""
     import code
 
@@ -63,13 +79,13 @@ def _embed_standard_shell(namespace={}, banner=""):
         readline.parse_and_bind("tab:complete")
 
     @wraps(_embed_standard_shell)
-    def wrapper(namespace=namespace, banner=""):
+    def wrapper(namespace: Dict[str, Any] = namespace, banner: str = "") -> None:
         code.interact(banner=banner, local=namespace)
 
     return wrapper
 
 
-DEFAULT_PYTHON_SHELLS = {
+DEFAULT_PYTHON_SHELLS: KnownShellsT = {
     "ptpython": _embed_ptpython_shell,
     "ipython": _embed_ipython_shell,
     "bpython": _embed_bpython_shell,
@@ -77,7 +93,9 @@ DEFAULT_PYTHON_SHELLS = {
 }
 
 
-def get_shell_embed_func(shells=None, known_shells=None):
+def get_shell_embed_func(
+    shells: Optional[Iterable[str]] = None, known_shells: Optional[KnownShellsT] = None
+) -> Any:
     """Return the first acceptable shell-embed function
     from a given list of shell names.
     """
@@ -95,7 +113,11 @@ def get_shell_embed_func(shells=None, known_shells=None):
                 continue
 
 
-def start_python_console(namespace=None, banner="", shells=None):
+def start_python_console(
+    namespace: Optional[Dict[str, Any]] = None,
+    banner: str = "",
+    shells: Optional[Iterable[str]] = None,
+) -> None:
     """Start Python console bound to the given namespace.
     Readline support and tab completion will be used on Unix, if available.
     """
diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index 63a484b42..a0b06f75c 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -3,24 +3,27 @@ This module provides some useful functions for working with
 scrapy.http.Response objects
 """
 
+from __future__ import annotations
+
 import os
 import re
 import tempfile
 import webbrowser
-from typing import Any, Callable, Iterable, Tuple, Union
+from typing import TYPE_CHECKING, Any, Callable, Iterable, Tuple, Union
 from weakref import WeakKeyDictionary
 
 from twisted.web import http
 from w3lib import html
 
-import scrapy
-from scrapy.http.response import Response
 from scrapy.utils.python import to_bytes, to_unicode
 
-_baseurl_cache: "WeakKeyDictionary[Response, str]" = WeakKeyDictionary()
+if TYPE_CHECKING:
+    from scrapy.http import Response, TextResponse
+
+_baseurl_cache: WeakKeyDictionary[Response, str] = WeakKeyDictionary()
 
 
-def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
+def get_base_url(response: TextResponse) -> str:
     """Return the base url of the given response, joined with the response url"""
     if response not in _baseurl_cache:
         text = response.text[0:4096]
@@ -30,13 +33,13 @@ def get_base_url(response: "scrapy.http.response.text.TextResponse") -> str:
     return _baseurl_cache[response]
 
 
-_metaref_cache: (
-    "WeakKeyDictionary[Response, Union[Tuple[None, None], Tuple[float, str]]]"
-) = WeakKeyDictionary()
+_metaref_cache: WeakKeyDictionary[
+    Response, Union[Tuple[None, None], Tuple[float, str]]
+] = WeakKeyDictionary()
 
 
 def get_meta_refresh(
-    response: "scrapy.http.response.text.TextResponse",
+    response: TextResponse,
     ignore_tags: Iterable[str] = ("script", "noscript"),
 ) -> Union[Tuple[None, None], Tuple[float, str]]:
     """Parse the http-equiv refresh parameter from the given response"""
@@ -68,10 +71,7 @@ def _remove_html_comments(body):
 
 
 def open_in_browser(
-    response: Union[
-        "scrapy.http.response.html.HtmlResponse",
-        "scrapy.http.response.text.TextResponse",
-    ],
+    response: TextResponse,
     _openfunc: Callable[[str], Any] = webbrowser.open,
 ) -> Any:
     """Open *response* in a local web browser, adjusting the `base tag`_ for

From 8985a04bd1328cd6156a7c33a5db74ad8c81802f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 8 Mar 2024 10:57:03 +0500
Subject: [PATCH 21/74] Full typing for scrapy/exporters.py. (#6275)

---
 scrapy/exporters.py             | 150 ++++++++++++++++++--------------
 scrapy/extensions/feedexport.py |  11 +--
 2 files changed, 89 insertions(+), 72 deletions(-)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 79fd4e56f..fb4998099 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -3,16 +3,18 @@ Item Exporters are used to export/serialize items into different formats.
 """
 
 import csv
-import io
 import marshal
 import pickle  # nosec
 import pprint
-from collections.abc import Mapping
+from io import BytesIO, TextIOWrapper
+from json import JSONEncoder
+from typing import Any, Callable, Dict, Iterable, Mapping, Optional, Tuple, Union
 from xml.sax.saxutils import XMLGenerator  # nosec
+from xml.sax.xmlreader import AttributesImpl  # nosec
 
 from itemadapter import ItemAdapter, is_item
 
-from scrapy.item import Item
+from scrapy.item import Field, Item
 from scrapy.utils.python import is_listlike, to_bytes, to_unicode
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
@@ -29,36 +31,42 @@ __all__ = [
 
 
 class BaseItemExporter:
-    def __init__(self, *, dont_fail=False, **kwargs):
-        self._kwargs = kwargs
+    def __init__(self, *, dont_fail: bool = False, **kwargs: Any):
+        self._kwargs: Dict[str, Any] = kwargs
         self._configure(kwargs, dont_fail=dont_fail)
 
-    def _configure(self, options, dont_fail=False):
+    def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
         """Configure the exporter by popping options from the ``options`` dict.
         If dont_fail is set, it won't raise an exception on unexpected options
         (useful for using with keyword arguments in subclasses ``__init__`` methods)
         """
-        self.encoding = options.pop("encoding", None)
-        self.fields_to_export = options.pop("fields_to_export", None)
-        self.export_empty_fields = options.pop("export_empty_fields", False)
-        self.indent = options.pop("indent", None)
+        self.encoding: Optional[str] = options.pop("encoding", None)
+        self.fields_to_export: Union[Mapping[str, str], Iterable[str], None] = (
+            options.pop("fields_to_export", None)
+        )
+        self.export_empty_fields: bool = options.pop("export_empty_fields", False)
+        self.indent: Optional[int] = options.pop("indent", None)
         if not dont_fail and options:
             raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         raise NotImplementedError
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", lambda x: x)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
         return serializer(value)
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         pass
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         pass
 
-    def _get_serialized_fields(self, item, default_value=None, include_empty=None):
+    def _get_serialized_fields(
+        self, item: Any, default_value: Any = None, include_empty: Optional[bool] = None
+    ) -> Iterable[Tuple[str, Any]]:
         """Return the fields to export as an iterable of tuples
         (name, serialized_value)
         """
@@ -100,22 +108,22 @@ class BaseItemExporter:
 
 
 class JsonLinesItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(dont_fail=True, **kwargs)
-        self.file = file
+        self.file: BytesIO = file
         self._kwargs.setdefault("ensure_ascii", not self.encoding)
-        self.encoder = ScrapyJSONEncoder(**self._kwargs)
+        self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         data = self.encoder.encode(itemdict) + "\n"
         self.file.write(to_bytes(data, self.encoding))
 
 
 class JsonItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(dont_fail=True, **kwargs)
-        self.file = file
+        self.file: BytesIO = file
         # there is a small difference between the behaviour or JsonItemExporter.indent
         # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent
         # the addition of newlines everywhere
@@ -127,26 +135,26 @@ class JsonItemExporter(BaseItemExporter):
         self.encoder = ScrapyJSONEncoder(**self._kwargs)
         self.first_item = True
 
-    def _beautify_newline(self):
+    def _beautify_newline(self) -> None:
         if self.indent is not None:
             self.file.write(b"\n")
 
-    def _add_comma_after_first(self):
+    def _add_comma_after_first(self) -> None:
         if self.first_item:
             self.first_item = False
         else:
             self.file.write(b",")
             self._beautify_newline()
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         self.file.write(b"[")
         self._beautify_newline()
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self._beautify_newline()
         self.file.write(b"]")
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         data = to_bytes(self.encoder.encode(itemdict), self.encoding)
         self._add_comma_after_first()
@@ -154,7 +162,7 @@ class JsonItemExporter(BaseItemExporter):
 
 
 class XmlItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         self.item_element = kwargs.pop("item_element", "item")
         self.root_element = kwargs.pop("root_element", "items")
         super().__init__(**kwargs)
@@ -162,22 +170,22 @@ class XmlItemExporter(BaseItemExporter):
             self.encoding = "utf-8"
         self.xg = XMLGenerator(file, encoding=self.encoding)
 
-    def _beautify_newline(self, new_item=False):
+    def _beautify_newline(self, new_item: bool = False) -> None:
         if self.indent is not None and (self.indent > 0 or new_item):
             self.xg.characters("\n")
 
-    def _beautify_indent(self, depth=1):
+    def _beautify_indent(self, depth: int = 1) -> None:
         if self.indent:
             self.xg.characters(" " * self.indent * depth)
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         self.xg.startDocument()
-        self.xg.startElement(self.root_element, {})
+        self.xg.startElement(self.root_element, AttributesImpl({}))
         self._beautify_newline(new_item=True)
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         self._beautify_indent(depth=1)
-        self.xg.startElement(self.item_element, {})
+        self.xg.startElement(self.item_element, AttributesImpl({}))
         self._beautify_newline()
         for name, value in self._get_serialized_fields(item, default_value=""):
             self._export_xml_field(name, value, depth=2)
@@ -185,13 +193,13 @@ class XmlItemExporter(BaseItemExporter):
         self.xg.endElement(self.item_element)
         self._beautify_newline(new_item=True)
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self.xg.endElement(self.root_element)
         self.xg.endDocument()
 
-    def _export_xml_field(self, name, serialized_value, depth):
+    def _export_xml_field(self, name: str, serialized_value: Any, depth: int) -> None:
         self._beautify_indent(depth=depth)
-        self.xg.startElement(name, {})
+        self.xg.startElement(name, AttributesImpl({}))
         if hasattr(serialized_value, "items"):
             self._beautify_newline()
             for subname, value in serialized_value.items():
@@ -213,17 +221,17 @@ class XmlItemExporter(BaseItemExporter):
 class CsvItemExporter(BaseItemExporter):
     def __init__(
         self,
-        file,
-        include_headers_line=True,
-        join_multivalued=",",
-        errors=None,
-        **kwargs,
+        file: BytesIO,
+        include_headers_line: bool = True,
+        join_multivalued: str = ",",
+        errors: Optional[str] = None,
+        **kwargs: Any,
     ):
         super().__init__(dont_fail=True, **kwargs)
         if not self.encoding:
             self.encoding = "utf-8"
         self.include_headers_line = include_headers_line
-        self.stream = io.TextIOWrapper(
+        self.stream = TextIOWrapper(
             file,
             line_buffering=False,
             write_through=True,
@@ -235,11 +243,13 @@ class CsvItemExporter(BaseItemExporter):
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", self._join_if_needed)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get("serializer", self._join_if_needed)
         return serializer(value)
 
-    def _join_if_needed(self, value):
+    def _join_if_needed(self, value: Any) -> Any:
         if isinstance(value, (list, tuple)):
             try:
                 return self._join_multivalued.join(value)
@@ -247,7 +257,7 @@ class CsvItemExporter(BaseItemExporter):
                 pass
         return value
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         if self._headers_not_written:
             self._headers_not_written = False
             self._write_headers_and_set_fields_to_export(item)
@@ -256,36 +266,38 @@ class CsvItemExporter(BaseItemExporter):
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         self.stream.detach()  # Avoid closing the wrapped file.
 
-    def _build_row(self, values):
+    def _build_row(self, values: Iterable[Any]) -> Iterable[Any]:
         for s in values:
             try:
                 yield to_unicode(s, self.encoding)
             except TypeError:
                 yield s
 
-    def _write_headers_and_set_fields_to_export(self, item):
+    def _write_headers_and_set_fields_to_export(self, item: Any) -> None:
         if self.include_headers_line:
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
+            fields: Iterable[str]
             if isinstance(self.fields_to_export, Mapping):
                 fields = self.fields_to_export.values()
             else:
+                assert self.fields_to_export
                 fields = self.fields_to_export
             row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
 class PickleItemExporter(BaseItemExporter):
-    def __init__(self, file, protocol=4, **kwargs):
+    def __init__(self, file: BytesIO, protocol: int = 4, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
-        self.protocol = protocol
+        self.file: BytesIO = file
+        self.protocol: int = protocol
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         d = dict(self._get_serialized_fields(item))
         pickle.dump(d, self.file, self.protocol)
 
@@ -299,20 +311,20 @@ class MarshalItemExporter(BaseItemExporter):
                  opened in binary mode, a :class:`~io.BytesIO` object, etc)
     """
 
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
+        self.file: BytesIO = file
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         marshal.dump(dict(self._get_serialized_fields(item)), self.file)
 
 
 class PprintItemExporter(BaseItemExporter):
-    def __init__(self, file, **kwargs):
+    def __init__(self, file: BytesIO, **kwargs: Any):
         super().__init__(**kwargs)
-        self.file = file
+        self.file: BytesIO = file
 
-    def export_item(self, item):
+    def export_item(self, item: Any) -> None:
         itemdict = dict(self._get_serialized_fields(item))
         self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
 
@@ -327,16 +339,20 @@ class PythonItemExporter(BaseItemExporter):
     .. _msgpack: https://pypi.org/project/msgpack/
     """
 
-    def _configure(self, options, dont_fail=False):
+    def _configure(self, options: Dict[str, Any], dont_fail: bool = False) -> None:
         super()._configure(options, dont_fail)
         if not self.encoding:
             self.encoding = "utf-8"
 
-    def serialize_field(self, field, name, value):
-        serializer = field.get("serializer", self._serialize_value)
+    def serialize_field(
+        self, field: Union[Mapping[str, Any], Field], name: str, value: Any
+    ) -> Any:
+        serializer: Callable[[Any], Any] = field.get(
+            "serializer", self._serialize_value
+        )
         return serializer(value)
 
-    def _serialize_value(self, value):
+    def _serialize_value(self, value: Any) -> Any:
         if isinstance(value, Item):
             return self.export_item(value)
         if is_item(value):
@@ -347,10 +363,10 @@ class PythonItemExporter(BaseItemExporter):
             return to_unicode(value, encoding=self.encoding)
         return value
 
-    def _serialize_item(self, item):
+    def _serialize_item(self, item: Any) -> Iterable[Tuple[Union[str, bytes], Any]]:
         for key, value in ItemAdapter(item).items():
             yield key, self._serialize_value(value)
 
-    def export_item(self, item):
-        result = dict(self._get_serialized_fields(item))
+    def export_item(self, item: Any) -> Dict[Union[str, bytes], Any]:  # type: ignore[override]
+        result: Dict[Union[str, bytes], Any] = dict(self._get_serialized_fields(item))
         return result
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e5e363b52..3b0dd804e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,7 +11,7 @@ import warnings
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Union
+from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union
 from urllib.parse import unquote, urlparse
 
 from twisted.internet import defer, threads
@@ -21,6 +21,7 @@ from zope.interface import Interface, implementer
 
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
+from scrapy.exporters import BaseItemExporter
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
@@ -324,12 +325,12 @@ class FeedSlot:
         filter,
         feed_options,
         spider,
-        exporters,
+        exporters: Dict[str, Type[BaseItemExporter]],
         settings,
         crawler,
     ):
         self.file = None
-        self.exporter = None
+        self.exporter: Optional[BaseItemExporter] = None
         self.storage = storage
         # feed params
         self.batch_id = batch_id
@@ -341,7 +342,7 @@ class FeedSlot:
         # exporter params
         self.feed_options = feed_options
         self.spider = spider
-        self.exporters = exporters
+        self.exporters: Dict[str, Type[BaseItemExporter]] = exporters
         self.settings = settings
         self.crawler = crawler
         # flags
@@ -373,7 +374,7 @@ class FeedSlot:
     def _get_instance(self, objcls, *args, **kwargs):
         return build_from_crawler(objcls, self.crawler, *args, **kwargs)
 
-    def _get_exporter(self, file, format, *args, **kwargs):
+    def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter:
         return self._get_instance(self.exporters[format], file, *args, **kwargs)
 
     def finish_exporting(self):

From 421e08dd4a4d5ed2acf3cd0fca4c7bfa5a6d3eb2 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 8 Mar 2024 10:59:56 +0500
Subject: [PATCH 22/74] Full typing for scrapy/extensions, part 1. (#6276)

---
 scrapy/extensions/closespider.py | 52 +++++++++++++++++++++-----------
 scrapy/extensions/corestats.py   | 33 +++++++++++++-------
 scrapy/extensions/debug.py       | 25 ++++++++++-----
 scrapy/extensions/logstats.py    | 50 ++++++++++++++++++------------
 scrapy/extensions/memdebug.py    | 22 ++++++++++----
 scrapy/extensions/memusage.py    | 51 ++++++++++++++++++++-----------
 scrapy/mail.py                   | 12 ++++++--
 scrapy/utils/engine.py           | 13 ++++----
 8 files changed, 171 insertions(+), 87 deletions(-)

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index 4307b4170..812b3553c 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -4,20 +4,31 @@ conditions are met.
 See documentation in docs/topics/extensions.rst
 """
 
+from __future__ import annotations
+
 import logging
 from collections import defaultdict
+from typing import TYPE_CHECKING, Any, DefaultDict, Dict
 
-from scrapy import signals
+from twisted.python.failure import Failure
+
+from scrapy import Request, Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
 
 class CloseSpider:
-    def __init__(self, crawler):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
 
-        self.close_on = {
+        self.close_on: Dict[str, Any] = {
             "timeout": crawler.settings.getfloat("CLOSESPIDER_TIMEOUT"),
             "itemcount": crawler.settings.getint("CLOSESPIDER_ITEMCOUNT"),
             "pagecount": crawler.settings.getint("CLOSESPIDER_PAGECOUNT"),
@@ -28,7 +39,7 @@ class CloseSpider:
         if not any(self.close_on.values()):
             raise NotConfigured
 
-        self.counter = defaultdict(int)
+        self.counter: DefaultDict[str, int] = defaultdict(int)
 
         if self.close_on.get("errorcount"):
             crawler.signals.connect(self.error_count, signal=signals.spider_error)
@@ -39,8 +50,8 @@ class CloseSpider:
         if self.close_on.get("itemcount"):
             crawler.signals.connect(self.item_scraped, signal=signals.item_scraped)
         if self.close_on.get("timeout_no_item"):
-            self.timeout_no_item = self.close_on["timeout_no_item"]
-            self.items_in_period = 0
+            self.timeout_no_item: int = self.close_on["timeout_no_item"]
+            self.items_in_period: int = 0
             crawler.signals.connect(
                 self.spider_opened_no_item, signal=signals.spider_opened
             )
@@ -50,22 +61,25 @@ class CloseSpider:
         crawler.signals.connect(self.spider_closed, signal=signals.spider_closed)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def error_count(self, failure, response, spider):
+    def error_count(self, failure: Failure, response: Response, spider: Spider) -> None:
         self.counter["errorcount"] += 1
         if self.counter["errorcount"] == self.close_on["errorcount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_errorcount")
 
-    def page_count(self, response, request, spider):
+    def page_count(self, response: Response, request: Request, spider: Spider) -> None:
         self.counter["pagecount"] += 1
         if self.counter["pagecount"] == self.close_on["pagecount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_pagecount")
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         from twisted.internet import reactor
 
+        assert self.crawler.engine
         self.task = reactor.callLater(
             self.close_on["timeout"],
             self.crawler.engine.close_spider,
@@ -73,21 +87,22 @@ class CloseSpider:
             reason="closespider_timeout",
         )
 
-    def item_scraped(self, item, spider):
+    def item_scraped(self, item: Any, spider: Spider) -> None:
         self.counter["itemcount"] += 1
         if self.counter["itemcount"] == self.close_on["itemcount"]:
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_itemcount")
 
-    def spider_closed(self, spider):
-        task = getattr(self, "task", False)
+    def spider_closed(self, spider: Spider) -> None:
+        task = getattr(self, "task", None)
         if task and task.active():
             task.cancel()
 
-        task_no_item = getattr(self, "task_no_item", False)
+        task_no_item = getattr(self, "task_no_item", None)
         if task_no_item and task_no_item.running:
             task_no_item.stop()
 
-    def spider_opened_no_item(self, spider):
+    def spider_opened_no_item(self, spider: Spider) -> None:
         from twisted.internet import task
 
         self.task_no_item = task.LoopingCall(self._count_items_produced, spider)
@@ -98,10 +113,10 @@ class CloseSpider:
             f"{self.timeout_no_item} seconds."
         )
 
-    def item_scraped_no_item(self, item, spider):
+    def item_scraped_no_item(self, item: Any, spider: Spider) -> None:
         self.items_in_period += 1
 
-    def _count_items_produced(self, spider):
+    def _count_items_produced(self, spider: Spider) -> None:
         if self.items_in_period >= 1:
             self.items_in_period = 0
         else:
@@ -109,4 +124,5 @@ class CloseSpider:
                 f"Closing spider since no items were produced in the last "
                 f"{self.timeout_no_item} seconds."
             )
+            assert self.crawler.engine
             self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")
diff --git a/scrapy/extensions/corestats.py b/scrapy/extensions/corestats.py
index 717c249d9..f3ac19623 100644
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@@ -2,18 +2,28 @@
 Extension for collecting core stats like items scraped and start/finish times
 """
 
-from datetime import datetime, timezone
+from __future__ import annotations
 
-from scrapy import signals
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, Optional
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class CoreStats:
-    def __init__(self, stats):
-        self.stats = stats
-        self.start_time = None
+    def __init__(self, stats: StatsCollector):
+        self.stats: StatsCollector = stats
+        self.start_time: Optional[datetime] = None
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
@@ -22,11 +32,12 @@ class CoreStats:
         crawler.signals.connect(o.response_received, signal=signals.response_received)
         return o
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         self.start_time = datetime.now(tz=timezone.utc)
         self.stats.set_value("start_time", self.start_time, spider=spider)
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
+        assert self.start_time is not None
         finish_time = datetime.now(tz=timezone.utc)
         elapsed_time = finish_time - self.start_time
         elapsed_time_seconds = elapsed_time.total_seconds()
@@ -36,13 +47,13 @@ class CoreStats:
         self.stats.set_value("finish_time", finish_time, spider=spider)
         self.stats.set_value("finish_reason", reason, spider=spider)
 
-    def item_scraped(self, item, spider):
+    def item_scraped(self, item: Any, spider: Spider) -> None:
         self.stats.inc_value("item_scraped_count", spider=spider)
 
-    def response_received(self, spider):
+    def response_received(self, spider: Spider) -> None:
         self.stats.inc_value("response_received_count", spider=spider)
 
-    def item_dropped(self, item, spider, exception):
+    def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:
         reason = exception.__class__.__name__
         self.stats.inc_value("item_dropped_count", spider=spider)
         self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)
diff --git a/scrapy/extensions/debug.py b/scrapy/extensions/debug.py
index 1b6c7777f..26726b662 100644
--- a/scrapy/extensions/debug.py
+++ b/scrapy/extensions/debug.py
@@ -4,22 +4,31 @@ Extensions for debugging Scrapy
 See documentation in docs/topics/extensions.rst
 """
 
+from __future__ import annotations
+
 import logging
 import signal
 import sys
 import threading
 import traceback
 from pdb import Pdb
+from types import FrameType
+from typing import TYPE_CHECKING, Optional
 
+from scrapy.crawler import Crawler
 from scrapy.utils.engine import format_engine_status
 from scrapy.utils.trackref import format_live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
 class StackTraceDump:
-    def __init__(self, crawler=None):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
         try:
             signal.signal(signal.SIGUSR2, self.dump_stacktrace)
             signal.signal(signal.SIGQUIT, self.dump_stacktrace)
@@ -28,10 +37,11 @@ class StackTraceDump:
             pass
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def dump_stacktrace(self, signum, frame):
+    def dump_stacktrace(self, signum: int, frame: Optional[FrameType]) -> None:
+        assert self.crawler.engine
         log_args = {
             "stackdumps": self._thread_stacks(),
             "enginestatus": format_engine_status(self.crawler.engine),
@@ -44,7 +54,7 @@ class StackTraceDump:
             extra={"crawler": self.crawler},
         )
 
-    def _thread_stacks(self):
+    def _thread_stacks(self) -> str:
         id2name = dict((th.ident, th.name) for th in threading.enumerate())
         dumps = ""
         for id_, frame in sys._current_frames().items():
@@ -55,12 +65,13 @@ class StackTraceDump:
 
 
 class Debugger:
-    def __init__(self):
+    def __init__(self) -> None:
         try:
             signal.signal(signal.SIGUSR2, self._enter_debugger)
         except AttributeError:
             # win32 platforms don't support SIGUSR signals
             pass
 
-    def _enter_debugger(self, signum, frame):
+    def _enter_debugger(self, signum: int, frame: Optional[FrameType]) -> None:
+        assert frame
         Pdb().set_trace(frame.f_back)
diff --git a/scrapy/extensions/logstats.py b/scrapy/extensions/logstats.py
index 9f63e9c4b..2388afa75 100644
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@@ -1,9 +1,18 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING, Optional, Tuple, Union
 
 from twisted.internet import task
 
-from scrapy import signals
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
@@ -14,30 +23,31 @@ class LogStats:
     * IPM - Items per Minute
     """
 
-    def __init__(self, stats, interval=60.0):
-        self.stats = stats
-        self.interval = interval
-        self.multiplier = 60.0 / self.interval
-        self.task = None
+    def __init__(self, stats: StatsCollector, interval: float = 60.0):
+        self.stats: StatsCollector = stats
+        self.interval: float = interval
+        self.multiplier: float = 60.0 / self.interval
+        self.task: Optional[task.LoopingCall] = None
 
     @classmethod
-    def from_crawler(cls, crawler):
-        interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
+        assert crawler.stats
         o = cls(crawler.stats, interval)
         crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_opened(self, spider):
-        self.pagesprev = 0
-        self.itemsprev = 0
+    def spider_opened(self, spider: Spider) -> None:
+        self.pagesprev: int = 0
+        self.itemsprev: int = 0
 
         self.task = task.LoopingCall(self.log, spider)
         self.task.start(self.interval)
 
-    def log(self, spider):
+    def log(self, spider: Spider) -> None:
         self.calculate_stats()
 
         msg = (
@@ -52,14 +62,14 @@ class LogStats:
         }
         logger.info(msg, log_args, extra={"spider": spider})
 
-    def calculate_stats(self):
-        self.items = self.stats.get_value("item_scraped_count", 0)
-        self.pages = self.stats.get_value("response_received_count", 0)
-        self.irate = (self.items - self.itemsprev) * self.multiplier
-        self.prate = (self.pages - self.pagesprev) * self.multiplier
+    def calculate_stats(self) -> None:
+        self.items: int = self.stats.get_value("item_scraped_count", 0)
+        self.pages: int = self.stats.get_value("response_received_count", 0)
+        self.irate: float = (self.items - self.itemsprev) * self.multiplier
+        self.prate: float = (self.pages - self.pagesprev) * self.multiplier
         self.pagesprev, self.itemsprev = self.pages, self.items
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         if self.task and self.task.running:
             self.task.stop()
 
@@ -67,7 +77,9 @@ class LogStats:
         self.stats.set_value("responses_per_minute", rpm_final)
         self.stats.set_value("items_per_minute", ipm_final)
 
-    def calculate_final_stats(self, spider):
+    def calculate_final_stats(
+        self, spider: Spider
+    ) -> Union[Tuple[None, None], Tuple[float, float]]:
         start_time = self.stats.get_value("start_time")
         finished_time = self.stats.get_value("finished_time")
 
diff --git a/scrapy/extensions/memdebug.py b/scrapy/extensions/memdebug.py
index 03ede0681..f304e1bf2 100644
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@@ -4,26 +4,36 @@ MemoryDebugger extension
 See documentation in docs/topics/extensions.rst
 """
 
-import gc
+from __future__ import annotations
 
-from scrapy import signals
+import gc
+from typing import TYPE_CHECKING
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.trackref import live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class MemoryDebugger:
-    def __init__(self, stats):
-        self.stats = stats
+    def __init__(self, stats: StatsCollector):
+        self.stats: StatsCollector = stats
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         if not crawler.settings.getbool("MEMDEBUG_ENABLED"):
             raise NotConfigured
+        assert crawler.stats
         o = cls(crawler.stats)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         gc.collect()
         self.stats.set_value(
             "memdebug/gc_garbage_count", len(gc.garbage), spider=spider
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 4d4501c44..9de06b24d 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -4,24 +4,32 @@ MemoryUsage extension
 See documentation in docs/topics/extensions.rst
 """
 
+from __future__ import annotations
+
 import logging
 import socket
 import sys
 from importlib import import_module
 from pprint import pformat
+from typing import TYPE_CHECKING, List
 
 from twisted.internet import task
 
 from scrapy import signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
 from scrapy.utils.engine import get_engine_status
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
 class MemoryUsage:
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("MEMUSAGE_ENABLED"):
             raise NotConfigured
         try:
@@ -30,32 +38,33 @@ class MemoryUsage:
         except ImportError:
             raise NotConfigured
 
-        self.crawler = crawler
-        self.warned = False
-        self.notify_mails = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
-        self.limit = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
-        self.warning = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
-        self.check_interval = crawler.settings.getfloat(
+        self.crawler: Crawler = crawler
+        self.warned: bool = False
+        self.notify_mails: List[str] = crawler.settings.getlist("MEMUSAGE_NOTIFY_MAIL")
+        self.limit: int = crawler.settings.getint("MEMUSAGE_LIMIT_MB") * 1024 * 1024
+        self.warning: int = crawler.settings.getint("MEMUSAGE_WARNING_MB") * 1024 * 1024
+        self.check_interval: float = crawler.settings.getfloat(
             "MEMUSAGE_CHECK_INTERVAL_SECONDS"
         )
-        self.mail = MailSender.from_settings(crawler.settings)
+        self.mail: MailSender = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def get_virtual_size(self):
-        size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
+    def get_virtual_size(self) -> int:
+        size: int = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
         if sys.platform != "darwin":
             # on macOS ru_maxrss is in bytes, on Linux it is in KB
             size *= 1024
         return size
 
-    def engine_started(self):
+    def engine_started(self) -> None:
+        assert self.crawler.stats
         self.crawler.stats.set_value("memusage/startup", self.get_virtual_size())
-        self.tasks = []
+        self.tasks: List[task.LoopingCall] = []
         tsk = task.LoopingCall(self.update)
         self.tasks.append(tsk)
         tsk.start(self.check_interval, now=True)
@@ -68,15 +77,18 @@ class MemoryUsage:
             self.tasks.append(tsk)
             tsk.start(self.check_interval, now=True)
 
-    def engine_stopped(self):
+    def engine_stopped(self) -> None:
         for tsk in self.tasks:
             if tsk.running:
                 tsk.stop()
 
-    def update(self):
+    def update(self) -> None:
+        assert self.crawler.stats
         self.crawler.stats.max_value("memusage/max", self.get_virtual_size())
 
-    def _check_limit(self):
+    def _check_limit(self) -> None:
+        assert self.crawler.engine
+        assert self.crawler.stats
         peak_mem_usage = self.get_virtual_size()
         if peak_mem_usage > self.limit:
             self.crawler.stats.set_value("memusage/limit_reached", 1)
@@ -106,9 +118,10 @@ class MemoryUsage:
                 {"virtualsize": peak_mem_usage / 1024 / 1024},
             )
 
-    def _check_warning(self):
+    def _check_warning(self) -> None:
         if self.warned:  # warn only once
             return
+        assert self.crawler.stats
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value("memusage/warning_reached", 1)
             mem = self.warning / 1024 / 1024
@@ -126,8 +139,10 @@ class MemoryUsage:
                 self.crawler.stats.set_value("memusage/warning_notified", 1)
             self.warned = True
 
-    def _send_report(self, rcpts, subject):
+    def _send_report(self, rcpts: List[str], subject: str) -> None:
         """send notification mail with some additional useful info"""
+        assert self.crawler.engine
+        assert self.crawler.stats
         stats = self.crawler.stats
         s = f"Memory usage at engine startup : {stats.get_value('memusage/startup') / 1024 / 1024}M\r\n"
         s += f"Maximum memory usage          : {stats.get_value('memusage/max') / 1024 / 1024}M\r\n"
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 4b18b6003..dce33fcdf 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -4,6 +4,8 @@ Mail sending helpers
 See documentation in docs/topics/email.rst
 """
 
+from __future__ import annotations
+
 import logging
 from email import encoders as Encoders
 from email.mime.base import MIMEBase
@@ -12,14 +14,20 @@ from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
+from typing import TYPE_CHECKING
 
 from twisted import version as twisted_version
 from twisted.internet import defer, ssl
 from twisted.python.versions import Version
 
+from scrapy.settings import BaseSettings
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +64,7 @@ class MailSender:
         self.debug = debug
 
     @classmethod
-    def from_settings(cls, settings):
+    def from_settings(cls, settings: BaseSettings) -> Self:
         return cls(
             smtphost=settings["MAIL_HOST"],
             mailfrom=settings["MAIL_FROM"],
@@ -203,7 +211,7 @@ class MailSender:
             to_addrs,
             msg,
             d,
-            **factory_keywords
+            **factory_keywords,
         )
         factory.noisy = False
         return factory
diff --git a/scrapy/utils/engine.py b/scrapy/utils/engine.py
index 0b2722663..fdcf484d4 100644
--- a/scrapy/utils/engine.py
+++ b/scrapy/utils/engine.py
@@ -1,14 +1,15 @@
 """Some debugging functions for working with the Scrapy engine"""
 
+from __future__ import annotations
+
 # used in global tests code
 from time import time  # noqa: F401
-from typing import TYPE_CHECKING, Any, List, Tuple
+from typing import Any, List, Tuple
 
-if TYPE_CHECKING:
-    from scrapy.core.engine import ExecutionEngine
+from scrapy.core.engine import ExecutionEngine
 
 
-def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
+def get_engine_status(engine: ExecutionEngine) -> List[Tuple[str, Any]]:
     """Return a report of the current engine status"""
     tests = [
         "time()-engine.start_time",
@@ -37,7 +38,7 @@ def get_engine_status(engine: "ExecutionEngine") -> List[Tuple[str, Any]]:
     return checks
 
 
-def format_engine_status(engine: "ExecutionEngine") -> str:
+def format_engine_status(engine: ExecutionEngine) -> str:
     checks = get_engine_status(engine)
     s = "Execution engine status\n\n"
     for test, result in checks:
@@ -47,5 +48,5 @@ def format_engine_status(engine: "ExecutionEngine") -> str:
     return s
 
 
-def print_engine_status(engine: "ExecutionEngine") -> None:
+def print_engine_status(engine: ExecutionEngine) -> None:
     print(format_engine_status(engine))

From 6e84648c0717642b069249225857019a87de54b9 Mon Sep 17 00:00:00 2001
From: pengqiseven <134899215+pengqiseven@users.noreply.github.com>
Date: Mon, 11 Mar 2024 17:03:06 +0800
Subject: [PATCH 23/74] Fix some comments (#6285)

Signed-off-by: pengqiseven <i@sohu.com>
Co-authored-by: pengqiseven <i@sohu.com>
---
 sep/sep-018.rst            | 2 +-
 tests/test_utils_python.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/sep/sep-018.rst b/sep/sep-018.rst
index 9ac62c090..13ab501ed 100644
--- a/sep/sep-018.rst
+++ b/sep/sep-018.rst
@@ -140,7 +140,7 @@ Example:
 
 The data flow with Spider Middleware v2 is as follows:
 
-1. When a response arrives from the engine, it it passed through all the spider
+1. When a response arrives from the engine, it is passed through all the spider
    middlewares (in descending order). The result of each middleware
    ``process_response`` is kept and then returned along with the spider
    callback result
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 80d2e8da1..1d1d19146 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -175,7 +175,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertTrue(equal_attributes(a, b, ["x", "y"]))
 
         a.y = 1
-        # differente attributes
+        # different attributes
         self.assertFalse(equal_attributes(a, b, ["x", "y"]))
 
         # test callable

From 642af407049a5ce8e76b7999c68333670c8c8622 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 11 Mar 2024 14:09:09 +0500
Subject: [PATCH 24/74] Full typing for scrapy/extensions, part 2. (#6279)

---
 docs/topics/telnetconsole.rst       |  4 +-
 scrapy/extensions/periodic_log.py   | 89 +++++++++++++++++------------
 scrapy/extensions/postprocessing.py |  4 +-
 scrapy/extensions/spiderstate.py    | 26 ++++++---
 scrapy/extensions/statsmailer.py    | 31 +++++++---
 scrapy/extensions/telnet.py         | 37 +++++++-----
 scrapy/extensions/throttle.py       | 49 ++++++++++------
 scrapy/mail.py                      | 12 ++--
 scrapy/utils/reactor.py             |  8 +--
 9 files changed, 165 insertions(+), 95 deletions(-)

diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 832829b75..0e4a8fa6c 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -172,8 +172,8 @@ TELNETCONSOLE_PORT
 
 Default: ``[6023, 6073]``
 
-The port range to use for the telnet console. If set to ``None`` or ``0``, a
-dynamically assigned port is used.
+The port range to use for the telnet console. If set to ``None``, a dynamically
+assigned port is used.
 
 
 .. setting:: TELNETCONSOLE_HOST
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index 2d557f123..9567f948a 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -1,12 +1,22 @@
+from __future__ import annotations
+
 import logging
 from datetime import datetime, timezone
+from json import JSONEncoder
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 from twisted.internet import task
 
-from scrapy import signals
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.statscollectors import StatsCollector
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
@@ -15,32 +25,34 @@ class PeriodicLog:
 
     def __init__(
         self,
-        stats,
-        interval=60.0,
-        ext_stats={},
-        ext_delta={},
-        ext_timing_enabled=False,
+        stats: StatsCollector,
+        interval: float = 60.0,
+        ext_stats: Dict[str, Any] = {},
+        ext_delta: Dict[str, Any] = {},
+        ext_timing_enabled: bool = False,
     ):
-        self.stats = stats
-        self.interval = interval
-        self.multiplier = 60.0 / self.interval
-        self.task = None
-        self.encoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
-        self.ext_stats_enabled = bool(ext_stats)
-        self.ext_stats_include = ext_stats.get("include", [])
-        self.ext_stats_exclude = ext_stats.get("exclude", [])
-        self.ext_delta_enabled = bool(ext_delta)
-        self.ext_delta_include = ext_delta.get("include", [])
-        self.ext_delta_exclude = ext_delta.get("exclude", [])
-        self.ext_timing_enabled = ext_timing_enabled
+        self.stats: StatsCollector = stats
+        self.interval: float = interval
+        self.multiplier: float = 60.0 / self.interval
+        self.task: Optional[task.LoopingCall] = None
+        self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
+        self.ext_stats_enabled: bool = bool(ext_stats)
+        self.ext_stats_include: List[str] = ext_stats.get("include", [])
+        self.ext_stats_exclude: List[str] = ext_stats.get("exclude", [])
+        self.ext_delta_enabled: bool = bool(ext_delta)
+        self.ext_delta_include: List[str] = ext_delta.get("include", [])
+        self.ext_delta_exclude: List[str] = ext_delta.get("exclude", [])
+        self.ext_timing_enabled: bool = ext_timing_enabled
 
     @classmethod
-    def from_crawler(cls, crawler):
-        interval = crawler.settings.getfloat("LOGSTATS_INTERVAL")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        interval: float = crawler.settings.getfloat("LOGSTATS_INTERVAL")
         if not interval:
             raise NotConfigured
         try:
-            ext_stats = crawler.settings.getdict("PERIODIC_LOG_STATS")
+            ext_stats: Optional[Dict[str, Any]] = crawler.settings.getdict(
+                "PERIODIC_LOG_STATS"
+            )
         except (TypeError, ValueError):
             ext_stats = (
                 {"enabled": True}
@@ -48,7 +60,9 @@ class PeriodicLog:
                 else None
             )
         try:
-            ext_delta = crawler.settings.getdict("PERIODIC_LOG_DELTA")
+            ext_delta: Optional[Dict[str, Any]] = crawler.settings.getdict(
+                "PERIODIC_LOG_DELTA"
+            )
         except (TypeError, ValueError):
             ext_delta = (
                 {"enabled": True}
@@ -56,11 +70,14 @@ class PeriodicLog:
                 else None
             )
 
-        ext_timing_enabled = crawler.settings.getbool(
+        ext_timing_enabled: bool = crawler.settings.getbool(
             "PERIODIC_LOG_TIMING_ENABLED", False
         )
         if not (ext_stats or ext_delta or ext_timing_enabled):
             raise NotConfigured
+        assert crawler.stats
+        assert ext_stats is not None
+        assert ext_delta is not None
         o = cls(
             crawler.stats,
             interval,
@@ -72,16 +89,16 @@ class PeriodicLog:
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_opened(self, spider):
-        self.time_prev = datetime.now(tz=timezone.utc)
-        self.delta_prev = {}
-        self.stats_prev = {}
+    def spider_opened(self, spider: Spider) -> None:
+        self.time_prev: datetime = datetime.now(tz=timezone.utc)
+        self.delta_prev: Dict[str, Union[int, float]] = {}
+        self.stats_prev: Dict[str, Union[int, float]] = {}
 
         self.task = task.LoopingCall(self.log)
         self.task.start(self.interval)
 
-    def log(self):
-        data = {}
+    def log(self) -> None:
+        data: Dict[str, Any] = {}
         if self.ext_timing_enabled:
             data.update(self.log_timing())
         if self.ext_delta_enabled:
@@ -90,8 +107,8 @@ class PeriodicLog:
             data.update(self.log_crawler_stats())
         logger.info(self.encoder.encode(data))
 
-    def log_delta(self):
-        num_stats = {
+    def log_delta(self) -> Dict[str, Any]:
+        num_stats: Dict[str, Union[int, float]] = {
             k: v
             for k, v in self.stats._stats.items()
             if isinstance(v, (int, float))
@@ -101,7 +118,7 @@ class PeriodicLog:
         self.delta_prev = num_stats
         return {"delta": delta}
 
-    def log_timing(self):
+    def log_timing(self) -> Dict[str, Any]:
         now = datetime.now(tz=timezone.utc)
         time = {
             "log_interval": self.interval,
@@ -113,7 +130,7 @@ class PeriodicLog:
         self.time_prev = now
         return {"time": time}
 
-    def log_crawler_stats(self):
+    def log_crawler_stats(self) -> Dict[str, Any]:
         stats = {
             k: v
             for k, v in self.stats._stats.items()
@@ -121,7 +138,9 @@ class PeriodicLog:
         }
         return {"stats": stats}
 
-    def param_allowed(self, stat_name, include, exclude):
+    def param_allowed(
+        self, stat_name: str, include: List[str], exclude: List[str]
+    ) -> bool:
         if not include and not exclude:
             return True
         for p in exclude:
@@ -134,7 +153,7 @@ class PeriodicLog:
                 return True
         return False
 
-    def spider_closed(self, spider, reason):
+    def spider_closed(self, spider: Spider, reason: str) -> None:
         self.log()
         if self.task and self.task.running:
             self.task.stop()
diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py
index f8b59827b..7ffbd8bc3 100644
--- a/scrapy/extensions/postprocessing.py
+++ b/scrapy/extensions/postprocessing.py
@@ -6,7 +6,7 @@ from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase
 from lzma import LZMAFile
-from typing import Any, BinaryIO, Dict, List
+from typing import Any, BinaryIO, Dict, List, cast
 
 from scrapy.utils.misc import load_object
 
@@ -142,7 +142,7 @@ class PostProcessingManager(IOBase):
         :return: returns number of bytes written
         :rtype: int
         """
-        return self.head_plugin.write(data)
+        return cast(int, self.head_plugin.write(data))
 
     def tell(self) -> int:
         return self.file.tell()
diff --git a/scrapy/extensions/spiderstate.py b/scrapy/extensions/spiderstate.py
index 43359401b..c6eb20277 100644
--- a/scrapy/extensions/spiderstate.py
+++ b/scrapy/extensions/spiderstate.py
@@ -1,19 +1,27 @@
+from __future__ import annotations
+
 import pickle  # nosec
 from pathlib import Path
+from typing import TYPE_CHECKING, Optional
 
-from scrapy import signals
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.job import job_dir
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class SpiderState:
     """Store and load spider state during a scraping job"""
 
-    def __init__(self, jobdir=None):
-        self.jobdir = jobdir
+    def __init__(self, jobdir: Optional[str] = None):
+        self.jobdir: Optional[str] = jobdir
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         jobdir = job_dir(crawler.settings)
         if not jobdir:
             raise NotConfigured
@@ -23,18 +31,20 @@ class SpiderState:
         crawler.signals.connect(obj.spider_opened, signal=signals.spider_opened)
         return obj
 
-    def spider_closed(self, spider):
+    def spider_closed(self, spider: Spider) -> None:
         if self.jobdir:
             with Path(self.statefn).open("wb") as f:
+                assert hasattr(spider, "state")  # set in spider_opened
                 pickle.dump(spider.state, f, protocol=4)
 
-    def spider_opened(self, spider):
+    def spider_opened(self, spider: Spider) -> None:
         if self.jobdir and Path(self.statefn).exists():
             with Path(self.statefn).open("rb") as f:
-                spider.state = pickle.load(f)  # nosec
+                spider.state = pickle.load(f)  # type: ignore[attr-defined]  # nosec
         else:
-            spider.state = {}
+            spider.state = {}  # type: ignore[attr-defined]
 
     @property
     def statefn(self) -> str:
+        assert self.jobdir
         return str(Path(self.jobdir, "spider.state"))
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index 58610c25e..20b8f910c 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -4,28 +4,41 @@ StatsMailer extension sends an email when a spider finishes scraping.
 Use STATSMAILER_RCPTS setting to enable and give the recipient mail address
 """
 
-from scrapy import signals
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List, Optional
+
+from twisted.internet.defer import Deferred
+
+from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
+from scrapy.statscollectors import StatsCollector
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class StatsMailer:
-    def __init__(self, stats, recipients, mail):
-        self.stats = stats
-        self.recipients = recipients
-        self.mail = mail
+    def __init__(self, stats: StatsCollector, recipients: List[str], mail: MailSender):
+        self.stats: StatsCollector = stats
+        self.recipients: List[str] = recipients
+        self.mail: MailSender = mail
 
     @classmethod
-    def from_crawler(cls, crawler):
-        recipients = crawler.settings.getlist("STATSMAILER_RCPTS")
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        recipients: List[str] = crawler.settings.getlist("STATSMAILER_RCPTS")
         if not recipients:
             raise NotConfigured
-        mail = MailSender.from_settings(crawler.settings)
+        mail: MailSender = MailSender.from_settings(crawler.settings)
+        assert crawler.stats
         o = cls(crawler.stats, recipients, mail)
         crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
         return o
 
-    def spider_closed(self, spider):
+    def spider_closed(self, spider: Spider) -> Optional[Deferred]:
         spider_stats = self.stats.get_stats(spider)
         body = "Global stats\n\n"
         body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
diff --git a/scrapy/extensions/telnet.py b/scrapy/extensions/telnet.py
index c92b7f5fe..00c69434c 100644
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@@ -4,13 +4,17 @@ Scrapy Telnet Console extension
 See documentation in docs/topics/telnetconsole.rst
 """
 
+from __future__ import annotations
+
 import binascii
 import logging
 import os
 import pprint
 import traceback
+from typing import TYPE_CHECKING, Any, Dict, List
 
 from twisted.internet import protocol
+from twisted.internet.tcp import Port
 
 try:
     from twisted.conch import manhole, telnet
@@ -22,12 +26,16 @@ except (ImportError, SyntaxError):
     TWISTED_CONCH_AVAILABLE = False
 
 from scrapy import signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.decorators import defers
 from scrapy.utils.engine import print_engine_status
 from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.trackref import print_live_refs
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 logger = logging.getLogger(__name__)
 
 # signal to update telnet variables
@@ -36,7 +44,7 @@ update_telnet_vars = object()
 
 
 class TelnetConsole(protocol.ServerFactory):
-    def __init__(self, crawler):
+    def __init__(self, crawler: Crawler):
         if not crawler.settings.getbool("TELNETCONSOLE_ENABLED"):
             raise NotConfigured
         if not TWISTED_CONCH_AVAILABLE:
@@ -44,14 +52,14 @@ class TelnetConsole(protocol.ServerFactory):
                 "TELNETCONSOLE_ENABLED setting is True but required twisted "
                 "modules failed to import:\n" + _TWISTED_CONCH_TRACEBACK
             )
-        self.crawler = crawler
-        self.noisy = False
-        self.portrange = [
+        self.crawler: Crawler = crawler
+        self.noisy: bool = False
+        self.portrange: List[int] = [
             int(x) for x in crawler.settings.getlist("TELNETCONSOLE_PORT")
         ]
-        self.host = crawler.settings["TELNETCONSOLE_HOST"]
-        self.username = crawler.settings["TELNETCONSOLE_USERNAME"]
-        self.password = crawler.settings["TELNETCONSOLE_PASSWORD"]
+        self.host: str = crawler.settings["TELNETCONSOLE_HOST"]
+        self.username: str = crawler.settings["TELNETCONSOLE_USERNAME"]
+        self.password: str = crawler.settings["TELNETCONSOLE_PASSWORD"]
 
         if not self.password:
             self.password = binascii.hexlify(os.urandom(8)).decode("utf8")
@@ -61,11 +69,11 @@ class TelnetConsole(protocol.ServerFactory):
         self.crawler.signals.connect(self.stop_listening, signals.engine_stopped)
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def start_listening(self):
-        self.port = listen_tcp(self.portrange, self.host, self)
+    def start_listening(self) -> None:
+        self.port: Port = listen_tcp(self.portrange, self.host, self)
         h = self.port.getHost()
         logger.info(
             "Telnet console listening on %(host)s:%(port)d",
@@ -73,10 +81,10 @@ class TelnetConsole(protocol.ServerFactory):
             extra={"crawler": self.crawler},
         )
 
-    def stop_listening(self):
+    def stop_listening(self) -> None:
         self.port.stopListening()
 
-    def protocol(self):
+    def protocol(self) -> telnet.TelnetTransport:  # type: ignore[override]
         class Portal:
             """An implementation of IPortal"""
 
@@ -95,9 +103,10 @@ class TelnetConsole(protocol.ServerFactory):
 
         return telnet.TelnetTransport(telnet.AuthenticatingTelnetProtocol, Portal())
 
-    def _get_telnet_vars(self):
+    def _get_telnet_vars(self) -> Dict[str, Any]:
         # Note: if you add entries here also update topics/telnetconsole.rst
-        telnet_vars = {
+        assert self.crawler.engine
+        telnet_vars: Dict[str, Any] = {
             "engine": self.crawler.engine,
             "spider": self.crawler.engine.spider,
             "slot": self.crawler.engine.slot,
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index d217c7a69..bf4e6bb63 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -1,19 +1,29 @@
-import logging
+from __future__ import annotations
 
-from scrapy import signals
+import logging
+from typing import TYPE_CHECKING, Optional, Tuple
+
+from scrapy import Request, Spider, signals
+from scrapy.core.downloader import Slot
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 logger = logging.getLogger(__name__)
 
 
 class AutoThrottle:
-    def __init__(self, crawler):
-        self.crawler = crawler
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
         if not crawler.settings.getbool("AUTOTHROTTLE_ENABLED"):
             raise NotConfigured
 
-        self.debug = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
-        self.target_concurrency = crawler.settings.getfloat(
+        self.debug: bool = crawler.settings.getbool("AUTOTHROTTLE_DEBUG")
+        self.target_concurrency: float = crawler.settings.getfloat(
             "AUTOTHROTTLE_TARGET_CONCURRENCY"
         )
         if self.target_concurrency <= 0.0:
@@ -27,27 +37,29 @@ class AutoThrottle:
         )
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def _spider_opened(self, spider):
+    def _spider_opened(self, spider: Spider) -> None:
         self.mindelay = self._min_delay(spider)
         self.maxdelay = self._max_delay(spider)
-        spider.download_delay = self._start_delay(spider)
+        spider.download_delay = self._start_delay(spider)  # type: ignore[attr-defined]
 
-    def _min_delay(self, spider):
+    def _min_delay(self, spider: Spider) -> float:
         s = self.crawler.settings
         return getattr(spider, "download_delay", s.getfloat("DOWNLOAD_DELAY"))
 
-    def _max_delay(self, spider):
+    def _max_delay(self, spider: Spider) -> float:
         return self.crawler.settings.getfloat("AUTOTHROTTLE_MAX_DELAY")
 
-    def _start_delay(self, spider):
+    def _start_delay(self, spider: Spider) -> float:
         return max(
             self.mindelay, self.crawler.settings.getfloat("AUTOTHROTTLE_START_DELAY")
         )
 
-    def _response_downloaded(self, response, request, spider):
+    def _response_downloaded(
+        self, response: Response, request: Request, spider: Spider
+    ) -> None:
         key, slot = self._get_slot(request, spider)
         latency = request.meta.get("download_latency")
         if latency is None or slot is None:
@@ -74,11 +86,16 @@ class AutoThrottle:
                 extra={"spider": spider},
             )
 
-    def _get_slot(self, request, spider):
-        key = request.meta.get("download_slot")
+    def _get_slot(
+        self, request: Request, spider: Spider
+    ) -> Tuple[Optional[str], Optional[Slot]]:
+        key: Optional[str] = request.meta.get("download_slot")
+        if key is None:
+            return None, None
+        assert self.crawler.engine
         return key, self.crawler.engine.downloader.slots.get(key)
 
-    def _adjust_delay(self, slot, latency, response):
+    def _adjust_delay(self, slot: Slot, latency: float, response: Response) -> None:
         """Define delay adjustment policy"""
 
         # If a server needs `latency` seconds to respond then
diff --git a/scrapy/mail.py b/scrapy/mail.py
index dce33fcdf..7cb5ef454 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -14,10 +14,11 @@ from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional
 
 from twisted import version as twisted_version
-from twisted.internet import defer, ssl
+from twisted.internet import ssl
+from twisted.internet.defer import Deferred
 from twisted.python.versions import Version
 
 from scrapy.settings import BaseSettings
@@ -85,9 +86,10 @@ class MailSender:
         mimetype="text/plain",
         charset=None,
         _callback=None,
-    ):
+    ) -> Optional[Deferred]:
         from twisted.internet import reactor
 
+        msg: MIMEBase
         if attachs:
             msg = MIMEMultipart()
         else:
@@ -134,7 +136,7 @@ class MailSender:
                     "mailattachs": len(attachs),
                 },
             )
-            return
+            return None
 
         dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
         dfd.addCallbacks(
@@ -178,7 +180,7 @@ class MailSender:
         from twisted.internet import reactor
 
         msg = BytesIO(msg)
-        d = defer.Deferred()
+        d = Deferred()
 
         factory = self._create_sender_factory(to_addrs, msg, d)
 
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ad3d1d8bc..6cde49bfe 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -2,17 +2,19 @@ import asyncio
 import sys
 from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
-from typing import Any, Callable, Dict, Optional, Sequence, Type
+from typing import Any, Callable, Dict, List, Optional, Sequence, Type
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
 from twisted.internet.base import DelayedCall
+from twisted.internet.protocol import ServerFactory
+from twisted.internet.tcp import Port
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
 
 
-def listen_tcp(portrange, host, factory):
+def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]
     """Like reactor.listenTCP but tries different ports in a range."""
     from twisted.internet import reactor
 
@@ -20,8 +22,6 @@ def listen_tcp(portrange, host, factory):
         raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
         return reactor.listenTCP(0, factory, interface=host)
-    if not hasattr(portrange, "__iter__"):
-        return reactor.listenTCP(portrange, factory, interface=host)
     if len(portrange) == 1:
         return reactor.listenTCP(portrange[0], factory, interface=host)
     for x in range(portrange[0], portrange[1] + 1):

From ab5ea32ffd9cbea22d0fb10ece5258cea207dd61 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 11 Mar 2024 23:26:19 +0100
Subject: [PATCH 25/74] Fix WindowsRunSpiderCommandTest skip outside Windows
 for older Twisted.

---
 tests/test_commands.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index febad21da..ff308c5ac 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -987,10 +987,14 @@ class MySpider(scrapy.Spider):
         self.assertIn("The value of FOO is 42", log)
 
 
-@skipIf(platform.system() != "Windows", "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     spider_filename = "myspider.pyw"
 
+    def setUp(self):
+        # https://github.com/scrapy/scrapy/issues/6286
+        if platform.system() != "Windows":
+            raise unittest.SkipTest("Windows required for .pyw files")
+
     def test_start_requests_errors(self):
         log = self.get_log(self.badspider, name="badspider.pyw")
         self.assertIn("start_requests", log)

From 188d9a8bb363ab3ff37dbb6020354afbc72ec02d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 11 Mar 2024 23:28:57 +0100
Subject: [PATCH 26/74] Remove unnecessary comment

---
 tests/test_commands.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index ff308c5ac..ae8289ba7 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -991,7 +991,6 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     spider_filename = "myspider.pyw"
 
     def setUp(self):
-        # https://github.com/scrapy/scrapy/issues/6286
         if platform.system() != "Windows":
             raise unittest.SkipTest("Windows required for .pyw files")
 

From e72de11f55dc5f37d449385f20d2ce4c504914d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Mar 2024 09:29:10 +0100
Subject: [PATCH 27/74] Add super

---
 tests/test_commands.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_commands.py b/tests/test_commands.py
index ae8289ba7..b9d468c66 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -993,6 +993,7 @@ class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
     def setUp(self):
         if platform.system() != "Windows":
             raise unittest.SkipTest("Windows required for .pyw files")
+        return super().setUp()
 
     def test_start_requests_errors(self):
         log = self.get_log(self.badspider, name="badspider.pyw")

From d7581c6b41e97fc09c011b089cf34ddd62f41876 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Tue, 12 Mar 2024 09:44:29 +0100
Subject: [PATCH 28/74] Allow disabling the AutoThrottle extension for a given
 slot (#6246)

---
 docs/topics/autothrottle.rst          | 12 +++++++++++
 docs/topics/settings.rst              | 12 +++++++++--
 scrapy/core/downloader/__init__.py    | 19 ++++++++++++++----
 scrapy/extensions/throttle.py         |  2 +-
 tests/test_core_downloader.py         |  3 ++-
 tests/test_downloaderslotssettings.py | 29 ++++++++++++++++++++++++++-
 tests/test_extension_throttle.py      | 19 ++++++++++--------
 7 files changed, 79 insertions(+), 17 deletions(-)

diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 5370d77b3..8a13b8976 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -47,6 +47,18 @@ effect, but there are some important differences:
 
 AutoThrottle doesn't have these issues.
 
+Disabling throttling on a downloader slot
+=========================================
+
+It is possible to disable AutoThrottle for a specific download slot at run time
+by setting its ``throttle`` attribute to ``False``, e.g. using
+:setting:`DOWNLOAD_SLOTS`.
+
+Note, however, that AutoThrottle still determines the starting delay of every
+slot by setting the ``download_delay`` attribute on the running spider. You
+might want to set a custom value for the ``delay`` attribute of the slot, e.g.
+using :setting:`DOWNLOAD_SLOTS`.
+
 Throttling algorithm
 ====================
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 439aedc18..2bd9cf1ed 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
 .. setting:: DOWNLOAD_SLOTS
 
 DOWNLOAD_SLOTS
-----------------
+--------------
 
 Default: ``{}``
 
@@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     .. code-block:: python
 
         DOWNLOAD_SLOTS = {
-            "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+            "quotes.toscrape.com": {
+                "concurrency": 1,
+                "delay": 2,
+                "randomize_delay": False,
+                "throttle": False,
+            },
             "books.toscrape.com": {"delay": 3, "randomize_delay": False},
         }
 
@@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
     -   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
     -   :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
 
+    There is no global setting for ``throttle``, whose default value is
+    ``None``.
+
 
 .. setting:: DOWNLOAD_TIMEOUT
 
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 666282856..ecd3e8b56 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -2,7 +2,7 @@ import random
 from collections import deque
 from datetime import datetime
 from time import time
-from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
+from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
 
 from twisted.internet import task
 from twisted.internet.defer import Deferred
@@ -24,10 +24,18 @@ if TYPE_CHECKING:
 class Slot:
     """Downloader slot"""
 
-    def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
+    def __init__(
+        self,
+        concurrency: int,
+        delay: float,
+        randomize_delay: bool,
+        *,
+        throttle: Optional[bool] = None,
+    ):
         self.concurrency: int = concurrency
         self.delay: float = delay
         self.randomize_delay: bool = randomize_delay
+        self.throttle = throttle
 
         self.active: Set[Request] = set()
         self.queue: Deque[Tuple[Request, Deferred]] = deque()
@@ -52,13 +60,15 @@ class Slot:
         return (
             f"{cls_name}(concurrency={self.concurrency!r}, "
             f"delay={self.delay:.2f}, "
-            f"randomize_delay={self.randomize_delay!r})"
+            f"randomize_delay={self.randomize_delay!r}, "
+            f"throttle={self.throttle!r})"
         )
 
     def __str__(self) -> str:
         return (
             f"<downloader.Slot concurrency={self.concurrency!r} "
             f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
+            f"throttle={self.throttle!r} "
             f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
             f"len(transferring)={len(self.transferring)} "
             f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@@ -127,7 +137,8 @@ class Downloader:
                 slot_settings.get("delay", delay),
             )
             randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
-            new_slot = Slot(conc, delay, randomize_delay)
+            throttle = slot_settings.get("throttle", None)
+            new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
             self.slots[key] = new_slot
 
         return key, self.slots[key]
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index bf4e6bb63..217e61a81 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -62,7 +62,7 @@ class AutoThrottle:
     ) -> None:
         key, slot = self._get_slot(request, spider)
         latency = request.meta.get("download_latency")
-        if latency is None or slot is None:
+        if latency is None or slot is None or slot.throttle is False:
             return
 
         olddelay = slot.delay
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 9a6e9e4ff..81cff4947 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase):
     def test_repr(self):
         slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
         self.assertEqual(
-            repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)"
+            repr(slot),
+            "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
         )
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 9d4072d19..ea8c5b4f0 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -3,8 +3,10 @@ import time
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
+from scrapy import Request
+from scrapy.core.downloader import Downloader, Slot
 from scrapy.crawler import CrawlerRunner
-from scrapy.http import Request
+from scrapy.utils.test import get_crawler
 from tests.mockserver import MockServer
 from tests.spiders import MetaSpider
 
@@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
                 "concurrency": 1,
                 "delay": 2,
                 "randomize_delay": False,
+                "throttle": False,
             },
             "books.toscrape.com": {"delay": 3, "randomize_delay": False},
         },
@@ -70,3 +73,27 @@ class CrawlTestCase(TestCase):
         }
 
         self.assertTrue(max(list(error_delta.values())) < tolerance)
+
+
+def test_params():
+    params = {
+        "concurrency": 1,
+        "delay": 2,
+        "randomize_delay": False,
+        "throttle": False,
+    }
+    settings = {
+        "DOWNLOAD_SLOTS": {
+            "example.com": params,
+        },
+    }
+    crawler = get_crawler(settings_dict=settings)
+    downloader = Downloader(crawler)
+    downloader._slot_gc_loop.stop()  # Prevent an unclean reactor.
+    request = Request("https://example.com")
+    _, actual = downloader._get_slot(request, spider=None)
+    expected = Slot(**params)
+    for param in params:
+        assert getattr(expected, param) == getattr(
+            actual, param
+        ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}"
diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py
index dae4ea966..722a05c26 100644
--- a/tests/test_extension_throttle.py
+++ b/tests/test_extension_throttle.py
@@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
 
 
 @pytest.mark.parametrize(
-    ("meta", "slot"),
+    ("meta", "slot", "throttle"),
     (
-        ({}, None),
-        ({"download_latency": 1.0}, None),
-        ({"download_slot": "foo"}, None),
-        ({"download_slot": "foo"}, "foo"),
-        ({"download_latency": 1.0, "download_slot": "foo"}, None),
+        ({}, None, None),
+        ({"download_latency": 1.0}, None, None),
+        ({"download_slot": "foo"}, None, None),
+        ({"download_slot": "foo"}, "foo", None),
+        ({"download_latency": 1.0, "download_slot": "foo"}, None, None),
+        ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),
     ),
 )
-def test_skipped(meta, slot):
+def test_skipped(meta, slot, throttle):
     crawler = get_crawler()
     at = build_from_crawler(AutoThrottle, crawler)
     spider = TestSpider()
@@ -177,7 +178,9 @@ def test_skipped(meta, slot):
     crawler.engine.downloader = Mock()
     crawler.engine.downloader.slots = {}
     if slot is not None:
-        crawler.engine.downloader.slots[slot] = object()
+        _slot = Mock()
+        _slot.throttle = throttle
+        crawler.engine.downloader.slots[slot] = _slot
     at._adjust_delay = None  # Raise exception if called.
 
     at._response_downloaded(None, request, spider)

From 4460d3ed9631f8409c24f78f1abb36345967b5d5 Mon Sep 17 00:00:00 2001
From: Lucas Belo <144740771+lucas-belo@users.noreply.github.com>
Date: Wed, 13 Mar 2024 03:22:48 -0300
Subject: [PATCH 29/74] Remove tests/requirements.txt and refactor extra deps
 (#6272)

Co-authored-by: lucasbelo777 <lucas.belo@cialdnb.com>
---
 conftest.py            | 18 ++++++------------
 tests/requirements.txt | 17 -----------------
 tox.ini                | 32 ++++++++++++++++++++++++++++----
 3 files changed, 34 insertions(+), 33 deletions(-)
 delete mode 100644 tests/requirements.txt

diff --git a/conftest.py b/conftest.py
index 2bfa46f5a..2ab3dffd4 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,10 +1,6 @@
-import platform
-import sys
 from pathlib import Path
 
 import pytest
-from twisted import version as twisted_version
-from twisted.python.versions import Version
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import install_reactor
@@ -85,14 +81,12 @@ def only_not_asyncio(request, reactor_pytest):
 def requires_uvloop(request):
     if not request.node.get_closest_marker("requires_uvloop"):
         return
-    if sys.implementation.name == "pypy":
-        pytest.skip("uvloop does not support pypy properly")
-    if platform.system() == "Windows":
-        pytest.skip("uvloop does not support Windows")
-    if twisted_version == Version("twisted", 21, 2, 0):
-        pytest.skip("https://twistedmatrix.com/trac/ticket/10106")
-    if sys.version_info >= (3, 12):
-        pytest.skip("uvloop doesn't support Python 3.12 yet")
+    try:
+        import uvloop
+
+        del uvloop
+    except ImportError:
+        pytest.skip("uvloop is not installed")
 
 
 def pytest_configure(config):
diff --git a/tests/requirements.txt b/tests/requirements.txt
deleted file mode 100644
index ca5f6ddbd..000000000
--- a/tests/requirements.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-# Tests requirements
-attrs
-pexpect >= 4.8.0
-pyftpdlib >= 1.5.8
-pytest
-pytest-cov==4.0.0
-pytest-xdist
-sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
-testfixtures
-uvloop; platform_system != "Windows"
-
-bpython  # optional for shell wrapper tests
-brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
-brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
-zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
-ipython
-pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index 237aa489c..b5effb527 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,9 +7,23 @@
 envlist = pre-commit,pylint,typing,py
 minversion = 1.7.0
 
+[test-requirements]
+deps =
+    attrs
+    pexpect >= 4.8.0
+    pyftpdlib >= 1.5.8
+    pygments
+    pytest
+    pytest-cov==4.0.0
+    pytest-xdist
+    sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
+    testfixtures
+    pywin32; sys_platform == "win32"
+
 [testenv]
 deps =
-    -rtests/requirements.txt
+    {[test-requirements]deps}
+
     # mitmproxy does not support PyPy
     mitmproxy; implementation_name != 'pypy'
 passenv =
@@ -81,7 +95,7 @@ deps =
     w3lib==1.17.0
     zope.interface==5.1.0
     lxml==4.4.1
-    -rtests/requirements.txt
+    {[test-requirements]deps}
 
     # mitmproxy 8.0.0 requires upgrading some of the pinned dependencies
     # above, hence we do not install it in pinned environments at the moment
@@ -124,8 +138,12 @@ deps =
     robotexclusionrulesparser
     Pillow
     Twisted[http2]
-    brotli
-    zstandard
+    uvloop; platform_system != "Windows"
+    bpython  # optional for shell wrapper tests
+    brotli; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
+    brotlicffi; implementation_name == 'pypy'  # optional for HTTP compress downloader middleware tests
+    zstandard; implementation_name != 'pypy'  # optional for HTTP compress downloader middleware tests
+    ipython
 
 [testenv:extra-deps-pinned]
 basepython = python3.8
@@ -136,6 +154,12 @@ deps =
     Pillow==7.1.0
     robotexclusionrulesparser==1.6.2
     brotlipy
+    uvloop==0.14.0; platform_system != "Windows"
+    bpython==0.7.1
+    zstandard==0.1; implementation_name != 'pypy'
+    ipython==2.0.0
+    brotli==0.5.2; implementation_name != 'pypy'
+    brotlicffi==0.8.0; implementation_name == 'pypy'
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}

From 8d917c0b55cbeebd284f7446ccc0f90af60729ac Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 Mar 2024 17:44:57 +0500
Subject: [PATCH 30/74] Run black.

---
 tests/test_command_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index effafae54..b0f1cd38a 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -1,6 +1,6 @@
 import sys
 from io import StringIO
-from unittest.mock import Mock, call, PropertyMock, patch
+from unittest.mock import Mock, PropertyMock, call, patch
 
 from scrapy.commands.check import Command, TextTestResult
 from tests.test_commands import CommandTest

From f7bf3f726e3f19bf68b5e7e460f116850896eb42 Mon Sep 17 00:00:00 2001
From: igeni <kublin@it8.ru>
Date: Mon, 1 Apr 2024 16:37:23 +0300
Subject: [PATCH 31/74] modified string's concat to f-strings (#6296)

---
 tests/test_selector.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/test_selector.py b/tests/test_selector.py
index 85527bba9..1b5f3f018 100644
--- a/tests/test_selector.py
+++ b/tests/test_selector.py
@@ -80,10 +80,10 @@ class SelectorTestCase(unittest.TestCase):
         meta = (
             '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">'
         )
-        head = "<head>" + meta + "</head>"
+        head = f"<head>{meta}</head>"
         body_content = '<span id="blank">\xa3</span>'
-        body = "<body>" + body_content + "</body>"
-        html = "<html>" + head + body + "</html>"
+        body = f"<body>{body_content}</body>"
+        html = f"<html>{head}{body}</html>"
         encoding = "utf-8"
         html_utf8 = html.encode(encoding)
 

From 7b37dcd80d3783d3a21ff524e572019ff2b9e0ed Mon Sep 17 00:00:00 2001
From: Lorenzo Verardo <lorenzo.verardo@outlook.it>
Date: Thu, 4 Apr 2024 12:22:50 +0200
Subject: [PATCH 32/74] Handle robots.txt files not UTF-8 encoded

---
 scrapy/robotstxt.py               |  2 +-
 tests/test_robotstxt_interface.py | 21 +++++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 6ea2bfd97..ad06137e2 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -23,7 +23,7 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
         if to_native_str_type:
             robotstxt_body = to_unicode(robotstxt_body)
         else:
-            robotstxt_body = robotstxt_body.decode("utf-8")
+            robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index d7a923085..6ad30deed 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,5 +1,7 @@
 from twisted.trial import unittest
 
+from scrapy.robotstxt import decode_robotstxt
+
 
 def reppy_available():
     # check if reppy parser is installed
@@ -141,6 +143,25 @@ class BaseRobotParserTest:
         )
 
 
+class DecodeRobotsTxtTest(unittest.TestCase):
+    def test_native_string_conversion(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(
+            robotstxt_body, spider=None, to_native_str_type=True
+        )
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_utf8(self):
+        robotstxt_body = "User-agent: *\nDisallow: /\n".encode("utf-8")
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+    def test_decode_non_utf8(self):
+        robotstxt_body = b"User-agent: *\n\xFFDisallow: /\n"
+        decoded_content = decode_robotstxt(robotstxt_body, spider=None)
+        self.assertEqual(decoded_content, "User-agent: *\nDisallow: /\n")
+
+
 class PythonRobotParserTest(BaseRobotParserTest, unittest.TestCase):
     def setUp(self):
         from scrapy.robotstxt import PythonRobotParser

From 48c5a8c98f545e35708a580e724d0b8e1ada5e6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADctor=20Ruiz?= <victorruiz@zyte.com>
Date: Mon, 8 Apr 2024 11:47:53 +0200
Subject: [PATCH 33/74] Fix WrappedRequest.get_header raising TypeError if
 default is None (#6310)

---
 scrapy/http/cookies.py     | 3 ++-
 tests/test_http_cookies.py | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 2595f328e..72855bad5 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -170,7 +170,8 @@ class WrappedRequest:
         return name in self.request.headers
 
     def get_header(self, name, default=None):
-        return to_unicode(self.request.headers.get(name, default), errors="replace")
+        value = self.request.headers.get(name, default)
+        return to_unicode(value, errors="replace") if value is not None else None
 
     def header_items(self):
         return [
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index 8b5554914..932644320 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -43,6 +43,13 @@ class WrappedRequestTest(TestCase):
     def test_get_header(self):
         self.assertEqual(self.wrapped.get_header("content-type"), "text/html")
         self.assertEqual(self.wrapped.get_header("xxxxx", "def"), "def")
+        self.assertEqual(self.wrapped.get_header("xxxxx"), None)
+        wrapped = WrappedRequest(
+            Request(
+                "http://www.example.com/page.html", headers={"empty-binary-header": b""}
+            )
+        )
+        self.assertEqual(wrapped.get_header("empty-binary-header"), "")
 
     def test_header_items(self):
         self.assertEqual(self.wrapped.header_items(), [("Content-Type", ["text/html"])])

From 1d11ea3a54607b436f9a88f07911902a4882f0e8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 11 Apr 2024 12:19:32 +0200
Subject: [PATCH 34/74] Update practices.rst

---
 docs/topics/practices.rst | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index b1b8c9e9c..cd359b147 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -289,7 +289,8 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
   services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
   super proxy that you can attach your own proxies to.
 * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
-  plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__
+  plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional 
+  features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__
 
 If you are still unable to prevent your bot getting banned, consider contacting
 `commercial support`_.

From 5f67c01d1d2cc8a0104361b0323d755e22ed93dc Mon Sep 17 00:00:00 2001
From: TechVest <166724172+TechVest@users.noreply.github.com>
Date: Wed, 17 Apr 2024 16:56:26 +0800
Subject: [PATCH 35/74] chore: fix some typos in comments (#6317)

Signed-off-by: TechVest <techdashen@qq.com>
---
 .git-blame-ignore-revs       | 4 ++--
 docs/topics/broad-crawls.rst | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs
index dbcebfa0a..a9fc3dd68 100644
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@@ -1,7 +1,7 @@
 # .git-blame-ignore-revs
 # adding black formatter to all the code
 e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
-# re applying black to the code with default line length
+# reapplying black to the code with default line length
 303f0a70fcf8067adf0a909c2096a5009162383a
-# reaplying black again and removing line length on pre-commit black config
+# reapplying black again and removing line length on pre-commit black config
 c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
\ No newline at end of file
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index 8be89feb2..750aae554 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -116,7 +116,7 @@ Reduce log level
 When doing broad crawls you are often only interested in the crawl rates you
 get and any errors found. These stats are reported by Scrapy when using the
 ``INFO`` log level. In order to save CPU (and log storage requirements) you
-should not use ``DEBUG`` log level when preforming large broad crawls in
+should not use ``DEBUG`` log level when performing large broad crawls in
 production. Using ``DEBUG`` level when developing your (broad) crawler may be
 fine though.
 

From b1fe97dc6c8509d58b29c61cf7801eeee1b409a9 Mon Sep 17 00:00:00 2001
From: kokobhara <146670393+kokobhara@users.noreply.github.com>
Date: Wed, 17 Apr 2024 16:44:57 +0530
Subject: [PATCH 36/74] Fix test expectations (#6316)

---
 tests/test_utils_python.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 1d1d19146..4c60deafe 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -239,8 +239,11 @@ class UtilsPythonTestCase(unittest.TestCase):
         self.assertEqual(get_func_args(" ".join, stripself=True), ["iterable"])
 
         if platform.python_implementation() == "CPython":
-            # doesn't work on CPython: https://bugs.python.org/issue42785
-            self.assertEqual(get_func_args(operator.itemgetter(2)), [])
+            # This didn't work on older versions of CPython: https://github.com/python/cpython/issues/86951
+            self.assertIn(
+                get_func_args(operator.itemgetter(2), stripself=True),
+                [[], ["args", "kwargs"]],
+            )
         elif platform.python_implementation() == "PyPy":
             self.assertEqual(
                 get_func_args(operator.itemgetter(2), stripself=True), ["obj"]

From a166e9739962ec7cca8a655e5f20a18a1bce7d14 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 12:39:22 +0500
Subject: [PATCH 37/74] Remove the auto-generated copyright years from the docs
 footer. (#6322)

---
 docs/conf.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index 399078010..dcd2c9a3a 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -10,7 +10,6 @@
 # serve to show the default.
 
 import sys
-from datetime import datetime
 from pathlib import Path
 
 # If your extensions are in another directory, add it here. If the directory
@@ -48,7 +47,7 @@ master_doc = "index"
 
 # General information about the project.
 project = "Scrapy"
-copyright = f"2008–{datetime.now().year}, Scrapy developers"
+copyright = "Scrapy developers"
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

From 57acad3c38602f4399c307c2c002f9eddde97cbc Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 12:43:45 +0500
Subject: [PATCH 38/74] Full typing for scrapy/extensions, part 3. (#6325)

---
 scrapy/extensions/feedexport.py     | 331 +++++++++++++++++++---------
 scrapy/extensions/httpcache.py      | 153 ++++++++-----
 scrapy/extensions/postprocessing.py |   4 +-
 scrapy/utils/iterators.py           |   6 +-
 tox.ini                             |  14 +-
 5 files changed, 328 insertions(+), 180 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 3b0dd804e..97f39afe7 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -4,6 +4,8 @@ Feed Exports extension
 See documentation in docs/topics/feed-exports.rst
 """
 
+from __future__ import annotations
+
 import logging
 import re
 import sys
@@ -11,18 +13,36 @@ import warnings
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, Any, Callable, Dict, List, Optional, Tuple, Type, Union
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    TypeVar,
+    Union,
+    cast,
+)
 from urllib.parse import unquote, urlparse
 
-from twisted.internet import defer, threads
-from twisted.internet.defer import DeferredList
+from twisted.internet import threads
+from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
+from twisted.python.failure import Failure
 from w3lib.url import file_uri_to_path
 from zope.interface import Interface, implementer
 
 from scrapy import Spider, signals
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import BaseItemExporter
 from scrapy.extensions.postprocessing import PostProcessingManager
+from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -32,6 +52,12 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
 
+if TYPE_CHECKING:
+    from _typeshed import OpenBinaryMode
+
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 try:
@@ -41,8 +67,19 @@ try:
 except ImportError:
     IS_BOTO3_AVAILABLE = False
 
+UriParamsCallableT = Callable[[Dict[str, Any], Spider], Optional[Dict[str, Any]]]
 
-def build_storage(builder, uri, *args, feed_options=None, preargs=(), **kwargs):
+_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
+
+
+def build_storage(
+    builder: Callable[..., _StorageT],
+    uri: str,
+    *args: Any,
+    feed_options: Optional[Dict[str, Any]] = None,
+    preargs: Iterable[Any] = (),
+    **kwargs: Any,
+) -> _StorageT:
     kwargs["feed_options"] = feed_options
     return builder(*preargs, uri, *args, **kwargs)
 
@@ -56,10 +93,10 @@ class ItemFilter:
     :type feed_options: dict
     """
 
-    feed_options: Optional[dict]
-    item_classes: Tuple
+    feed_options: Optional[Dict[str, Any]]
+    item_classes: Tuple[type, ...]
 
-    def __init__(self, feed_options: Optional[dict]) -> None:
+    def __init__(self, feed_options: Optional[Dict[str, Any]]) -> None:
         self.feed_options = feed_options
         if feed_options is not None:
             self.item_classes = tuple(
@@ -98,28 +135,49 @@ class IFeedStorage(Interface):
         """Store the given file stream"""
 
 
+class FeedStorageProtocol(Protocol):
+    """Reimplementation of ``IFeedStorage`` that can be used in type hints."""
+
+    def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None):
+        """Initialize the storage with the parameters given in the URI and the
+        feed-specific options (see :setting:`FEEDS`)"""
+
+    def open(self, spider: Spider) -> IO[bytes]:
+        """Open the storage for the given spider. It must return a file-like
+        object that will be used for the exporters"""
+
+    def store(self, file: IO[bytes]) -> Optional[Deferred]:
+        """Store the given file stream"""
+
+
 @implementer(IFeedStorage)
 class BlockingFeedStorage:
-    def open(self, spider):
+    def open(self, spider: Spider) -> IO[bytes]:
         path = spider.crawler.settings["FEED_TEMPDIR"]
         if path and not Path(path).is_dir():
             raise OSError("Not a Directory: " + str(path))
 
         return NamedTemporaryFile(prefix="feed-", dir=path)
 
-    def store(self, file):
+    def store(self, file: IO[bytes]) -> Optional[Deferred]:
         return threads.deferToThread(self._store_in_thread, file)
 
-    def _store_in_thread(self, file):
+    def _store_in_thread(self, file: IO[bytes]) -> None:
         raise NotImplementedError
 
 
 @implementer(IFeedStorage)
 class StdoutFeedStorage:
-    def __init__(self, uri, _stdout=None, *, feed_options=None):
+    def __init__(
+        self,
+        uri: str,
+        _stdout: Optional[IO[bytes]] = None,
+        *,
+        feed_options: Optional[Dict[str, Any]] = None,
+    ):
         if not _stdout:
             _stdout = sys.stdout.buffer
-        self._stdout = _stdout
+        self._stdout: IO[bytes] = _stdout
         if feed_options and feed_options.get("overwrite", False) is True:
             logger.warning(
                 "Standard output (stdout) storage does not support "
@@ -128,54 +186,58 @@ class StdoutFeedStorage:
                 "it to False."
             )
 
-    def open(self, spider):
+    def open(self, spider: Spider) -> IO[bytes]:
         return self._stdout
 
-    def store(self, file):
+    def store(self, file: IO[bytes]) -> Optional[Deferred]:
         pass
 
 
 @implementer(IFeedStorage)
 class FileFeedStorage:
-    def __init__(self, uri, *, feed_options=None):
-        self.path = file_uri_to_path(uri)
+    def __init__(self, uri: str, *, feed_options: Optional[Dict[str, Any]] = None):
+        self.path: str = file_uri_to_path(uri)
         feed_options = feed_options or {}
-        self.write_mode = "wb" if feed_options.get("overwrite", False) else "ab"
+        self.write_mode: OpenBinaryMode = (
+            "wb" if feed_options.get("overwrite", False) else "ab"
+        )
 
-    def open(self, spider) -> IO[Any]:
+    def open(self, spider: Spider) -> IO[bytes]:
         dirname = Path(self.path).parent
         if dirname and not dirname.exists():
             dirname.mkdir(parents=True)
         return Path(self.path).open(self.write_mode)
 
-    def store(self, file):
+    def store(self, file: IO[bytes]) -> Optional[Deferred]:
         file.close()
+        return None
 
 
 class S3FeedStorage(BlockingFeedStorage):
     def __init__(
         self,
-        uri,
-        access_key=None,
-        secret_key=None,
-        acl=None,
-        endpoint_url=None,
+        uri: str,
+        access_key: Optional[str] = None,
+        secret_key: Optional[str] = None,
+        acl: Optional[str] = None,
+        endpoint_url: Optional[str] = None,
         *,
-        feed_options=None,
-        session_token=None,
-        region_name=None,
+        feed_options: Optional[Dict[str, Any]] = None,
+        session_token: Optional[str] = None,
+        region_name: Optional[str] = None,
     ):
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
         u = urlparse(uri)
-        self.bucketname = u.hostname
-        self.access_key = u.username or access_key
-        self.secret_key = u.password or secret_key
-        self.session_token = session_token
-        self.keyname = u.path[1:]  # remove first "/"
-        self.acl = acl
-        self.endpoint_url = endpoint_url
-        self.region_name = region_name
+        assert u.hostname
+        self.bucketname: str = u.hostname
+        self.access_key: Optional[str] = u.username or access_key
+        self.secret_key: Optional[str] = u.password or secret_key
+        self.session_token: Optional[str] = session_token
+        self.keyname: str = u.path[1:]  # remove first "/"
+        self.acl: Optional[str] = acl
+        self.endpoint_url: Optional[str] = endpoint_url
+        self.region_name: Optional[str] = region_name
 
         if IS_BOTO3_AVAILABLE:
             import boto3.session
@@ -218,7 +280,13 @@ class S3FeedStorage(BlockingFeedStorage):
             )
 
     @classmethod
-    def from_crawler(cls, crawler, uri, *, feed_options=None):
+    def from_crawler(
+        cls,
+        crawler: Crawler,
+        uri: str,
+        *,
+        feed_options: Optional[Dict[str, Any]] = None,
+    ) -> Self:
         return build_storage(
             cls,
             uri,
@@ -231,8 +299,9 @@ class S3FeedStorage(BlockingFeedStorage):
             feed_options=feed_options,
         )
 
-    def _store_in_thread(self, file):
+    def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
+        kwargs: Dict[str, Any]
         if IS_BOTO3_AVAILABLE:
             kwargs = {"ExtraArgs": {"ACL": self.acl}} if self.acl else {}
             self.s3_client.upload_fileobj(
@@ -247,22 +316,23 @@ class S3FeedStorage(BlockingFeedStorage):
 
 
 class GCSFeedStorage(BlockingFeedStorage):
-    def __init__(self, uri, project_id, acl):
-        self.project_id = project_id
-        self.acl = acl
+    def __init__(self, uri: str, project_id: Optional[str], acl: Optional[str]):
+        self.project_id: Optional[str] = project_id
+        self.acl: Optional[str] = acl
         u = urlparse(uri)
-        self.bucket_name = u.hostname
-        self.blob_name = u.path[1:]  # remove first "/"
+        assert u.hostname
+        self.bucket_name: str = u.hostname
+        self.blob_name: str = u.path[1:]  # remove first "/"
 
     @classmethod
-    def from_crawler(cls, crawler, uri):
+    def from_crawler(cls, crawler: Crawler, uri: str) -> Self:
         return cls(
             uri,
             crawler.settings["GCS_PROJECT_ID"],
             crawler.settings["FEED_STORAGE_GCS_ACL"] or None,
         )
 
-    def _store_in_thread(self, file):
+    def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
         from google.cloud.storage import Client
 
@@ -292,7 +362,13 @@ class FTPFeedStorage(BlockingFeedStorage):
         self.overwrite: bool = not feed_options or feed_options.get("overwrite", True)
 
     @classmethod
-    def from_crawler(cls, crawler, uri, *, feed_options=None):
+    def from_crawler(
+        cls,
+        crawler: Crawler,
+        uri: str,
+        *,
+        feed_options: Optional[Dict[str, Any]] = None,
+    ) -> Self:
         return build_storage(
             cls,
             uri,
@@ -300,7 +376,7 @@ class FTPFeedStorage(BlockingFeedStorage):
             feed_options=feed_options,
         )
 
-    def _store_in_thread(self, file):
+    def _store_in_thread(self, file: IO[bytes]) -> None:
         ftp_store_file(
             path=self.path,
             file=file,
@@ -316,46 +392,51 @@ class FTPFeedStorage(BlockingFeedStorage):
 class FeedSlot:
     def __init__(
         self,
-        storage,
-        uri,
-        format,
-        store_empty,
-        batch_id,
-        uri_template,
-        filter,
-        feed_options,
-        spider,
+        storage: FeedStorageProtocol,
+        uri: str,
+        format: str,
+        store_empty: bool,
+        batch_id: int,
+        uri_template: str,
+        filter: ItemFilter,
+        feed_options: Dict[str, Any],
+        spider: Spider,
         exporters: Dict[str, Type[BaseItemExporter]],
-        settings,
-        crawler,
+        settings: BaseSettings,
+        crawler: Crawler,
     ):
-        self.file = None
+        self.file: Optional[IO[bytes]] = None
         self.exporter: Optional[BaseItemExporter] = None
-        self.storage = storage
+        self.storage: FeedStorageProtocol = storage
         # feed params
-        self.batch_id = batch_id
-        self.format = format
-        self.store_empty = store_empty
-        self.uri_template = uri_template
-        self.uri = uri
-        self.filter = filter
+        self.batch_id: int = batch_id
+        self.format: str = format
+        self.store_empty: bool = store_empty
+        self.uri_template: str = uri_template
+        self.uri: str = uri
+        self.filter: ItemFilter = filter
         # exporter params
-        self.feed_options = feed_options
-        self.spider = spider
+        self.feed_options: Dict[str, Any] = feed_options
+        self.spider: Spider = spider
         self.exporters: Dict[str, Type[BaseItemExporter]] = exporters
-        self.settings = settings
-        self.crawler = crawler
+        self.settings: BaseSettings = settings
+        self.crawler: Crawler = crawler
         # flags
-        self.itemcount = 0
-        self._exporting = False
-        self._fileloaded = False
+        self.itemcount: int = 0
+        self._exporting: bool = False
+        self._fileloaded: bool = False
 
-    def start_exporting(self):
+    def start_exporting(self) -> None:
         if not self._fileloaded:
             self.file = self.storage.open(self.spider)
             if "postprocessing" in self.feed_options:
-                self.file = PostProcessingManager(
-                    self.feed_options["postprocessing"], self.file, self.feed_options
+                self.file = cast(
+                    IO[bytes],
+                    PostProcessingManager(
+                        self.feed_options["postprocessing"],
+                        self.file,
+                        self.feed_options,
+                    ),
                 )
             self.exporter = self._get_exporter(
                 file=self.file,
@@ -368,17 +449,23 @@ class FeedSlot:
             self._fileloaded = True
 
         if not self._exporting:
+            assert self.exporter
             self.exporter.start_exporting()
             self._exporting = True
 
-    def _get_instance(self, objcls, *args, **kwargs):
+    def _get_instance(
+        self, objcls: Type[BaseItemExporter], *args: Any, **kwargs: Any
+    ) -> BaseItemExporter:
         return build_from_crawler(objcls, self.crawler, *args, **kwargs)
 
-    def _get_exporter(self, file, format, *args, **kwargs) -> BaseItemExporter:
+    def _get_exporter(
+        self, file: IO[bytes], format: str, *args: Any, **kwargs: Any
+    ) -> BaseItemExporter:
         return self._get_instance(self.exporters[format], file, *args, **kwargs)
 
-    def finish_exporting(self):
+    def finish_exporting(self) -> None:
         if self._exporting:
+            assert self.exporter
             self.exporter.finish_exporting()
             self._exporting = False
 
@@ -390,22 +477,22 @@ _FeedSlot = create_deprecated_class(
 
 
 class FeedExporter:
-    _pending_deferreds: List[defer.Deferred] = []
+    _pending_deferreds: List[Deferred] = []
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         exporter = cls(crawler)
         crawler.signals.connect(exporter.open_spider, signals.spider_opened)
         crawler.signals.connect(exporter.close_spider, signals.spider_closed)
         crawler.signals.connect(exporter.item_scraped, signals.item_scraped)
         return exporter
 
-    def __init__(self, crawler):
-        self.crawler = crawler
-        self.settings = crawler.settings
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
+        self.settings: Settings = crawler.settings
         self.feeds = {}
-        self.slots = []
-        self.filters = {}
+        self.slots: List[FeedSlot] = []
+        self.filters: Dict[str, ItemFilter] = {}
 
         if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
             raise NotConfigured
@@ -437,8 +524,12 @@ class FeedExporter:
             )
             self.filters[uri] = self._load_filter(feed_options)
 
-        self.storages = self._load_components("FEED_STORAGES")
-        self.exporters = self._load_components("FEED_EXPORTERS")
+        self.storages: Dict[str, Type[FeedStorageProtocol]] = self._load_components(
+            "FEED_STORAGES"
+        )
+        self.exporters: Dict[str, Type[BaseItemExporter]] = self._load_components(
+            "FEED_EXPORTERS"
+        )
         for uri, feed_options in self.feeds.items():
             if not self._storage_supported(uri, feed_options):
                 raise NotConfigured
@@ -447,7 +538,7 @@ class FeedExporter:
             if not self._exporter_supported(feed_options["format"]):
                 raise NotConfigured
 
-    def open_spider(self, spider):
+    def open_spider(self, spider: Spider) -> None:
         for uri, feed_options in self.feeds.items():
             uri_params = self._get_uri_params(spider, feed_options["uri_params"])
             self.slots.append(
@@ -460,7 +551,7 @@ class FeedExporter:
                 )
             )
 
-    async def close_spider(self, spider):
+    async def close_spider(self, spider: Spider) -> None:
         for slot in self.slots:
             self._close_slot(slot, spider)
 
@@ -473,8 +564,9 @@ class FeedExporter:
             self.crawler.signals.send_catch_log_deferred(signals.feed_exporter_closed)
         )
 
-    def _close_slot(self, slot, spider):
-        def get_file(slot_):
+    def _close_slot(self, slot: FeedSlot, spider: Spider) -> Optional[Deferred]:
+        def get_file(slot_: FeedSlot) -> IO[bytes]:
+            assert slot_.file
             if isinstance(slot_.file, PostProcessingManager):
                 slot_.file.close()
                 return slot_.file.file
@@ -492,7 +584,7 @@ class FeedExporter:
             return None
 
         logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
-        d = defer.maybeDeferred(slot.storage.store, get_file(slot))
+        d: Deferred = maybeDeferred(slot.storage.store, get_file(slot))
 
         d.addCallback(
             self._handle_store_success, logmsg, spider, type(slot.storage).__name__
@@ -510,20 +602,33 @@ class FeedExporter:
 
         return d
 
-    def _handle_store_error(self, f, logmsg, spider, slot_type):
+    def _handle_store_error(
+        self, f: Failure, logmsg: str, spider: Spider, slot_type: str
+    ) -> None:
         logger.error(
             "Error storing %s",
             logmsg,
             exc_info=failure_to_exc_info(f),
             extra={"spider": spider},
         )
+        assert self.crawler.stats
         self.crawler.stats.inc_value(f"feedexport/failed_count/{slot_type}")
 
-    def _handle_store_success(self, f, logmsg, spider, slot_type):
+    def _handle_store_success(
+        self, f: Failure, logmsg: str, spider: Spider, slot_type: str
+    ) -> None:
         logger.info("Stored %s", logmsg, extra={"spider": spider})
+        assert self.crawler.stats
         self.crawler.stats.inc_value(f"feedexport/success_count/{slot_type}")
 
-    def _start_new_batch(self, batch_id, uri, feed_options, spider, uri_template):
+    def _start_new_batch(
+        self,
+        batch_id: int,
+        uri: str,
+        feed_options: Dict[str, Any],
+        spider: Spider,
+        uri_template: str,
+    ) -> FeedSlot:
         """
         Redirect the output data stream to a new file.
         Execute multiple times if FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
@@ -546,11 +651,11 @@ class FeedExporter:
             spider=spider,
             exporters=self.exporters,
             settings=self.settings,
-            crawler=getattr(self, "crawler", None),
+            crawler=self.crawler,
         )
         return slot
 
-    def item_scraped(self, item, spider):
+    def item_scraped(self, item: Any, spider: Spider) -> None:
         slots = []
         for slot in self.slots:
             if not slot.filter.accepts(item):
@@ -560,6 +665,7 @@ class FeedExporter:
                 continue
 
             slot.start_exporting()
+            assert slot.exporter
             slot.exporter.export_item(item)
             slot.itemcount += 1
             # create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
@@ -584,7 +690,7 @@ class FeedExporter:
                 slots.append(slot)
         self.slots = slots
 
-    def _load_components(self, setting_prefix):
+    def _load_components(self, setting_prefix: str) -> Dict[str, Any]:
         conf = without_none_values(self.settings.getwithbase(setting_prefix))
         d = {}
         for k, v in conf.items():
@@ -594,12 +700,13 @@ class FeedExporter:
                 pass
         return d
 
-    def _exporter_supported(self, format):
+    def _exporter_supported(self, format: str) -> bool:
         if format in self.exporters:
             return True
         logger.error("Unknown feed format: %(format)s", {"format": format})
+        return False
 
-    def _settings_are_valid(self):
+    def _settings_are_valid(self) -> bool:
         """
         If FEED_EXPORT_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
         %(batch_time)s or %(batch_id)d to distinguish different files of partial output
@@ -617,7 +724,7 @@ class FeedExporter:
                 return False
         return True
 
-    def _storage_supported(self, uri, feed_options):
+    def _storage_supported(self, uri: str, feed_options: Dict[str, Any]) -> bool:
         scheme = urlparse(uri).scheme
         if scheme in self.storages or PureWindowsPath(uri).drive:
             try:
@@ -630,8 +737,11 @@ class FeedExporter:
                 )
         else:
             logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})
+        return False
 
-    def _get_storage(self, uri, feed_options):
+    def _get_storage(
+        self, uri: str, feed_options: Dict[str, Any]
+    ) -> FeedStorageProtocol:
         """Fork of create_instance specific to feed storage classes
 
         It supports not passing the *feed_options* parameters to classes that
@@ -640,11 +750,14 @@ class FeedExporter:
         feedcls = self.storages.get(urlparse(uri).scheme, self.storages["file"])
         crawler = getattr(self, "crawler", None)
 
-        def build_instance(builder, *preargs):
+        def build_instance(
+            builder: Type[FeedStorageProtocol], *preargs: Any
+        ) -> FeedStorageProtocol:
             return build_storage(
                 builder, uri, feed_options=feed_options, preargs=preargs
             )
 
+        instance: FeedStorageProtocol
         if crawler and hasattr(feedcls, "from_crawler"):
             instance = build_instance(feedcls.from_crawler, crawler)
             method_name = "from_crawler"
@@ -661,9 +774,9 @@ class FeedExporter:
     def _get_uri_params(
         self,
         spider: Spider,
-        uri_params_function: Optional[Union[str, Callable[[dict, Spider], dict]]],
+        uri_params_function: Union[str, UriParamsCallableT, None],
         slot: Optional[FeedSlot] = None,
-    ) -> dict:
+    ) -> Dict[str, Any]:
         params = {}
         for k in dir(spider):
             params[k] = getattr(spider, k)
@@ -671,7 +784,7 @@ class FeedExporter:
         params["time"] = utc_now.replace(microsecond=0).isoformat().replace(":", "-")
         params["batch_time"] = utc_now.isoformat().replace(":", "-")
         params["batch_id"] = slot.batch_id + 1 if slot is not None else 1
-        uripar_function = (
+        uripar_function: UriParamsCallableT = (
             load_object(uri_params_function)
             if uri_params_function
             else lambda params, _: params
@@ -679,7 +792,9 @@ class FeedExporter:
         new_params = uripar_function(params, spider)
         return new_params if new_params is not None else params
 
-    def _load_filter(self, feed_options):
+    def _load_filter(self, feed_options: Dict[str, Any]) -> ItemFilter:
         # load the item filter if declared else load the default filter class
-        item_filter_class = load_object(feed_options.get("item_filter", ItemFilter))
+        item_filter_class: Type[ItemFilter] = load_object(
+            feed_options.get("item_filter", ItemFilter)
+        )
         return item_filter_class(feed_options)
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 335728502..dd5bce24f 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -1,10 +1,13 @@
 import gzip
 import logging
+import os
 import pickle  # nosec
 from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
 from time import time
+from types import ModuleType
+from typing import IO, TYPE_CHECKING, Any, Callable, Dict, List, Optional, Union, cast
 from weakref import WeakKeyDictionary
 
 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@@ -12,49 +15,65 @@ from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
 from scrapy.http import Headers, Response
 from scrapy.http.request import Request
 from scrapy.responsetypes import responsetypes
+from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
+from scrapy.utils.request import RequestFingerprinter
+
+if TYPE_CHECKING:
+    # typing.Concatenate requires Python 3.10
+    from typing_extensions import Concatenate
+
 
 logger = logging.getLogger(__name__)
 
 
 class DummyPolicy:
-    def __init__(self, settings):
-        self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
-        self.ignore_http_codes = [
+    def __init__(self, settings: BaseSettings):
+        self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
+        self.ignore_http_codes: List[int] = [
             int(x) for x in settings.getlist("HTTPCACHE_IGNORE_HTTP_CODES")
         ]
 
-    def should_cache_request(self, request):
+    def should_cache_request(self, request: Request) -> bool:
         return urlparse_cached(request).scheme not in self.ignore_schemes
 
-    def should_cache_response(self, response, request):
+    def should_cache_response(self, response: Response, request: Request) -> bool:
         return response.status not in self.ignore_http_codes
 
-    def is_cached_response_fresh(self, cachedresponse, request):
+    def is_cached_response_fresh(
+        self, cachedresponse: Response, request: Request
+    ) -> bool:
         return True
 
-    def is_cached_response_valid(self, cachedresponse, response, request):
+    def is_cached_response_valid(
+        self, cachedresponse: Response, response: Response, request: Request
+    ) -> bool:
         return True
 
 
 class RFC2616Policy:
     MAXAGE = 3600 * 24 * 365  # one year
 
-    def __init__(self, settings):
-        self.always_store = settings.getbool("HTTPCACHE_ALWAYS_STORE")
-        self.ignore_schemes = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
-        self._cc_parsed = WeakKeyDictionary()
-        self.ignore_response_cache_controls = [
+    def __init__(self, settings: BaseSettings):
+        self.always_store: bool = settings.getbool("HTTPCACHE_ALWAYS_STORE")
+        self.ignore_schemes: List[str] = settings.getlist("HTTPCACHE_IGNORE_SCHEMES")
+        self._cc_parsed: WeakKeyDictionary[
+            Union[Request, Response], Dict[bytes, Optional[bytes]]
+        ] = WeakKeyDictionary()
+        self.ignore_response_cache_controls: List[bytes] = [
             to_bytes(cc)
             for cc in settings.getlist("HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS")
         ]
 
-    def _parse_cachecontrol(self, r):
+    def _parse_cachecontrol(
+        self, r: Union[Request, Response]
+    ) -> Dict[bytes, Optional[bytes]]:
         if r not in self._cc_parsed:
             cch = r.headers.get(b"Cache-Control", b"")
+            assert cch is not None
             parsed = parse_cachecontrol(cch)
             if isinstance(r, Response):
                 for key in self.ignore_response_cache_controls:
@@ -62,7 +81,7 @@ class RFC2616Policy:
             self._cc_parsed[r] = parsed
         return self._cc_parsed[r]
 
-    def should_cache_request(self, request):
+    def should_cache_request(self, request: Request) -> bool:
         if urlparse_cached(request).scheme in self.ignore_schemes:
             return False
         cc = self._parse_cachecontrol(request)
@@ -72,7 +91,7 @@ class RFC2616Policy:
         # Any other is eligible for caching
         return True
 
-    def should_cache_response(self, response, request):
+    def should_cache_response(self, response: Response, request: Request) -> bool:
         # What is cacheable - https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9.1
         # Response cacheability - https://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html#sec13.4
         # Status code 206 is not included because cache can not deal with partial contents
@@ -100,7 +119,9 @@ class RFC2616Policy:
         # info and can not be revalidated
         return False
 
-    def is_cached_response_fresh(self, cachedresponse, request):
+    def is_cached_response_fresh(
+        self, cachedresponse: Response, request: Request
+    ) -> bool:
         cc = self._parse_cachecontrol(cachedresponse)
         ccreq = self._parse_cachecontrol(request)
         if b"no-cache" in cc or b"no-cache" in ccreq:
@@ -141,7 +162,9 @@ class RFC2616Policy:
         self._set_conditional_validators(request, cachedresponse)
         return False
 
-    def is_cached_response_valid(self, cachedresponse, response, request):
+    def is_cached_response_valid(
+        self, cachedresponse: Response, response: Response, request: Request
+    ) -> bool:
         # Use the cached response if the new response is a server error,
         # as long as the old response didn't specify must-revalidate.
         if response.status >= 500:
@@ -152,7 +175,9 @@ class RFC2616Policy:
         # Use the cached response if the server says it hasn't changed.
         return response.status == 304
 
-    def _set_conditional_validators(self, request, cachedresponse):
+    def _set_conditional_validators(
+        self, request: Request, cachedresponse: Response
+    ) -> None:
         if b"Last-Modified" in cachedresponse.headers:
             request.headers[b"If-Modified-Since"] = cachedresponse.headers[
                 b"Last-Modified"
@@ -161,13 +186,15 @@ class RFC2616Policy:
         if b"ETag" in cachedresponse.headers:
             request.headers[b"If-None-Match"] = cachedresponse.headers[b"ETag"]
 
-    def _get_max_age(self, cc):
+    def _get_max_age(self, cc: Dict[bytes, Optional[bytes]]) -> Optional[int]:
         try:
-            return max(0, int(cc[b"max-age"]))
+            return max(0, int(cc[b"max-age"]))  # type: ignore[arg-type]
         except (KeyError, ValueError):
             return None
 
-    def _compute_freshness_lifetime(self, response, request, now):
+    def _compute_freshness_lifetime(
+        self, response: Response, request: Request, now: float
+    ) -> float:
         # Reference nsHttpResponseHead::ComputeFreshnessLifetime
         # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#706
         cc = self._parse_cachecontrol(response)
@@ -198,10 +225,12 @@ class RFC2616Policy:
         # Insufficient information to compute freshness lifetime
         return 0
 
-    def _compute_current_age(self, response, request, now):
+    def _compute_current_age(
+        self, response: Response, request: Request, now: float
+    ) -> float:
         # Reference nsHttpResponseHead::ComputeCurrentAge
         # https://dxr.mozilla.org/mozilla-central/source/netwerk/protocol/http/nsHttpResponseHead.cpp#658
-        currentage = 0
+        currentage: float = 0
         # If Date header is not set we assume it is a fast connection, and
         # clock is in sync with the server
         date = rfc1123_to_epoch(response.headers.get(b"Date")) or now
@@ -210,7 +239,7 @@ class RFC2616Policy:
 
         if b"Age" in response.headers:
             try:
-                age = int(response.headers[b"Age"])
+                age = int(response.headers[b"Age"])  # type: ignore[arg-type]
                 currentage = max(currentage, age)
             except ValueError:
                 pass
@@ -219,13 +248,13 @@ class RFC2616Policy:
 
 
 class DbmCacheStorage:
-    def __init__(self, settings):
-        self.cachedir = data_path(settings["HTTPCACHE_DIR"], createdir=True)
-        self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
-        self.dbmodule = import_module(settings["HTTPCACHE_DBM_MODULE"])
-        self.db = None
+    def __init__(self, settings: BaseSettings):
+        self.cachedir: str = data_path(settings["HTTPCACHE_DIR"], createdir=True)
+        self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
+        self.dbmodule: ModuleType = import_module(settings["HTTPCACHE_DBM_MODULE"])
+        self.db: Any = None  # the real type is private
 
-    def open_spider(self, spider: Spider):
+    def open_spider(self, spider: Spider) -> None:
         dbpath = Path(self.cachedir, f"{spider.name}.db")
         self.db = self.dbmodule.open(str(dbpath), "c")
 
@@ -235,15 +264,16 @@ class DbmCacheStorage:
             extra={"spider": spider},
         )
 
-        self._fingerprinter = spider.crawler.request_fingerprinter
+        assert spider.crawler.request_fingerprinter
+        self._fingerprinter: RequestFingerprinter = spider.crawler.request_fingerprinter
 
-    def close_spider(self, spider):
+    def close_spider(self, spider: Spider) -> None:
         self.db.close()
 
-    def retrieve_response(self, spider, request):
+    def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]:
         data = self._read_data(spider, request)
         if data is None:
-            return  # not cached
+            return None  # not cached
         url = data["url"]
         status = data["status"]
         headers = Headers(data["headers"])
@@ -252,7 +282,9 @@ class DbmCacheStorage:
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
-    def store_response(self, spider, request, response):
+    def store_response(
+        self, spider: Spider, request: Request, response: Response
+    ) -> None:
         key = self._fingerprinter.fingerprint(request).hex()
         data = {
             "status": response.status,
@@ -263,28 +295,31 @@ class DbmCacheStorage:
         self.db[f"{key}_data"] = pickle.dumps(data, protocol=4)
         self.db[f"{key}_time"] = str(time())
 
-    def _read_data(self, spider, request):
+    def _read_data(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]:
         key = self._fingerprinter.fingerprint(request).hex()
         db = self.db
         tkey = f"{key}_time"
         if tkey not in db:
-            return  # not found
+            return None  # not found
 
         ts = db[tkey]
         if 0 < self.expiration_secs < time() - float(ts):
-            return  # expired
+            return None  # expired
 
-        return pickle.loads(db[f"{key}_data"])  # nosec
+        return cast(Dict[str, Any], pickle.loads(db[f"{key}_data"]))  # nosec
 
 
 class FilesystemCacheStorage:
-    def __init__(self, settings):
-        self.cachedir = data_path(settings["HTTPCACHE_DIR"])
-        self.expiration_secs = settings.getint("HTTPCACHE_EXPIRATION_SECS")
-        self.use_gzip = settings.getbool("HTTPCACHE_GZIP")
-        self._open = gzip.open if self.use_gzip else open
+    def __init__(self, settings: BaseSettings):
+        self.cachedir: str = data_path(settings["HTTPCACHE_DIR"])
+        self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
+        self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
+        # https://github.com/python/mypy/issues/10740
+        self._open: Callable[Concatenate[Union[str, os.PathLike], str, ...], IO] = (
+            gzip.open if self.use_gzip else open  # type: ignore[assignment]
+        )
 
-    def open_spider(self, spider: Spider):
+    def open_spider(self, spider: Spider) -> None:
         logger.debug(
             "Using filesystem cache storage in %(cachedir)s",
             {"cachedir": self.cachedir},
@@ -294,27 +329,29 @@ class FilesystemCacheStorage:
         assert spider.crawler.request_fingerprinter
         self._fingerprinter = spider.crawler.request_fingerprinter
 
-    def close_spider(self, spider):
+    def close_spider(self, spider: Spider) -> None:
         pass
 
-    def retrieve_response(self, spider: Spider, request: Request):
+    def retrieve_response(self, spider: Spider, request: Request) -> Optional[Response]:
         """Return response if present in cache, or None otherwise."""
         metadata = self._read_meta(spider, request)
         if metadata is None:
-            return  # not cached
+            return None  # not cached
         rpath = Path(self._get_request_path(spider, request))
         with self._open(rpath / "response_body", "rb") as f:
             body = f.read()
         with self._open(rpath / "response_headers", "rb") as f:
             rawheaders = f.read()
-        url = metadata.get("response_url")
+        url = metadata["response_url"]
         status = metadata["status"]
         headers = Headers(headers_raw_to_dict(rawheaders))
         respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
-    def store_response(self, spider: Spider, request: Request, response):
+    def store_response(
+        self, spider: Spider, request: Request, response: Response
+    ) -> None:
         """Store the given response in the cache."""
         rpath = Path(self._get_request_path(spider, request))
         if not rpath.exists():
@@ -343,19 +380,19 @@ class FilesystemCacheStorage:
         key = self._fingerprinter.fingerprint(request).hex()
         return str(Path(self.cachedir, spider.name, key[0:2], key))
 
-    def _read_meta(self, spider: Spider, request: Request):
+    def _read_meta(self, spider: Spider, request: Request) -> Optional[Dict[str, Any]]:
         rpath = Path(self._get_request_path(spider, request))
         metapath = rpath / "pickled_meta"
         if not metapath.exists():
-            return  # not found
+            return None  # not found
         mtime = metapath.stat().st_mtime
         if 0 < self.expiration_secs < time() - mtime:
-            return  # expired
+            return None  # expired
         with self._open(metapath, "rb") as f:
-            return pickle.load(f)  # nosec
+            return cast(Dict[str, Any], pickle.load(f))  # nosec
 
 
-def parse_cachecontrol(header):
+def parse_cachecontrol(header: bytes) -> Dict[bytes, Optional[bytes]]:
     """Parse Cache-Control header
 
     https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9
@@ -375,9 +412,9 @@ def parse_cachecontrol(header):
     return directives
 
 
-def rfc1123_to_epoch(date_str):
+def rfc1123_to_epoch(date_str: Union[str, bytes, None]) -> Optional[int]:
     try:
-        date_str = to_unicode(date_str, encoding="ascii")
-        return mktime_tz(parsedate_tz(date_str))
+        date_str = to_unicode(date_str, encoding="ascii")  # type: ignore[arg-type]
+        return mktime_tz(parsedate_tz(date_str))  # type: ignore[arg-type]
     except Exception:
         return None
diff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.py
index 7ffbd8bc3..ac12ad829 100644
--- a/scrapy/extensions/postprocessing.py
+++ b/scrapy/extensions/postprocessing.py
@@ -6,7 +6,7 @@ from bz2 import BZ2File
 from gzip import GzipFile
 from io import IOBase
 from lzma import LZMAFile
-from typing import Any, BinaryIO, Dict, List, cast
+from typing import IO, Any, BinaryIO, Dict, List, cast
 
 from scrapy.utils.misc import load_object
 
@@ -126,7 +126,7 @@ class PostProcessingManager(IOBase):
     """
 
     def __init__(
-        self, plugins: List[Any], file: BinaryIO, feed_options: Dict[str, Any]
+        self, plugins: List[Any], file: IO[bytes], feed_options: Dict[str, Any]
     ) -> None:
         self.plugins = self._load_plugins(plugins)
         self.file = file
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 49493e9c6..cd6e9d04e 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -3,7 +3,6 @@ import logging
 import re
 from io import StringIO
 from typing import (
-    TYPE_CHECKING,
     Any,
     Callable,
     Dict,
@@ -25,9 +24,6 @@ from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import re_rsearch, to_unicode
 
-if TYPE_CHECKING:
-    from lxml._types import SupportsReadClose  # nosec
-
 logger = logging.getLogger(__name__)
 
 
@@ -98,7 +94,7 @@ def xmliter_lxml(
     reader = _StreamReader(obj)
     tag = f"{{{namespace}}}{nodename}" if namespace else nodename
     iterable = etree.iterparse(
-        cast("SupportsReadClose[bytes]", reader),
+        reader,
         encoding=reader.encoding,
         events=("end", "start-ns"),
         resolve_entities=False,
diff --git a/tox.ini b/tox.ini
index b5effb527..d7527bb04 100644
--- a/tox.ini
+++ b/tox.ini
@@ -43,14 +43,14 @@ install_command =
 [testenv:typing]
 basepython = python3
 deps =
-    mypy==1.8.0
-    typing-extensions==4.10.0
+    mypy==1.10.0
+    typing-extensions==4.11.0
     types-attrs==19.1.0
-    types-lxml==2024.2.9
-    types-Pillow==10.2.0.20240213
-    types-Pygments==2.17.0.20240106
-    types-pyOpenSSL==24.0.0.20240130
-    types-setuptools==69.1.0.20240223
+    types-lxml==2024.4.14
+    types-Pillow==10.2.0.20240423
+    types-Pygments==2.17.0.20240310
+    types-pyOpenSSL==24.0.0.20240417
+    types-setuptools==69.5.0.20240423
     # 2.1.2 fixes a typing bug: https://github.com/scrapy/w3lib/pull/211
     w3lib >= 2.1.2
 commands =

From d7da298e0637d105dcec379f6cbb3196e752ae72 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 19:14:59 +0500
Subject: [PATCH 39/74] Typing for build_from_*. (#6326)

---
 scrapy/core/engine.py    |  3 ++-
 scrapy/core/scheduler.py |  2 ++
 scrapy/crawler.py        |  4 +++-
 scrapy/utils/misc.py     | 28 +++++++++++++++++++---------
 4 files changed, 26 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 2db085081..93a0c51bc 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -366,7 +366,8 @@ class ExecutionEngine:
         self.slot = Slot(start_requests, close_if_idle, nextcall, scheduler)
         self.spider = spider
         if hasattr(scheduler, "open"):
-            yield scheduler.open(spider)
+            if d := scheduler.open(spider):
+                yield d
         yield self.scraper.open_spider(spider)
         assert self.crawler.stats
         self.crawler.stats.open_spider(spider)
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index f41b83a67..b2209e53f 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -322,6 +322,7 @@ class Scheduler(BaseScheduler):
 
     def _mq(self):
         """Create a new priority queue instance, with in-memory storage"""
+        assert self.crawler
         return build_from_crawler(
             self.pqclass,
             self.crawler,
@@ -331,6 +332,7 @@ class Scheduler(BaseScheduler):
 
     def _dq(self):
         """Create a new priority queue instance, with disk storage"""
+        assert self.crawler
         assert self.dqdir
         state = self._read_dqs_state(self.dqdir)
         q = build_from_crawler(
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 1db9ace28..ccfe78891 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -445,7 +445,9 @@ class CrawlerProcess(CrawlerRunner):
             d.addBoth(self._stop_reactor)
 
         resolver_class = load_object(self.settings["DNS_RESOLVER"])
-        resolver = build_from_crawler(resolver_class, self, reactor=reactor)
+        # We pass self, which is CrawlerProcess, instead of Crawler here,
+        # which works because the default resolvers only use crawler.settings.
+        resolver = build_from_crawler(resolver_class, self, reactor=reactor)  # type: ignore[arg-type]
         resolver.install_on_reactor()
         tp = reactor.getThreadPool()
         tp.adjustPoolsize(maxthreads=self.settings.getint("REACTOR_THREADPOOL_MAXSIZE"))
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 7f83d06fb..faf52e44a 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -1,5 +1,7 @@
 """Helper functions which don't fit anywhere else"""
 
+from __future__ import annotations
+
 import ast
 import hashlib
 import inspect
@@ -22,6 +24,8 @@ from typing import (
     Iterable,
     List,
     Optional,
+    Type,
+    TypeVar,
     Union,
     cast,
 )
@@ -32,9 +36,11 @@ from scrapy.utils.datatypes import LocalWeakReferencedCache
 
 if TYPE_CHECKING:
     from scrapy import Spider
-
+    from scrapy.crawler import Crawler
+    from scrapy.settings import BaseSettings
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
+T = TypeVar("T")
 
 
 def arg_to_iter(arg: Any) -> Iterable[Any]:
@@ -177,7 +183,9 @@ def create_instance(objcls, settings, crawler, *args, **kwargs):
     return instance
 
 
-def build_from_crawler(objcls, crawler, /, *args, **kwargs):
+def build_from_crawler(
+    objcls: Type[T], crawler: Crawler, /, *args: Any, **kwargs: Any
+) -> T:
     """Construct a class instance using its ``from_crawler`` constructor.
 
     ``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -185,20 +193,22 @@ def build_from_crawler(objcls, crawler, /, *args, **kwargs):
     Raises ``TypeError`` if the resulting instance is ``None``.
     """
     if hasattr(objcls, "from_crawler"):
-        instance = objcls.from_crawler(crawler, *args, **kwargs)
+        instance = objcls.from_crawler(crawler, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_crawler"
     elif hasattr(objcls, "from_settings"):
-        instance = objcls.from_settings(crawler.settings, *args, **kwargs)
+        instance = objcls.from_settings(crawler.settings, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_settings"
     else:
         instance = objcls(*args, **kwargs)
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return instance
+    return cast(T, instance)
 
 
-def build_from_settings(objcls, settings, /, *args, **kwargs):
+def build_from_settings(
+    objcls: Type[T], settings: BaseSettings, /, *args: Any, **kwargs: Any
+) -> T:
     """Construct a class instance using its ``from_settings`` constructor.
 
     ``*args`` and ``**kwargs`` are forwarded to the constructor.
@@ -206,14 +216,14 @@ def build_from_settings(objcls, settings, /, *args, **kwargs):
     Raises ``TypeError`` if the resulting instance is ``None``.
     """
     if hasattr(objcls, "from_settings"):
-        instance = objcls.from_settings(settings, *args, **kwargs)
+        instance = objcls.from_settings(settings, *args, **kwargs)  # type: ignore[attr-defined]
         method_name = "from_settings"
     else:
         instance = objcls(*args, **kwargs)
         method_name = "__new__"
     if instance is None:
         raise TypeError(f"{objcls.__qualname__}.{method_name} returned None")
-    return instance
+    return cast(T, instance)
 
 
 @contextmanager
@@ -290,7 +300,7 @@ def is_generator_with_return_value(callable: Callable) -> bool:
     return bool(_generator_callbacks_cache[callable])
 
 
-def warn_on_generator_with_return_value(spider: "Spider", callable: Callable) -> None:
+def warn_on_generator_with_return_value(spider: Spider, callable: Callable) -> None:
     """
     Logs a warning if a callable is a generator function and includes
     a 'return' statement with a value different than None

From 9eea22fb0ca99193b7f38f9f9398b278d64ea977 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 18:59:31 +0500
Subject: [PATCH 40/74] Full typing for scrapy/cmdline.py.

---
 scrapy/cmdline.py | 47 ++++++++++++++++++++++++++++++++---------------
 1 file changed, 32 insertions(+), 15 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 6580ba9ce..4df5698a6 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -4,18 +4,22 @@ import inspect
 import os
 import sys
 from importlib.metadata import entry_points
+from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
 from scrapy.crawler import CrawlerProcess
 from scrapy.exceptions import UsageError
+from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 
 
 class ScrapyArgumentParser(argparse.ArgumentParser):
-    def _parse_optional(self, arg_string):
+    def _parse_optional(
+        self, arg_string: str
+    ) -> Optional[Tuple[Optional[argparse.Action], str, Optional[str]]]:
         # if starts with -: it means that is a parameter not a argument
         if arg_string[:2] == "-:":
             return None
@@ -23,7 +27,7 @@ class ScrapyArgumentParser(argparse.ArgumentParser):
         return super()._parse_optional(arg_string)
 
 
-def _iter_command_classes(module_name):
+def _iter_command_classes(module_name: str) -> Iterable[Type[ScrapyCommand]]:
     # TODO: add `name` attribute to commands and merge this function with
     # scrapy.utils.spider.iter_spider_classes
     for module in walk_modules(module_name):
@@ -37,8 +41,8 @@ def _iter_command_classes(module_name):
                 yield obj
 
 
-def _get_commands_from_module(module, inproject):
-    d = {}
+def _get_commands_from_module(module: str, inproject: bool) -> Dict[str, ScrapyCommand]:
+    d: Dict[str, ScrapyCommand] = {}
     for cmd in _iter_command_classes(module):
         if inproject or not cmd.requires_project:
             cmdname = cmd.__module__.split(".")[-1]
@@ -46,8 +50,10 @@ def _get_commands_from_module(module, inproject):
     return d
 
 
-def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
-    cmds = {}
+def _get_commands_from_entry_points(
+    inproject: bool, group: str = "scrapy.commands"
+) -> Dict[str, ScrapyCommand]:
+    cmds: Dict[str, ScrapyCommand] = {}
     if sys.version_info >= (3, 10):
         eps = entry_points(group=group)
     else:
@@ -61,7 +67,9 @@ def _get_commands_from_entry_points(inproject, group="scrapy.commands"):
     return cmds
 
 
-def _get_commands_dict(settings, inproject):
+def _get_commands_dict(
+    settings: BaseSettings, inproject: bool
+) -> Dict[str, ScrapyCommand]:
     cmds = _get_commands_from_module("scrapy.commands", inproject)
     cmds.update(_get_commands_from_entry_points(inproject))
     cmds_module = settings["COMMANDS_MODULE"]
@@ -70,16 +78,17 @@ def _get_commands_dict(settings, inproject):
     return cmds
 
 
-def _pop_command_name(argv):
+def _pop_command_name(argv: List[str]) -> Optional[str]:
     i = 0
     for arg in argv[1:]:
         if not arg.startswith("-"):
             del argv[i]
             return arg
         i += 1
+    return None
 
 
-def _print_header(settings, inproject):
+def _print_header(settings: BaseSettings, inproject: bool) -> None:
     version = scrapy.__version__
     if inproject:
         print(f"Scrapy {version} - active project: {settings['BOT_NAME']}\n")
@@ -88,7 +97,7 @@ def _print_header(settings, inproject):
         print(f"Scrapy {version} - no active project\n")
 
 
-def _print_commands(settings, inproject):
+def _print_commands(settings: BaseSettings, inproject: bool) -> None:
     _print_header(settings, inproject)
     print("Usage:")
     print("  scrapy <command> [options] [args]\n")
@@ -103,13 +112,17 @@ def _print_commands(settings, inproject):
     print('Use "scrapy <command> -h" to see more info about a command')
 
 
-def _print_unknown_command(settings, cmdname, inproject):
+def _print_unknown_command(
+    settings: BaseSettings, cmdname: str, inproject: bool
+) -> None:
     _print_header(settings, inproject)
     print(f"Unknown command: {cmdname}\n")
     print('Use "scrapy" to see available commands')
 
 
-def _run_print_help(parser, func, *a, **kw):
+def _run_print_help(
+    parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any
+) -> None:
     try:
         func(*a, **kw)
     except UsageError as e:
@@ -120,7 +133,9 @@ def _run_print_help(parser, func, *a, **kw):
         sys.exit(2)
 
 
-def execute(argv=None, settings=None):
+def execute(
+    argv: Optional[List[str]] = None, settings: Optional[Settings] = None
+) -> None:
     if argv is None:
         argv = sys.argv
 
@@ -162,14 +177,16 @@ def execute(argv=None, settings=None):
     sys.exit(cmd.exitcode)
 
 
-def _run_command(cmd, args, opts):
+def _run_command(cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace) -> None:
     if opts.profile:
         _run_command_profiled(cmd, args, opts)
     else:
         cmd.run(args, opts)
 
 
-def _run_command_profiled(cmd, args, opts):
+def _run_command_profiled(
+    cmd: ScrapyCommand, args: List[str], opts: argparse.Namespace
+) -> None:
     if opts.profile:
         sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
     loc = locals()

From fc1a83e7c42dc5142eb9190fdca887385254a7a6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 19:07:31 +0500
Subject: [PATCH 41/74] Full typing for scrapy/item.py.

---
 scrapy/item.py | 40 ++++++++++++++++++++++++----------------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index d3eb90b7b..e04e994ef 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -4,14 +4,20 @@ Scrapy Item
 See documentation in docs/topics/item.rst
 """
 
+from __future__ import annotations
+
 from abc import ABCMeta
 from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
-from typing import Dict
+from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple
 
 from scrapy.utils.trackref import object_ref
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class Field(dict):
     """Container of field metadata"""
@@ -23,7 +29,9 @@ class ItemMeta(ABCMeta):
     .. _metaclass: https://realpython.com/python-metaclasses
     """
 
-    def __new__(mcs, class_name, bases, attrs):
+    def __new__(
+        mcs, class_name: str, bases: Tuple[type, ...], attrs: Dict[str, Any]
+    ) -> ItemMeta:
         classcell = attrs.pop("__classcell__", None)
         new_bases = tuple(base._class for base in bases if hasattr(base, "_class"))
         _class = super().__new__(mcs, "x_" + class_name, new_bases, attrs)
@@ -44,7 +52,7 @@ class ItemMeta(ABCMeta):
         return super().__new__(mcs, class_name, bases, new_attrs)
 
 
-class Item(MutableMapping, object_ref, metaclass=ItemMeta):
+class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
     """
     Base class for scraped items.
 
@@ -69,51 +77,51 @@ class Item(MutableMapping, object_ref, metaclass=ItemMeta):
 
     fields: Dict[str, Field]
 
-    def __init__(self, *args, **kwargs):
-        self._values = {}
+    def __init__(self, *args: Any, **kwargs: Any):
+        self._values: Dict[str, Any] = {}
         if args or kwargs:  # avoid creating dict for most common case
             for k, v in dict(*args, **kwargs).items():
                 self[k] = v
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         return self._values[key]
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any) -> None:
         if key in self.fields:
             self._values[key] = value
         else:
             raise KeyError(f"{self.__class__.__name__} does not support field: {key}")
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         del self._values[key]
 
-    def __getattr__(self, name):
+    def __getattr__(self, name: str) -> NoReturn:
         if name in self.fields:
             raise AttributeError(f"Use item[{name!r}] to get field value")
         raise AttributeError(name)
 
-    def __setattr__(self, name, value):
+    def __setattr__(self, name: str, value: Any) -> None:
         if not name.startswith("_"):
             raise AttributeError(f"Use item[{name!r}] = {value!r} to set field value")
         super().__setattr__(name, value)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self._values)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[str]:
         return iter(self._values)
 
     __hash__ = object_ref.__hash__
 
-    def keys(self):
+    def keys(self) -> KeysView[str]:
         return self._values.keys()
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return pformat(dict(self))
 
-    def copy(self):
+    def copy(self) -> Self:
         return self.__class__(self)
 
-    def deepcopy(self):
+    def deepcopy(self) -> Self:
         """Return a :func:`~copy.deepcopy` of this item."""
         return deepcopy(self)

From 08a265b6ff9bc47774173238b06715154b39e534 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 19:10:00 +0500
Subject: [PATCH 42/74] Full typing for scrapy/extension.py.

---
 scrapy/extension.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/scrapy/extension.py b/scrapy/extension.py
index 6be14450c..8221b675e 100644
--- a/scrapy/extension.py
+++ b/scrapy/extension.py
@@ -4,7 +4,10 @@ The Extension Manager
 See documentation in docs/topics/extensions.rst
 """
 
+from typing import Any, List
+
 from scrapy.middleware import MiddlewareManager
+from scrapy.settings import Settings
 from scrapy.utils.conf import build_component_list
 
 
@@ -12,5 +15,5 @@ class ExtensionManager(MiddlewareManager):
     component_name = "extension"
 
     @classmethod
-    def _get_mwlist_from_settings(cls, settings):
+    def _get_mwlist_from_settings(cls, settings: Settings) -> List[Any]:
         return build_component_list(settings.getwithbase("EXTENSIONS"))

From 38020e0b0481d2b15792757669272d6d3bf4b14f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 20:12:30 +0500
Subject: [PATCH 43/74] Full typing for scrapy/mail.py.

---
 scrapy/mail.py | 98 +++++++++++++++++++++++++++++++-------------------
 1 file changed, 62 insertions(+), 36 deletions(-)

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 7cb5ef454..56adba934 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -14,11 +14,23 @@ from email.mime.nonmultipart import MIMENonMultipart
 from email.mime.text import MIMEText
 from email.utils import formatdate
 from io import BytesIO
-from typing import TYPE_CHECKING, Optional
+from typing import (
+    IO,
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
 
 from twisted import version as twisted_version
 from twisted.internet import ssl
 from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
 from twisted.python.versions import Version
 
 from scrapy.settings import BaseSettings
@@ -26,6 +38,9 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
+    # imports twisted.internet.reactor
+    from twisted.mail.smtp import ESMTPSenderFactory
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
@@ -37,7 +52,7 @@ logger = logging.getLogger(__name__)
 COMMASPACE = ", "
 
 
-def _to_bytes_or_none(text):
+def _to_bytes_or_none(text: Union[str, bytes, None]) -> Optional[bytes]:
     if text is None:
         return None
     return to_bytes(text)
@@ -46,23 +61,23 @@ def _to_bytes_or_none(text):
 class MailSender:
     def __init__(
         self,
-        smtphost="localhost",
-        mailfrom="scrapy@localhost",
-        smtpuser=None,
-        smtppass=None,
-        smtpport=25,
-        smtptls=False,
-        smtpssl=False,
-        debug=False,
+        smtphost: str = "localhost",
+        mailfrom: str = "scrapy@localhost",
+        smtpuser: Optional[str] = None,
+        smtppass: Optional[str] = None,
+        smtpport: int = 25,
+        smtptls: bool = False,
+        smtpssl: bool = False,
+        debug: bool = False,
     ):
-        self.smtphost = smtphost
-        self.smtpport = smtpport
-        self.smtpuser = _to_bytes_or_none(smtpuser)
-        self.smtppass = _to_bytes_or_none(smtppass)
-        self.smtptls = smtptls
-        self.smtpssl = smtpssl
-        self.mailfrom = mailfrom
-        self.debug = debug
+        self.smtphost: str = smtphost
+        self.smtpport: int = smtpport
+        self.smtpuser: Optional[bytes] = _to_bytes_or_none(smtpuser)
+        self.smtppass: Optional[bytes] = _to_bytes_or_none(smtppass)
+        self.smtptls: bool = smtptls
+        self.smtpssl: bool = smtpssl
+        self.mailfrom: str = mailfrom
+        self.debug: bool = debug
 
     @classmethod
     def from_settings(cls, settings: BaseSettings) -> Self:
@@ -78,14 +93,14 @@ class MailSender:
 
     def send(
         self,
-        to,
-        subject,
-        body,
-        cc=None,
-        attachs=(),
-        mimetype="text/plain",
-        charset=None,
-        _callback=None,
+        to: Union[str, List[str]],
+        subject: str,
+        body: str,
+        cc: Union[str, List[str], None] = None,
+        attachs: Sequence[Tuple[str, str, IO]] = (),
+        mimetype: str = "text/plain",
+        charset: Optional[str] = None,
+        _callback: Optional[Callable[..., None]] = None,
     ) -> Optional[Deferred]:
         from twisted.internet import reactor
 
@@ -142,13 +157,15 @@ class MailSender:
         dfd.addCallbacks(
             callback=self._sent_ok,
             errback=self._sent_failed,
-            callbackArgs=[to, cc, subject, len(attachs)],
-            errbackArgs=[to, cc, subject, len(attachs)],
+            callbackArgs=(to, cc, subject, len(attachs)),
+            errbackArgs=(to, cc, subject, len(attachs)),
         )
         reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
         return dfd
 
-    def _sent_ok(self, result, to, cc, subject, nattachs):
+    def _sent_ok(
+        self, result: Any, to: List[str], cc: List[str], subject: str, nattachs: int
+    ) -> None:
         logger.info(
             "Mail sent OK: To=%(mailto)s Cc=%(mailcc)s "
             'Subject="%(mailsubject)s" Attachs=%(mailattachs)d',
@@ -160,7 +177,14 @@ class MailSender:
             },
         )
 
-    def _sent_failed(self, failure, to, cc, subject, nattachs):
+    def _sent_failed(
+        self,
+        failure: Failure,
+        to: List[str],
+        cc: List[str],
+        subject: str,
+        nattachs: int,
+    ) -> Failure:
         errstr = str(failure.value)
         logger.error(
             "Unable to send mail: To=%(mailto)s Cc=%(mailcc)s "
@@ -176,13 +200,13 @@ class MailSender:
         )
         return failure
 
-    def _sendmail(self, to_addrs, msg):
+    def _sendmail(self, to_addrs: List[str], msg: bytes) -> Deferred:
         from twisted.internet import reactor
 
-        msg = BytesIO(msg)
-        d = Deferred()
+        msg_io = BytesIO(msg)
+        d: Deferred = Deferred()
 
-        factory = self._create_sender_factory(to_addrs, msg, d)
+        factory = self._create_sender_factory(to_addrs, msg_io, d)
 
         if self.smtpssl:
             reactor.connectSSL(
@@ -193,10 +217,12 @@ class MailSender:
 
         return d
 
-    def _create_sender_factory(self, to_addrs, msg, d):
+    def _create_sender_factory(
+        self, to_addrs: List[str], msg: IO, d: Deferred
+    ) -> ESMTPSenderFactory:
         from twisted.mail.smtp import ESMTPSenderFactory
 
-        factory_keywords = {
+        factory_keywords: Dict[str, Any] = {
             "heloFallback": True,
             "requireAuthentication": False,
             "requireTransportSecurity": self.smtptls,

From 0c8e21b8acfcac2d6d057f3c67b2252d0fa660e3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 20:57:31 +0500
Subject: [PATCH 44/74] Full typing for scrapy/pqueues.py.

---
 scrapy/core/downloader/__init__.py |   2 +-
 scrapy/pqueues.py                  | 138 +++++++++++++++++++++--------
 2 files changed, 100 insertions(+), 40 deletions(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index ecd3e8b56..f88da41ea 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -143,7 +143,7 @@ class Downloader:
 
         return key, self.slots[key]
 
-    def _get_slot_key(self, request: Request, spider: Spider) -> str:
+    def _get_slot_key(self, request: Request, spider: Any) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
             return cast(str, request.meta[self.DOWNLOAD_SLOT])
 
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 593667f1f..213ad590d 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -1,12 +1,32 @@
+from __future__ import annotations
+
 import hashlib
 import logging
+from typing import (
+    TYPE_CHECKING,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Protocol,
+    Tuple,
+    Type,
+    cast,
+)
 
+from scrapy import Request
+from scrapy.core.downloader import Downloader
+from scrapy.crawler import Crawler
 from scrapy.utils.misc import build_from_crawler
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
-def _path_safe(text):
+def _path_safe(text: str) -> str:
     """
     Return a filesystem-safe version of a string ``text``
 
@@ -24,6 +44,18 @@ def _path_safe(text):
     return "-".join([pathable_slot, unique_slot])
 
 
+class QueueProtocol(Protocol):
+    """Protocol for downstream queues of ``ScrapyPriorityQueue``."""
+
+    def push(self, request: Request) -> None: ...
+
+    def pop(self) -> Optional[Request]: ...
+
+    def close(self) -> None: ...
+
+    def __len__(self) -> int: ...
+
+
 class ScrapyPriorityQueue:
     """A priority queue implemented using multiple internal queues (typically,
     FIFO queues). It uses one internal queue for each priority value. The internal
@@ -51,18 +83,30 @@ class ScrapyPriorityQueue:
     """
 
     @classmethod
-    def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+    def from_crawler(
+        cls,
+        crawler: Crawler,
+        downstream_queue_cls: Type[QueueProtocol],
+        key: str,
+        startprios: Iterable[int] = (),
+    ) -> Self:
         return cls(crawler, downstream_queue_cls, key, startprios)
 
-    def __init__(self, crawler, downstream_queue_cls, key, startprios=()):
-        self.crawler = crawler
-        self.downstream_queue_cls = downstream_queue_cls
-        self.key = key
-        self.queues = {}
-        self.curprio = None
+    def __init__(
+        self,
+        crawler: Crawler,
+        downstream_queue_cls: Type[QueueProtocol],
+        key: str,
+        startprios: Iterable[int] = (),
+    ):
+        self.crawler: Crawler = crawler
+        self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
+        self.key: str = key
+        self.queues: Dict[int, QueueProtocol] = {}
+        self.curprio: Optional[int] = None
         self.init_prios(startprios)
 
-    def init_prios(self, startprios):
+    def init_prios(self, startprios: Iterable[int]) -> None:
         if not startprios:
             return
 
@@ -71,17 +115,17 @@ class ScrapyPriorityQueue:
 
         self.curprio = min(startprios)
 
-    def qfactory(self, key):
+    def qfactory(self, key: int) -> QueueProtocol:
         return build_from_crawler(
             self.downstream_queue_cls,
             self.crawler,
             self.key + "/" + str(key),
         )
 
-    def priority(self, request):
+    def priority(self, request: Request) -> int:
         return -request.priority
 
-    def push(self, request):
+    def push(self, request: Request) -> None:
         priority = self.priority(request)
         if priority not in self.queues:
             self.queues[priority] = self.qfactory(priority)
@@ -90,9 +134,9 @@ class ScrapyPriorityQueue:
         if self.curprio is None or priority < self.curprio:
             self.curprio = priority
 
-    def pop(self):
+    def pop(self) -> Optional[Request]:
         if self.curprio is None:
-            return
+            return None
         q = self.queues[self.curprio]
         m = q.pop()
         if not q:
@@ -102,7 +146,7 @@ class ScrapyPriorityQueue:
             self.curprio = min(prios) if prios else None
         return m
 
-    def peek(self):
+    def peek(self) -> Optional[Request]:
         """Returns the next object to be returned by :meth:`pop`,
         but without removing it from the queue.
 
@@ -112,30 +156,32 @@ class ScrapyPriorityQueue:
         if self.curprio is None:
             return None
         queue = self.queues[self.curprio]
-        return queue.peek()
+        # Protocols can't declare optional members
+        return cast(Request, queue.peek())  # type: ignore[attr-defined]
 
-    def close(self):
-        active = []
+    def close(self) -> List[int]:
+        active: List[int] = []
         for p, q in self.queues.items():
             active.append(p)
             q.close()
         return active
 
-    def __len__(self):
+    def __len__(self) -> int:
         return sum(len(x) for x in self.queues.values()) if self.queues else 0
 
 
 class DownloaderInterface:
-    def __init__(self, crawler):
-        self.downloader = crawler.engine.downloader
+    def __init__(self, crawler: Crawler):
+        assert crawler.engine
+        self.downloader: Downloader = crawler.engine.downloader
 
-    def stats(self, possible_slots):
+    def stats(self, possible_slots: Iterable[str]) -> List[Tuple[int, str]]:
         return [(self._active_downloads(slot), slot) for slot in possible_slots]
 
-    def get_slot_key(self, request):
+    def get_slot_key(self, request: Request) -> str:
         return self.downloader._get_slot_key(request, None)
 
-    def _active_downloads(self, slot):
+    def _active_downloads(self, slot: str) -> int:
         """Return a number of requests in a Downloader for a given slot"""
         if slot not in self.downloader.slots:
             return 0
@@ -149,10 +195,22 @@ class DownloaderAwarePriorityQueue:
     """
 
     @classmethod
-    def from_crawler(cls, crawler, downstream_queue_cls, key, startprios=()):
+    def from_crawler(
+        cls,
+        crawler: Crawler,
+        downstream_queue_cls: Type[QueueProtocol],
+        key: str,
+        startprios: Optional[Dict[str, Iterable[int]]] = None,
+    ) -> Self:
         return cls(crawler, downstream_queue_cls, key, startprios)
 
-    def __init__(self, crawler, downstream_queue_cls, key, slot_startprios=()):
+    def __init__(
+        self,
+        crawler: Crawler,
+        downstream_queue_cls: Type[QueueProtocol],
+        key: str,
+        slot_startprios: Optional[Dict[str, Iterable[int]]] = None,
+    ):
         if crawler.settings.getint("CONCURRENT_REQUESTS_PER_IP") != 0:
             raise ValueError(
                 f'"{self.__class__}" does not support CONCURRENT_REQUESTS_PER_IP'
@@ -169,16 +227,18 @@ class DownloaderAwarePriorityQueue:
                 "queue class can be resumed."
             )
 
-        self._downloader_interface = DownloaderInterface(crawler)
-        self.downstream_queue_cls = downstream_queue_cls
-        self.key = key
-        self.crawler = crawler
+        self._downloader_interface: DownloaderInterface = DownloaderInterface(crawler)
+        self.downstream_queue_cls: Type[QueueProtocol] = downstream_queue_cls
+        self.key: str = key
+        self.crawler: Crawler = crawler
 
-        self.pqueues = {}  # slot -> priority queue
+        self.pqueues: Dict[str, ScrapyPriorityQueue] = {}  # slot -> priority queue
         for slot, startprios in (slot_startprios or {}).items():
             self.pqueues[slot] = self.pqfactory(slot, startprios)
 
-    def pqfactory(self, slot, startprios=()):
+    def pqfactory(
+        self, slot: str, startprios: Iterable[int] = ()
+    ) -> ScrapyPriorityQueue:
         return ScrapyPriorityQueue(
             self.crawler,
             self.downstream_queue_cls,
@@ -186,11 +246,11 @@ class DownloaderAwarePriorityQueue:
             startprios,
         )
 
-    def pop(self):
+    def pop(self) -> Optional[Request]:
         stats = self._downloader_interface.stats(self.pqueues)
 
         if not stats:
-            return
+            return None
 
         slot = min(stats)[1]
         queue = self.pqueues[slot]
@@ -199,14 +259,14 @@ class DownloaderAwarePriorityQueue:
             del self.pqueues[slot]
         return request
 
-    def push(self, request):
+    def push(self, request: Request) -> None:
         slot = self._downloader_interface.get_slot_key(request)
         if slot not in self.pqueues:
             self.pqueues[slot] = self.pqfactory(slot)
         queue = self.pqueues[slot]
         queue.push(request)
 
-    def peek(self):
+    def peek(self) -> Optional[Request]:
         """Returns the next object to be returned by :meth:`pop`,
         but without removing it from the queue.
 
@@ -220,13 +280,13 @@ class DownloaderAwarePriorityQueue:
         queue = self.pqueues[slot]
         return queue.peek()
 
-    def close(self):
+    def close(self) -> Dict[str, List[int]]:
         active = {slot: queue.close() for slot, queue in self.pqueues.items()}
         self.pqueues.clear()
         return active
 
-    def __len__(self):
+    def __len__(self) -> int:
         return sum(len(x) for x in self.pqueues.values()) if self.pqueues else 0
 
-    def __contains__(self, slot):
+    def __contains__(self, slot: str) -> bool:
         return slot in self.pqueues

From 21fa0761818c158ae9fc35b49ea8d2300f0fa510 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 22:41:58 +0500
Subject: [PATCH 45/74] Fix MutableMapping import for Python 3.8.

---
 scrapy/item.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/scrapy/item.py b/scrapy/item.py
index e04e994ef..2daea64cc 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -7,10 +7,18 @@ See documentation in docs/topics/item.rst
 from __future__ import annotations
 
 from abc import ABCMeta
-from collections.abc import MutableMapping
 from copy import deepcopy
 from pprint import pformat
-from typing import TYPE_CHECKING, Any, Dict, Iterator, KeysView, NoReturn, Tuple
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterator,
+    KeysView,
+    MutableMapping,
+    NoReturn,
+    Tuple,
+)
 
 from scrapy.utils.trackref import object_ref
 

From ad35ffdb0da052d0df194ce5dc1ba7e8d823190f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 23:10:03 +0500
Subject: [PATCH 46/74] Full typing for scrapy/resolver.py.

---
 scrapy/resolver.py | 67 +++++++++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 28 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index e2e8beff4..ba7cd716b 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -1,8 +1,12 @@
-from typing import Any
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Type
 
 from twisted.internet import defer
-from twisted.internet.base import ThreadedResolver
+from twisted.internet.base import ReactorBase, ThreadedResolver
+from twisted.internet.defer import Deferred
 from twisted.internet.interfaces import (
+    IAddress,
     IHostnameResolver,
     IHostResolution,
     IResolutionReceiver,
@@ -12,6 +16,12 @@ from zope.interface.declarations import implementer, provider
 
 from scrapy.utils.datatypes import LocalCache
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+    from scrapy.crawler import Crawler
+
 # TODO: cache misses
 dnscache: LocalCache[str, Any] = LocalCache(10000)
 
@@ -22,65 +32,66 @@ class CachingThreadedResolver(ThreadedResolver):
     Default caching resolver. IPv4 only, supports setting a timeout value for DNS requests.
     """
 
-    def __init__(self, reactor, cache_size, timeout):
+    def __init__(self, reactor: ReactorBase, cache_size: int, timeout: float):
         super().__init__(reactor)
         dnscache.limit = cache_size
         self.timeout = timeout
 
     @classmethod
-    def from_crawler(cls, crawler, reactor):
+    def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self:
         if crawler.settings.getbool("DNSCACHE_ENABLED"):
             cache_size = crawler.settings.getint("DNSCACHE_SIZE")
         else:
             cache_size = 0
         return cls(reactor, cache_size, crawler.settings.getfloat("DNS_TIMEOUT"))
 
-    def install_on_reactor(self):
+    def install_on_reactor(self) -> None:
         self.reactor.installResolver(self)
 
-    def getHostByName(self, name: str, timeout=None):
+    def getHostByName(self, name: str, timeout: Sequence[int] = ()) -> Deferred[str]:
         if name in dnscache:
             return defer.succeed(dnscache[name])
         # in Twisted<=16.6, getHostByName() is always called with
         # a default timeout of 60s (actually passed as (1, 3, 11, 45) tuple),
         # so the input argument above is simply overridden
         # to enforce Scrapy's DNS_TIMEOUT setting's value
-        timeout = (self.timeout,)
+        # The timeout arg is typed as Sequence[int] but supports floats.
+        timeout = (self.timeout,)  # type: ignore[assignment]
         d = super().getHostByName(name, timeout)
         if dnscache.limit:
             d.addCallback(self._cache_result, name)
         return d
 
-    def _cache_result(self, result, name):
+    def _cache_result(self, result: Any, name: str) -> Any:
         dnscache[name] = result
         return result
 
 
 @implementer(IHostResolution)
 class HostResolution:
-    def __init__(self, name):
-        self.name = name
+    def __init__(self, name: str):
+        self.name: str = name
 
-    def cancel(self):
+    def cancel(self) -> None:
         raise NotImplementedError()
 
 
 @provider(IResolutionReceiver)
 class _CachingResolutionReceiver:
-    def __init__(self, resolutionReceiver, hostName):
-        self.resolutionReceiver = resolutionReceiver
-        self.hostName = hostName
-        self.addresses = []
+    def __init__(self, resolutionReceiver: IResolutionReceiver, hostName: str):
+        self.resolutionReceiver: IResolutionReceiver = resolutionReceiver
+        self.hostName: str = hostName
+        self.addresses: List[IAddress] = []
 
-    def resolutionBegan(self, resolution):
+    def resolutionBegan(self, resolution: IHostResolution) -> None:
         self.resolutionReceiver.resolutionBegan(resolution)
         self.resolution = resolution
 
-    def addressResolved(self, address):
+    def addressResolved(self, address: IAddress) -> None:
         self.resolutionReceiver.addressResolved(address)
         self.addresses.append(address)
 
-    def resolutionComplete(self):
+    def resolutionComplete(self) -> None:
         self.resolutionReceiver.resolutionComplete()
         if self.addresses:
             dnscache[self.hostName] = self.addresses
@@ -93,30 +104,30 @@ class CachingHostnameResolver:
     does not support setting a timeout value for DNS requests.
     """
 
-    def __init__(self, reactor, cache_size):
-        self.reactor = reactor
-        self.original_resolver = reactor.nameResolver
+    def __init__(self, reactor: ReactorBase, cache_size: int):
+        self.reactor: ReactorBase = reactor
+        self.original_resolver: IHostnameResolver = reactor.nameResolver
         dnscache.limit = cache_size
 
     @classmethod
-    def from_crawler(cls, crawler, reactor):
+    def from_crawler(cls, crawler: Crawler, reactor: ReactorBase) -> Self:
         if crawler.settings.getbool("DNSCACHE_ENABLED"):
             cache_size = crawler.settings.getint("DNSCACHE_SIZE")
         else:
             cache_size = 0
         return cls(reactor, cache_size)
 
-    def install_on_reactor(self):
+    def install_on_reactor(self) -> None:
         self.reactor.installNameResolver(self)
 
     def resolveHostName(
         self,
-        resolutionReceiver,
+        resolutionReceiver: IResolutionReceiver,
         hostName: str,
-        portNumber=0,
-        addressTypes=None,
-        transportSemantics="TCP",
-    ):
+        portNumber: int = 0,
+        addressTypes: Optional[Sequence[Type[IAddress]]] = None,
+        transportSemantics: str = "TCP",
+    ) -> IHostResolution:
         try:
             addresses = dnscache[hostName]
         except KeyError:

From b749db92e5b15c974b0d77280c22b63000ad4263 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 23:17:59 +0500
Subject: [PATCH 47/74] Full typing for scrapy/robotstxt.py.

---
 scrapy/robotstxt.py | 65 +++++++++++++++++++++++----------------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index ad06137e2..a33f73306 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,9 +3,10 @@ from __future__ import annotations
 import logging
 import sys
 from abc import ABCMeta, abstractmethod
-from typing import TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, Optional, Union
 from warnings import warn
 
+from scrapy import Spider
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import to_unicode
 
@@ -18,12 +19,14 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
+def decode_robotstxt(
+    robotstxt_body: bytes, spider: Optional[Spider], to_native_str_type: bool = False
+) -> str:
     try:
         if to_native_str_type:
-            robotstxt_body = to_unicode(robotstxt_body)
+            body_decoded = to_unicode(robotstxt_body)
         else:
-            robotstxt_body = robotstxt_body.decode("utf-8", errors="ignore")
+            body_decoded = robotstxt_body.decode("utf-8", errors="ignore")
     except UnicodeDecodeError:
         # If we found garbage or robots.txt in an encoding other than UTF-8, disregard it.
         # Switch to 'allow all' state.
@@ -33,8 +36,8 @@ def decode_robotstxt(robotstxt_body, spider, to_native_str_type=False):
             exc_info=sys.exc_info(),
             extra={"spider": spider},
         )
-        robotstxt_body = ""
-    return robotstxt_body
+        body_decoded = ""
+    return body_decoded
 
 
 class RobotParser(metaclass=ABCMeta):
@@ -66,82 +69,80 @@ class RobotParser(metaclass=ABCMeta):
 
 
 class PythonRobotParser(RobotParser):
-    def __init__(self, robotstxt_body, spider):
+    def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
         from urllib.robotparser import RobotFileParser
 
-        self.spider = spider
-        robotstxt_body = decode_robotstxt(
-            robotstxt_body, spider, to_native_str_type=True
-        )
-        self.rp = RobotFileParser()
-        self.rp.parse(robotstxt_body.splitlines())
+        self.spider: Optional[Spider] = spider
+        body_decoded = decode_robotstxt(robotstxt_body, spider, to_native_str_type=True)
+        self.rp: RobotFileParser = RobotFileParser()
+        self.rp.parse(body_decoded.splitlines())
 
     @classmethod
-    def from_crawler(cls, crawler, robotstxt_body):
+    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
         spider = None if not crawler else crawler.spider
         o = cls(robotstxt_body, spider)
         return o
 
-    def allowed(self, url, user_agent):
+    def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.can_fetch(user_agent, url)
 
 
 class ReppyRobotParser(RobotParser):
-    def __init__(self, robotstxt_body, spider):
+    def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
         warn("ReppyRobotParser is deprecated.", ScrapyDeprecationWarning, stacklevel=2)
         from reppy.robots import Robots
 
-        self.spider = spider
+        self.spider: Optional[Spider] = spider
         self.rp = Robots.parse("", robotstxt_body)
 
     @classmethod
-    def from_crawler(cls, crawler, robotstxt_body):
+    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
         spider = None if not crawler else crawler.spider
         o = cls(robotstxt_body, spider)
         return o
 
-    def allowed(self, url, user_agent):
+    def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
         return self.rp.allowed(url, user_agent)
 
 
 class RerpRobotParser(RobotParser):
-    def __init__(self, robotstxt_body, spider):
+    def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
         from robotexclusionrulesparser import RobotExclusionRulesParser
 
-        self.spider = spider
-        self.rp = RobotExclusionRulesParser()
-        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
-        self.rp.parse(robotstxt_body)
+        self.spider: Optional[Spider] = spider
+        self.rp: RobotExclusionRulesParser = RobotExclusionRulesParser()
+        body_decoded = decode_robotstxt(robotstxt_body, spider)
+        self.rp.parse(body_decoded)
 
     @classmethod
-    def from_crawler(cls, crawler, robotstxt_body):
+    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
         spider = None if not crawler else crawler.spider
         o = cls(robotstxt_body, spider)
         return o
 
-    def allowed(self, url, user_agent):
+    def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.is_allowed(user_agent, url)
 
 
 class ProtegoRobotParser(RobotParser):
-    def __init__(self, robotstxt_body, spider):
+    def __init__(self, robotstxt_body: bytes, spider: Optional[Spider]):
         from protego import Protego
 
-        self.spider = spider
-        robotstxt_body = decode_robotstxt(robotstxt_body, spider)
-        self.rp = Protego.parse(robotstxt_body)
+        self.spider: Optional[Spider] = spider
+        body_decoded = decode_robotstxt(robotstxt_body, spider)
+        self.rp = Protego.parse(body_decoded)
 
     @classmethod
-    def from_crawler(cls, crawler, robotstxt_body):
+    def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
         spider = None if not crawler else crawler.spider
         o = cls(robotstxt_body, spider)
         return o
 
-    def allowed(self, url, user_agent):
+    def allowed(self, url: Union[str, bytes], user_agent: Union[str, bytes]) -> bool:
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
         return self.rp.can_fetch(url, user_agent)

From 5f7fd2a653407da3eb3e53c853209d9bfb6b275f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 23:39:31 +0500
Subject: [PATCH 48/74] Full typing for scrapy/squeues.py.

---
 scrapy/squeues.py | 58 +++++++++++++++++++++++++++++++----------------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index e20f60f06..4676b058e 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -2,20 +2,28 @@
 Scheduler queues
 """
 
+from __future__ import annotations
+
 import marshal
 import pickle  # nosec
 from os import PathLike
 from pathlib import Path
-from typing import Union
+from typing import TYPE_CHECKING, Any, Callable, Optional, Type, Union
 
 from queuelib import queue
 
+from scrapy import Request
+from scrapy.crawler import Crawler
 from scrapy.utils.request import request_from_dict
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
-def _with_mkdir(queue_class):
+
+def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
     class DirectoriesCreated(queue_class):
-        def __init__(self, path: Union[str, PathLike], *args, **kwargs):
+        def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
             dirname = Path(path).parent
             if not dirname.exists():
                 dirname.mkdir(parents=True, exist_ok=True)
@@ -24,18 +32,23 @@ def _with_mkdir(queue_class):
     return DirectoriesCreated
 
 
-def _serializable_queue(queue_class, serialize, deserialize):
+def _serializable_queue(
+    queue_class: Type[queue.BaseQueue],
+    serialize: Callable[[Any], bytes],
+    deserialize: Callable[[bytes], Any],
+) -> Type[queue.BaseQueue]:
     class SerializableQueue(queue_class):
-        def push(self, obj):
+        def push(self, obj: Any) -> None:
             s = serialize(obj)
             super().push(s)
 
-        def pop(self):
+        def pop(self) -> Optional[Any]:
             s = super().pop()
             if s:
                 return deserialize(s)
+            return None
 
-        def peek(self):
+        def peek(self) -> Optional[Any]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -50,31 +63,36 @@ def _serializable_queue(queue_class, serialize, deserialize):
                 ) from ex
             if s:
                 return deserialize(s)
+            return None
 
     return SerializableQueue
 
 
-def _scrapy_serialization_queue(queue_class):
+def _scrapy_serialization_queue(
+    queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
     class ScrapyRequestQueue(queue_class):
-        def __init__(self, crawler, key):
+        def __init__(self, crawler: Crawler, key: str):
             self.spider = crawler.spider
             super().__init__(key)
 
         @classmethod
-        def from_crawler(cls, crawler, key, *args, **kwargs):
+        def from_crawler(
+            cls, crawler: Crawler, key: str, *args: Any, **kwargs: Any
+        ) -> Self:
             return cls(crawler, key)
 
-        def push(self, request):
-            request = request.to_dict(spider=self.spider)
-            return super().push(request)
+        def push(self, request: Request) -> None:
+            request_dict = request.to_dict(spider=self.spider)
+            super().push(request_dict)
 
-        def pop(self):
+        def pop(self) -> Optional[Request]:
             request = super().pop()
             if not request:
                 return None
             return request_from_dict(request, spider=self.spider)
 
-        def peek(self):
+        def peek(self) -> Optional[Request]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -89,13 +107,15 @@ def _scrapy_serialization_queue(queue_class):
     return ScrapyRequestQueue
 
 
-def _scrapy_non_serialization_queue(queue_class):
+def _scrapy_non_serialization_queue(
+    queue_class: Type[queue.BaseQueue],
+) -> Type[queue.BaseQueue]:
     class ScrapyRequestQueue(queue_class):
         @classmethod
-        def from_crawler(cls, crawler, *args, **kwargs):
+        def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
             return cls()
 
-        def peek(self):
+        def peek(self) -> Optional[Any]:
             """Returns the next object to be returned by :meth:`pop`,
             but without removing it from the queue.
 
@@ -113,7 +133,7 @@ def _scrapy_non_serialization_queue(queue_class):
     return ScrapyRequestQueue
 
 
-def _pickle_serialize(obj):
+def _pickle_serialize(obj: Any) -> bytes:
     try:
         return pickle.dumps(obj, protocol=4)
     # Both pickle.PicklingError and AttributeError can be raised by pickle.dump(s)

From 203fa9667fb69f6251c0a44b14cf0450ce769a32 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 29 Apr 2024 23:47:55 +0500
Subject: [PATCH 49/74] Add queue typing to scrapy/core/scheduler.py.

---
 scrapy/core/scheduler.py | 33 +++++++++++++++++----------------
 scrapy/pqueues.py        |  3 ++-
 2 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index b2209e53f..ab59c0d14 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -4,13 +4,15 @@ import json
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, Optional, Type, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Optional, Type, cast
 
+from queuelib.queue import BaseQueue
 from twisted.internet.defer import Deferred
 
 from scrapy.crawler import Crawler
 from scrapy.dupefilters import BaseDupeFilter
 from scrapy.http.request import Request
+from scrapy.pqueues import ScrapyPriorityQueue
 from scrapy.spiders import Spider
 from scrapy.statscollectors import StatsCollector
 from scrapy.utils.job import job_dir
@@ -121,9 +123,6 @@ class BaseScheduler(metaclass=BaseSchedulerMeta):
         raise NotImplementedError()
 
 
-SchedulerTV = TypeVar("SchedulerTV", bound="Scheduler")
-
-
 class Scheduler(BaseScheduler):
     """
     Default Scrapy scheduler. This implementation also handles duplication
@@ -179,24 +178,24 @@ class Scheduler(BaseScheduler):
         self,
         dupefilter: BaseDupeFilter,
         jobdir: Optional[str] = None,
-        dqclass=None,
-        mqclass=None,
+        dqclass: Optional[Type[BaseQueue]] = None,
+        mqclass: Optional[Type[BaseQueue]] = None,
         logunser: bool = False,
         stats: Optional[StatsCollector] = None,
-        pqclass=None,
+        pqclass: Optional[Type[ScrapyPriorityQueue]] = None,
         crawler: Optional[Crawler] = None,
     ):
         self.df: BaseDupeFilter = dupefilter
         self.dqdir: Optional[str] = self._dqdir(jobdir)
-        self.pqclass = pqclass
-        self.dqclass = dqclass
-        self.mqclass = mqclass
+        self.pqclass: Optional[Type[ScrapyPriorityQueue]] = pqclass
+        self.dqclass: Optional[Type[BaseQueue]] = dqclass
+        self.mqclass: Optional[Type[BaseQueue]] = mqclass
         self.logunser: bool = logunser
         self.stats: Optional[StatsCollector] = stats
         self.crawler: Optional[Crawler] = crawler
 
     @classmethod
-    def from_crawler(cls: Type[SchedulerTV], crawler: Crawler) -> SchedulerTV:
+    def from_crawler(cls, crawler: Crawler) -> Self:
         """
         Factory method, initializes the scheduler with arguments taken from the crawl settings
         """
@@ -221,9 +220,9 @@ class Scheduler(BaseScheduler):
         (2) initialize the disk queue if the ``jobdir`` attribute is a valid directory
         (3) return the result of the dupefilter's ``open`` method
         """
-        self.spider = spider
-        self.mqs = self._mq()
-        self.dqs = self._dq() if self.dqdir else None
+        self.spider: Spider = spider
+        self.mqs: ScrapyPriorityQueue = self._mq()
+        self.dqs: Optional[ScrapyPriorityQueue] = self._dq() if self.dqdir else None
         return self.df.open()
 
     def close(self, reason: str) -> Optional[Deferred]:
@@ -320,9 +319,10 @@ class Scheduler(BaseScheduler):
             return self.dqs.pop()
         return None
 
-    def _mq(self):
+    def _mq(self) -> ScrapyPriorityQueue:
         """Create a new priority queue instance, with in-memory storage"""
         assert self.crawler
+        assert self.pqclass
         return build_from_crawler(
             self.pqclass,
             self.crawler,
@@ -330,10 +330,11 @@ class Scheduler(BaseScheduler):
             key="",
         )
 
-    def _dq(self):
+    def _dq(self) -> ScrapyPriorityQueue:
         """Create a new priority queue instance, with disk storage"""
         assert self.crawler
         assert self.dqdir
+        assert self.pqclass
         state = self._read_dqs_state(self.dqdir)
         q = build_from_crawler(
             self.pqclass,
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 213ad590d..773825c5e 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -16,13 +16,14 @@ from typing import (
 
 from scrapy import Request
 from scrapy.core.downloader import Downloader
-from scrapy.crawler import Crawler
 from scrapy.utils.misc import build_from_crawler
 
 if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
+
 logger = logging.getLogger(__name__)
 
 

From bd0d4cee885744c7ea38185ec42f0137e7632b79 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 16:12:44 +0500
Subject: [PATCH 50/74] Fixes for queuelib.

---
 scrapy/core/scheduler.py |  4 +++-
 scrapy/squeues.py        | 21 +++++++++++----------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index ab59c0d14..f30a5d9c9 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -6,7 +6,6 @@ from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Optional, Type, cast
 
-from queuelib.queue import BaseQueue
 from twisted.internet.defer import Deferred
 
 from scrapy.crawler import Crawler
@@ -19,6 +18,9 @@ from scrapy.utils.job import job_dir
 from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
+    # requires queuelib >= 1.6.2
+    from queuelib.queue import BaseQueue
+
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 4676b058e..6f80ee388 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -22,7 +22,7 @@ if TYPE_CHECKING:
 
 
 def _with_mkdir(queue_class: Type[queue.BaseQueue]) -> Type[queue.BaseQueue]:
-    class DirectoriesCreated(queue_class):
+    class DirectoriesCreated(queue_class):  # type: ignore[valid-type,misc]
         def __init__(self, path: Union[str, PathLike], *args: Any, **kwargs: Any):
             dirname = Path(path).parent
             if not dirname.exists():
@@ -37,7 +37,7 @@ def _serializable_queue(
     serialize: Callable[[Any], bytes],
     deserialize: Callable[[bytes], Any],
 ) -> Type[queue.BaseQueue]:
-    class SerializableQueue(queue_class):
+    class SerializableQueue(queue_class):  # type: ignore[valid-type,misc]
         def push(self, obj: Any) -> None:
             s = serialize(obj)
             super().push(s)
@@ -71,7 +71,7 @@ def _serializable_queue(
 def _scrapy_serialization_queue(
     queue_class: Type[queue.BaseQueue],
 ) -> Type[queue.BaseQueue]:
-    class ScrapyRequestQueue(queue_class):
+    class ScrapyRequestQueue(queue_class):  # type: ignore[valid-type,misc]
         def __init__(self, crawler: Crawler, key: str):
             self.spider = crawler.spider
             super().__init__(key)
@@ -110,7 +110,7 @@ def _scrapy_serialization_queue(
 def _scrapy_non_serialization_queue(
     queue_class: Type[queue.BaseQueue],
 ) -> Type[queue.BaseQueue]:
-    class ScrapyRequestQueue(queue_class):
+    class ScrapyRequestQueue(queue_class):  # type: ignore[valid-type,misc]
         @classmethod
         def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
             return cls()
@@ -142,17 +142,18 @@ def _pickle_serialize(obj: Any) -> bytes:
         raise ValueError(str(e)) from e
 
 
+# queue.*Queue aren't subclasses of queue.BaseQueue
 _PickleFifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads
+    _with_mkdir(queue.FifoDiskQueue), _pickle_serialize, pickle.loads  # type: ignore[arg-type]
 )
 _PickleLifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads
+    _with_mkdir(queue.LifoDiskQueue), _pickle_serialize, pickle.loads  # type: ignore[arg-type]
 )
 _MarshalFifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads
+    _with_mkdir(queue.FifoDiskQueue), marshal.dumps, marshal.loads  # type: ignore[arg-type]
 )
 _MarshalLifoSerializationDiskQueue = _serializable_queue(
-    _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads
+    _with_mkdir(queue.LifoDiskQueue), marshal.dumps, marshal.loads  # type: ignore[arg-type]
 )
 
 # public queue classes
@@ -160,5 +161,5 @@ PickleFifoDiskQueue = _scrapy_serialization_queue(_PickleFifoSerializationDiskQu
 PickleLifoDiskQueue = _scrapy_serialization_queue(_PickleLifoSerializationDiskQueue)
 MarshalFifoDiskQueue = _scrapy_serialization_queue(_MarshalFifoSerializationDiskQueue)
 MarshalLifoDiskQueue = _scrapy_serialization_queue(_MarshalLifoSerializationDiskQueue)
-FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)
-LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)
+FifoMemoryQueue = _scrapy_non_serialization_queue(queue.FifoMemoryQueue)  # type: ignore[arg-type]
+LifoMemoryQueue = _scrapy_non_serialization_queue(queue.LifoMemoryQueue)  # type: ignore[arg-type]

From 1f394306e14ccab9d14ffb9adc64a7c5c08d9af6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 20:37:30 +0500
Subject: [PATCH 51/74] Use the Self type hint in from_crawler/from_settings.

---
 scrapy/core/downloader/contextfactory.py  |  7 +++++-
 scrapy/core/downloader/handlers/ftp.py    | 10 +++++++-
 scrapy/core/downloader/handlers/http10.py | 10 +++++++-
 scrapy/core/downloader/handlers/http11.py | 10 +++++++-
 scrapy/core/downloader/handlers/http2.py  | 14 +++++------
 scrapy/core/downloader/handlers/s3.py     | 10 +++++++-
 scrapy/http/request/__init__.py           | 13 ++++++----
 scrapy/pipelines/files.py                 | 20 ++++++++++++----
 scrapy/pipelines/images.py                | 29 ++++++++++++++++++-----
 scrapy/pipelines/media.py                 | 12 ++++++++--
 scrapy/spiders/crawl.py                   | 12 +++++++---
 scrapy/spiders/sitemap.py                 |  4 +++-
 scrapy/utils/request.py                   |  9 +++++--
 13 files changed, 124 insertions(+), 36 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index dba4d8cdc..6a82634f1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import warnings
 from typing import TYPE_CHECKING, Any, List, Optional
 
@@ -25,6 +27,9 @@ from scrapy.utils.misc import build_from_crawler, load_object
 if TYPE_CHECKING:
     from twisted.internet._sslverify import ClientTLSOptions
 
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 @implementer(IPolicyForHTTPS)
 class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
@@ -62,7 +67,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         method: int = SSL.SSLv23_METHOD,
         *args: Any,
         **kwargs: Any,
-    ):
+    ) -> Self:
         tls_verbose_logging: bool = settings.getbool(
             "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
         )
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 4081545ce..69add8558 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -28,18 +28,26 @@ In case of status 200 request, response.headers will come with two keys:
     'Size' - with size of the downloaded data
 """
 
+from __future__ import annotations
+
 import re
 from io import BytesIO
+from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
 from twisted.internet.protocol import ClientCreator, Protocol
 from twisted.protocols.ftp import CommandFailed, FTPClient
 
+from scrapy.crawler import Crawler
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class ReceivedDataProtocol(Protocol):
     def __init__(self, filename=None):
@@ -76,7 +84,7 @@ class FTPDownloadHandler:
         self.passive_mode = settings["FTP_PASSIVE_MODE"]
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings)
 
     def download_request(self, request, spider):
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index d168c2b2e..256dc36a1 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -1,9 +1,17 @@
 """Download handlers for http and https schemes
 """
 
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class HTTP10DownloadHandler:
     lazy = False
@@ -17,7 +25,7 @@ class HTTP10DownloadHandler:
         self._crawler = crawler
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler) -> Self:
         return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index c3704de3d..15f8abc64 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -1,11 +1,14 @@
 """Download handlers for http and https schemes"""
 
+from __future__ import annotations
+
 import ipaddress
 import logging
 import re
 from contextlib import suppress
 from io import BytesIO
 from time import time
+from typing import TYPE_CHECKING
 from urllib.parse import urldefrag, urlunparse
 
 from twisted.internet import defer, protocol, ssl
@@ -32,6 +35,11 @@ from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.python import to_bytes, to_unicode
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -56,7 +64,7 @@ class HTTP11DownloadHandler:
         self._disconnect_timeout = 1
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler) -> Self:
         return cls(crawler.settings, crawler)
 
     def download_request(self, request, spider):
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index b2579362c..e9a6b6fa3 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 from time import time
-from typing import Optional, Type, TypeVar
+from typing import TYPE_CHECKING, Optional
 from urllib.parse import urldefrag
 
 from twisted.internet.base import DelayedCall
@@ -16,9 +18,9 @@ from scrapy.settings import Settings
 from scrapy.spiders import Spider
 from scrapy.utils.python import to_bytes
 
-H2DownloadHandlerOrSubclass = TypeVar(
-    "H2DownloadHandlerOrSubclass", bound="H2DownloadHandler"
-)
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 class H2DownloadHandler:
@@ -31,9 +33,7 @@ class H2DownloadHandler:
         self._context_factory = load_context_factory_from_settings(settings, crawler)
 
     @classmethod
-    def from_crawler(
-        cls: Type[H2DownloadHandlerOrSubclass], crawler: Crawler
-    ) -> H2DownloadHandlerOrSubclass:
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
     def download_request(self, request: Request, spider: Spider) -> Deferred:
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 1f7533759..99fbb49ce 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,17 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class S3DownloadHandler:
     def __init__(
@@ -57,7 +65,7 @@ class S3DownloadHandler:
         self._download_http = _http_handler.download_request
 
     @classmethod
-    def from_crawler(cls, crawler, **kwargs):
+    def from_crawler(cls, crawler, **kwargs) -> Self:
         return cls(crawler.settings, crawler=crawler, **kwargs)
 
     def download_request(self, request, spider):
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 6269ee86a..191b3cef4 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -5,8 +5,11 @@ requests in Scrapy.
 See documentation in docs/topics/request-response.rst
 """
 
+from __future__ import annotations
+
 import inspect
 from typing import (
+    TYPE_CHECKING,
     Any,
     AnyStr,
     Callable,
@@ -17,8 +20,6 @@ from typing import (
     NoReturn,
     Optional,
     Tuple,
-    Type,
-    TypeVar,
     Union,
     cast,
 )
@@ -32,7 +33,9 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import escape_ajax
 
-RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
 
 
 def NO_CALLBACK(*args: Any, **kwargs: Any) -> NoReturn:
@@ -186,11 +189,11 @@ class Request(object_ref):
 
     @classmethod
     def from_curl(
-        cls: Type[RequestTypeVar],
+        cls,
         curl_command: str,
         ignore_unknown_options: bool = True,
         **kwargs: Any,
-    ) -> RequestTypeVar:
+    ) -> Self:
         """Create a Request object from a string containing a `cURL
         <https://curl.haxx.se/>`_ command. It populates the HTTP method, the
         URL, the headers, the cookies and the body. It accepts the same
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index d04218089..d00f44502 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -4,6 +4,8 @@ Files Pipeline
 See documentation in topics/media-pipeline.rst
 """
 
+from __future__ import annotations
+
 import base64
 import functools
 import hashlib
@@ -16,7 +18,7 @@ from ftplib import FTP
 from io import BytesIO
 from os import PathLike
 from pathlib import Path
-from typing import IO, DefaultDict, Optional, Set, Union
+from typing import IO, TYPE_CHECKING, DefaultDict, Optional, Set, Type, Union, cast
 from urllib.parse import urlparse
 
 from itemadapter import ItemAdapter
@@ -34,6 +36,10 @@ from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.python import to_bytes
 from scrapy.utils.request import referer_str
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 logger = logging.getLogger(__name__)
 
 
@@ -385,8 +391,8 @@ class FilesPipeline(MediaPipeline):
         super().__init__(download_func=download_func, settings=settings)
 
     @classmethod
-    def from_settings(cls, settings):
-        s3store = cls.STORE_SCHEMES["s3"]
+    def from_settings(cls, settings) -> Self:
+        s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
         s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
         s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
         s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
@@ -396,11 +402,15 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_VERIFY = settings["AWS_VERIFY"]
         s3store.POLICY = settings["FILES_STORE_S3_ACL"]
 
-        gcs_store = cls.STORE_SCHEMES["gs"]
+        gcs_store: Type[GCSFilesStore] = cast(
+            Type[GCSFilesStore], cls.STORE_SCHEMES["gs"]
+        )
         gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
         gcs_store.POLICY = settings["FILES_STORE_GCS_ACL"] or None
 
-        ftp_store = cls.STORE_SCHEMES["ftp"]
+        ftp_store: Type[FTPFilesStore] = cast(
+            Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]
+        )
         ftp_store.FTP_USERNAME = settings["FTP_USER"]
         ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
         ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 137aa7a9a..e7ef06fb3 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -4,25 +4,38 @@ Images Pipeline
 See documentation in topics/media-pipeline.rst
 """
 
+from __future__ import annotations
+
 import functools
 import hashlib
 import warnings
 from contextlib import suppress
 from io import BytesIO
 from os import PathLike
-from typing import Dict, Tuple, Union
+from typing import TYPE_CHECKING, Dict, Tuple, Type, Union, cast
 
 from itemadapter import ItemAdapter
 
 from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.files import FileException, FilesPipeline, _md5sum
+from scrapy.pipelines.files import (
+    FileException,
+    FilesPipeline,
+    FTPFilesStore,
+    GCSFilesStore,
+    S3FilesStore,
+    _md5sum,
+)
 
 # TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.python import get_func_args, to_bytes
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class NoimagesDrop(DropItem):
     """Product with no images exception"""
@@ -96,8 +109,8 @@ class ImagesPipeline(FilesPipeline):
         self._deprecated_convert_image = None
 
     @classmethod
-    def from_settings(cls, settings):
-        s3store = cls.STORE_SCHEMES["s3"]
+    def from_settings(cls, settings) -> Self:
+        s3store: Type[S3FilesStore] = cast(Type[S3FilesStore], cls.STORE_SCHEMES["s3"])
         s3store.AWS_ACCESS_KEY_ID = settings["AWS_ACCESS_KEY_ID"]
         s3store.AWS_SECRET_ACCESS_KEY = settings["AWS_SECRET_ACCESS_KEY"]
         s3store.AWS_SESSION_TOKEN = settings["AWS_SESSION_TOKEN"]
@@ -107,11 +120,15 @@ class ImagesPipeline(FilesPipeline):
         s3store.AWS_VERIFY = settings["AWS_VERIFY"]
         s3store.POLICY = settings["IMAGES_STORE_S3_ACL"]
 
-        gcs_store = cls.STORE_SCHEMES["gs"]
+        gcs_store: Type[GCSFilesStore] = cast(
+            Type[GCSFilesStore], cls.STORE_SCHEMES["gs"]
+        )
         gcs_store.GCS_PROJECT_ID = settings["GCS_PROJECT_ID"]
         gcs_store.POLICY = settings["IMAGES_STORE_GCS_ACL"] or None
 
-        ftp_store = cls.STORE_SCHEMES["ftp"]
+        ftp_store: Type[FTPFilesStore] = cast(
+            Type[FTPFilesStore], cls.STORE_SCHEMES["ftp"]
+        )
         ftp_store.FTP_USERNAME = settings["FTP_USER"]
         ftp_store.FTP_PASSWORD = settings["FTP_PASSWORD"]
         ftp_store.USE_ACTIVE_MODE = settings.getbool("FEED_STORAGE_FTP_ACTIVE")
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index fc156ab41..fd5e70cb9 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -1,6 +1,9 @@
+from __future__ import annotations
+
 import functools
 import logging
 from collections import defaultdict
+from typing import TYPE_CHECKING
 
 from twisted.internet.defer import Deferred, DeferredList
 from twisted.python.failure import Failure
@@ -12,6 +15,11 @@ from scrapy.utils.defer import defer_result, mustbe_deferred
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -67,9 +75,9 @@ class MediaPipeline:
         return formatted_key
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler) -> Self:
         try:
-            pipe = cls.from_settings(crawler.settings)
+            pipe = cls.from_settings(crawler.settings)  # type: ignore[attr-defined]
         except AttributeError:
             pipe = cls()
         pipe.crawler = crawler
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 2a3913da5..ba8b7b366 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -5,8 +5,10 @@ for scraping typical web sites that requires crawling pages.
 See documentation in docs/topics/spiders.rst
 """
 
+from __future__ import annotations
+
 import copy
-from typing import AsyncIterable, Awaitable, Sequence
+from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence
 
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.linkextractors import LinkExtractor
@@ -14,6 +16,10 @@ from scrapy.spiders import Spider
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.spider import iterate_spider_output
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 def _identity(x):
     return x
@@ -140,9 +146,9 @@ class CrawlSpider(Spider):
             self._rules[-1]._compile(self)
 
     @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
+    def from_crawler(cls, crawler, *args, **kwargs) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool(
+        spider._follow_links = crawler.settings.getbool(  # type: ignore[attr-defined]
             "CRAWLSPIDER_FOLLOW_LINKS", True
         )
         return spider
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index cd83a1464..f0e630c42 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 import re
 from typing import TYPE_CHECKING, Any
@@ -26,7 +28,7 @@ class SitemapSpider(Spider):
     _warn_size: int
 
     @classmethod
-    def from_crawler(cls, crawler: "Crawler", *args: Any, **kwargs: Any) -> "Self":
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
         spider._max_size = getattr(
             spider, "download_maxsize", spider.settings.getint("DOWNLOAD_MAXSIZE")
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 1f07d58eb..c86f9fe39 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -3,6 +3,8 @@ This module provides some useful functions for working with
 scrapy.http.Request objects
 """
 
+from __future__ import annotations
+
 import hashlib
 import json
 import warnings
@@ -32,6 +34,9 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
     from scrapy.crawler import Crawler
 
 
@@ -133,10 +138,10 @@ class RequestFingerprinter:
     """
 
     @classmethod
-    def from_crawler(cls, crawler):
+    def from_crawler(cls, crawler) -> Self:
         return cls(crawler)
 
-    def __init__(self, crawler: Optional["Crawler"] = None):
+    def __init__(self, crawler: Optional[Crawler] = None):
         if crawler:
             implementation = crawler.settings.get(
                 "REQUEST_FINGERPRINTER_IMPLEMENTATION"

From 8a08283580176049cb539423795830b9faea91a9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 5 May 2024 22:32:46 +0500
Subject: [PATCH 52/74] Full typing for scrapy/http/cookies.py.

---
 scrapy/http/cookies.py       | 123 +++++++++++++++++++++--------------
 scrapy/http/response/text.py |   4 +-
 2 files changed, 76 insertions(+), 51 deletions(-)

diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 72855bad5..8af89c74f 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -1,36 +1,56 @@
+from __future__ import annotations
+
 import re
 import time
 from http.cookiejar import Cookie
 from http.cookiejar import CookieJar as _CookieJar
-from http.cookiejar import DefaultCookiePolicy
-from typing import Sequence
+from http.cookiejar import CookiePolicy, DefaultCookiePolicy
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Dict,
+    Iterator,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    cast,
+)
 
 from scrapy import Request
 from scrapy.http import Response
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
 
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 # Defined in the http.cookiejar module, but undocumented:
 # https://github.com/python/cpython/blob/v3.9.0/Lib/http/cookiejar.py#L527
 IPV4_RE = re.compile(r"\.\d+$", re.ASCII)
 
 
 class CookieJar:
-    def __init__(self, policy=None, check_expired_frequency=10000):
-        self.policy = policy or DefaultCookiePolicy()
-        self.jar = _CookieJar(self.policy)
-        self.jar._cookies_lock = _DummyLock()
-        self.check_expired_frequency = check_expired_frequency
-        self.processed = 0
+    def __init__(
+        self,
+        policy: Optional[CookiePolicy] = None,
+        check_expired_frequency: int = 10000,
+    ):
+        self.policy: CookiePolicy = policy or DefaultCookiePolicy()
+        self.jar: _CookieJar = _CookieJar(self.policy)
+        self.jar._cookies_lock = _DummyLock()  # type: ignore[attr-defined]
+        self.check_expired_frequency: int = check_expired_frequency
+        self.processed: int = 0
 
-    def extract_cookies(self, response, request):
+    def extract_cookies(self, response: Response, request: Request) -> None:
         wreq = WrappedRequest(request)
         wrsp = WrappedResponse(response)
-        return self.jar.extract_cookies(wrsp, wreq)
+        self.jar.extract_cookies(wrsp, wreq)  # type: ignore[arg-type]
 
     def add_cookie_header(self, request: Request) -> None:
         wreq = WrappedRequest(request)
-        self.policy._now = self.jar._now = int(time.time())
+        self.policy._now = self.jar._now = int(time.time())  # type: ignore[attr-defined]
 
         # the cookiejar implementation iterates through all domains
         # instead we restrict to potential matches on the domain
@@ -47,10 +67,10 @@ class CookieJar:
 
         cookies = []
         for host in hosts:
-            if host in self.jar._cookies:
-                cookies += self.jar._cookies_for_domain(host, wreq)
+            if host in self.jar._cookies:  # type: ignore[attr-defined]
+                cookies += self.jar._cookies_for_domain(host, wreq)  # type: ignore[attr-defined]
 
-        attrs = self.jar._cookie_attrs(cookies)
+        attrs = self.jar._cookie_attrs(cookies)  # type: ignore[attr-defined]
         if attrs:
             if not wreq.has_header("Cookie"):
                 wreq.add_unredirected_header("Cookie", "; ".join(attrs))
@@ -61,37 +81,42 @@ class CookieJar:
             self.jar.clear_expired_cookies()
 
     @property
-    def _cookies(self):
-        return self.jar._cookies
+    def _cookies(self) -> Dict[str, Dict[str, Dict[str, Cookie]]]:
+        return self.jar._cookies  # type: ignore[attr-defined,no-any-return]
 
-    def clear_session_cookies(self, *args, **kwargs):
-        return self.jar.clear_session_cookies(*args, **kwargs)
+    def clear_session_cookies(self) -> None:
+        return self.jar.clear_session_cookies()
 
-    def clear(self, domain=None, path=None, name=None):
-        return self.jar.clear(domain, path, name)
+    def clear(
+        self,
+        domain: Optional[str] = None,
+        path: Optional[str] = None,
+        name: Optional[str] = None,
+    ) -> None:
+        self.jar.clear(domain, path, name)
 
-    def __iter__(self):
+    def __iter__(self) -> Iterator[Cookie]:
         return iter(self.jar)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return len(self.jar)
 
-    def set_policy(self, pol):
-        return self.jar.set_policy(pol)
+    def set_policy(self, pol: CookiePolicy) -> None:
+        self.jar.set_policy(pol)
 
     def make_cookies(self, response: Response, request: Request) -> Sequence[Cookie]:
         wreq = WrappedRequest(request)
         wrsp = WrappedResponse(response)
-        return self.jar.make_cookies(wrsp, wreq)
+        return self.jar.make_cookies(wrsp, wreq)  # type: ignore[arg-type]
 
-    def set_cookie(self, cookie):
+    def set_cookie(self, cookie: Cookie) -> None:
         self.jar.set_cookie(cookie)
 
     def set_cookie_if_ok(self, cookie: Cookie, request: Request) -> None:
-        self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))
+        self.jar.set_cookie_if_ok(cookie, WrappedRequest(request))  # type: ignore[arg-type]
 
 
-def potential_domain_matches(domain):
+def potential_domain_matches(domain: str) -> List[str]:
     """Potential domain matches for a cookie
 
     >>> potential_domain_matches('www.example.com')
@@ -111,10 +136,10 @@ def potential_domain_matches(domain):
 
 
 class _DummyLock:
-    def acquire(self):
+    def acquire(self) -> None:
         pass
 
-    def release(self):
+    def release(self) -> None:
         pass
 
 
@@ -124,19 +149,19 @@ class WrappedRequest:
     see http://docs.python.org/library/urllib2.html#urllib2.Request
     """
 
-    def __init__(self, request):
+    def __init__(self, request: Request):
         self.request = request
 
-    def get_full_url(self):
+    def get_full_url(self) -> str:
         return self.request.url
 
-    def get_host(self):
+    def get_host(self) -> str:
         return urlparse_cached(self.request).netloc
 
-    def get_type(self):
+    def get_type(self) -> str:
         return urlparse_cached(self.request).scheme
 
-    def is_unverifiable(self):
+    def is_unverifiable(self) -> bool:
         """Unverifiable should indicate whether the request is unverifiable, as defined by RFC 2965.
 
         It defaults to False. An unverifiable request is one whose URL the user did not have the
@@ -144,36 +169,36 @@ class WrappedRequest:
         HTML document, and the user had no option to approve the automatic
         fetching of the image, this should be true.
         """
-        return self.request.meta.get("is_unverifiable", False)
+        return cast(bool, self.request.meta.get("is_unverifiable", False))
 
     @property
-    def full_url(self):
+    def full_url(self) -> str:
         return self.get_full_url()
 
     @property
-    def host(self):
+    def host(self) -> str:
         return self.get_host()
 
     @property
-    def type(self):
+    def type(self) -> str:
         return self.get_type()
 
     @property
-    def unverifiable(self):
+    def unverifiable(self) -> bool:
         return self.is_unverifiable()
 
     @property
-    def origin_req_host(self):
-        return urlparse_cached(self.request).hostname
+    def origin_req_host(self) -> str:
+        return cast(str, urlparse_cached(self.request).hostname)
 
-    def has_header(self, name):
+    def has_header(self, name: str) -> bool:
         return name in self.request.headers
 
-    def get_header(self, name, default=None):
+    def get_header(self, name: str, default: Optional[str] = None) -> Optional[str]:
         value = self.request.headers.get(name, default)
         return to_unicode(value, errors="replace") if value is not None else None
 
-    def header_items(self):
+    def header_items(self) -> List[Tuple[str, List[str]]]:
         return [
             (
                 to_unicode(k, errors="replace"),
@@ -182,18 +207,18 @@ class WrappedRequest:
             for k, v in self.request.headers.items()
         ]
 
-    def add_unredirected_header(self, name, value):
+    def add_unredirected_header(self, name: str, value: str) -> None:
         self.request.headers.appendlist(name, value)
 
 
 class WrappedResponse:
-    def __init__(self, response):
+    def __init__(self, response: Response):
         self.response = response
 
-    def info(self):
+    def info(self) -> Self:
         return self
 
-    def get_all(self, name, default=None):
+    def get_all(self, name: str, default: Any = None) -> List[str]:
         return [
             to_unicode(v, errors="replace") for v in self.response.headers.getlist(name)
         ]
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 2816610fb..522ffc0d5 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -159,12 +159,12 @@ class TextResponse(Response):
     def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
         from scrapy.selector import SelectorList
 
-        if not hasattr(self.selector, "jmespath"):  # type: ignore[attr-defined]
+        if not hasattr(self.selector, "jmespath"):
             raise AttributeError(
                 "Please install parsel >= 1.8.1 to get jmespath support"
             )
 
-        return cast(SelectorList, self.selector.jmespath(query, **kwargs))  # type: ignore[attr-defined]
+        return cast(SelectorList, self.selector.jmespath(query, **kwargs))
 
     def xpath(self, query: str, **kwargs: Any) -> SelectorList:
         from scrapy.selector import SelectorList

From c76dfc383f34514a5a2841d2e32ac4e0c8c751a3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 5 May 2024 22:52:15 +0500
Subject: [PATCH 53/74] Full typing for scrapy/linkextractors.

---
 scrapy/commands/bench.py          |   3 +-
 scrapy/linkextractors/__init__.py |   8 +-
 scrapy/linkextractors/lxmlhtml.py | 149 +++++++++++++++++-------------
 3 files changed, 91 insertions(+), 69 deletions(-)

diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 2e6bb5d86..7523f3cfe 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -8,7 +8,7 @@ from urllib.parse import urlencode
 import scrapy
 from scrapy import Request
 from scrapy.commands import ScrapyCommand
-from scrapy.http import Response
+from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
 
 
@@ -61,5 +61,6 @@ class _BenchSpider(scrapy.Spider):
         return [scrapy.Request(url, dont_filter=True)]
 
     def parse(self, response: Response) -> Any:  # type: ignore[override]
+        assert isinstance(Response, TextResponse)
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 73a63651c..38dbe8135 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -7,6 +7,7 @@ For more info see docs/topics/link-extractors.rst
 """
 
 import re
+from typing import Iterable
 
 # common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
@@ -110,14 +111,11 @@ IGNORED_EXTENSIONS = [
 ]
 
 
-_re_type = type(re.compile("", 0))
-
-
-def _matches(url, regexs):
+def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool:
     return any(r.search(url) for r in regexs)
 
 
-def _is_valid_url(url):
+def _is_valid_url(url: str) -> bool:
     return url.split("://", 1)[0] in {"http", "https", "file", "ftp"}
 
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 71c6d08fc..3fa7d1e3a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -5,21 +5,19 @@ Link extractor based on lxml.html
 import logging
 import operator
 from functools import partial
+from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast
 from urllib.parse import urljoin, urlparse
 
 from lxml import etree  # nosec
+from lxml.html import HtmlElement  # nosec
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 from w3lib.url import canonicalize_url, safe_url_string
 
+from scrapy import Selector
+from scrapy.http import TextResponse
 from scrapy.link import Link
-from scrapy.linkextractors import (
-    IGNORED_EXTENSIONS,
-    _is_valid_url,
-    _matches,
-    _re_type,
-    re,
-)
+from scrapy.linkextractors import IGNORED_EXTENSIONS, _is_valid_url, _matches, re
 from scrapy.utils.misc import arg_to_iter, rel_has_nofollow
 from scrapy.utils.python import unique as unique_list
 from scrapy.utils.response import get_base_url
@@ -33,41 +31,56 @@ XHTML_NAMESPACE = "http://www.w3.org/1999/xhtml"
 _collect_string_content = etree.XPath("string()")
 
 
-def _nons(tag):
+def _nons(tag: Any) -> Any:
     if isinstance(tag, str):
         if tag[0] == "{" and tag[1 : len(XHTML_NAMESPACE) + 1] == XHTML_NAMESPACE:
             return tag.split("}")[-1]
     return tag
 
 
-def _identity(x):
+def _identity(x: Any) -> Any:
     return x
 
 
-def _canonicalize_link_url(link):
+def _canonicalize_link_url(link: Link) -> str:
     return canonicalize_url(link.url, keep_fragments=True)
 
 
 class LxmlParserLinkExtractor:
     def __init__(
         self,
-        tag="a",
-        attr="href",
-        process=None,
-        unique=False,
-        strip=True,
-        canonicalized=False,
+        tag: Union[str, Callable[[str], bool]] = "a",
+        attr: Union[str, Callable[[str], bool]] = "href",
+        process: Optional[Callable[[Any], Any]] = None,
+        unique: bool = False,
+        strip: bool = True,
+        canonicalized: bool = False,
     ):
-        self.scan_tag = tag if callable(tag) else partial(operator.eq, tag)
-        self.scan_attr = attr if callable(attr) else partial(operator.eq, attr)
-        self.process_attr = process if callable(process) else _identity
-        self.unique = unique
-        self.strip = strip
-        self.link_key = (
-            operator.attrgetter("url") if canonicalized else _canonicalize_link_url
+        # mypy doesn't infer types for operator.* and also for partial()
+        self.scan_tag: Callable[[str], bool] = (
+            tag
+            if callable(tag)
+            else cast(Callable[[str], bool], partial(operator.eq, tag))
+        )
+        self.scan_attr: Callable[[str], bool] = (
+            attr
+            if callable(attr)
+            else cast(Callable[[str], bool], partial(operator.eq, attr))
+        )
+        self.process_attr: Callable[[Any], Any] = (
+            process if callable(process) else _identity
+        )
+        self.unique: bool = unique
+        self.strip: bool = strip
+        self.link_key: Callable[[Link], str] = (
+            cast(Callable[[Link], str], operator.attrgetter("url"))
+            if canonicalized
+            else _canonicalize_link_url
         )
 
-    def _iter_links(self, document):
+    def _iter_links(
+        self, document: HtmlElement
+    ) -> Iterable[Tuple[HtmlElement, str, str]]:
         for el in document.iter(etree.Element):
             if not self.scan_tag(_nons(el.tag)):
                 continue
@@ -75,10 +88,16 @@ class LxmlParserLinkExtractor:
             for attrib in attribs:
                 if not self.scan_attr(attrib):
                     continue
-                yield (el, attrib, attribs[attrib])
+                yield el, attrib, attribs[attrib]
 
-    def _extract_links(self, selector, response_url, response_encoding, base_url):
-        links = []
+    def _extract_links(
+        self,
+        selector: Selector,
+        response_url: str,
+        response_encoding: str,
+        base_url: str,
+    ) -> List[Link]:
+        links: List[Link] = []
         # hacky way to get the underlying lxml parsed document
         for el, attr, attr_val in self._iter_links(selector.root):
             # pseudo lxml.html.HtmlElement.make_links_absolute(base_url)
@@ -108,44 +127,48 @@ class LxmlParserLinkExtractor:
             links.append(link)
         return self._deduplicate_if_needed(links)
 
-    def extract_links(self, response):
+    def extract_links(self, response: TextResponse) -> List[Link]:
         base_url = get_base_url(response)
         return self._extract_links(
             response.selector, response.url, response.encoding, base_url
         )
 
-    def _process_links(self, links):
+    def _process_links(self, links: List[Link]) -> List[Link]:
         """Normalize and filter extracted links
 
         The subclass should override it if necessary
         """
         return self._deduplicate_if_needed(links)
 
-    def _deduplicate_if_needed(self, links):
+    def _deduplicate_if_needed(self, links: List[Link]) -> List[Link]:
         if self.unique:
             return unique_list(links, key=self.link_key)
         return links
 
 
+_RegexT = Union[str, re.Pattern[str]]
+_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
+
+
 class LxmlLinkExtractor:
     _csstranslator = HTMLTranslator()
 
     def __init__(
         self,
-        allow=(),
-        deny=(),
-        allow_domains=(),
-        deny_domains=(),
-        restrict_xpaths=(),
-        tags=("a", "area"),
-        attrs=("href",),
-        canonicalize=False,
-        unique=True,
-        process_value=None,
-        deny_extensions=None,
-        restrict_css=(),
-        strip=True,
-        restrict_text=None,
+        allow: _RegexOrSeveralT = (),
+        deny: _RegexOrSeveralT = (),
+        allow_domains: Union[str, Iterable[str]] = (),
+        deny_domains: Union[str, Iterable[str]] = (),
+        restrict_xpaths: Union[str, Iterable[str]] = (),
+        tags: Union[str, Iterable[str]] = ("a", "area"),
+        attrs: Union[str, Iterable[str]] = ("href",),
+        canonicalize: bool = False,
+        unique: bool = True,
+        process_value: Optional[Callable[[Any], Any]] = None,
+        deny_extensions: Union[str, Iterable[str], None] = None,
+        restrict_css: Union[str, Iterable[str]] = (),
+        strip: bool = True,
+        restrict_text: Optional[_RegexOrSeveralT] = None,
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
         self.link_extractor = LxmlParserLinkExtractor(
@@ -156,31 +179,31 @@ class LxmlLinkExtractor:
             strip=strip,
             canonicalized=not canonicalize,
         )
-        self.allow_res = [
-            x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(allow)
-        ]
-        self.deny_res = [
-            x if isinstance(x, _re_type) else re.compile(x) for x in arg_to_iter(deny)
-        ]
+        self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow)
+        self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny)
 
-        self.allow_domains = set(arg_to_iter(allow_domains))
-        self.deny_domains = set(arg_to_iter(deny_domains))
+        self.allow_domains: Set[str] = set(arg_to_iter(allow_domains))
+        self.deny_domains: Set[str] = set(arg_to_iter(deny_domains))
 
-        self.restrict_xpaths = tuple(arg_to_iter(restrict_xpaths))
+        self.restrict_xpaths: Tuple[str, ...] = tuple(arg_to_iter(restrict_xpaths))
         self.restrict_xpaths += tuple(
             map(self._csstranslator.css_to_xpath, arg_to_iter(restrict_css))
         )
 
         if deny_extensions is None:
             deny_extensions = IGNORED_EXTENSIONS
-        self.canonicalize = canonicalize
-        self.deny_extensions = {"." + e for e in arg_to_iter(deny_extensions)}
-        self.restrict_text = [
-            x if isinstance(x, _re_type) else re.compile(x)
-            for x in arg_to_iter(restrict_text)
+        self.canonicalize: bool = canonicalize
+        self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)}
+        self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text)
+
+    @staticmethod
+    def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]:
+        return [
+            x if isinstance(x, re.Pattern) else re.compile(x)
+            for x in arg_to_iter(value)
         ]
 
-    def _link_allowed(self, link):
+    def _link_allowed(self, link: Link) -> bool:
         if not _is_valid_url(link.url):
             return False
         if self.allow_res and not _matches(link.url, self.allow_res):
@@ -202,7 +225,7 @@ class LxmlLinkExtractor:
             return False
         return True
 
-    def matches(self, url):
+    def matches(self, url: str) -> bool:
         if self.allow_domains and not url_is_from_any_domain(url, self.allow_domains):
             return False
         if self.deny_domains and url_is_from_any_domain(url, self.deny_domains):
@@ -216,7 +239,7 @@ class LxmlLinkExtractor:
         denied = (regex.search(url) for regex in self.deny_res) if self.deny_res else []
         return any(allowed) and not any(denied)
 
-    def _process_links(self, links):
+    def _process_links(self, links: List[Link]) -> List[Link]:
         links = [x for x in links if self._link_allowed(x)]
         if self.canonicalize:
             for link in links:
@@ -224,10 +247,10 @@ class LxmlLinkExtractor:
         links = self.link_extractor._process_links(links)
         return links
 
-    def _extract_links(self, *args, **kwargs):
+    def _extract_links(self, *args: Any, **kwargs: Any) -> List[Link]:
         return self.link_extractor._extract_links(*args, **kwargs)
 
-    def extract_links(self, response):
+    def extract_links(self, response: TextResponse) -> List[Link]:
         """Returns a list of :class:`~scrapy.link.Link` objects from the
         specified :class:`response <scrapy.http.Response>`.
 

From 4b47a5dc32232431494b38f3a1d9ddb3dbaa6247 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 5 May 2024 22:55:21 +0500
Subject: [PATCH 54/74] Skip coverage checks for TYPE_CHECKING blocks.

---
 .coveragerc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.coveragerc b/.coveragerc
index ad0ee0f6c..f9ad353d5 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -4,3 +4,9 @@ include = scrapy/*
 omit =
   tests/*
 disable_warnings = include-ignored
+
+[report]
+# https://github.com/nedbat/coveragepy/issues/831#issuecomment-517778185
+exclude_lines =
+    pragma: no cover
+    if TYPE_CHECKING:

From 40e4a5960477299e36a0f7363db4a3c03576f64e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 5 May 2024 23:00:16 +0500
Subject: [PATCH 55/74] Fix Python 3.8.

---
 scrapy/linkextractors/__init__.py |  4 ++--
 scrapy/linkextractors/lxmlhtml.py | 23 +++++++++++++++++------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/scrapy/linkextractors/__init__.py b/scrapy/linkextractors/__init__.py
index 38dbe8135..d59005edd 100644
--- a/scrapy/linkextractors/__init__.py
+++ b/scrapy/linkextractors/__init__.py
@@ -7,7 +7,7 @@ For more info see docs/topics/link-extractors.rst
 """
 
 import re
-from typing import Iterable
+from typing import Iterable, Pattern
 
 # common file extensions that are not followed if they occur in links
 IGNORED_EXTENSIONS = [
@@ -111,7 +111,7 @@ IGNORED_EXTENSIONS = [
 ]
 
 
-def _matches(url: str, regexs: Iterable[re.Pattern[str]]) -> bool:
+def _matches(url: str, regexs: Iterable[Pattern[str]]) -> bool:
     return any(r.search(url) for r in regexs)
 
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 3fa7d1e3a..33a10cd6c 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -5,7 +5,18 @@ Link extractor based on lxml.html
 import logging
 import operator
 from functools import partial
-from typing import Any, Callable, Iterable, List, Optional, Set, Tuple, Union, cast
+from typing import (
+    Any,
+    Callable,
+    Iterable,
+    List,
+    Optional,
+    Pattern,
+    Set,
+    Tuple,
+    Union,
+    cast,
+)
 from urllib.parse import urljoin, urlparse
 
 from lxml import etree  # nosec
@@ -146,7 +157,7 @@ class LxmlParserLinkExtractor:
         return links
 
 
-_RegexT = Union[str, re.Pattern[str]]
+_RegexT = Union[str, Pattern[str]]
 _RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
 
 
@@ -179,8 +190,8 @@ class LxmlLinkExtractor:
             strip=strip,
             canonicalized=not canonicalize,
         )
-        self.allow_res: List[re.Pattern[str]] = self._compile_regexes(allow)
-        self.deny_res: List[re.Pattern[str]] = self._compile_regexes(deny)
+        self.allow_res: List[Pattern[str]] = self._compile_regexes(allow)
+        self.deny_res: List[Pattern[str]] = self._compile_regexes(deny)
 
         self.allow_domains: Set[str] = set(arg_to_iter(allow_domains))
         self.deny_domains: Set[str] = set(arg_to_iter(deny_domains))
@@ -194,10 +205,10 @@ class LxmlLinkExtractor:
             deny_extensions = IGNORED_EXTENSIONS
         self.canonicalize: bool = canonicalize
         self.deny_extensions: Set[str] = {"." + e for e in arg_to_iter(deny_extensions)}
-        self.restrict_text: List[re.Pattern[str]] = self._compile_regexes(restrict_text)
+        self.restrict_text: List[Pattern[str]] = self._compile_regexes(restrict_text)
 
     @staticmethod
-    def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[re.Pattern[str]]:
+    def _compile_regexes(value: Optional[_RegexOrSeveralT]) -> List[Pattern[str]]:
         return [
             x if isinstance(x, re.Pattern) else re.compile(x)
             for x in arg_to_iter(value)

From aa025d7eacb461ccb0c724584532d402bb400bd1 Mon Sep 17 00:00:00 2001
From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com>
Date: Mon, 6 May 2024 14:59:35 +0530
Subject: [PATCH 56/74] Indicate that Selector.type can be json (#6334)

---
 scrapy/selector/unified.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index aa9581fcd..e852aadc7 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -47,7 +47,7 @@ class Selector(_ParselSelector, object_ref):
     ``response`` isn't available. Using ``text`` and ``response`` together is
     undefined behavior.
 
-    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"``
     or ``None`` (default).
 
     If ``type`` is ``None``, the selector automatically chooses the best type

From 2cba7896d26dda51ff2e598300363531ebd328b8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 6 May 2024 14:31:24 +0500
Subject: [PATCH 57/74] Small fix for _get_slot_key().

---
 scrapy/core/downloader/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index f88da41ea..98e1af6fb 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -143,7 +143,7 @@ class Downloader:
 
         return key, self.slots[key]
 
-    def _get_slot_key(self, request: Request, spider: Any) -> str:
+    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
             return cast(str, request.meta[self.DOWNLOAD_SLOT])
 

From c4d2748ff572adad8150f1bc09b2d52e61d9dfc8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 17:07:19 +0500
Subject: [PATCH 58/74] Small typing improvements in
 scrapy/core/downloader/contextfactory.py.

---
 scrapy/core/downloader/contextfactory.py  | 7 +++++--
 scrapy/core/downloader/handlers/http10.py | 2 +-
 scrapy/core/downloader/handlers/http11.py | 2 +-
 scrapy/core/downloader/handlers/http2.py  | 2 +-
 4 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 6a82634f1..0e77cd2fe 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -21,6 +21,7 @@ from scrapy.core.downloader.tls import (
     ScrapyClientTLSOptions,
     openssl_methods,
 )
+from scrapy.crawler import Crawler
 from scrapy.settings import BaseSettings
 from scrapy.utils.misc import build_from_crawler, load_object
 
@@ -102,7 +103,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     # kept for old-style HTTP/1.0 downloader context twisted calls,
     # e.g. connectSSL()
     def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
-        ctx = self.getCertificateOptions().getContext()
+        ctx: SSL.Context = self.getCertificateOptions().getContext()
         ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
         return ctx
 
@@ -165,7 +166,9 @@ class AcceptableProtocolsContextFactory:
         return options
 
 
-def load_context_factory_from_settings(settings, crawler):
+def load_context_factory_from_settings(
+    settings: BaseSettings, crawler: Crawler
+) -> IPolicyForHTTPS:
     ssl_method = openssl_methods[settings.get("DOWNLOADER_CLIENT_TLS_METHOD")]
     context_factory_cls = load_object(settings["DOWNLOADER_CLIENTCONTEXTFACTORY"])
     # try method-aware context factory
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 256dc36a1..2507a4231 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -16,7 +16,7 @@ if TYPE_CHECKING:
 class HTTP10DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None):
+    def __init__(self, settings, crawler):
         self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
         self.ClientContextFactory = load_object(
             settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 15f8abc64..52561f7e6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -46,7 +46,7 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler=None):
+    def __init__(self, settings, crawler):
         self._crawler = crawler
 
         from twisted.internet import reactor
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index e9a6b6fa3..efe45c459 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -24,7 +24,7 @@ if TYPE_CHECKING:
 
 
 class H2DownloadHandler:
-    def __init__(self, settings: Settings, crawler: Optional[Crawler] = None):
+    def __init__(self, settings: Settings, crawler: Crawler):
         self._crawler = crawler
 
         from twisted.internet import reactor

From ec4d40702227f5486c184de9439da4896121a33b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 17:36:16 +0500
Subject: [PATCH 59/74] Full typing for smaller download handlers.

---
 scrapy/core/downloader/handlers/file.py   |  4 +++-
 scrapy/core/downloader/handlers/http11.py |  6 ++---
 scrapy/core/downloader/handlers/http2.py  |  3 ++-
 scrapy/core/downloader/handlers/s3.py     | 27 ++++++++++++++---------
 4 files changed, 25 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.py
index 4824167da..17dd7483b 100644
--- a/scrapy/core/downloader/handlers/file.py
+++ b/scrapy/core/downloader/handlers/file.py
@@ -2,6 +2,8 @@ from pathlib import Path
 
 from w3lib.url import file_uri_to_path
 
+from scrapy import Request, Spider
+from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.decorators import defers
 
@@ -10,7 +12,7 @@ class FileDownloadHandler:
     lazy = False
 
     @defers
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         filepath = file_uri_to_path(request.url)
         body = Path(filepath).read_bytes()
         respcls = responsetypes.from_args(filename=filepath, body=body)
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 52561f7e6..2e7ea559a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -27,11 +27,11 @@ from twisted.web.http_headers import Headers as TxHeaders
 from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from zope.interface import implementer
 
-from scrapy import signals
+from scrapy import Request, Spider, signals
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
 from scrapy.exceptions import StopDownload
-from scrapy.http import Headers
+from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.python import to_bytes, to_unicode
 
@@ -67,7 +67,7 @@ class HTTP11DownloadHandler:
     def from_crawler(cls, crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index efe45c459..16fc1e3ae 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -8,6 +8,7 @@ from twisted.internet.base import DelayedCall
 from twisted.internet.defer import Deferred
 from twisted.internet.error import TimeoutError
 from twisted.web.client import URI
+from twisted.web.iweb import IPolicyForHTTPS
 
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
@@ -54,7 +55,7 @@ class ScrapyH2Agent:
 
     def __init__(
         self,
-        context_factory,
+        context_factory: IPolicyForHTTPS,
         pool: H2ConnectionPool,
         connect_timeout: int = 10,
         bind_address: Optional[bytes] = None,
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 99fbb49ce..c88dd2cdc 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,13 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Optional, Type
 
+from scrapy import Request, Spider
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
+from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
+from scrapy.http import Response
+from scrapy.settings import BaseSettings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler
@@ -16,14 +20,14 @@ if TYPE_CHECKING:
 class S3DownloadHandler:
     def __init__(
         self,
-        settings,
+        settings: BaseSettings,
         *,
-        crawler=None,
-        aws_access_key_id=None,
-        aws_secret_access_key=None,
-        aws_session_token=None,
-        httpdownloadhandler=HTTPDownloadHandler,
-        **kw,
+        crawler: Crawler,
+        aws_access_key_id: Optional[str] = None,
+        aws_secret_access_key: Optional[str] = None,
+        aws_session_token: Optional[str] = None,
+        httpdownloadhandler: Type[HTTPDownloadHandler] = HTTPDownloadHandler,
+        **kw: Any,
     ):
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
@@ -51,6 +55,8 @@ class S3DownloadHandler:
         if kw:
             raise TypeError(f"Unexpected keyword arguments: {kw}")
         if not self.anon:
+            assert aws_access_key_id is not None
+            assert aws_secret_access_key is not None
             SignerCls = botocore.auth.AUTH_TYPE_MAPS["s3"]
             self._signer = SignerCls(
                 botocore.credentials.Credentials(
@@ -65,10 +71,10 @@ class S3DownloadHandler:
         self._download_http = _http_handler.download_request
 
     @classmethod
-    def from_crawler(cls, crawler, **kwargs) -> Self:
+    def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
         return cls(crawler.settings, crawler=crawler, **kwargs)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Response:
         p = urlparse_cached(request)
         scheme = "https" if request.meta.get("is_secure") else "http"
         bucket = p.hostname
@@ -85,6 +91,7 @@ class S3DownloadHandler:
                 headers=request.headers.to_unicode_dict(),
                 data=request.body,
             )
+            assert self._signer
             self._signer.add_auth(awsrequest)
             request = request.replace(url=url, headers=awsrequest.headers.items())
         return self._download_http(request, spider)

From e8e13ebb78dd1d2db425b285335f31dba7c1fd39 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 18:03:06 +0500
Subject: [PATCH 60/74] Full typing for scrapy/core/downloader/handlers/ftp.py.

---
 scrapy/core/downloader/handlers/ftp.py | 40 +++++++++++++++-----------
 1 file changed, 24 insertions(+), 16 deletions(-)

diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 69add8558..ed94b2221 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -32,15 +32,19 @@ from __future__ import annotations
 
 import re
 from io import BytesIO
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, BinaryIO, Dict, Optional
 from urllib.parse import unquote
 
+from twisted.internet.defer import Deferred
 from twisted.internet.protocol import ClientCreator, Protocol
 from twisted.protocols.ftp import CommandFailed, FTPClient
+from twisted.python.failure import Failure
 
+from scrapy import Request, Spider
 from scrapy.crawler import Crawler
 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
+from scrapy.settings import BaseSettings
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes
 
@@ -50,20 +54,20 @@ if TYPE_CHECKING:
 
 
 class ReceivedDataProtocol(Protocol):
-    def __init__(self, filename=None):
-        self.__filename = filename
-        self.body = open(filename, "wb") if filename else BytesIO()
-        self.size = 0
+    def __init__(self, filename: Optional[str] = None):
+        self.__filename: Optional[str] = filename
+        self.body: BinaryIO = open(filename, "wb") if filename else BytesIO()
+        self.size: int = 0
 
-    def dataReceived(self, data):
+    def dataReceived(self, data: bytes) -> None:
         self.body.write(data)
         self.size += len(data)
 
     @property
-    def filename(self):
+    def filename(self) -> Optional[str]:
         return self.__filename
 
-    def close(self):
+    def close(self) -> None:
         self.body.close() if self.filename else self.body.seek(0)
 
 
@@ -73,12 +77,12 @@ _CODE_RE = re.compile(r"\d+")
 class FTPDownloadHandler:
     lazy = False
 
-    CODE_MAPPING = {
+    CODE_MAPPING: Dict[str, int] = {
         "550": 404,
         "default": 503,
     }
 
-    def __init__(self, settings):
+    def __init__(self, settings: BaseSettings):
         self.default_user = settings["FTP_USER"]
         self.default_password = settings["FTP_PASSWORD"]
         self.passive_mode = settings["FTP_PASSIVE_MODE"]
@@ -87,7 +91,7 @@ class FTPDownloadHandler:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         from twisted.internet import reactor
 
         parsed_url = urlparse_cached(request)
@@ -99,10 +103,10 @@ class FTPDownloadHandler:
         creator = ClientCreator(
             reactor, FTPClient, user, password, passive=passive_mode
         )
-        dfd = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
+        dfd: Deferred = creator.connectTCP(parsed_url.hostname, parsed_url.port or 21)
         return dfd.addCallback(self.gotClient, request, unquote(parsed_url.path))
 
-    def gotClient(self, client, request, filepath):
+    def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
         return client.retrieveFile(filepath, protocol).addCallbacks(
@@ -112,15 +116,18 @@ class FTPDownloadHandler:
             errbackArgs=(request,),
         )
 
-    def _build_response(self, result, request, protocol):
+    def _build_response(
+        self, result: Any, request: Request, protocol: ReceivedDataProtocol
+    ) -> Response:
         self.result = result
         protocol.close()
         headers = {"local filename": protocol.filename or "", "size": protocol.size}
         body = to_bytes(protocol.filename or protocol.body.read())
         respcls = responsetypes.from_args(url=request.url, body=body)
-        return respcls(url=request.url, status=200, body=body, headers=headers)
+        # hints for Headers-related types may need to be fixed to not use AnyStr
+        return respcls(url=request.url, status=200, body=body, headers=headers)  # type: ignore[arg-type]
 
-    def _failed(self, result, request):
+    def _failed(self, result: Failure, request: Request) -> Response:
         message = result.getErrorMessage()
         if result.type == CommandFailed:
             m = _CODE_RE.search(message)
@@ -130,4 +137,5 @@ class FTPDownloadHandler:
                 return Response(
                     url=request.url, status=httpcode, body=to_bytes(message)
                 )
+        assert result.type
         raise result.type(result.value)

From af3e38ab1f3dc7095ae27f572cbde0ea652d7664 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 18:09:25 +0500
Subject: [PATCH 61/74] Full typing for
 scrapy/core/downloader/handlers/http10.py.

---
 scrapy/core/downloader/handlers/http10.py | 27 +++++++++++++++--------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 2507a4231..9117cb818 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -3,8 +3,13 @@
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Type
 
+from twisted.internet.defer import Deferred
+
+from scrapy import Request, Spider
+from scrapy.crawler import Crawler
+from scrapy.settings import BaseSettings
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import to_unicode
 
@@ -12,29 +17,33 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
+    from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
 
 class HTTP10DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler):
-        self.HTTPClientFactory = load_object(settings["DOWNLOADER_HTTPCLIENTFACTORY"])
-        self.ClientContextFactory = load_object(
+    def __init__(self, settings: BaseSettings, crawler: Crawler):
+        self.HTTPClientFactory: Type[ScrapyHTTPClientFactory] = load_object(
+            settings["DOWNLOADER_HTTPCLIENTFACTORY"]
+        )
+        self.ClientContextFactory: Type[ScrapyClientContextFactory] = load_object(
             settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
         )
-        self._settings = settings
-        self._crawler = crawler
+        self._settings: BaseSettings = settings
+        self._crawler: Crawler = crawler
 
     @classmethod
-    def from_crawler(cls, crawler) -> Self:
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request, spider):
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         """Return a deferred for the HTTP download"""
         factory = self.HTTPClientFactory(request)
         self._connect(factory)
         return factory.deferred
 
-    def _connect(self, factory):
+    def _connect(self, factory: ScrapyHTTPClientFactory) -> Deferred:
         from twisted.internet import reactor
 
         host, port = to_unicode(factory.host), factory.port

From 045387e07faba4a18ecff919457d69938e89b710 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sat, 4 May 2024 19:56:23 +0500
Subject: [PATCH 62/74] More typing for
 scrapy/core/downloader/handlers/http11.py.

---
 scrapy/core/downloader/handlers/http11.py | 286 +++++++++++++---------
 scrapy/core/downloader/handlers/s3.py     |   5 +-
 2 files changed, 169 insertions(+), 122 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 2e7ea559a..40ae1921a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -8,31 +8,33 @@ import re
 from contextlib import suppress
 from io import BytesIO
 from time import time
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 from urllib.parse import urldefrag, urlunparse
 
-from twisted.internet import defer, protocol, ssl
+from twisted.internet import ssl
+from twisted.internet.base import ReactorBase
+from twisted.internet.defer import CancelledError, Deferred, succeed
 from twisted.internet.endpoints import TCP4ClientEndpoint
 from twisted.internet.error import TimeoutError
+from twisted.internet.interfaces import IConsumer
+from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
-from twisted.web.client import (
-    URI,
-    Agent,
-    HTTPConnectionPool,
-    ResponseDone,
-    ResponseFailed,
-)
+from twisted.web.client import URI, Agent, HTTPConnectionPool
+from twisted.web.client import Response as TxResponse
+from twisted.web.client import ResponseDone, ResponseFailed
 from twisted.web.http import PotentialDataLoss, _DataLoss
 from twisted.web.http_headers import Headers as TxHeaders
-from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IPolicyForHTTPS
 from zope.interface import implementer
 
 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
 from scrapy.core.downloader.webclient import _parse
+from scrapy.crawler import Crawler
 from scrapy.exceptions import StopDownload
 from scrapy.http import Headers, Response
 from scrapy.responsetypes import responsetypes
+from scrapy.settings import BaseSettings
 from scrapy.utils.python import to_bytes, to_unicode
 
 if TYPE_CHECKING:
@@ -46,28 +48,30 @@ logger = logging.getLogger(__name__)
 class HTTP11DownloadHandler:
     lazy = False
 
-    def __init__(self, settings, crawler):
+    def __init__(self, settings: BaseSettings, crawler: Crawler):
         self._crawler = crawler
 
         from twisted.internet import reactor
 
-        self._pool = HTTPConnectionPool(reactor, persistent=True)
+        self._pool: HTTPConnectionPool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint(
             "CONCURRENT_REQUESTS_PER_DOMAIN"
         )
         self._pool._factory.noisy = False
 
-        self._contextFactory = load_context_factory_from_settings(settings, crawler)
-        self._default_maxsize = settings.getint("DOWNLOAD_MAXSIZE")
-        self._default_warnsize = settings.getint("DOWNLOAD_WARNSIZE")
-        self._fail_on_dataloss = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
-        self._disconnect_timeout = 1
+        self._contextFactory: IPolicyForHTTPS = load_context_factory_from_settings(
+            settings, crawler
+        )
+        self._default_maxsize: int = settings.getint("DOWNLOAD_MAXSIZE")
+        self._default_warnsize: int = settings.getint("DOWNLOAD_WARNSIZE")
+        self._fail_on_dataloss: bool = settings.getbool("DOWNLOAD_FAIL_ON_DATALOSS")
+        self._disconnect_timeout: int = 1
 
     @classmethod
-    def from_crawler(cls, crawler) -> Self:
+    def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler.settings, crawler)
 
-    def download_request(self, request: Request, spider: Spider) -> Response:
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         """Return a deferred for the HTTP download"""
         agent = ScrapyAgent(
             contextFactory=self._contextFactory,
@@ -79,10 +83,10 @@ class HTTP11DownloadHandler:
         )
         return agent.download_request(request)
 
-    def close(self):
+    def close(self) -> Deferred:
         from twisted.internet import reactor
 
-        d = self._pool.closeCachedConnections()
+        d: Deferred = self._pool.closeCachedConnections()
         # closeCachedConnections will hang on network or server issues, so
         # we'll manually timeout the deferred.
         #
@@ -93,7 +97,7 @@ class HTTP11DownloadHandler:
         # issue a callback after `_disconnect_timeout` seconds.
         delayed_call = reactor.callLater(self._disconnect_timeout, d.callback, [])
 
-        def cancel_delayed_call(result):
+        def cancel_delayed_call(result: Any) -> Any:
             if delayed_call.active():
                 delayed_call.cancel()
             return result
@@ -123,39 +127,41 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
 
     def __init__(
         self,
-        reactor,
-        host,
-        port,
-        proxyConf,
-        contextFactory,
-        timeout=30,
-        bindAddress=None,
+        reactor: ReactorBase,
+        host: str,
+        port: int,
+        proxyConf: Tuple[str, int, Optional[bytes]],
+        contextFactory: IPolicyForHTTPS,
+        timeout: float = 30,
+        bindAddress: Optional[Tuple[str, int]] = None,
     ):
         proxyHost, proxyPort, self._proxyAuthHeader = proxyConf
         super().__init__(reactor, proxyHost, proxyPort, timeout, bindAddress)
-        self._tunnelReadyDeferred = defer.Deferred()
-        self._tunneledHost = host
-        self._tunneledPort = port
-        self._contextFactory = contextFactory
-        self._connectBuffer = bytearray()
+        self._tunnelReadyDeferred: Deferred = Deferred()
+        self._tunneledHost: str = host
+        self._tunneledPort: int = port
+        self._contextFactory: IPolicyForHTTPS = contextFactory
+        self._connectBuffer: bytearray = bytearray()
 
-    def requestTunnel(self, protocol):
+    def requestTunnel(self, protocol: Protocol) -> Protocol:
         """Asks the proxy to open a tunnel."""
+        assert protocol.transport
         tunnelReq = tunnel_request_data(
             self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
         )
         protocol.transport.write(tunnelReq)
         self._protocolDataReceived = protocol.dataReceived
-        protocol.dataReceived = self.processProxyResponse
+        protocol.dataReceived = self.processProxyResponse  # type: ignore[method-assign]
         self._protocol = protocol
         return protocol
 
-    def processProxyResponse(self, rcvd_bytes):
+    def processProxyResponse(self, data: bytes) -> None:
         """Processes the response from the proxy. If the tunnel is successfully
         created, notifies the client that we are ready to send requests. If not
         raises a TunnelError.
         """
-        self._connectBuffer += rcvd_bytes
+        assert self._protocol.transport
+        self._connectBuffer += data
         # make sure that enough (all) bytes are consumed
         # and that we've got all HTTP headers (ending with a blank line)
         # from the proxy so that we don't send those bytes to the TLS layer
@@ -163,23 +169,24 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         # see https://github.com/scrapy/scrapy/issues/2491
         if b"\r\n\r\n" not in self._connectBuffer:
             return
-        self._protocol.dataReceived = self._protocolDataReceived
+        self._protocol.dataReceived = self._protocolDataReceived  # type: ignore[method-assign]
         respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
         if respm and int(respm.group("status")) == 200:
             # set proper Server Name Indication extension
-            sslOptions = self._contextFactory.creatorForNetloc(
+            sslOptions = self._contextFactory.creatorForNetloc(  # type: ignore[call-arg,misc]
                 self._tunneledHost, self._tunneledPort
             )
             self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
         else:
+            extra: Any
             if respm:
                 extra = {
                     "status": int(respm.group("status")),
                     "reason": respm.group("reason").strip(),
                 }
             else:
-                extra = rcvd_bytes[: self._truncatedLength]
+                extra = data[: self._truncatedLength]
             self._tunnelReadyDeferred.errback(
                 TunnelError(
                     "Could not open CONNECT tunnel with proxy "
@@ -187,11 +194,11 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
                 )
             )
 
-    def connectFailed(self, reason):
+    def connectFailed(self, reason: Failure) -> None:
         """Propagates the errback to the appropriate deferred."""
         self._tunnelReadyDeferred.errback(reason)
 
-    def connect(self, protocolFactory):
+    def connect(self, protocolFactory: Factory) -> Deferred:
         self._protocolFactory = protocolFactory
         connectDeferred = super().connect(protocolFactory)
         connectDeferred.addCallback(self.requestTunnel)
@@ -199,7 +206,9 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
-def tunnel_request_data(host, port, proxy_auth_header=None):
+def tunnel_request_data(
+    host: str, port: int, proxy_auth_header: Optional[bytes] = None
+) -> bytes:
     r"""
     Return binary content of a CONNECT request.
 
@@ -230,18 +239,20 @@ class TunnelingAgent(Agent):
 
     def __init__(
         self,
-        reactor,
-        proxyConf,
-        contextFactory=None,
-        connectTimeout=None,
-        bindAddress=None,
-        pool=None,
+        reactor: ReactorBase,
+        proxyConf: Tuple[str, int, Optional[bytes]],
+        contextFactory: Optional[IPolicyForHTTPS] = None,
+        connectTimeout: Optional[float] = None,
+        bindAddress: Optional[bytes] = None,
+        pool: Optional[HTTPConnectionPool] = None,
     ):
+        # TODO make this arg required instead
+        assert contextFactory is not None
         super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
-        self._proxyConf = proxyConf
-        self._contextFactory = contextFactory
+        self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf
+        self._contextFactory: IPolicyForHTTPS = contextFactory
 
-    def _getEndpoint(self, uri):
+    def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:
         return TunnelingTCP4ClientEndpoint(
             reactor=self._reactor,
             host=uri.host,
@@ -253,8 +264,15 @@ class TunnelingAgent(Agent):
         )
 
     def _requestWithEndpoint(
-        self, key, endpoint, method, parsedURI, headers, bodyProducer, requestPath
-    ):
+        self,
+        key: Any,
+        endpoint: TCP4ClientEndpoint,
+        method: bytes,
+        parsedURI: bytes,
+        headers: Optional[TxHeaders],
+        bodyProducer: Optional[IBodyProducer],
+        requestPath: bytes,
+    ) -> Deferred:
         # proxy host and port are required for HTTP pool `key`
         # otherwise, same remote host connection request could reuse
         # a cached tunneled connection to a different proxy
@@ -272,7 +290,12 @@ class TunnelingAgent(Agent):
 
 class ScrapyProxyAgent(Agent):
     def __init__(
-        self, reactor, proxyURI, connectTimeout=None, bindAddress=None, pool=None
+        self,
+        reactor: ReactorBase,
+        proxyURI: bytes,
+        connectTimeout: Optional[float] = None,
+        bindAddress: Optional[bytes] = None,
+        pool: Optional[HTTPConnectionPool] = None,
     ):
         super().__init__(
             reactor=reactor,
@@ -280,9 +303,15 @@ class ScrapyProxyAgent(Agent):
             bindAddress=bindAddress,
             pool=pool,
         )
-        self._proxyURI = URI.fromBytes(proxyURI)
+        self._proxyURI: URI = URI.fromBytes(proxyURI)
 
-    def request(self, method, uri, headers=None, bodyProducer=None):
+    def request(
+        self,
+        method: bytes,
+        uri: bytes,
+        headers: Optional[TxHeaders] = None,
+        bodyProducer: Optional[IBodyProducer] = None,
+    ) -> Deferred:
         """
         Issue a new request via the configured proxy.
         """
@@ -306,26 +335,29 @@ class ScrapyAgent:
 
     def __init__(
         self,
-        contextFactory=None,
-        connectTimeout=10,
-        bindAddress=None,
-        pool=None,
-        maxsize=0,
-        warnsize=0,
-        fail_on_dataloss=True,
-        crawler=None,
+        contextFactory: Optional[IPolicyForHTTPS] = None,
+        connectTimeout: float = 10,
+        bindAddress: Optional[bytes] = None,
+        pool: Optional[HTTPConnectionPool] = None,
+        maxsize: int = 0,
+        warnsize: int = 0,
+        fail_on_dataloss: bool = True,
+        crawler: Optional[Crawler] = None,
     ):
-        self._contextFactory = contextFactory
-        self._connectTimeout = connectTimeout
-        self._bindAddress = bindAddress
-        self._pool = pool
-        self._maxsize = maxsize
-        self._warnsize = warnsize
-        self._fail_on_dataloss = fail_on_dataloss
-        self._txresponse = None
-        self._crawler = crawler
+        # TODO make these args required instead
+        assert contextFactory is not None
+        assert crawler is not None
+        self._contextFactory: IPolicyForHTTPS = contextFactory
+        self._connectTimeout: float = connectTimeout
+        self._bindAddress: Optional[bytes] = bindAddress
+        self._pool: Optional[HTTPConnectionPool] = pool
+        self._maxsize: int = maxsize
+        self._warnsize: int = warnsize
+        self._fail_on_dataloss: bool = fail_on_dataloss
+        self._txresponse: Optional[TxResponse] = None
+        self._crawler: Crawler = crawler
 
-    def _get_agent(self, request, timeout):
+    def _get_agent(self, request: Request, timeout: float) -> Agent:
         from twisted.internet import reactor
 
         bindaddress = request.meta.get("bindaddress") or self._bindAddress
@@ -333,10 +365,10 @@ class ScrapyAgent:
         if proxy:
             proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)
             scheme = _parse(request.url)[0]
-            proxyHost = to_unicode(proxyHost)
+            proxyHost_str = to_unicode(proxyHost)
             if scheme == b"https":
                 proxyAuth = request.headers.get(b"Proxy-Authorization", None)
-                proxyConf = (proxyHost, proxyPort, proxyAuth)
+                proxyConf = (proxyHost_str, proxyPort, proxyAuth)
                 return self._TunnelingAgent(
                     reactor=reactor,
                     proxyConf=proxyConf,
@@ -346,7 +378,9 @@ class ScrapyAgent:
                     pool=self._pool,
                 )
             proxyScheme = proxyScheme or b"http"
-            proxyURI = urlunparse((proxyScheme, proxyNetloc, proxyParams, "", "", ""))
+            proxyURI = urlunparse(
+                (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"")
+            )
             return self._ProxyAgent(
                 reactor=reactor,
                 proxyURI=to_bytes(proxyURI, encoding="ascii"),
@@ -363,7 +397,7 @@ class ScrapyAgent:
             pool=self._pool,
         )
 
-    def download_request(self, request):
+    def download_request(self, request: Request) -> Deferred:
         from twisted.internet import reactor
 
         timeout = request.meta.get("download_timeout") or self._connectTimeout
@@ -380,7 +414,7 @@ class ScrapyAgent:
         else:
             bodyproducer = None
         start_time = time()
-        d = agent.request(
+        d: Deferred = agent.request(
             method, to_bytes(url, encoding="ascii"), headers, bodyproducer
         )
         # set download latency
@@ -393,7 +427,9 @@ class ScrapyAgent:
         d.addBoth(self._cb_timeout, request, url, timeout)
         return d
 
-    def _cb_timeout(self, result, request, url, timeout):
+    def _cb_timeout(
+        self, result: Any, request: Request, url: str, timeout: float
+    ) -> Any:
         if self._timeout_cl.active():
             self._timeout_cl.cancel()
             return result
@@ -404,19 +440,21 @@ class ScrapyAgent:
 
         raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
 
-    def _cb_latency(self, result, request, start_time):
+    def _cb_latency(self, result: Any, request: Request, start_time: float) -> Any:
         request.meta["download_latency"] = time() - start_time
         return result
 
     @staticmethod
-    def _headers_from_twisted_response(response):
+    def _headers_from_twisted_response(response: TxResponse) -> Headers:
         headers = Headers()
         if response.length != UNKNOWN_LENGTH:
             headers[b"Content-Length"] = str(response.length).encode()
         headers.update(response.headers.getAllRawHeaders())
         return headers
 
-    def _cb_bodyready(self, txresponse, request):
+    def _cb_bodyready(
+        self, txresponse: TxResponse, request: Request
+    ) -> Union[Dict[str, Any], Deferred]:
         headers_received_result = self._crawler.signals.send_catch_log(
             signal=signals.headers_received,
             headers=self._headers_from_twisted_response(txresponse),
@@ -472,7 +510,7 @@ class ScrapyAgent:
             logger.warning(warning_msg, warning_args)
 
             txresponse._transport.loseConnection()
-            raise defer.CancelledError(warning_msg % warning_args)
+            raise CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
             logger.warning(
@@ -481,11 +519,11 @@ class ScrapyAgent:
                 {"size": expected_size, "warnsize": warnsize, "request": request},
             )
 
-        def _cancel(_):
+        def _cancel(_: Any) -> None:
             # Abort connection immediately.
             txresponse._transport._producer.abortConnection()
 
-        d = defer.Deferred(_cancel)
+        d: Deferred = Deferred(_cancel)
         txresponse.deliverBody(
             _ResponseReader(
                 finished=d,
@@ -503,7 +541,9 @@ class ScrapyAgent:
 
         return d
 
-    def _cb_bodydone(self, result, request, url):
+    def _cb_bodydone(
+        self, result: Dict[str, Any], request: Request, url: str
+    ) -> Union[Response, Failure]:
         headers = self._headers_from_twisted_response(result["txresponse"])
         respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
         try:
@@ -523,53 +563,57 @@ class ScrapyAgent:
         )
         if result.get("failure"):
             result["failure"].value.response = response
-            return result["failure"]
+            return cast(Failure, result["failure"])
         return response
 
 
 @implementer(IBodyProducer)
 class _RequestBodyProducer:
-    def __init__(self, body):
+    def __init__(self, body: bytes):
         self.body = body
         self.length = len(body)
 
-    def startProducing(self, consumer):
+    def startProducing(self, consumer: IConsumer) -> Deferred:
         consumer.write(self.body)
-        return defer.succeed(None)
+        return succeed(None)
 
-    def pauseProducing(self):
+    def pauseProducing(self) -> None:
         pass
 
-    def stopProducing(self):
+    def stopProducing(self) -> None:
         pass
 
 
-class _ResponseReader(protocol.Protocol):
+class _ResponseReader(Protocol):
     def __init__(
         self,
-        finished,
-        txresponse,
-        request,
-        maxsize,
-        warnsize,
-        fail_on_dataloss,
-        crawler,
+        finished: Deferred,
+        txresponse: TxResponse,
+        request: Request,
+        maxsize: int,
+        warnsize: int,
+        fail_on_dataloss: bool,
+        crawler: Crawler,
     ):
-        self._finished = finished
-        self._txresponse = txresponse
-        self._request = request
-        self._bodybuf = BytesIO()
-        self._maxsize = maxsize
-        self._warnsize = warnsize
-        self._fail_on_dataloss = fail_on_dataloss
-        self._fail_on_dataloss_warned = False
-        self._reached_warnsize = False
-        self._bytes_received = 0
-        self._certificate = None
-        self._ip_address = None
-        self._crawler = crawler
+        self._finished: Deferred = finished
+        self._txresponse: TxResponse = txresponse
+        self._request: Request = request
+        self._bodybuf: BytesIO = BytesIO()
+        self._maxsize: int = maxsize
+        self._warnsize: int = warnsize
+        self._fail_on_dataloss: bool = fail_on_dataloss
+        self._fail_on_dataloss_warned: bool = False
+        self._reached_warnsize: bool = False
+        self._bytes_received: int = 0
+        self._certificate: Optional[ssl.Certificate] = None
+        self._ip_address: Union[ipaddress.IPv4Address, ipaddress.IPv6Address, None] = (
+            None
+        )
+        self._crawler: Crawler = crawler
 
-    def _finish_response(self, flags=None, failure=None):
+    def _finish_response(
+        self, flags: Optional[List[str]] = None, failure: Optional[Failure] = None
+    ) -> None:
         self._finished.callback(
             {
                 "txresponse": self._txresponse,
@@ -581,7 +625,8 @@ class _ResponseReader(protocol.Protocol):
             }
         )
 
-    def connectionMade(self):
+    def connectionMade(self) -> None:
+        assert self.transport
         if self._certificate is None:
             with suppress(AttributeError):
                 self._certificate = ssl.Certificate(
@@ -593,11 +638,12 @@ class _ResponseReader(protocol.Protocol):
                 self.transport._producer.getPeer().host
             )
 
-    def dataReceived(self, bodyBytes):
+    def dataReceived(self, bodyBytes: bytes) -> None:
         # This maybe called several times after cancel was called with buffered data.
         if self._finished.called:
             return
 
+        assert self.transport
         self._bodybuf.write(bodyBytes)
         self._bytes_received += len(bodyBytes)
 
@@ -644,7 +690,7 @@ class _ResponseReader(protocol.Protocol):
                 {"warnsize": self._warnsize, "request": self._request},
             )
 
-    def connectionLost(self, reason):
+    def connectionLost(self, reason: Failure = connectionDone) -> None:
         if self._finished.called:
             return
 
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index c88dd2cdc..9a0811a50 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -2,11 +2,12 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any, Optional, Type
 
+from twisted.internet.defer import Deferred
+
 from scrapy import Request, Spider
 from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from scrapy.http import Response
 from scrapy.settings import BaseSettings
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
@@ -74,7 +75,7 @@ class S3DownloadHandler:
     def from_crawler(cls, crawler: Crawler, **kwargs: Any) -> Self:
         return cls(crawler.settings, crawler=crawler, **kwargs)
 
-    def download_request(self, request: Request, spider: Spider) -> Response:
+    def download_request(self, request: Request, spider: Spider) -> Deferred:
         p = urlparse_cached(request)
         scheme = "https" if request.meta.get("is_secure") else "http"
         bucket = p.hostname

From a828da98c3834ae70a1258278890485803ac7a5c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 6 May 2024 22:34:36 +0500
Subject: [PATCH 63/74] Re-run pre-commit.

---
 scrapy/core/downloader/handlers/http10.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
index 9117cb818..da9559525 100644
--- a/scrapy/core/downloader/handlers/http10.py
+++ b/scrapy/core/downloader/handlers/http10.py
@@ -20,6 +20,7 @@ if TYPE_CHECKING:
     from scrapy.core.downloader.contextfactory import ScrapyClientContextFactory
     from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
 
+
 class HTTP10DownloadHandler:
     lazy = False
 

From 6bbfb537f9f1ba5dd1c51fc860022adb1d326117 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 8 May 2024 00:39:05 +0500
Subject: [PATCH 64/74] Update MANIFEST.in.

---
 MANIFEST.in | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/MANIFEST.in b/MANIFEST.in
index 4920dc0c3..06971e39c 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,9 +1,8 @@
-include README.rst
-include AUTHORS
-include INSTALL
-include LICENSE
-include MANIFEST.in
+include CODE_OF_CONDUCT.md
+include CONTRIBUTING.md
+include INSTALL.md
 include NEWS
+include SECURITY.md
 
 include scrapy/VERSION
 include scrapy/mime.types
@@ -12,16 +11,13 @@ include scrapy/py.typed
 include codecov.yml
 include conftest.py
 include pytest.ini
-include requirements-*.txt
 include tox.ini
 
 recursive-include scrapy/templates *
-recursive-include scrapy license.txt
 recursive-include docs *
 prune docs/build
 
 recursive-include extras *
-recursive-include bin *
 recursive-include tests *
 
 global-exclude __pycache__ *.py[cod]

From 180bc9bad7aceb3a9e10c1411212914bc32fb721 Mon Sep 17 00:00:00 2001
From: aisha-partha <153170327+aisha-partha@users.noreply.github.com>
Date: Wed, 8 May 2024 22:06:46 +0530
Subject: [PATCH 65/74] =?UTF-8?q?Closes=20#6342.=20Setting=20METAREFRESH?=
 =?UTF-8?q?=5FIGNORE=5FTAGS=20to=20[=E2=80=98noscript=E2=80=99]=20by=20def?=
 =?UTF-8?q?ault?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docs/topics/downloader-middleware.rst       | 4 ++--
 scrapy/settings/default_settings.py         | 2 +-
 tests/test_downloadermiddleware_redirect.py | 5 ++---
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 1abbc4968..3f90cf2ed 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -876,13 +876,13 @@ Whether the Meta Refresh middleware will be enabled.
 METAREFRESH_IGNORE_TAGS
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Default: ``[]``
+Default: ``['noscript']``
 
 Meta tags within these tags are ignored.
 
 .. versionchanged:: 2.0
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
-   ``['script', 'noscript']`` to ``[]``.
+   ``[]`` to ``['noscript']``.
 
 .. setting:: METAREFRESH_MAXDELAY
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 2b3d95a0e..d7ac7ec35 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -239,7 +239,7 @@ MEMUSAGE_NOTIFY_MAIL = []
 MEMUSAGE_WARNING_MB = 0
 
 METAREFRESH_ENABLED = True
-METAREFRESH_IGNORE_TAGS = []
+METAREFRESH_IGNORE_TAGS = ["noscript"]
 METAREFRESH_MAXDELAY = 100
 
 NEWSPIDER_MODULE = ""
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 10b8ca9af..83ff25982 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -395,9 +395,8 @@ class MetaRefreshMiddlewareTest(unittest.TestCase):
             """content="0;URL='http://example.org/newpage'"></noscript>"""
         )
         rsp = HtmlResponse(req.url, body=body.encode())
-        req2 = self.mw.process_response(req, rsp, self.spider)
-        assert isinstance(req2, Request)
-        self.assertEqual(req2.url, "http://example.org/newpage")
+        response = self.mw.process_response(req, rsp, self.spider)
+        assert isinstance(response, Response)
 
     def test_ignore_tags_1_x_list(self):
         """Test that Scrapy 1.x behavior remains possible"""

From 3590a1f66b30edfc836b95bbd0a3611eee9371e2 Mon Sep 17 00:00:00 2001
From: aisha-partha <153170327+aisha-partha@users.noreply.github.com>
Date: Wed, 8 May 2024 23:23:17 +0530
Subject: [PATCH 66/74] Closes #6342. Update documentation on
 METAREFRESH_IGNORE_TAGS default value change

---
 docs/topics/downloader-middleware.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3f90cf2ed..2663a3cf3 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -881,6 +881,10 @@ Default: ``['noscript']``
 Meta tags within these tags are ignored.
 
 .. versionchanged:: 2.0
+   The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
+   ``['script', 'noscript']`` to ``[]``.
+
+.. versionchanges:: VERSION
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
    ``[]`` to ``['noscript']``.
 

From 2e13a9b8e19ae1c0f595435d7d3ce096c3b29ced Mon Sep 17 00:00:00 2001
From: aisha-partha <153170327+aisha-partha@users.noreply.github.com>
Date: Wed, 8 May 2024 23:33:13 +0530
Subject: [PATCH 67/74] Update METAREFRESH_IGNORE_TAGS default value in
 documentation

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 2663a3cf3..01bde772c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -876,7 +876,7 @@ Whether the Meta Refresh middleware will be enabled.
 METAREFRESH_IGNORE_TAGS
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Default: ``['noscript']``
+Default: ``[]``
 
 Meta tags within these tags are ignored.
 

From fe163d98ea81b4aff82bb9b194c4b0268c44f1db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 8 May 2024 20:41:20 +0200
Subject: [PATCH 68/74] Fix typo

---
 docs/topics/downloader-middleware.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 01bde772c..d4cd062fe 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -884,7 +884,7 @@ Meta tags within these tags are ignored.
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
    ``['script', 'noscript']`` to ``[]``.
 
-.. versionchanges:: VERSION
+.. versionchanged:: VERSION
    The default value of :setting:`METAREFRESH_IGNORE_TAGS` changed from
    ``[]`` to ``['noscript']``.
 

From ae7bb849f50af0b91eea4f022d93ad201e545c06 Mon Sep 17 00:00:00 2001
From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com>
Date: Fri, 10 May 2024 15:13:49 +0530
Subject: [PATCH 69/74] Make certain args of ScrapyAgent and TunnelingAgent
 required (#6349)

---
 scrapy/core/downloader/handlers/http11.py | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 40ae1921a..5e84be6ba 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -239,15 +239,14 @@ class TunnelingAgent(Agent):
 
     def __init__(
         self,
+        *,
         reactor: ReactorBase,
         proxyConf: Tuple[str, int, Optional[bytes]],
-        contextFactory: Optional[IPolicyForHTTPS] = None,
+        contextFactory: IPolicyForHTTPS,
         connectTimeout: Optional[float] = None,
         bindAddress: Optional[bytes] = None,
         pool: Optional[HTTPConnectionPool] = None,
     ):
-        # TODO make this arg required instead
-        assert contextFactory is not None
         super().__init__(reactor, contextFactory, connectTimeout, bindAddress, pool)
         self._proxyConf: Tuple[str, int, Optional[bytes]] = proxyConf
         self._contextFactory: IPolicyForHTTPS = contextFactory
@@ -335,18 +334,16 @@ class ScrapyAgent:
 
     def __init__(
         self,
-        contextFactory: Optional[IPolicyForHTTPS] = None,
+        *,
+        contextFactory: IPolicyForHTTPS,
         connectTimeout: float = 10,
         bindAddress: Optional[bytes] = None,
         pool: Optional[HTTPConnectionPool] = None,
         maxsize: int = 0,
         warnsize: int = 0,
         fail_on_dataloss: bool = True,
-        crawler: Optional[Crawler] = None,
+        crawler: Crawler,
     ):
-        # TODO make these args required instead
-        assert contextFactory is not None
-        assert crawler is not None
         self._contextFactory: IPolicyForHTTPS = contextFactory
         self._connectTimeout: float = connectTimeout
         self._bindAddress: Optional[bytes] = bindAddress

From c9ef5209365bb820ba8f2a3cd9df9fdeca0c9591 Mon Sep 17 00:00:00 2001
From: Sanchay Kumar <51812506+kumar-sanchay@users.noreply.github.com>
Date: Fri, 10 May 2024 17:56:45 +0530
Subject: [PATCH 70/74] Add Downloader.get_slot_key() without a spider
 parameter (#6352)

---
 docs/news.rst                      | 14 ++++++++++++++
 scrapy/core/downloader/__init__.py | 14 ++++++++++++--
 scrapy/pqueues.py                  |  2 +-
 tests/test_scheduler.py            |  6 +++---
 4 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index fafea0bf8..7db4e59a1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,20 @@
 Release notes
 =============
 
+
+.. _release-VERSION:
+
+Scrapy VERSION (YYYY-MM-DD)
+---------------------------
+
+Deprecations
+~~~~~~~~~~~~
+
+-   :func:`scrapy.core.downloader.Downloader._get_slot_key` is now deprecated.
+    Consider using its corresponding public method get_slot_key() instead.
+    (:issue:`6340`)
+
+
 .. _release-2.11.1:
 
 Scrapy 2.11.1 (2024-02-14)
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 98e1af6fb..0ab3bdb77 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -1,4 +1,5 @@
 import random
+import warnings
 from collections import deque
 from datetime import datetime
 from time import time
@@ -10,6 +11,7 @@ from twisted.internet.defer import Deferred
 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
 from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response
 from scrapy.resolver import dnscache
 from scrapy.settings import BaseSettings
@@ -125,7 +127,7 @@ class Downloader:
         return len(self.active) >= self.total_concurrency
 
     def _get_slot(self, request: Request, spider: Spider) -> Tuple[str, Slot]:
-        key = self._get_slot_key(request, spider)
+        key = self.get_slot_key(request)
         if key not in self.slots:
             slot_settings = self.per_slot_settings.get(key, {})
             conc = (
@@ -143,7 +145,7 @@ class Downloader:
 
         return key, self.slots[key]
 
-    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+    def get_slot_key(self, request: Request) -> str:
         if self.DOWNLOAD_SLOT in request.meta:
             return cast(str, request.meta[self.DOWNLOAD_SLOT])
 
@@ -153,6 +155,14 @@ class Downloader:
 
         return key
 
+    def _get_slot_key(self, request: Request, spider: Optional[Spider]) -> str:
+        warnings.warn(
+            "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return self.get_slot_key(request)
+
     def _enqueue_request(self, request: Request, spider: Spider) -> Deferred:
         key, slot = self._get_slot(request, spider)
         request.meta[self.DOWNLOAD_SLOT] = key
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 773825c5e..58a47ef0f 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -180,7 +180,7 @@ class DownloaderInterface:
         return [(self._active_downloads(slot), slot) for slot in possible_slots]
 
     def get_slot_key(self, request: Request) -> str:
-        return self.downloader._get_slot_key(request, None)
+        return self.downloader.get_slot_key(request)
 
     def _active_downloads(self, slot: str) -> int:
         """Return a number of requests in a Downloader for a given slot"""
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 37099dae6..02b50baa3 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -25,7 +25,7 @@ class MockDownloader:
     def __init__(self):
         self.slots = {}
 
-    def _get_slot_key(self, request, spider):
+    def get_slot_key(self, request):
         if Downloader.DOWNLOAD_SLOT in request.meta:
             return request.meta[Downloader.DOWNLOAD_SLOT]
 
@@ -273,14 +273,14 @@ class DownloaderAwareSchedulerTestMixin:
         while self.scheduler.has_pending_requests():
             request = self.scheduler.next_request()
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             dequeued_slots.append(slot)
             downloader.increment(slot)
             requests.append(request)
 
         for request in requests:
             # pylint: disable=protected-access
-            slot = downloader._get_slot_key(request, None)
+            slot = downloader.get_slot_key(request)
             downloader.decrement(slot)
 
         self.assertTrue(

From 93f06285309bd46e96fd147bf41e564c94b5bf2b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 13 May 2024 13:55:45 +0400
Subject: [PATCH 71/74] Improve typing for Spider.parse(). (#6274)

---
 .github/workflows/checks.yml           |  3 ++
 scrapy/commands/bench.py               |  2 +-
 scrapy/spiders/__init__.py             | 19 +++++--
 tests_typing/test_spiders.mypy-testing | 68 ++++++++++++++++++++++++++
 tox.ini                                |  9 ++++
 5 files changed, 95 insertions(+), 6 deletions(-)
 create mode 100644 tests_typing/test_spiders.mypy-testing

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index d6fc0f6c5..ed1629b67 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -18,6 +18,9 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: typing
+        - python-version: 3.8
+          env:
+            TOXENV: typing-tests
         - python-version: "3.11"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
diff --git a/scrapy/commands/bench.py b/scrapy/commands/bench.py
index 7523f3cfe..0c4ebcd23 100644
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@@ -60,7 +60,7 @@ class _BenchSpider(scrapy.Spider):
         url = f"{self.baseurl}?{urlencode(qargs, doseq=True)}"
         return [scrapy.Request(url, dont_filter=True)]
 
-    def parse(self, response: Response) -> Any:  # type: ignore[override]
+    def parse(self, response: Response) -> Any:
         assert isinstance(Response, TextResponse)
         for link in self.link_extractor.extract_links(response):
             yield scrapy.Request(link.url, callback=self.parse)
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 72c2aaba7..2416d2a4d 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -17,12 +17,17 @@ from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 
 if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    # typing.Concatenate requires Python 3.10
     # typing.Self requires Python 3.11
-    from typing_extensions import Self
+    from typing_extensions import Concatenate, Self
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
 
+    CallbackT = Callable[Concatenate[Response, ...], Any]
+
 
 class Spider(object_ref):
     """Base class for scrapy spiders. All spiders must inherit from this
@@ -79,10 +84,14 @@ class Spider(object_ref):
     def _parse(self, response: Response, **kwargs: Any) -> Any:
         return self.parse(response, **kwargs)
 
-    def parse(self, response: Response, **kwargs: Any) -> Any:
-        raise NotImplementedError(
-            f"{self.__class__.__name__}.parse callback is not defined"
-        )
+    if TYPE_CHECKING:
+        parse: CallbackT
+    else:
+
+        def parse(self, response: Response, **kwargs: Any) -> Any:
+            raise NotImplementedError(
+                f"{self.__class__.__name__}.parse callback is not defined"
+            )
 
     @classmethod
     def update_settings(cls, settings: BaseSettings) -> None:
diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing
new file mode 100644
index 000000000..162e31d0c
--- /dev/null
+++ b/tests_typing/test_spiders.mypy-testing
@@ -0,0 +1,68 @@
+from typing import Any
+
+import pytest
+
+from scrapy.http import HtmlResponse, Response
+from scrapy.spiders import Spider
+
+
+class SimpleSpider(Spider):
+    pass
+
+
+class SameOverrideSpider(Spider):
+    def parse(self, response: Response, **kwargs: Any) -> Any:
+        pass
+
+
+class NoKwargsSpider(Spider):
+    def parse(self, response: Response) -> Any:
+        pass
+
+
+class SpecificKwargsSpider(Spider):
+    def parse(self, response: Response, page: int) -> Any:
+        pass
+
+
+class NarrowOverrideSpider(Spider):
+    # without type: ignore this produces several note lines in addition to an error line,
+    # which is unsupported by pytest-mypy-testing
+    def parse(self, response: HtmlResponse, **kwargs: Any) -> Any:  # type: ignore[override]
+        pass
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse() -> None:
+    spider = Spider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_simple() -> None:
+    spider = SimpleSpider()
+    reveal_type(spider.parse)  # R: def (scrapy.http.response.Response, *Any, **Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_same() -> None:
+    spider = SameOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, **kwargs: Any) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_no_kwargs() -> None:
+    spider = NoKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_specific_kwargs() -> None:
+    spider = SpecificKwargsSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+
+
+@pytest.mark.mypy_testing
+def test_spider_parse_override_narrow() -> None:
+    spider = NarrowOverrideSpider()
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.html.HtmlResponse, **kwargs: Any) -> Any
diff --git a/tox.ini b/tox.ini
index d7527bb04..ede139756 100644
--- a/tox.ini
+++ b/tox.ini
@@ -56,6 +56,15 @@ deps =
 commands =
     mypy {posargs: scrapy tests}
 
+[testenv:typing-tests]
+basepython = python3.8
+deps =
+    {[test-requirements]deps}
+    {[testenv:typing]deps}
+    pytest-mypy-testing==0.1.3
+commands =
+    pytest {posargs: tests_typing}
+
 [testenv:pre-commit]
 basepython = python3
 deps =

From 4ed5c5ae91318768efa338680df67337ed0f67fd Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 13 May 2024 14:01:52 +0400
Subject: [PATCH 72/74] Use ParamSpec for callables. (#6353)

---
 scrapy/cmdline.py                      |  15 ++-
 scrapy/core/downloader/handlers/ftp.py |  10 +-
 scrapy/core/engine.py                  |   2 +-
 scrapy/core/scraper.py                 |  10 +-
 scrapy/core/spidermw.py                |   6 +-
 scrapy/mail.py                         |   8 +-
 scrapy/pipelines/files.py              |   3 +-
 scrapy/pipelines/media.py              |  27 ++---
 scrapy/shell.py                        |   4 +-
 scrapy/utils/decorators.py             |  29 ++++--
 scrapy/utils/defer.py                  | 135 +++++++++++++++++--------
 scrapy/utils/python.py                 |  23 ++++-
 scrapy/utils/reactor.py                |  33 ++++--
 13 files changed, 204 insertions(+), 101 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 4df5698a6..da0e51386 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -1,10 +1,12 @@
+from __future__ import annotations
+
 import argparse
 import cProfile
 import inspect
 import os
 import sys
 from importlib.metadata import entry_points
-from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, Type
+from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Optional, Tuple, Type
 
 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@@ -15,6 +17,12 @@ from scrapy.utils.misc import walk_modules
 from scrapy.utils.project import get_project_settings, inside_project
 from scrapy.utils.python import garbage_collect
 
+if TYPE_CHECKING:
+    # typing.ParamSpec requires Python 3.10
+    from typing_extensions import ParamSpec
+
+    _P = ParamSpec("_P")
+
 
 class ScrapyArgumentParser(argparse.ArgumentParser):
     def _parse_optional(
@@ -121,7 +129,10 @@ def _print_unknown_command(
 
 
 def _run_print_help(
-    parser: argparse.ArgumentParser, func: Callable, *a: Any, **kw: Any
+    parser: argparse.ArgumentParser,
+    func: Callable[_P, None],
+    *a: _P.args,
+    **kw: _P.kwargs,
 ) -> None:
     try:
         func(*a, **kw)
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index ed94b2221..77dcf3c38 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -109,12 +109,10 @@ class FTPDownloadHandler:
     def gotClient(self, client: FTPClient, request: Request, filepath: str) -> Deferred:
         self.client = client
         protocol = ReceivedDataProtocol(request.meta.get("ftp_local_filename"))
-        return client.retrieveFile(filepath, protocol).addCallbacks(
-            callback=self._build_response,
-            callbackArgs=(request, protocol),
-            errback=self._failed,
-            errbackArgs=(request,),
-        )
+        d = client.retrieveFile(filepath, protocol)
+        d.addCallback(self._build_response, request, protocol)
+        d.addErrback(self._failed, request)
+        return d
 
     def _build_response(
         self, result: Any, request: Request, protocol: ReceivedDataProtocol
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 93a0c51bc..6bf3f3e26 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -347,7 +347,7 @@ class ExecutionEngine:
 
         assert self.spider is not None
         dwld = self.downloader.fetch(request, self.spider)
-        dwld.addCallbacks(_on_success)
+        dwld.addCallback(_on_success)
         dwld.addBoth(_on_complete)
         return dwld
 
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 272841e01..566e6628b 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -8,7 +8,6 @@ from collections import deque
 from typing import (
     TYPE_CHECKING,
     Any,
-    AsyncGenerator,
     AsyncIterable,
     Deque,
     Generator,
@@ -18,6 +17,7 @@ from typing import (
     Tuple,
     Type,
     Union,
+    cast,
 )
 
 from itemadapter import is_item
@@ -184,7 +184,9 @@ class Scraper:
             result, request, spider
         )  # returns spider's processed output
         dfd.addErrback(self.handle_spider_error, request, result, spider)
-        dfd.addCallback(self.handle_spider_output, request, result, spider)
+        dfd.addCallback(
+            self.handle_spider_output, request, cast(Response, result), spider
+        )
         return dfd
 
     def _scrape2(
@@ -256,12 +258,12 @@ class Scraper:
         self,
         result: Union[Iterable, AsyncIterable],
         request: Request,
-        response: Union[Response, Failure],
+        response: Response,
         spider: Spider,
     ) -> Deferred:
         if not result:
             return defer_succeed(None)
-        it: Union[Generator, AsyncGenerator]
+        it: Union[Iterable, AsyncIterable]
         if isinstance(result, AsyncIterable):
             it = aiter_errback(
                 result, self.handle_spider_error, request, response, spider
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 1ccfd08a2..2cef2e1dd 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -303,10 +303,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
         dfd = mustbe_deferred(
             self._process_spider_input, scrape_func, response, request, spider
         )
-        dfd.addCallbacks(
-            callback=deferred_f_from_coro_f(process_callback_output),
-            errback=process_spider_exception,
-        )
+        dfd.addCallback(deferred_f_from_coro_f(process_callback_output))
+        dfd.addErrback(process_spider_exception)
         return dfd
 
     def process_start_requests(
diff --git a/scrapy/mail.py b/scrapy/mail.py
index 56adba934..fd6302550 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -154,12 +154,8 @@ class MailSender:
             return None
 
         dfd = self._sendmail(rcpts, msg.as_string().encode(charset or "utf-8"))
-        dfd.addCallbacks(
-            callback=self._sent_ok,
-            errback=self._sent_failed,
-            callbackArgs=(to, cc, subject, len(attachs)),
-            errbackArgs=(to, cc, subject, len(attachs)),
-        )
+        dfd.addCallback(self._sent_ok, to, cc, subject, len(attachs))
+        dfd.addErrback(self._sent_failed, to, cc, subject, len(attachs))
         reactor.addSystemEventTrigger("before", "shutdown", lambda: dfd)
         return dfd
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index d00f44502..47457f2a8 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -459,7 +459,8 @@ class FilesPipeline(MediaPipeline):
 
         path = self.file_path(request, info=info, item=item)
         dfd = defer.maybeDeferred(self.store.stat_file, path, info)
-        dfd.addCallbacks(_onsuccess, lambda _: None)
+        dfd.addCallback(_onsuccess)
+        dfd.addErrback(lambda _: None)
         dfd.addErrback(
             lambda f: logger.error(
                 self.__class__.__name__ + ".store.stat_file",
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index fd5e70cb9..5f6c5cb07 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -106,10 +106,17 @@ class MediaPipeline:
 
         # Return cached result if request was already seen
         if fp in info.downloaded:
-            return defer_result(info.downloaded[fp]).addCallbacks(cb, eb)
+            d = defer_result(info.downloaded[fp])
+            d.addCallback(cb)
+            if eb:
+                d.addErrback(eb)
+            return d
 
         # Otherwise, wait for result
-        wad = Deferred().addCallbacks(cb, eb)
+        wad = Deferred()
+        wad.addCallback(cb)
+        if eb:
+            wad.addErrback(eb)
         info.waiting[fp].append(wad)
 
         # Check if request is downloading right now to avoid doing it twice
@@ -140,23 +147,11 @@ class MediaPipeline:
         if self.download_func:
             # this ugly code was left only to support tests. TODO: remove
             dfd = mustbe_deferred(self.download_func, request, info.spider)
-            dfd.addCallbacks(
-                callback=self.media_downloaded,
-                callbackArgs=(request, info),
-                callbackKeywords={"item": item},
-                errback=self.media_failed,
-                errbackArgs=(request, info),
-            )
         else:
             self._modify_media_request(request)
             dfd = self.crawler.engine.download(request)
-            dfd.addCallbacks(
-                callback=self.media_downloaded,
-                callbackArgs=(request, info),
-                callbackKeywords={"item": item},
-                errback=self.media_failed,
-                errbackArgs=(request, info),
-            )
+        dfd.addCallback(self.media_downloaded, request, info, item=item)
+        dfd.addErrback(self.media_failed, request, info)
         return dfd
 
     def _cache_result_and_execute_waiters(self, result, fp, info):
diff --git a/scrapy/shell.py b/scrapy/shell.py
index fac42e8a2..2c22d3d8f 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -231,7 +231,9 @@ def _request_deferred(request: Request) -> defer.Deferred:
     d: defer.Deferred = defer.Deferred()
     d.addBoth(_restore_callbacks)
     if request.callback:
-        d.addCallbacks(request.callback, request.errback)
+        d.addCallback(request.callback)
+    if request.errback:
+        d.addErrback(request.errback)
 
     request.callback, request.errback = d.callback, d.errback
     return d
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 04186559f..7e82dd519 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -1,21 +1,34 @@
+from __future__ import annotations
+
 import warnings
 from functools import wraps
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any, Callable, TypeVar
 
 from twisted.internet import defer, threads
 from twisted.internet.defer import Deferred
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
+if TYPE_CHECKING:
+    # typing.ParamSpec requires Python 3.10
+    from typing_extensions import ParamSpec
 
-def deprecated(use_instead: Any = None) -> Callable:
+    _P = ParamSpec("_P")
+
+
+_T = TypeVar("_T")
+
+
+def deprecated(
+    use_instead: Any = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used."""
 
-    def deco(func: Callable) -> Callable:
+    def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:
         @wraps(func)
-        def wrapped(*args: Any, **kwargs: Any) -> Any:
+        def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> Any:
             message = f"Call to deprecated function {func.__name__}."
             if use_instead:
                 message += f" Use {use_instead} instead."
@@ -30,23 +43,23 @@ def deprecated(use_instead: Any = None) -> Callable:
     return deco
 
 
-def defers(func: Callable) -> Callable[..., Deferred]:
+def defers(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
     """Decorator to make sure a function always returns a deferred"""
 
     @wraps(func)
-    def wrapped(*a: Any, **kw: Any) -> Deferred:
+    def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
         return defer.maybeDeferred(func, *a, **kw)
 
     return wrapped
 
 
-def inthread(func: Callable) -> Callable[..., Deferred]:
+def inthread(func: Callable[_P, _T]) -> Callable[_P, Deferred[_T]]:
     """Decorator to call a function in a thread and return a deferred with the
     result
     """
 
     @wraps(func)
-    def wrapped(*a: Any, **kw: Any) -> Deferred:
+    def wrapped(*a: _P.args, **kw: _P.kwargs) -> Deferred[_T]:
         return threads.deferToThread(func, *a, **kw)
 
     return wrapped
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index c391db9fd..abb7e1726 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -2,21 +2,22 @@
 Helper functions for dealing with Twisted deferreds
 """
 
+from __future__ import annotations
+
 import asyncio
 import inspect
 from asyncio import Future
 from functools import wraps
 from types import CoroutineType
 from typing import (
+    TYPE_CHECKING,
     Any,
-    AsyncGenerator,
     AsyncIterable,
     AsyncIterator,
     Awaitable,
     Callable,
     Coroutine,
     Dict,
-    Generator,
     Iterable,
     Iterator,
     List,
@@ -37,6 +38,14 @@ from twisted.python.failure import Failure
 from scrapy.exceptions import IgnoreRequest
 from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed
 
+if TYPE_CHECKING:
+    # typing.Concatenate and typing.ParamSpec require Python 3.10
+    from typing_extensions import Concatenate, ParamSpec
+
+    _P = ParamSpec("_P")
+
+_T = TypeVar("_T")
+
 
 def defer_fail(_failure: Failure) -> Deferred:
     """Same as twisted.internet.defer.fail but delay calling errback until
@@ -74,7 +83,31 @@ def defer_result(result: Any) -> Deferred:
     return defer_succeed(result)
 
 
-def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred:
+@overload
+def mustbe_deferred(
+    f: Callable[_P, Deferred[_T]], *args: _P.args, **kw: _P.kwargs
+) -> Deferred[_T]: ...
+
+
+@overload
+def mustbe_deferred(
+    f: Callable[_P, Coroutine[Deferred[Any], Any, _T]],
+    *args: _P.args,
+    **kw: _P.kwargs,
+) -> Deferred[_T]: ...
+
+
+@overload
+def mustbe_deferred(
+    f: Callable[_P, _T], *args: _P.args, **kw: _P.kwargs
+) -> Deferred[_T]: ...
+
+
+def mustbe_deferred(
+    f: Callable[_P, Union[Deferred[_T], Coroutine[Deferred[Any], Any, _T], _T]],
+    *args: _P.args,
+    **kw: _P.kwargs,
+) -> Deferred[_T]:
     """Same as twisted.internet.defer.maybeDeferred, but delay calling
     callback/errback to next reactor loop
     """
@@ -92,7 +125,11 @@ def mustbe_deferred(f: Callable, *args: Any, **kw: Any) -> Deferred:
 
 
 def parallel(
-    iterable: Iterable, count: int, callable: Callable, *args: Any, **named: Any
+    iterable: Iterable[_T],
+    count: int,
+    callable: Callable[Concatenate[_T, _P], Any],
+    *args: _P.args,
+    **named: _P.kwargs,
 ) -> Deferred:
     """Execute a callable over the objects in the given iterable, in parallel,
     using no more than ``count`` concurrent calls.
@@ -104,7 +141,7 @@ def parallel(
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-class _AsyncCooperatorAdapter(Iterator):
+class _AsyncCooperatorAdapter(Iterator[Deferred]):
     """A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more
@@ -152,28 +189,30 @@ class _AsyncCooperatorAdapter(Iterator):
 
     def __init__(
         self,
-        aiterable: AsyncIterable,
-        callable: Callable,
-        *callable_args: Any,
-        **callable_kwargs: Any,
+        aiterable: AsyncIterable[_T],
+        callable: Callable[Concatenate[_T, _P], Any],
+        *callable_args: _P.args,
+        **callable_kwargs: _P.kwargs,
     ):
-        self.aiterator: AsyncIterator = aiterable.__aiter__()
-        self.callable: Callable = callable
+        self.aiterator: AsyncIterator[_T] = aiterable.__aiter__()
+        self.callable: Callable[Concatenate[_T, _P], Any] = callable
         self.callable_args: Tuple[Any, ...] = callable_args
         self.callable_kwargs: Dict[str, Any] = callable_kwargs
         self.finished: bool = False
         self.waiting_deferreds: List[Deferred] = []
-        self.anext_deferred: Optional[Deferred] = None
+        self.anext_deferred: Optional[Deferred[_T]] = None
 
-    def _callback(self, result: Any) -> None:
+    def _callback(self, result: _T) -> None:
         # This gets called when the result from aiterator.__anext__() is available.
         # It calls the callable on it and sends the result to the oldest waiting Deferred
         # (by chaining if the result is a Deferred too or by firing if not).
         self.anext_deferred = None
-        result = self.callable(result, *self.callable_args, **self.callable_kwargs)
+        callable_result = self.callable(
+            result, *self.callable_args, **self.callable_kwargs
+        )
         d = self.waiting_deferreds.pop(0)
-        if isinstance(result, Deferred):
-            result.chainDeferred(d)
+        if isinstance(callable_result, Deferred):
+            callable_result.chainDeferred(d)
         else:
             d.callback(None)
         if self.waiting_deferreds:
@@ -207,11 +246,11 @@ class _AsyncCooperatorAdapter(Iterator):
 
 
 def parallel_async(
-    async_iterable: AsyncIterable,
+    async_iterable: AsyncIterable[_T],
     count: int,
-    callable: Callable,
-    *args: Any,
-    **named: Any,
+    callable: Callable[Concatenate[_T, _P], Any],
+    *args: _P.args,
+    **named: _P.kwargs,
 ) -> Deferred:
     """Like parallel but for async iterators"""
     coop = Cooperator()
@@ -221,7 +260,10 @@ def parallel_async(
 
 
 def process_chain(
-    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+    callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
+    input: Any,
+    *a: _P.args,
+    **kw: _P.kwargs,
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
     d: Deferred = Deferred()
@@ -232,23 +274,17 @@ def process_chain(
 
 
 def process_chain_both(
-    callbacks: Iterable[Callable],
-    errbacks: Iterable[Callable],
+    callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
+    errbacks: Iterable[Callable[Concatenate[Failure, _P], Any]],
     input: Any,
-    *a: Any,
-    **kw: Any,
+    *a: _P.args,
+    **kw: _P.kwargs,
 ) -> Deferred:
     """Return a Deferred built by chaining the given callbacks and errbacks"""
     d: Deferred = Deferred()
     for cb, eb in zip(callbacks, errbacks):
-        d.addCallbacks(
-            callback=cb,
-            errback=eb,
-            callbackArgs=a,
-            callbackKeywords=kw,
-            errbackArgs=a,
-            errbackKeywords=kw,
-        )
+        d.addCallback(cb, *a, **kw)
+        d.addErrback(eb, *a, **kw)
     if isinstance(input, failure.Failure):
         d.errback(input)
     else:
@@ -257,20 +293,27 @@ def process_chain_both(
 
 
 def process_parallel(
-    callbacks: Iterable[Callable], input: Any, *a: Any, **kw: Any
+    callbacks: Iterable[Callable[Concatenate[_T, _P], Any]],
+    input: Any,
+    *a: _P.args,
+    **kw: _P.kwargs,
 ) -> Deferred:
     """Return a Deferred with the output of all successful calls to the given
     callbacks
     """
     dfds = [defer.succeed(input).addCallback(x, *a, **kw) for x in callbacks]
     d: Deferred = DeferredList(dfds, fireOnOneErrback=True, consumeErrors=True)
-    d.addCallbacks(lambda r: [x[1] for x in r], lambda f: f.value.subFailure)
+    d.addCallback(lambda r: [x[1] for x in r])
+    d.addErrback(lambda f: f.value.subFailure)
     return d
 
 
 def iter_errback(
-    iterable: Iterable, errback: Callable, *a: Any, **kw: Any
-) -> Generator:
+    iterable: Iterable[_T],
+    errback: Callable[Concatenate[Failure, _P], Any],
+    *a: _P.args,
+    **kw: _P.kwargs,
+) -> Iterable[_T]:
     """Wraps an iterable calling an errback if an error is caught while
     iterating it.
     """
@@ -285,8 +328,11 @@ def iter_errback(
 
 
 async def aiter_errback(
-    aiterable: AsyncIterable, errback: Callable, *a: Any, **kw: Any
-) -> AsyncGenerator:
+    aiterable: AsyncIterable[_T],
+    errback: Callable[Concatenate[Failure, _P], Any],
+    *a: _P.args,
+    **kw: _P.kwargs,
+) -> AsyncIterable[_T]:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
     """
@@ -301,7 +347,6 @@ async def aiter_errback(
 
 
 _CT = TypeVar("_CT", bound=Union[Awaitable, CoroutineType, Future])
-_T = TypeVar("_T")
 
 
 @overload
@@ -327,7 +372,9 @@ def deferred_from_coro(o: _T) -> Union[Deferred, _T]:
     return o
 
 
-def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
+def deferred_f_from_coro_f(
+    coro_f: Callable[_P, Coroutine[Any, Any, _T]]
+) -> Callable[_P, Deferred[_T]]:
     """Converts a coroutine function into a function that returns a Deferred.
 
     The coroutine function will be called at the time when the wrapper is called. Wrapper args will be passed to it.
@@ -335,13 +382,15 @@ def deferred_f_from_coro_f(coro_f: Callable[..., Coroutine]) -> Callable:
     """
 
     @wraps(coro_f)
-    def f(*coro_args: Any, **coro_kwargs: Any) -> Any:
+    def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Any:
         return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
 
     return f
 
 
-def maybeDeferred_coro(f: Callable, *args: Any, **kw: Any) -> Deferred:
+def maybeDeferred_coro(
+    f: Callable[_P, Any], *args: _P.args, **kw: _P.kwargs
+) -> Deferred:
     """Copy of defer.maybeDeferred that also converts coroutines to Deferreds."""
     try:
         result = f(*args, **kw)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 1e7364e49..5d2d490b2 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -2,6 +2,8 @@
 This module contains essential stuff that should've come with Python itself ;)
 """
 
+from __future__ import annotations
+
 import collections.abc
 import gc
 import inspect
@@ -11,6 +13,7 @@ import weakref
 from functools import partial, wraps
 from itertools import chain
 from typing import (
+    TYPE_CHECKING,
     Any,
     AsyncGenerator,
     AsyncIterable,
@@ -25,12 +28,21 @@ from typing import (
     Optional,
     Pattern,
     Tuple,
+    TypeVar,
     Union,
     overload,
 )
 
 from scrapy.utils.asyncgen import as_async_generator
 
+if TYPE_CHECKING:
+    # typing.Concatenate and typing.ParamSpec require Python 3.10
+    from typing_extensions import Concatenate, ParamSpec
+
+    _P = ParamSpec("_P")
+
+_T = TypeVar("_T")
+
 
 def flatten(x: Iterable) -> list:
     """flatten(sequence) -> list
@@ -169,14 +181,19 @@ def re_rsearch(
     return None
 
 
-def memoizemethod_noargs(method: Callable) -> Callable:
+_SelfT = TypeVar("_SelfT")
+
+
+def memoizemethod_noargs(
+    method: Callable[Concatenate[_SelfT, _P], _T]
+) -> Callable[Concatenate[_SelfT, _P], _T]:
     """Decorator to cache the result of a method (without arguments) using a
     weak reference to its object
     """
-    cache: weakref.WeakKeyDictionary[Any, Any] = weakref.WeakKeyDictionary()
+    cache: weakref.WeakKeyDictionary[_SelfT, _T] = weakref.WeakKeyDictionary()
 
     @wraps(method)
-    def new_method(self: Any, *args: Any, **kwargs: Any) -> Any:
+    def new_method(self: _SelfT, *args: _P.args, **kwargs: _P.kwargs) -> _T:
         if self not in cache:
             cache[self] = method(self, *args, **kwargs)
         return cache[self]
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 6cde49bfe..5af6d22eb 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -1,8 +1,21 @@
+from __future__ import annotations
+
 import asyncio
 import sys
 from asyncio import AbstractEventLoop, AbstractEventLoopPolicy
 from contextlib import suppress
-from typing import Any, Callable, Dict, List, Optional, Sequence, Type
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Generic,
+    List,
+    Optional,
+    Tuple,
+    Type,
+    TypeVar,
+)
 from warnings import catch_warnings, filterwarnings, warn
 
 from twisted.internet import asyncioreactor, error
@@ -13,6 +26,14 @@ from twisted.internet.tcp import Port
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.misc import load_object
 
+if TYPE_CHECKING:
+    # typing.ParamSpec requires Python 3.10
+    from typing_extensions import ParamSpec
+
+    _P = ParamSpec("_P")
+
+_T = TypeVar("_T")
+
 
 def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port:  # type: ignore[return]
     """Like reactor.listenTCP but tries different ports in a range."""
@@ -32,14 +53,14 @@ def listen_tcp(portrange: List[int], host: str, factory: ServerFactory) -> Port:
                 raise
 
 
-class CallLaterOnce:
+class CallLaterOnce(Generic[_T]):
     """Schedule a function to be called in the next reactor loop, but only if
     it hasn't been already scheduled since the last time it ran.
     """
 
-    def __init__(self, func: Callable, *a: Any, **kw: Any):
-        self._func: Callable = func
-        self._a: Sequence[Any] = a
+    def __init__(self, func: Callable[_P, _T], *a: _P.args, **kw: _P.kwargs):
+        self._func: Callable[_P, _T] = func
+        self._a: Tuple[Any, ...] = a
         self._kw: Dict[str, Any] = kw
         self._call: Optional[DelayedCall] = None
 
@@ -53,7 +74,7 @@ class CallLaterOnce:
         if self._call:
             self._call.cancel()
 
-    def __call__(self) -> Any:
+    def __call__(self) -> _T:
         self._call = None
         return self._func(*self._a, **self._kw)
 

From b8e333c8ce78c195e265fa453e7bc0b11c5ca4af Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 13 May 2024 14:07:51 +0400
Subject: [PATCH 73/74] Full typing for scrapy/spiders. (#6356)

---
 scrapy/spiders/__init__.py |   6 +-
 scrapy/spiders/crawl.py    | 138 +++++++++++++++++++++++++------------
 scrapy/spiders/feed.py     |  48 ++++++++-----
 scrapy/spiders/init.py     |  14 ++--
 scrapy/spiders/sitemap.py  |  49 ++++++++-----
 scrapy/utils/sitemap.py    |   4 +-
 6 files changed, 172 insertions(+), 87 deletions(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 2416d2a4d..370801f28 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -39,7 +39,7 @@ class Spider(object_ref):
 
     def __init__(self, name: Optional[str] = None, **kwargs: Any):
         if name is not None:
-            self.name = name
+            self.name: str = name
         elif not getattr(self, "name", None):
             raise ValueError(f"{type(self).__name__} must have a name")
         self.__dict__.update(kwargs)
@@ -67,8 +67,8 @@ class Spider(object_ref):
         return spider
 
     def _set_crawler(self, crawler: Crawler) -> None:
-        self.crawler = crawler
-        self.settings = crawler.settings
+        self.crawler: Crawler = crawler
+        self.settings: BaseSettings = crawler.settings
         crawler.signals.connect(self.close, signals.spider_closed)
 
     def start_requests(self) -> Iterable[Request]:
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index ba8b7b366..48c830d2a 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -8,9 +8,27 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import copy
-from typing import TYPE_CHECKING, AsyncIterable, Awaitable, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterable,
+    Awaitable,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Set,
+    TypeVar,
+    Union,
+    cast,
+)
+
+from twisted.python.failure import Failure
 
 from scrapy.http import HtmlResponse, Request, Response
+from scrapy.link import Link
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.utils.asyncgen import collect_asyncgen
@@ -20,20 +38,32 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
+    from scrapy.crawler import Crawler
 
-def _identity(x):
+
+_T = TypeVar("_T")
+ProcessLinksT = Callable[[List[Link]], List[Link]]
+ProcessRequestT = Callable[[Request, Response], Optional[Request]]
+
+
+def _identity(x: _T) -> _T:
     return x
 
 
-def _identity_process_request(request, response):
+def _identity_process_request(
+    request: Request, response: Response
+) -> Optional[Request]:
     return request
 
 
-def _get_method(method, spider):
+def _get_method(
+    method: Union[Callable, str, None], spider: Spider
+) -> Optional[Callable]:
     if callable(method):
         return method
     if isinstance(method, str):
         return getattr(spider, method, None)
+    return None
 
 
 _default_link_extractor = LinkExtractor()
@@ -42,37 +72,46 @@ _default_link_extractor = LinkExtractor()
 class Rule:
     def __init__(
         self,
-        link_extractor=None,
-        callback=None,
-        cb_kwargs=None,
-        follow=None,
-        process_links=None,
-        process_request=None,
-        errback=None,
+        link_extractor: Optional[LinkExtractor] = None,
+        callback: Union[Callable, str, None] = None,
+        cb_kwargs: Optional[Dict[str, Any]] = None,
+        follow: Optional[bool] = None,
+        process_links: Union[ProcessLinksT, str, None] = None,
+        process_request: Union[ProcessRequestT, str, None] = None,
+        errback: Union[Callable[[Failure], Any], str, None] = None,
     ):
-        self.link_extractor = link_extractor or _default_link_extractor
-        self.callback = callback
-        self.errback = errback
-        self.cb_kwargs = cb_kwargs or {}
-        self.process_links = process_links or _identity
-        self.process_request = process_request or _identity_process_request
-        self.follow = follow if follow is not None else not callback
+        self.link_extractor: LinkExtractor = link_extractor or _default_link_extractor
+        self.callback: Union[Callable, str, None] = callback
+        self.errback: Union[Callable[[Failure], Any], str, None] = errback
+        self.cb_kwargs: Dict[str, Any] = cb_kwargs or {}
+        self.process_links: Union[ProcessLinksT, str] = process_links or _identity
+        self.process_request: Union[ProcessRequestT, str] = (
+            process_request or _identity_process_request
+        )
+        self.follow: bool = follow if follow is not None else not callback
 
-    def _compile(self, spider):
+    def _compile(self, spider: Spider) -> None:
+        # this replaces method names with methods and we can't express this in type hints
         self.callback = _get_method(self.callback, spider)
-        self.errback = _get_method(self.errback, spider)
-        self.process_links = _get_method(self.process_links, spider)
-        self.process_request = _get_method(self.process_request, spider)
+        self.errback = cast(Callable[[Failure], Any], _get_method(self.errback, spider))
+        self.process_links = cast(
+            ProcessLinksT, _get_method(self.process_links, spider)
+        )
+        self.process_request = cast(
+            ProcessRequestT, _get_method(self.process_request, spider)
+        )
 
 
 class CrawlSpider(Spider):
     rules: Sequence[Rule] = ()
+    _rules: List[Rule]
+    _follow_links: bool
 
-    def __init__(self, *a, **kw):
+    def __init__(self, *a: Any, **kw: Any):
         super().__init__(*a, **kw)
         self._compile_rules()
 
-    def _parse(self, response, **kwargs):
+    def _parse(self, response: Response, **kwargs: Any) -> Any:
         return self._parse_response(
             response=response,
             callback=self.parse_start_url,
@@ -80,13 +119,13 @@ class CrawlSpider(Spider):
             follow=True,
         )
 
-    def parse_start_url(self, response, **kwargs):
+    def parse_start_url(self, response: Response, **kwargs: Any) -> Any:
         return []
 
-    def process_results(self, response: Response, results: list):
+    def process_results(self, response: Response, results: Any) -> Any:
         return results
 
-    def _build_request(self, rule_index, link):
+    def _build_request(self, rule_index: int, link: Link) -> Request:
         return Request(
             url=link.url,
             callback=self._callback,
@@ -94,32 +133,43 @@ class CrawlSpider(Spider):
             meta={"rule": rule_index, "link_text": link.text},
         )
 
-    def _requests_to_follow(self, response):
+    def _requests_to_follow(self, response: Response) -> Iterable[Optional[Request]]:
         if not isinstance(response, HtmlResponse):
             return
-        seen = set()
+        seen: Set[Link] = set()
         for rule_index, rule in enumerate(self._rules):
-            links = [
+            links: List[Link] = [
                 lnk
                 for lnk in rule.link_extractor.extract_links(response)
                 if lnk not in seen
             ]
-            for link in rule.process_links(links):
+            for link in cast(ProcessLinksT, rule.process_links)(links):
                 seen.add(link)
                 request = self._build_request(rule_index, link)
-                yield rule.process_request(request, response)
+                yield cast(ProcessRequestT, rule.process_request)(request, response)
 
-    def _callback(self, response, **cb_kwargs):
-        rule = self._rules[response.meta["rule"]]
+    def _callback(self, response: Response, **cb_kwargs: Any) -> Any:
+        rule = self._rules[cast(int, response.meta["rule"])]
         return self._parse_response(
-            response, rule.callback, {**rule.cb_kwargs, **cb_kwargs}, rule.follow
+            response,
+            cast(Callable, rule.callback),
+            {**rule.cb_kwargs, **cb_kwargs},
+            rule.follow,
         )
 
-    def _errback(self, failure):
-        rule = self._rules[failure.request.meta["rule"]]
-        return self._handle_failure(failure, rule.errback)
+    def _errback(self, failure: Failure) -> Iterable[Any]:
+        rule = self._rules[cast(int, failure.request.meta["rule"])]  # type: ignore[attr-defined]
+        return self._handle_failure(
+            failure, cast(Callable[[Failure], Any], rule.errback)
+        )
 
-    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(
+        self,
+        response: Response,
+        callback: Optional[Callable],
+        cb_kwargs: Dict[str, Any],
+        follow: bool = True,
+    ) -> AsyncIterable[Any]:
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
             if isinstance(cb_res, AsyncIterable):
@@ -134,21 +184,23 @@ class CrawlSpider(Spider):
             for request_or_item in self._requests_to_follow(response):
                 yield request_or_item
 
-    def _handle_failure(self, failure, errback):
+    def _handle_failure(
+        self, failure: Failure, errback: Optional[Callable[[Failure], Any]]
+    ) -> Iterable[Any]:
         if errback:
             results = errback(failure) or ()
             yield from iterate_spider_output(results)
 
-    def _compile_rules(self):
+    def _compile_rules(self) -> None:
         self._rules = []
         for rule in self.rules:
             self._rules.append(copy.copy(rule))
             self._rules[-1]._compile(self)
 
     @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs) -> Self:
+    def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
         spider = super().from_crawler(crawler, *args, **kwargs)
-        spider._follow_links = crawler.settings.getbool(  # type: ignore[attr-defined]
+        spider._follow_links = crawler.settings.getbool(
             "CRAWLSPIDER_FOLLOW_LINKS", True
         )
         return spider
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 5caf8c79e..9dd8a5d68 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -5,7 +5,10 @@ for scraping from an XML feed.
 See documentation in docs/topics/spiders.rst
 """
 
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
 from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.spiders import Spider
 from scrapy.utils.iterators import csviter, xmliter_lxml
@@ -22,11 +25,13 @@ class XMLFeedSpider(Spider):
     use iternodes, since it's a faster and cleaner.
     """
 
-    iterator = "iternodes"
-    itertag = "item"
-    namespaces = ()
+    iterator: str = "iternodes"
+    itertag: str = "item"
+    namespaces: Sequence[Tuple[str, str]] = ()
 
-    def process_results(self, response, results):
+    def process_results(
+        self, response: Response, results: Iterable[Any]
+    ) -> Iterable[Any]:
         """This overridable method is called for each result (item or request)
         returned by the spider, and it's intended to perform any last time
         processing required before returning the results to the framework core,
@@ -36,20 +41,20 @@ class XMLFeedSpider(Spider):
         """
         return results
 
-    def adapt_response(self, response):
+    def adapt_response(self, response: Response) -> Response:
         """You can override this function in order to make any changes you want
         to into the feed before parsing it. This function must return a
         response.
         """
         return response
 
-    def parse_node(self, response, selector):
+    def parse_node(self, response: Response, selector: Selector) -> Any:
         """This method must be overridden with your custom spider functionality"""
         if hasattr(self, "parse_item"):  # backward compatibility
             return self.parse_item(response, selector)
         raise NotImplementedError
 
-    def parse_nodes(self, response, nodes):
+    def parse_nodes(self, response: Response, nodes: Iterable[Selector]) -> Any:
         """This method is called for the nodes matching the provided tag name
         (itertag). Receives the response and an Selector for each node.
         Overriding this method is mandatory. Otherwise, you spider won't work.
@@ -61,20 +66,25 @@ class XMLFeedSpider(Spider):
             ret = iterate_spider_output(self.parse_node(response, selector))
             yield from self.process_results(response, ret)
 
-    def _parse(self, response, **kwargs):
+    def _parse(self, response: Response, **kwargs: Any) -> Any:
         if not hasattr(self, "parse_node"):
             raise NotConfigured(
                 "You must define parse_node method in order to scrape this XML feed"
             )
 
         response = self.adapt_response(response)
+        nodes: Iterable[Selector]
         if self.iterator == "iternodes":
             nodes = self._iternodes(response)
         elif self.iterator == "xml":
+            if not isinstance(response, TextResponse):
+                raise ValueError("Response content isn't text")
             selector = Selector(response, type="xml")
             self._register_namespaces(selector)
             nodes = selector.xpath(f"//{self.itertag}")
         elif self.iterator == "html":
+            if not isinstance(response, TextResponse):
+                raise ValueError("Response content isn't text")
             selector = Selector(response, type="html")
             self._register_namespaces(selector)
             nodes = selector.xpath(f"//{self.itertag}")
@@ -83,12 +93,12 @@ class XMLFeedSpider(Spider):
 
         return self.parse_nodes(response, nodes)
 
-    def _iternodes(self, response):
+    def _iternodes(self, response: Response) -> Iterable[Selector]:
         for node in xmliter_lxml(response, self.itertag):
             self._register_namespaces(node)
             yield node
 
-    def _register_namespaces(self, selector):
+    def _register_namespaces(self, selector: Selector) -> None:
         for prefix, uri in self.namespaces:
             selector.register_namespace(prefix, uri)
 
@@ -102,27 +112,29 @@ class CSVFeedSpider(Spider):
     and the file's headers.
     """
 
-    delimiter = (
+    delimiter: Optional[str] = (
         None  # When this is None, python's csv module's default delimiter is used
     )
-    quotechar = (
+    quotechar: Optional[str] = (
         None  # When this is None, python's csv module's default quotechar is used
     )
-    headers = None
+    headers: Optional[List[str]] = None
 
-    def process_results(self, response, results):
+    def process_results(
+        self, response: Response, results: Iterable[Any]
+    ) -> Iterable[Any]:
         """This method has the same purpose as the one in XMLFeedSpider"""
         return results
 
-    def adapt_response(self, response):
+    def adapt_response(self, response: Response) -> Response:
         """This method has the same purpose as the one in XMLFeedSpider"""
         return response
 
-    def parse_row(self, response, row):
+    def parse_row(self, response: Response, row: Dict[str, str]) -> Any:
         """This method must be overridden with your custom spider functionality"""
         raise NotImplementedError
 
-    def parse_rows(self, response):
+    def parse_rows(self, response: Response) -> Any:
         """Receives a response and a dict (representing each row) with a key for
         each provided (or detected) header of the CSV file.  This spider also
         gives the opportunity to override adapt_response and
@@ -135,7 +147,7 @@ class CSVFeedSpider(Spider):
             ret = iterate_spider_output(self.parse_row(response, row))
             yield from self.process_results(response, ret)
 
-    def _parse(self, response, **kwargs):
+    def _parse(self, response: Response, **kwargs: Any) -> Any:
         if not hasattr(self, "parse_row"):
             raise NotConfigured(
                 "You must define parse_row method in order to scrape this CSV feed"
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
index 3cb215b0f..a0898a0cf 100644
--- a/scrapy/spiders/init.py
+++ b/scrapy/spiders/init.py
@@ -1,3 +1,7 @@
+from typing import Any, Iterable, Optional, cast
+
+from scrapy import Request
+from scrapy.http import Response
 from scrapy.spiders import Spider
 from scrapy.utils.spider import iterate_spider_output
 
@@ -5,17 +9,17 @@ from scrapy.utils.spider import iterate_spider_output
 class InitSpider(Spider):
     """Base Spider with initialization facilities"""
 
-    def start_requests(self):
-        self._postinit_reqs = super().start_requests()
-        return iterate_spider_output(self.init_request())
+    def start_requests(self) -> Iterable[Request]:
+        self._postinit_reqs: Iterable[Request] = super().start_requests()
+        return cast(Iterable[Request], iterate_spider_output(self.init_request()))
 
-    def initialized(self, response=None):
+    def initialized(self, response: Optional[Response] = None) -> Any:
         """This method must be set as the callback of your last initialization
         request. See self.init_request() docstring for more info.
         """
         return self.__dict__.pop("_postinit_reqs")
 
-    def init_request(self):
+    def init_request(self) -> Any:
         """This function should return one initialization request, with the
         self.initialized method as callback. When the self.initialized method
         is called this spider is considered initialized. If you need to perform
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index f0e630c42..d082fbfdb 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -2,9 +2,21 @@ from __future__ import annotations
 
 import logging
 import re
-from typing import TYPE_CHECKING, Any
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Callable,
+    Dict,
+    Iterable,
+    List,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+    cast,
+)
 
-from scrapy.http import Request, XmlResponse
+from scrapy.http import Request, Response, XmlResponse
 from scrapy.spiders import Spider
 from scrapy.utils._compression import _DecompressionMaxSizeExceeded
 from scrapy.utils.gz import gunzip, gzip_magic_number
@@ -20,10 +32,12 @@ logger = logging.getLogger(__name__)
 
 
 class SitemapSpider(Spider):
-    sitemap_urls = ()
-    sitemap_rules = [("", "parse")]
-    sitemap_follow = [""]
-    sitemap_alternate_links = False
+    sitemap_urls: Sequence[str] = ()
+    sitemap_rules: Sequence[
+        Tuple[Union[re.Pattern[str], str], Union[str, Callable]]
+    ] = [("", "parse")]
+    sitemap_follow: Sequence[Union[re.Pattern[str], str]] = [""]
+    sitemap_alternate_links: bool = False
     _max_size: int
     _warn_size: int
 
@@ -38,27 +52,29 @@ class SitemapSpider(Spider):
         )
         return spider
 
-    def __init__(self, *a, **kw):
+    def __init__(self, *a: Any, **kw: Any):
         super().__init__(*a, **kw)
-        self._cbs = []
+        self._cbs: List[Tuple[re.Pattern[str], Callable]] = []
         for r, c in self.sitemap_rules:
             if isinstance(c, str):
-                c = getattr(self, c)
+                c = cast(Callable, getattr(self, c))
             self._cbs.append((regex(r), c))
-        self._follow = [regex(x) for x in self.sitemap_follow]
+        self._follow: List[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
 
-    def start_requests(self):
+    def start_requests(self) -> Iterable[Request]:
         for url in self.sitemap_urls:
             yield Request(url, self._parse_sitemap)
 
-    def sitemap_filter(self, entries):
+    def sitemap_filter(
+        self, entries: Iterable[Dict[str, Any]]
+    ) -> Iterable[Dict[str, Any]]:
         """This method can be used to filter sitemap entries by their
         attributes, for example, you can filter locs with lastmod greater
         than a given date (see docs).
         """
         yield from entries
 
-    def _parse_sitemap(self, response):
+    def _parse_sitemap(self, response: Response) -> Iterable[Request]:
         if response.url.endswith("/robots.txt"):
             for url in sitemap_urls_from_robots(response.text, base_url=response.url):
                 yield Request(url, callback=self._parse_sitemap)
@@ -86,7 +102,7 @@ class SitemapSpider(Spider):
                             yield Request(loc, callback=c)
                             break
 
-    def _get_sitemap_body(self, response):
+    def _get_sitemap_body(self, response: Response) -> Optional[bytes]:
         """Return the sitemap body contained in the given response,
         or None if the response is not a sitemap.
         """
@@ -117,15 +133,16 @@ class SitemapSpider(Spider):
         # in other word, here, we have plain XML
         if response.url.endswith(".xml") or response.url.endswith(".xml.gz"):
             return response.body
+        return None
 
 
-def regex(x):
+def regex(x: Union[re.Pattern[str], str]) -> re.Pattern[str]:
     if isinstance(x, str):
         return re.compile(x)
     return x
 
 
-def iterloc(it, alt=False):
+def iterloc(it: Iterable[Dict[str, Any]], alt: bool = False) -> Iterable[str]:
     for d in it:
         yield d["loc"]
 
diff --git a/scrapy/utils/sitemap.py b/scrapy/utils/sitemap.py
index 7dcee3a2f..cf429043d 100644
--- a/scrapy/utils/sitemap.py
+++ b/scrapy/utils/sitemap.py
@@ -5,7 +5,7 @@ Note: The main purpose of this module is to provide support for the
 SitemapSpider, its API is subject to change without notice.
 """
 
-from typing import Any, Dict, Generator, Iterator, Optional
+from typing import Any, Dict, Generator, Iterator, Optional, Union
 from urllib.parse import urljoin
 
 import lxml.etree  # nosec
@@ -15,7 +15,7 @@ class Sitemap:
     """Class to parse Sitemap (type=urlset) and Sitemap Index
     (type=sitemapindex) files"""
 
-    def __init__(self, xmltext: str):
+    def __init__(self, xmltext: Union[str, bytes]):
         xmlp = lxml.etree.XMLParser(
             recover=True, remove_comments=True, resolve_entities=False
         )

From b88f22c6c5de4ca8828b2abe860516c2468c4fe3 Mon Sep 17 00:00:00 2001
From: Bagaudin Magomedov <51474632+bloodforcream@users.noreply.github.com>
Date: Mon, 13 May 2024 15:33:24 +0300
Subject: [PATCH 74/74] Add SpiderLoggerAdapter, change Spider.logger to return
 SpiderLoggerAdapter (#6324)

---
 scrapy/spiders/__init__.py |   7 +-
 scrapy/utils/log.py        |  25 ++++-
 tests/spiders.py           |  23 +++++
 tests/test_utils_log.py    | 185 +++++++++++++++++++++++++++++++++++++
 4 files changed, 237 insertions(+), 3 deletions(-)

diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 370801f28..bef041325 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -25,6 +25,7 @@ if TYPE_CHECKING:
 
     from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
+    from scrapy.utils.log import SpiderLoggerAdapter
 
     CallbackT = Callable[Concatenate[Response, ...], Any]
 
@@ -47,9 +48,11 @@ class Spider(object_ref):
             self.start_urls: List[str] = []
 
     @property
-    def logger(self) -> logging.LoggerAdapter:
+    def logger(self) -> SpiderLoggerAdapter:
+        from scrapy.utils.log import SpiderLoggerAdapter
+
         logger = logging.getLogger(self.name)
-        return logging.LoggerAdapter(logger, {"spider": self})
+        return SpiderLoggerAdapter(logger, {"spider": self})
 
     def log(self, message: Any, level: int = logging.DEBUG, **kw: Any) -> None:
         """Log the given message at the given log level
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 2a38f151a..430a91e95 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -4,7 +4,17 @@ import logging
 import sys
 from logging.config import dictConfig
 from types import TracebackType
-from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Type, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    List,
+    MutableMapping,
+    Optional,
+    Tuple,
+    Type,
+    Union,
+    cast,
+)
 
 from twisted.python import log as twisted_log
 from twisted.python.failure import Failure
@@ -238,3 +248,16 @@ def logformatter_adapter(logkws: dict) -> Tuple[int, str, dict]:
     args = logkws if not logkws.get("args") else logkws["args"]
 
     return (level, message, args)
+
+
+class SpiderLoggerAdapter(logging.LoggerAdapter):
+    def process(
+        self, msg: str, kwargs: MutableMapping[str, Any]
+    ) -> Tuple[str, MutableMapping[str, Any]]:
+        """Method that augments logging with additional 'extra' data"""
+        if isinstance(kwargs.get("extra"), MutableMapping):
+            kwargs["extra"].update(self.extra)
+        else:
+            kwargs["extra"] = self.extra
+
+        return msg, kwargs
diff --git a/tests/spiders.py b/tests/spiders.py
index 94969db99..ea419afbd 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -4,6 +4,7 @@ Some spiders used for testing and benchmarking
 
 import asyncio
 import time
+from typing import Optional
 from urllib.parse import urlencode
 
 from twisted.internet import defer
@@ -78,6 +79,28 @@ class DelaySpider(MetaSpider):
         self.t2_err = time.time()
 
 
+class LogSpider(MetaSpider):
+    name = "log_spider"
+
+    def log_debug(self, message: str, extra: Optional[dict] = None):
+        self.logger.debug(message, extra=extra)
+
+    def log_info(self, message: str, extra: Optional[dict] = None):
+        self.logger.info(message, extra=extra)
+
+    def log_warning(self, message: str, extra: Optional[dict] = None):
+        self.logger.warning(message, extra=extra)
+
+    def log_error(self, message: str, extra: Optional[dict] = None):
+        self.logger.error(message, extra=extra)
+
+    def log_critical(self, message: str, extra: Optional[dict] = None):
+        self.logger.critical(message, extra=extra)
+
+    def parse(self, response):
+        pass
+
+
 class SlowSpider(DelaySpider):
     name = "slow"
 
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index eae744df5..a8d080822 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -1,18 +1,26 @@
+import json
 import logging
+import re
 import sys
 import unittest
+from io import StringIO
+from typing import Any, Dict, Mapping, MutableMapping
+from unittest import TestCase
 
+import pytest
 from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.extensions import telnet
 from scrapy.utils.log import (
     LogCounterHandler,
+    SpiderLoggerAdapter,
     StreamLogger,
     TopLevelFormatter,
     failure_to_exc_info,
 )
 from scrapy.utils.test import get_crawler
+from tests.spiders import LogSpider
 
 
 class FailureToExcInfoTest(unittest.TestCase):
@@ -106,3 +114,180 @@ class StreamLoggerTest(unittest.TestCase):
         with LogCapture() as log:
             print("test log msg")
         log.check(("test", "ERROR", "test log msg"))
+
+
+@pytest.mark.parametrize(
+    ("base_extra", "log_extra", "expected_extra"),
+    (
+        (
+            {"spider": "test"},
+            {"extra": {"log_extra": "info"}},
+            {"extra": {"log_extra": "info", "spider": "test"}},
+        ),
+        (
+            {"spider": "test"},
+            {"extra": None},
+            {"extra": {"spider": "test"}},
+        ),
+        (
+            {"spider": "test"},
+            {"extra": {"spider": "test2"}},
+            {"extra": {"spider": "test"}},
+        ),
+    ),
+)
+def test_spider_logger_adapter_process(
+    base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: Dict
+):
+    logger = logging.getLogger("test")
+    spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
+
+    log_message = "test_log_message"
+    result_message, result_kwargs = spider_logger_adapter.process(
+        log_message, log_extra
+    )
+
+    assert result_message == log_message
+    assert result_kwargs == expected_extra
+
+
+class LoggingTestCase(TestCase):
+    def setUp(self):
+        self.log_stream = StringIO()
+        handler = logging.StreamHandler(self.log_stream)
+        logger = logging.getLogger("log_spider")
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+        self.handler = handler
+        self.logger = logger
+        self.spider = LogSpider()
+
+    def tearDown(self):
+        self.logger.removeHandler(self.handler)
+
+    def test_debug_logging(self):
+        log_message = "Foo message"
+        self.spider.log_debug(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_info_logging(self):
+        log_message = "Bar message"
+        self.spider.log_info(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_warning_logging(self):
+        log_message = "Baz message"
+        self.spider.log_warning(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_error_logging(self):
+        log_message = "Foo bar message"
+        self.spider.log_error(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+    def test_critical_logging(self):
+        log_message = "Foo bar baz message"
+        self.spider.log_critical(log_message)
+        log_contents = self.log_stream.getvalue()
+
+        assert log_contents == f"{log_message}\n"
+
+
+class LoggingWithExtraTestCase(TestCase):
+    def setUp(self):
+        self.log_stream = StringIO()
+        handler = logging.StreamHandler(self.log_stream)
+        formatter = logging.Formatter(
+            '{"levelname": "%(levelname)s", "message": "%(message)s", "spider": "%(spider)s", "important_info": "%(important_info)s"}'
+        )
+        handler.setFormatter(formatter)
+        logger = logging.getLogger("log_spider")
+        logger.addHandler(handler)
+        logger.setLevel(logging.DEBUG)
+        self.handler = handler
+        self.logger = logger
+        self.spider = LogSpider()
+        self.regex_pattern = re.compile(r"^<LogSpider\s'log_spider'\sat\s[^>]+>$")
+
+    def tearDown(self):
+        self.logger.removeHandler(self.handler)
+
+    def test_debug_logging(self):
+        log_message = "Foo message"
+        extra = {"important_info": "foo"}
+        self.spider.log_debug(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "DEBUG"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_info_logging(self):
+        log_message = "Bar message"
+        extra = {"important_info": "bar"}
+        self.spider.log_info(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "INFO"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_warning_logging(self):
+        log_message = "Baz message"
+        extra = {"important_info": "baz"}
+        self.spider.log_warning(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "WARNING"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_error_logging(self):
+        log_message = "Foo bar message"
+        extra = {"important_info": "foo bar"}
+        self.spider.log_error(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "ERROR"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_critical_logging(self):
+        log_message = "Foo bar baz message"
+        extra = {"important_info": "foo bar baz"}
+        self.spider.log_critical(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "CRITICAL"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]
+
+    def test_overwrite_spider_extra(self):
+        log_message = "Foo message"
+        extra = {"important_info": "foo", "spider": "shouldn't change"}
+        self.spider.log_error(log_message, extra)
+        log_contents = self.log_stream.getvalue()
+        log_contents = json.loads(log_contents)
+
+        assert log_contents["levelname"] == "ERROR"
+        assert log_contents["message"] == log_message
+        assert self.regex_pattern.match(log_contents["spider"])
+        assert log_contents["important_info"] == extra["important_info"]