From fc5216f15611e40d795f5ab566acff8f9ca0af1e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 1 Jul 2026 11:50:32 +0500
Subject: [PATCH 01/62] Clarify/cleanup Selector.type (#7704)

---
 scrapy/selector/unified.py | 47 ++++++++++++++------------------------
 1 file changed, 17 insertions(+), 30 deletions(-)

diff --git a/scrapy/selector/unified.py b/scrapy/selector/unified.py
index 99b22aca9..f6334c32c 100644
--- a/scrapy/selector/unified.py
+++ b/scrapy/selector/unified.py
@@ -1,10 +1,6 @@
-"""
-XPath selectors based on lxml
-"""
-
 from __future__ import annotations
 
-from typing import Any
+from typing import Any, Literal
 
 from parsel import Selector as _ParselSelector
 
@@ -18,13 +14,10 @@ __all__ = ["Selector", "SelectorList"]
 _NOT_SET = object()
 
 
-def _st(response: TextResponse | None, st: str | None) -> str:
-    if st is None:
-        return "xml" if isinstance(response, XmlResponse) else "html"
-    return st
+SelectorType = Literal["html", "xml", "json", "text"]
 
 
-def _response_from_text(text: str | bytes, st: str | None) -> TextResponse:
+def _response_from_text(text: str | bytes, st: SelectorType | None) -> TextResponse:
     rt: type[TextResponse] = XmlResponse if st == "xml" else HtmlResponse
     return rt(url="about:blank", encoding="utf-8", body=to_bytes(text, "utf-8"))
 
@@ -49,23 +42,16 @@ class Selector(_ParselSelector, object_ref):
     ``response`` isn't available. Using ``text`` and ``response`` together is
     undefined behavior.
 
-    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``, ``"json"``
-    or ``None`` (default).
+    ``type`` defines the selector type, it can be ``"html"``, ``"xml"``,
+    ``"json"``, ``"text"`` or ``None`` (default). It's passed to
+    :class:`parsel.Selector` and its meaning is defined there. However, when
+    ``type`` is ``None``, it is set to ``"xml"`` for an
+    :class:`~scrapy.http.XmlResponse` and to ``"html"`` otherwise before
+    passing it to :class:`parsel.Selector`.
 
-    If ``type`` is ``None``, the selector automatically chooses the best type
-    based on ``response`` type (see below), or defaults to ``"html"`` in case it
-    is used together with ``text``.
-
-    If ``type`` is ``None`` and a ``response`` is passed, the selector type is
-    inferred from the response type as follows:
-
-    * ``"html"`` for :class:`~scrapy.http.HtmlResponse` type
-    * ``"xml"`` for :class:`~scrapy.http.XmlResponse` type
-    * ``"json"`` for :class:`~scrapy.http.TextResponse` type
-    * ``"html"`` for anything else
-
-    Otherwise, if ``type`` is set, the selector type will be forced and no
-    detection will occur.
+    .. note:: JSON selector support requires ``parsel`` 1.8.0 or higher. With
+       older versions setting ``type`` to ``"json"`` or ``"text"`` is not
+       supported.
     """
 
     __slots__ = ["response"]
@@ -75,7 +61,7 @@ class Selector(_ParselSelector, object_ref):
         self,
         response: TextResponse | None = None,
         text: str | None = None,
-        type: str | None = None,  # noqa: A002
+        type: SelectorType | None = None,  # noqa: A002
         root: Any | None = _NOT_SET,
         **kwargs: Any,
     ):
@@ -84,10 +70,11 @@ class Selector(_ParselSelector, object_ref):
                 f"{self.__class__.__name__}.__init__() received both response and text"
             )
 
-        st = _st(response, type)
+        if type is None:
+            type = "xml" if isinstance(response, XmlResponse) else "html"  # noqa: A001
 
         if text is not None:
-            response = _response_from_text(text, st)
+            response = _response_from_text(text, type)
 
         if response is not None:
             text = response.text
@@ -98,4 +85,4 @@ class Selector(_ParselSelector, object_ref):
         if root is not _NOT_SET:
             kwargs["root"] = root
 
-        super().__init__(text=text, type=st, **kwargs)
+        super().__init__(text=text, type=type, **kwargs)

From 361f689df785959a59cf939b9efaefc72079f037 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Wed, 1 Jul 2026 08:53:07 +0200
Subject: [PATCH 02/62] Improve test coverage for crawler.py (#7682)

* Improve test coverage for crawler.py

* Silence mypy warnings

* Improve test coverage for crawler.py
---
 ...yncio_enabled_reactor_same_loop_default.py |  31 ++++
 .../dns_resolver_deprecated.py                |  31 ++++
 .../reactorless_sleeping.py                   |   2 +-
 tests/AsyncCrawlerProcess/sleeping.py         |   2 +-
 .../CrawlerProcess/dns_resolver_deprecated.py |  31 ++++
 tests/CrawlerProcess/sleeping.py              |   2 +-
 tests/test_crawler.py                         | 163 +++++++++++++++++-
 tests/test_crawler_subprocess.py              |  30 +++-
 8 files changed, 286 insertions(+), 6 deletions(-)
 create mode 100644 tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py
 create mode 100644 tests/AsyncCrawlerProcess/dns_resolver_deprecated.py
 create mode 100644 tests/CrawlerProcess/dns_resolver_deprecated.py

diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py
new file mode 100644
index 000000000..c519e123b
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop_default.py
@@ -0,0 +1,31 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+
+if sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+loop = asyncio.SelectorEventLoop()
+asyncio.set_event_loop(loop)
+asyncioreactor.install(loop)
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+process = AsyncCrawlerProcess(
+    settings={
+        "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "ASYNCIO_EVENT_LOOP": "asyncio.SelectorEventLoop",
+    }
+)
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py
new file mode 100644
index 000000000..8c8df96bb
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/dns_resolver_deprecated.py
@@ -0,0 +1,31 @@
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.settings import Settings
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+settings = Settings()
+# The deprecated DNS_RESOLVER setting, set above its default priority so that
+# AsyncCrawlerProcess._setup_reactor() emits the deprecation warning.
+settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10)
+if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins":
+    # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the
+    # deprecated DNS_RESOLVER setting.
+    settings.set(
+        "TWISTED_DNS_RESOLVER",
+        "scrapy.resolver.CachingThreadedResolver",
+        priority=20,
+    )
+
+process = AsyncCrawlerProcess(settings)
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/AsyncCrawlerProcess/reactorless_sleeping.py b/tests/AsyncCrawlerProcess/reactorless_sleeping.py
index 12101d221..0d11a0996 100644
--- a/tests/AsyncCrawlerProcess/reactorless_sleeping.py
+++ b/tests/AsyncCrawlerProcess/reactorless_sleeping.py
@@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider):
 process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
 
 process.crawl(SleepingSpider)
-process.start()
+process.start(stop_after_crawl="--no-stop" not in sys.argv)
diff --git a/tests/AsyncCrawlerProcess/sleeping.py b/tests/AsyncCrawlerProcess/sleeping.py
index 88caf5032..dad6a3f20 100644
--- a/tests/AsyncCrawlerProcess/sleeping.py
+++ b/tests/AsyncCrawlerProcess/sleeping.py
@@ -17,4 +17,4 @@ class SleepingSpider(scrapy.Spider):
 process = AsyncCrawlerProcess(settings={})
 
 process.crawl(SleepingSpider)
-process.start()
+process.start(stop_after_crawl="--no-stop" not in sys.argv)
diff --git a/tests/CrawlerProcess/dns_resolver_deprecated.py b/tests/CrawlerProcess/dns_resolver_deprecated.py
new file mode 100644
index 000000000..b8cd24325
--- /dev/null
+++ b/tests/CrawlerProcess/dns_resolver_deprecated.py
@@ -0,0 +1,31 @@
+import sys
+
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from scrapy.settings import Settings
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+settings = Settings()
+# The deprecated DNS_RESOLVER setting, set above its default priority so that
+# CrawlerProcess._setup_reactor() emits the deprecation warning.
+settings.set("DNS_RESOLVER", "scrapy.resolver.CachingThreadedResolver", priority=10)
+if len(sys.argv) > 1 and sys.argv[1] == "twisted-wins":
+    # TWISTED_DNS_RESOLVER at a higher priority takes precedence over the
+    # deprecated DNS_RESOLVER setting.
+    settings.set(
+        "TWISTED_DNS_RESOLVER",
+        "scrapy.resolver.CachingThreadedResolver",
+        priority=20,
+    )
+
+process = CrawlerProcess(settings)
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
index cb8f869e1..a577b1909 100644
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@@ -23,4 +23,4 @@ class SleepingSpider(scrapy.Spider):
 process = CrawlerProcess(settings={})
 
 process.crawl(SleepingSpider)
-process.start()
+process.start(stop_after_crawl="--no-stop" not in sys.argv)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 0cddfd0ed..adac32df1 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -3,8 +3,11 @@ from __future__ import annotations
 import asyncio
 import logging
 import re
+import signal
+import threading
 from pathlib import Path
-from typing import Any, ClassVar
+from typing import TYPE_CHECKING, Any, ClassVar
+from unittest.mock import MagicMock
 
 import pytest
 from zope.interface.exceptions import MultipleInvalid
@@ -32,6 +35,9 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 BASE_SETTINGS: dict[str, Any] = {}
 
 
@@ -651,6 +657,145 @@ class TestAsyncCrawlerProcess(TestBaseCrawler):
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
 
+class TestAsyncCrawlerProcessReactorlessHelpers:
+    """Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
+
+    These cover defensive branches that guard against shutdown races and that
+    are not reachable through a full process run.
+    """
+
+    @staticmethod
+    def _bare_process(
+        monkeypatch: pytest.MonkeyPatch,
+    ) -> tuple[AsyncCrawlerProcess, list[Any]]:
+        # AsyncCrawlerProcess.__init__ has global side effects (it installs a
+        # reactor import hook and an asyncio event loop), so build a bare
+        # instance and set only the attributes these helpers read. The shutdown
+        # handlers installed by these helpers are recorded for assertions
+        # instead of touching the real process-wide signal handlers.
+        installed_handlers: list[Any] = []
+        monkeypatch.setattr(
+            "scrapy.crawler.install_shutdown_handlers",
+            lambda handler, *args, **kwargs: installed_handlers.append(handler),
+        )
+        return AsyncCrawlerProcess.__new__(AsyncCrawlerProcess), installed_handlers
+
+    @staticmethod
+    def _run_in_thread(target: Callable[[], None]) -> None:
+        # Run target in a dedicated thread so its event loop is not nested
+        # inside the event loop that may already be running the test session.
+        thread = threading.Thread(target=target)
+        thread.start()
+        thread.join()
+
+    def test_signal_shutdown_reactorless_without_loop(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        process, installed_handlers = self._bare_process(monkeypatch)
+        process._reactorless_loop = None
+        # No loop to schedule the shutdown task on, so it returns early, but it
+        # must still escalate the handler so a second signal forces a kill.
+        process._signal_shutdown_reactorless(signal.SIGINT, None)
+        assert installed_handlers == [process._signal_kill_reactorless]
+
+    def test_signal_kill_reactorless_without_loop(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        process, installed_handlers = self._bare_process(monkeypatch)
+        process._reactorless_loop = None
+        process._reactorless_main_task = None
+        # No loop to cancel the main task on, so it returns early, but it must
+        # still ignore any further signals.
+        process._signal_kill_reactorless(signal.SIGINT, None)
+        assert installed_handlers == [signal.SIG_IGN]
+
+    def test_signal_kill_reactorless_without_main_task(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        process, installed_handlers = self._bare_process(monkeypatch)
+        loop = MagicMock()
+        process._reactorless_loop = loop
+        process._reactorless_main_task = None
+        # No main task to cancel, so nothing is scheduled on the loop.
+        process._signal_kill_reactorless(signal.SIGINT, None)
+        assert installed_handlers == [signal.SIG_IGN]
+        loop.call_soon_threadsafe.assert_not_called()
+
+    def test_shutdown_graceful_reactorless_main_task_already_done(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        process, _ = self._bare_process(monkeypatch)
+        process._stop_after_crawl = False
+
+        async def noop() -> None:
+            return None
+
+        monkeypatch.setattr(process, "stop", noop)
+        monkeypatch.setattr(process, "join", noop)
+
+        def run() -> None:
+            loop = asyncio.new_event_loop()
+            try:
+                main_task: asyncio.Future[None] = loop.create_future()
+                main_task.set_result(None)
+                process._reactorless_main_task = main_task
+                # The main task is already done, so it is not cancelled.
+                loop.run_until_complete(process._shutdown_graceful_reactorless())
+                assert not main_task.cancelled()
+            finally:
+                loop.close()
+
+        self._run_in_thread(run)
+
+    def test_create_shutdown_task_closed_loop(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        process, _ = self._bare_process(monkeypatch)
+        loop = asyncio.new_event_loop()
+        loop.close()
+        process._reactorless_loop = loop
+        process._stop_after_crawl = True
+        # create_task() raises RuntimeError on a closed loop; the coroutine
+        # must be closed instead of leaking.
+        process._create_shutdown_task()
+
+    def test_cancel_all_tasks_logs_task_exception(self) -> None:
+        contexts: list[dict[str, Any]] = []
+        task_was_cancelled: list[bool] = []
+
+        def run() -> None:
+            loop = asyncio.new_event_loop()
+            loop.set_exception_handler(lambda _loop, context: contexts.append(context))
+
+            async def fail_on_cancel() -> None:
+                try:
+                    await asyncio.sleep(10)
+                except asyncio.CancelledError:
+                    raise RuntimeError("boom")
+
+            try:
+                task = loop.create_task(fail_on_cancel())
+                # Let the task start and suspend on the sleep so the
+                # cancellation is raised inside its body and turned into a
+                # RuntimeError rather than cancelling the task cleanly.
+                loop.run_until_complete(asyncio.sleep(0))
+                AsyncCrawlerProcess._cancel_all_tasks(loop)
+                task_was_cancelled.append(task.cancelled())
+            finally:
+                loop.close()
+
+        self._run_in_thread(run)
+
+        # The task raised instead of being cancelled, so its exception is
+        # reported to the loop exception handler.
+        assert task_was_cancelled == [False]
+        assert any(
+            context.get("message")
+            == "unhandled exception during AsyncCrawlerProcess shutdown"
+            for context in contexts
+        )
+
+
 @pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
 def test_runner_settings_applied_to_crawler_instance(
     runner_cls: type[CrawlerRunnerBase],
@@ -687,6 +832,22 @@ def test_create_crawler_instance_consistent_with_spider_class() -> None:
     assert pre_built.settings["FOO"] == "runner"
 
 
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_create_crawler_rejects_spider_object(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    runner = runner_cls()
+    with pytest.raises(ValueError, match="cannot be a spider object"):
+        runner.create_crawler(DefaultSpider())  # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
+    runner = runner_cls()
+    with pytest.raises(ValueError, match="cannot be a spider object"):
+        runner.crawl(DefaultSpider())  # type: ignore[arg-type]
+
+
 class ExceptionSpider(scrapy.Spider):
     name = "exception"
 
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index e3f9ac161..018a2b31b 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -126,6 +126,16 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "TimeoutError" not in log
         assert "scrapy.exceptions.CannotResolveHostError" not in log
 
+    def test_dns_resolver_deprecated(self) -> None:
+        log = self.run_script("dns_resolver_deprecated.py")
+        assert "Spider closed (finished)" in log
+        assert "The DNS_RESOLVER setting is deprecated" in log
+
+    def test_dns_resolver_deprecated_twisted_dns_resolver(self) -> None:
+        log = self.run_script("dns_resolver_deprecated.py", "twisted-wins")
+        assert "Spider closed (finished)" in log
+        assert "The DNS_RESOLVER setting is deprecated" in log
+
     def test_twisted_reactor_asyncio(self) -> None:
         log = self.run_script("twisted_reactor_asyncio.py")
         assert "Spider closed (finished)" in log
@@ -205,9 +215,11 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Spider closed (finished)" in log
         assert "The value of FOO is 42" in log
 
-    def _test_shutdown_graceful(self, script: str = "sleeping.py") -> None:
+    def _test_shutdown_graceful(
+        self, script: str = "sleeping.py", *extra_args: str
+    ) -> None:
         sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK  # type: ignore[attr-defined]
-        args = self.get_script_args(script, "3")
+        args = self.get_script_args(script, "3", *extra_args)
         p = PopenSpawn(args, timeout=5, env=get_script_run_env())
         p.expect_exact("Spider opened")
         p.expect_exact("Crawled (200)")
@@ -245,6 +257,9 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
     async def test_shutdown_forced(self) -> None:
         await self._test_shutdown_forced()
 
+    def test_shutdown_graceful_no_stop(self) -> None:
+        self._test_shutdown_graceful("sleeping.py", "--no-stop")
+
 
 class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
     @property
@@ -429,6 +444,17 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
     async def test_shutdown_forced(self) -> None:
         await self._test_shutdown_forced("reactorless_sleeping.py")
 
+    def test_shutdown_graceful_reactorless_no_stop(self) -> None:
+        self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop")
+
+    def test_asyncio_enabled_reactor_same_loop_default(self) -> None:
+        log = self.run_script("asyncio_enabled_reactor_same_loop_default.py")
+        assert "Spider closed (finished)" in log
+        assert (
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+            in log
+        )
+
 
 class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
     """Common tests between CrawlerRunner and AsyncCrawlerRunner,

From 870803b7fb1ed56c296eb6b51d0212f0460dfdda Mon Sep 17 00:00:00 2001
From: Fat-Coder-CN <cnmasker@gmail.com>
Date: Wed, 1 Jul 2026 15:16:01 +0800
Subject: [PATCH 03/62] fix-utf16-response-test-on-big-endian-systems (#7508)

---
 tests/test_http_response_text.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 507fd1864..5ef89fe4a 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -163,12 +163,13 @@ class TestTextResponse(TestResponse):
 
     def test_utf16(self):
         """Test utf-16 because UnicodeDammit is known to have problems with"""
+        body = b"\xff\xfeh\x00i\x00"
         r = self.response_class(
             "http://www.example.com",
-            body=b"\xff\xfeh\x00i\x00",
+            body=body,
             encoding="utf-16",
         )
-        self._assert_response_values(r, "utf-16", "hi")
+        self._assert_response_values(r, "utf-16", body)
 
     def test_invalid_utf8_encoded_body_with_valid_utf8_BOM(self):
         r6 = self.response_class(

From dd10cb8e9a982fe3d311078d6e1207596e272717 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Sun, 5 Jul 2026 13:46:22 +0200
Subject: [PATCH 04/62] LxmlLinkExtractor: add deny_attrs and deny_tags (#7679)

---
 docs/topics/link-extractors.rst   | 103 +---------------------
 scrapy/linkextractors/lxmlhtml.py | 142 +++++++++++++++++++++++++++++-
 tests/test_linkextractors.py      |  53 +++++++++++
 3 files changed, 194 insertions(+), 104 deletions(-)

diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 613e175da..3fc896507 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -47,108 +47,7 @@ LxmlLinkExtractor
    :synopsis: lxml's HTMLParser-based link extractors
 
 
-.. class:: LxmlLinkExtractor(allow=(), deny=(), allow_domains=(), deny_domains=(), deny_extensions=None, restrict_xpaths=(), restrict_css=(), tags=('a', 'area'), attrs=('href',), canonicalize=False, unique=True, process_value=None, strip=True)
-
-    LxmlLinkExtractor is the recommended link extractor with handy filtering
-    options. It is implemented using lxml's robust HTMLParser.
-
-    :param allow: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be extracted. If not
-        given (or empty), it will match all links.
-    :type allow: str or list
-
-    :param deny: a single regular expression (or list of regular expressions)
-        that the (absolute) urls must match in order to be excluded (i.e. not
-        extracted). It has precedence over the ``allow`` parameter. If not
-        given (or empty) it won't exclude any links.
-    :type deny: str or list
-
-    :param allow_domains: a single value or a list of string containing
-        domains which will be considered for extracting the links
-    :type allow_domains: str or list
-
-    :param deny_domains: a single value or a list of strings containing
-        domains which won't be considered for extracting the links
-    :type deny_domains: str or list
-
-    :param deny_extensions: a single value or list of strings containing
-        extensions that should be ignored when extracting links.
-        If not given, it will default to
-        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
-
-    :type deny_extensions: list
-
-    :param restrict_xpaths: is an XPath (or list of XPath's) which defines
-        regions inside the response where links should be extracted from.
-        If given, only the text selected by those XPath will be scanned for
-        links.
-    :type restrict_xpaths: str or list
-
-    :param restrict_css: a CSS selector (or list of selectors) which defines
-        regions inside the response where links should be extracted from.
-        Has the same behaviour as ``restrict_xpaths``.
-    :type restrict_css: str or list
-
-    :param restrict_text: a single regular expression (or list of regular expressions)
-        that the link's text must match in order to be extracted. If not
-        given (or empty), it will match all links. If a list of regular expressions is
-        given, the link will be extracted if it matches at least one.
-    :type restrict_text: str or list
-
-    :param tags: a tag or a list of tags to consider when extracting links.
-        Defaults to ``('a', 'area')``.
-    :type tags: str or list
-
-    :param attrs: an attribute or list of attributes which should be considered when looking
-        for links to extract (only for those tags specified in the ``tags``
-        parameter). Defaults to ``('href',)``
-    :type attrs: list
-
-    :param canonicalize: canonicalize each extracted url (using
-        w3lib.url.canonicalize_url). Defaults to ``False``.
-        Note that canonicalize_url is meant for duplicate checking;
-        it can change the URL visible at server side, so the response can be
-        different for requests with canonicalized and raw URLs. If you're
-        using LinkExtractor to follow links it is more robust to
-        keep the default ``canonicalize=False``.
-    :type canonicalize: bool
-
-    :param unique: whether duplicate filtering should be applied to extracted
-        links.
-    :type unique: bool
-
-    :param process_value: a function which receives each value extracted from
-        the tag and attributes scanned and can modify the value and return a
-        new one, or return ``None`` to ignore the link altogether. If not
-        given, ``process_value`` defaults to ``lambda x: x``.
-
-        .. highlight:: html
-
-        For example, to extract links from this code::
-
-            <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
-
-        .. highlight:: python
-
-        You can use the following function in ``process_value``:
-
-        .. code-block:: python
-
-            def process_value(value):
-                m = re.search(r"javascript:goToPage\('(.*?)'", value)
-                if m:
-                    return m.group(1)
-
-    :type process_value: collections.abc.Callable
-
-    :param strip: whether to strip whitespaces from extracted attributes.
-        According to HTML5 standard, leading and trailing whitespaces
-        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
-        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
-        elements, etc., so LinkExtractor strips space chars by default.
-        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
-        from elements or attributes which allow leading/trailing whitespaces).
-    :type strip: bool
+.. autoclass:: LxmlLinkExtractor
 
     .. automethod:: extract_links
 
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 11fd62fb2..2efe7cd8e 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -57,6 +57,18 @@ def _canonicalize_link_url(link: Link) -> str:
     return canonicalize_url(link.url, keep_fragments=True)
 
 
+def _name_matches(allowed: set[str], denied: set[str], name: str) -> bool:
+    """Return whether a tag or attribute *name* should be considered.
+
+    A name matches when it is allowed and not denied. A name is allowed if it is
+    listed in *allowed*, or if *allowed* contains the ``"*"`` wildcard, which
+    matches every name. *denied* has precedence over *allowed*.
+    """
+    if name in denied:
+        return False
+    return "*" in allowed or name in allowed
+
+
 class LxmlParserLinkExtractor:
     def __init__(
         self,
@@ -162,6 +174,126 @@ _RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]
 
 
 class LxmlLinkExtractor:
+    r"""LxmlLinkExtractor is the recommended link extractor with handy filtering
+    options. It is implemented using lxml's robust HTMLParser.
+
+    :param allow: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be extracted. If not
+        given (or empty), it will match all links.
+    :type allow: str or list
+
+    :param deny: a single regular expression (or list of regular expressions)
+        that the (absolute) urls must match in order to be excluded (i.e. not
+        extracted). It has precedence over the ``allow`` parameter. If not
+        given (or empty) it won't exclude any links.
+    :type deny: str or list
+
+    :param allow_domains: a single value or a list of string containing
+        domains which will be considered for extracting the links
+    :type allow_domains: str or list
+
+    :param deny_domains: a single value or a list of strings containing
+        domains which won't be considered for extracting the links
+    :type deny_domains: str or list
+
+    :param deny_extensions: a single value or list of strings containing
+        extensions that should be ignored when extracting links.
+        If not given, it will default to
+        :data:`scrapy.linkextractors.IGNORED_EXTENSIONS`.
+    :type deny_extensions: list
+
+    :param restrict_xpaths: is an XPath (or list of XPath's) which defines
+        regions inside the response where links should be extracted from.
+        If given, only the text selected by those XPath will be scanned for
+        links.
+    :type restrict_xpaths: str or list
+
+    :param restrict_css: a CSS selector (or list of selectors) which defines
+        regions inside the response where links should be extracted from.
+        Has the same behaviour as ``restrict_xpaths``.
+    :type restrict_css: str or list
+
+    :param restrict_text: a single regular expression (or list of regular
+        expressions) that the link's text must match in order to be extracted.
+        If not given (or empty), it will match all links. If a list of regular
+        expressions is given, the link will be extracted if it matches at least
+        one.
+    :type restrict_text: str or list
+
+    :param tags: a tag or a list of tags to consider when extracting links.
+        Defaults to ``('a', 'area')``. Use ``'*'`` to consider every tag.
+    :type tags: str or list
+
+    :param attrs: an attribute or list of attributes which should be considered
+        when looking for links to extract (only for those tags specified in the
+        ``tags`` parameter). Defaults to ``('href',)``. Use ``'*'`` to consider
+        every attribute.
+    :type attrs: list
+
+    :param deny_tags: a tag or a list of tags that should not be considered when
+        extracting links. It has precedence over the ``tags`` parameter, so it
+        can be combined with ``tags='*'`` to consider every tag except a few.
+        Defaults to ``()`` (no tag is excluded).
+
+        .. versionadded:: VERSION
+    :type deny_tags: str or list
+
+    :param deny_attrs: an attribute or a list of attributes that should not be
+        considered when looking for links to extract. It has precedence over the
+        ``attrs`` parameter, so it can be combined with ``attrs='*'`` to consider
+        every attribute except a few. Defaults to ``()`` (no attribute is
+        excluded).
+
+        .. versionadded:: VERSION
+    :type deny_attrs: str or list
+
+    :param canonicalize: canonicalize each extracted url (using
+        w3lib.url.canonicalize_url). Defaults to ``False``.
+        Note that canonicalize_url is meant for duplicate checking;
+        it can change the URL visible at server side, so the response can be
+        different for requests with canonicalized and raw URLs. If you're
+        using LinkExtractor to follow links it is more robust to
+        keep the default ``canonicalize=False``.
+    :type canonicalize: bool
+
+    :param unique: whether duplicate filtering should be applied to extracted
+        links.
+    :type unique: bool
+
+    :param process_value: a function which receives each value extracted from
+        the tag and attributes scanned and can modify the value and return a
+        new one, or return ``None`` to ignore the link altogether. If not
+        given, ``process_value`` defaults to ``lambda x: x``.
+
+        .. highlight:: html
+
+        For example, to extract links from this code::
+
+            <a href="javascript:goToPage('../other/page.html'); return false">Link text</a>
+
+        .. highlight:: python
+
+        You can use the following function in ``process_value``:
+
+        .. code-block:: python
+
+            def process_value(value):
+                m = re.search(r"javascript:goToPage\('(.*?)'", value)
+                if m:
+                    return m.group(1)
+
+    :type process_value: collections.abc.Callable
+
+    :param strip: whether to strip whitespaces from extracted attributes.
+        According to HTML5 standard, leading and trailing whitespaces
+        must be stripped from ``href`` attributes of ``<a>``, ``<area>``
+        and many other elements, ``src`` attribute of ``<img>``, ``<iframe>``
+        elements, etc., so LinkExtractor strips space chars by default.
+        Set ``strip=False`` to turn it off (e.g. if you're extracting urls
+        from elements or attributes which allow leading/trailing whitespaces).
+    :type strip: bool
+    """
+
     _csstranslator = HTMLTranslator()
 
     def __init__(
@@ -180,11 +312,17 @@ class LxmlLinkExtractor:
         restrict_css: str | Iterable[str] = (),
         strip: bool = True,
         restrict_text: _RegexOrSeveral | None = None,
+        deny_tags: str | Iterable[str] = (),
+        deny_attrs: str | Iterable[str] = (),
     ):
         tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
+        deny_tags, deny_attrs = (
+            set(arg_to_iter(deny_tags)),
+            set(arg_to_iter(deny_attrs)),
+        )
         self.link_extractor = LxmlParserLinkExtractor(
-            tag=partial(operator.contains, tags),
-            attr=partial(operator.contains, attrs),
+            tag=partial(_name_matches, tags, deny_tags),
+            attr=partial(_name_matches, attrs, deny_attrs),
             unique=unique,
             process=process_value,
             strip=strip,
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index 063b92dac..db23e9aef 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -499,6 +499,59 @@ class Base:
                 ),
             ]
 
+        def test_tags_attrs_wildcard_and_deny(self):
+            html = b"""
+            <html><body>
+            <a href="a.html">a</a>
+            <div data-url="div.html">div</div>
+            <span href="span.html">span</span>
+            <p data-url="p.html">p</p>
+            </body></html>
+            """
+            response = HtmlResponse("http://example.com/index.html", body=html)
+
+            def urls(**kwargs):
+                lx = self.extractor_cls(**kwargs)
+                return [link.url for link in lx.extract_links(response)]
+
+            # Default behavior is unchanged: only the listed tags and attributes.
+            assert urls() == ["http://example.com/a.html"]
+
+            # "*" as a tag matches every tag.
+            assert urls(tags="*") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+            ]
+
+            # "*" as an attribute matches every attribute.
+            assert urls(tags="*", attrs="*") == [
+                "http://example.com/a.html",
+                "http://example.com/div.html",
+                "http://example.com/span.html",
+                "http://example.com/p.html",
+            ]
+
+            # deny_tags excludes tags from the wildcard.
+            assert urls(tags="*", attrs="*", deny_tags="div") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+                "http://example.com/p.html",
+            ]
+
+            # deny_attrs excludes attributes from the wildcard.
+            assert urls(tags="*", attrs="*", deny_attrs="data-url") == [
+                "http://example.com/a.html",
+                "http://example.com/span.html",
+            ]
+
+            # deny_tags also applies when tags are listed explicitly.
+            assert urls(tags=("a", "span"), attrs="*", deny_tags="span") == [
+                "http://example.com/a.html",
+            ]
+
+            # The wildcard for one parameter is independent of the other.
+            assert urls(tags="a", attrs="*") == ["http://example.com/a.html"]
+
         def test_xhtml(self):
             xhtml = b"""
     <?xml version="1.0"?>

From 5ccc8dbe8a735b322bacca16848ab9dfed839b99 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 7 Jul 2026 14:57:42 +0500
Subject: [PATCH 05/62] Release notes for 2.17.0 (#7723)

* Cover 2.17.0 in the release notes up to dd10cb8.

* Fix a copypasted issue number.

* Address minor issues

* Minor fixes

* Cover additional deprecations introduced while improving test coverage

* versionadded/versionchanged improvements

* Add a versionadded to the give_up_log_level docstring

---------

Co-authored-by: Adrian Chaves <adrian@zyte.com>
---
 docs/news.rst                         | 308 +++++++++++++++++++++++++-
 docs/topics/autothrottle.rst          |   2 +
 docs/topics/download-handlers.rst     |   4 +-
 docs/topics/downloader-middleware.rst |   8 +
 docs/topics/feed-exports.rst          |   4 -
 docs/topics/request-response.rst      |  11 +-
 docs/topics/settings.rst              |   7 +
 scrapy/downloadermiddlewares/retry.py |   3 +
 scrapy/item.py                        |   2 +-
 scrapy/linkextractors/lxmlhtml.py     |   4 +-
 10 files changed, 341 insertions(+), 12 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index b8b976df9..6966e7763 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,310 @@
 Release notes
 =============
 
+.. _release-2.17.0:
+
+Scrapy 2.17.0 (unreleased)
+--------------------------
+
+Highlights:
+
+-   HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler``
+
+-   Improved settings for changing allowed TLS versions
+
+Deprecations
+~~~~~~~~~~~~
+
+-   The ``DOWNLOADER_CLIENT_TLS_METHOD`` setting is deprecated. You should use
+    the :setting:`DOWNLOAD_TLS_MIN_VERSION` and/or
+    :setting:`DOWNLOAD_TLS_MAX_VERSION` settings instead if you want to change
+    the TLS method selection.
+    (:issue:`3288`, :issue:`6546`)
+
+-   The following spider attributes are deprecated in favor of settings:
+
+    - ``http_user`` (use :setting:`HTTPAUTH_USER`)
+
+    - ``http_pass`` (use :setting:`HTTPAUTH_PASS`)
+
+    - ``http_auth_domain`` (use :setting:`HTTPAUTH_DOMAIN`)
+
+    (:issue:`7590`)
+
+-   The ``scrapy.commands.ScrapyCommand.help()`` method is deprecated. It was
+    never called by Scrapy.
+    (:issue:`7626`, :issue:`7633`)
+
+-   The following TLS-related functions and constants, intended for internal
+    use, are deprecated:
+
+    - ``scrapy.core.downloader.tls.METHOD_TLS``
+
+    - ``scrapy.core.downloader.tls.METHOD_TLSv10``
+
+    - ``scrapy.core.downloader.tls.METHOD_TLSv11``
+
+    - ``scrapy.core.downloader.tls.METHOD_TLSv12``
+
+    - ``scrapy.core.downloader.tls.openssl_methods``
+
+    - ``scrapy.core.downloader.tls.DEFAULT_CIPHERS``
+
+    - ``scrapy.utils.ssl.ffi_buf_to_string()``
+
+    - ``scrapy.utils.ssl.get_temp_key_info()``
+
+    - ``scrapy.utils.ssl.x509name_to_string()``
+
+    (:issue:`6546`, :issue:`7619`, :issue:`7665`)
+
+-   The ``CRAWLSPIDER_FOLLOW_LINKS`` setting is deprecated. You can set
+    ``follow=False`` in your rules to achieve the same effect.
+    (:issue:`7592`)
+
+-   Instantiating
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    without a ``crawler`` argument is deprecated.
+    (:issue:`7655`)
+
+-   Instantiating
+    :class:`~scrapy.spidermiddlewares.referer.RefererMiddleware` without a
+    ``settings`` argument is deprecated.
+    (:issue:`7664`)
+
+New features
+~~~~~~~~~~~~
+
+-   Added support for HTTP/2 requests to
+    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. It
+    requires setting the new :setting:`HTTPX_HTTP2_ENABLED` setting to
+    ``True``.
+    (:issue:`7575`)
+
+-   Added support for SOCKS proxies to
+    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
+    (:issue:`747`, :issue:`7575`)
+
+-   Added :setting:`DOWNLOAD_TLS_MIN_VERSION` and
+    :setting:`DOWNLOAD_TLS_MAX_VERSION` settings as replacements for the
+    ``DOWNLOADER_CLIENT_TLS_METHOD`` setting (which is now deprecated).
+    Compared to the old setting, they support specifying a range of allowed
+    versions and support newer TLS versions.
+    (:issue:`4821`, :issue:`6546`)
+
+-   Added :setting:`HTTPAUTH_USER`, :setting:`HTTPAUTH_PASS` and
+    :setting:`HTTPAUTH_DOMAIN` settings and :reqmeta:`http_user`,
+    :reqmeta:`http_pass` and :reqmeta:`http_auth_domain` meta keys as more
+    flexible ways to set HTTP authentication data.
+    (:issue:`7590`)
+
+-   Added a :reqmeta:`verbatim_url` meta key that can be set to ``True`` to
+    skip request URL canonicalization.
+    (:issue:`7473`)
+
+-   Added ``deny_tags`` and ``deny_attrs`` arguments to :class:`LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>`.
+    (:issue:`6321`, :issue:`7679`)
+
+-   :attr:`scrapy.Item.fields` now returns the fields in the definition order
+    instead of the alphabetical one.
+    (:issue:`7015`, :issue:`7694`)
+
+-   Added a :setting:`RETRY_GIVE_UP_LOG_LEVEL` setting, a
+    :reqmeta:`give_up_log_level` meta key and a ``give_up_log_level`` argument
+    of the
+    :func:`~scrapy.downloadermiddlewares.retry.get_retry_request` function that
+    allow changing the log level of the message logged when the retry limit has
+    been reached.
+    (:issue:`4622`, :issue:`5297`, :issue:`7567`)
+
+-   It's now possible to set :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` to
+    ``None`` to use the default ciphers of the underlying TLS implementation.
+    (:issue:`7499`, :issue:`7665`)
+
+Improvements
+~~~~~~~~~~~~
+
+-   :class:`~scrapy.FormRequest` is no longer deprecated, only its
+    ``from_response()`` method is still deprecated.
+    (:issue:`7561`, :issue:`7671`)
+
+-   Switched the item definition in the default project template from a
+    :class:`scrapy.item.Item` to a dataclass.
+    (:issue:`7493`, :issue:`7513`)
+
+-   Fixed deprecation warnings with pyOpenSSL 26.3.0.
+    (:issue:`7619`)
+
+-   Removed the runtime warnings for :attr:`Spider.allowed_domains
+    <scrapy.Spider.allowed_domains>` containing URLs or domains with ports
+    instead of just domains and for spider classes having a ``start_url``
+    attribute instead of :class:`~scrapy.spiders.Spider.start_urls`. Please use
+    :doc:`scrapy-lint <scrapy-lint:index>` to find mistakes in your spider code
+    instead.
+    (:issue:`4421`, :issue:`7627`)
+
+-   :func:`scrapy.utils.test.get_crawler` now disables
+    :setting:`TELNETCONSOLE_ENABLED` by default.
+    (:issue:`7644`)
+
+-   Other code refactoring and improvements.
+    (:issue:`7409`, :issue:`7593`, :issue:`7594`, :issue:`7611`, :issue:`7649`)
+
+Bug fixes
+~~~~~~~~~
+
+-   :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler` no
+    longer ignores proxy credentials for redirected or retried requests.
+    (:issue:`7601`, :issue:`7630`)
+
+-   :class:`~scrapy.extensions.feedexport.GCSFeedStorage` now closes the
+    temporary file after the upload.
+    (:issue:`7546`)
+
+-   Fixed ``scrapy shell <URL>`` running a full spider crawl when there is a
+    spider for the requested URL. This bug was introduced in Scrapy 2.13.0.
+    (:issue:`7552`, :issue:`7557`)
+
+-   The :setting:`IMAGES_STORE_S3_ACL` and :setting:`IMAGES_STORE_GCS_ACL`
+    settings are no longer ignored. This bug was introduced in Scrapy 2.12.0.
+    (:issue:`7597`, :issue:`7614`)
+
+-   :class:`~scrapy.core.downloader.handlers.ftp.FTPDownloadHandler` now closes
+    the connection after making the request.
+    (:issue:`7602`, :issue:`7667`)
+
+-   Removed the deprecated ``spider`` argument from the pipeline defined in the
+    default project template.
+    (:issue:`7676`)
+
+-   Fixed ``scrapy genspider --edit`` not working.
+    (:issue:`7260`, :issue:`7683`)
+
+-   When a :class:`~scrapy.crawler.Crawler` instance is passed to
+    :meth:`AsyncCrawlerRunner.create_crawler()
+    <scrapy.crawler.AsyncCrawlerRunner.create_crawler>` or
+    :meth:`CrawlerRunner.create_crawler()
+    <scrapy.crawler.CrawlerRunner.create_crawler>`, settings from both classes
+    are now merged, previously only the settings from the
+    :class:`~scrapy.crawler.Crawler` instance were used.
+    (:issue:`1280`, :issue:`7647`)
+
+-   Fixed several issues with cookie handling in
+    :func:`scrapy.utils.request.request_to_curl`.
+    (:issue:`7603`, :issue:`7675`, :issue:`7684`)
+
+-   Fixed :class:`scrapy.resolver.CachingThreadedResolver` not disabling the
+    cache when :setting:`DNSCACHE_ENABLED` is set to ``False``.
+    (:issue:`7663`)
+
+-   Fixed :func:`scrapy.utils.response.open_in_browser` not removing comments
+    when looking for the ``<base>`` tag.
+    (:issue:`7506`)
+
+-   Fixed checking for deprecated methods in custom :setting:`ITEM_PROCESSOR`
+    implementations.
+    (:issue:`7589`)
+
+-   Fixed :func:`scrapy.utils.url.strip_url` corrupting some URLs with
+    credentials.
+    (:issue:`7604`, :issue:`7605`)
+
+-   :func:`scrapy.utils.misc.rel_has_nofollow` now ignores the case when
+    looking for "nofollow" strings.
+    (:issue:`7632`)
+
+-   Fixed an exception in :class:`scrapy.utils.sitemap.Sitemap` when parsing
+    some malformed sitemaps.
+    (:issue:`7686`, :issue:`7687`)
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Mentioned :doc:`scrapy-lint <scrapy-lint:index>` in the docs.
+    (:issue:`4421`, :issue:`7627`)
+
+-   Added the docs about :ref:`security considerations <security>`.
+    (:issue:`7389`, :issue:`7678`)
+
+-   Improved the :ref:`item pipeline docs <topics-item-pipeline>`.
+    (:issue:`2350`, :issue:`7676`)
+
+-   Documented which stats are collected by
+    :class:`~scrapy.extensions.corestats.CoreStats`.
+    (:issue:`7421`)
+
+-   Switched documentation examples from using :class:`scrapy.item.Item` to
+    using dataclasses.
+    (:issue:`7493`, :issue:`7513`)
+
+-   Added feature comparison tables to the :ref:`download handler
+    <download-handlers-ref>` docs.
+    (:issue:`7575`)
+
+-   Improved the docs for :ref:`logging settings <logging-settings>`.
+    (:issue:`6909`, :issue:`7668`)
+
+-   Documented a way to :ref:`improve startup time and memory usage
+    <large-project-startup>` by using :setting:`SPIDER_MODULES`.
+    (:issue:`7576`, :issue:`7600`)
+
+-   Clarified handling of the ``type`` argument of :class:`~scrapy.Selector`.
+    (:issue:`7704`)
+
+-   Other documentation improvements and fixes.
+    (:issue:`4954`,
+    :issue:`6120`,
+    :issue:`7286`,
+    :issue:`7564`,
+    :issue:`7573`,
+    :issue:`7598`,
+    :issue:`7599`,
+    :issue:`7698`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Fixed deprecation warnings with pytest 9.1.0.
+    (:issue:`7621`)
+
+-   Type hints improvements and fixes.
+    (:issue:`6958`, :issue:`7586`)
+
+-   CI and test improvements and fixes.
+    (:issue:`5954`,
+    :issue:`7002`,
+    :issue:`7017`,
+    :issue:`7247`,
+    :issue:`7508`,
+    :issue:`7545`,
+    :issue:`7566`,
+    :issue:`7574`,
+    :issue:`7585`,
+    :issue:`7595`,
+    :issue:`7608`,
+    :issue:`7610`,
+    :issue:`7612`,
+    :issue:`7616`,
+    :issue:`7625`,
+    :issue:`7637`,
+    :issue:`7639`,
+    :issue:`7640`,
+    :issue:`7641`,
+    :issue:`7642`,
+    :issue:`7643`,
+    :issue:`7644`,
+    :issue:`7645`,
+    :issue:`7646`,
+    :issue:`7654`,
+    :issue:`7655`,
+    :issue:`7664`,
+    :issue:`7672`,
+    :issue:`7677`,
+    :issue:`7680`,
+    :issue:`7682`,
+    :issue:`7692`)
+
 .. _release-2.16.0:
 
 Scrapy 2.16.0 (2026-05-19)
@@ -6243,8 +6547,8 @@ Backward-incompatible changes
     ``429``, you must override :setting:`RETRY_HTTP_CODES` accordingly.
 
 *   :class:`~scrapy.crawler.Crawler`,
-    :class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
-    :class:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
+    :meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>` and
+    :meth:`CrawlerRunner.create_crawler <scrapy.crawler.CrawlerRunner.create_crawler>`
     no longer accept a :class:`~scrapy.spiders.Spider` subclass instance, they
     only accept a :class:`~scrapy.spiders.Spider` subclass now.
 
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index d0321c906..787ef842a 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -88,6 +88,8 @@ server) is, and this extension builds on that premise.
 Prevent specific requests from triggering slot delay adjustments
 ================================================================
 
+.. versionadded:: 2.12.0
+
 AutoThrottle adjusts the delay of download slots based on the latencies of
 responses that belong to that download slot. The only exceptions are non-200
 responses, which are only taken into account to increase that delay, but
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index c901e01e4..9bbfee9e2 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -329,13 +329,13 @@ Other limitations:
 HTTPX_HTTP2_ENABLED
 ^^^^^^^^^^^^^^^^^^^
 
+.. versionadded:: 2.17.0
+
 Default: ``False``
 
 Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
 needs to be installed if you want to enable this setting.
 
-.. versionadded:: VERSION
-
 Built-in non-HTTP download handlers reference
 =============================================
 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 5649453b1..054c1130c 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -351,6 +351,8 @@ HttpAuthMiddleware
 HTTPAUTH_USER
 ~~~~~~~~~~~~~
 
+.. versionadded:: 2.17.0
+
 Default: ``""``
 
 The username to use for HTTP basic authentication, applied to all requests
@@ -361,6 +363,8 @@ whose URL matches :setting:`HTTPAUTH_DOMAIN`.
 HTTPAUTH_PASS
 ~~~~~~~~~~~~~
 
+.. versionadded:: 2.17.0
+
 Default: ``""``
 
 The password to use for HTTP basic authentication.
@@ -370,6 +374,8 @@ The password to use for HTTP basic authentication.
 HTTPAUTH_DOMAIN
 ~~~~~~~~~~~~~~~
 
+.. versionadded:: 2.17.0
+
 Default: ``None``
 
 The domain (and its subdomains) to which HTTP basic authentication credentials
@@ -1104,6 +1110,8 @@ exception propagation, see
 RETRY_GIVE_UP_LOG_LEVEL
 ^^^^^^^^^^^^^^^^^^^^^^^
 
+.. versionadded:: 2.17.0
+
 Default: ``"ERROR"``
 
 :ref:`Logging level <levels>` used for the message logged when a request
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 6ce5255b9..9aa1b8d5f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -528,10 +528,6 @@ safe numeric encoding (``\uXXXX`` sequences) for historic reasons.
 
 Use ``"utf-8"`` if you want UTF-8 for JSON too.
 
-.. versionchanged:: 2.8
-   The :command:`startproject` command now sets this setting to
-   ``"utf-8"`` in the generated ``settings.py`` file.
-
 .. setting:: FEED_EXPORT_FIELDS
 
 FEED_EXPORT_FIELDS
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f2fcfb0b5..44bc09b7f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -806,6 +806,8 @@ Whether or not to fail on broken responses. See:
 give_up_log_level
 -----------------
 
+.. versionadded:: 2.17.0
+
 :ref:`Logging level <levels>` used for the message logged when a request
 exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 
@@ -814,6 +816,8 @@ exceeds its retries. See :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 http_auth_domain
 ----------------
 
+.. versionadded:: 2.17.0
+
 Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
 
 .. reqmeta:: http_pass
@@ -821,6 +825,8 @@ Overrides :setting:`HTTPAUTH_DOMAIN` for this request.
 http_pass
 ---------
 
+.. versionadded:: 2.17.0
+
 Overrides :setting:`HTTPAUTH_PASS` for this request.
 
 .. reqmeta:: http_user
@@ -828,6 +834,8 @@ Overrides :setting:`HTTPAUTH_PASS` for this request.
 http_user
 ---------
 
+.. versionadded:: 2.17.0
+
 Overrides :setting:`HTTPAUTH_USER` for this request.
 
 .. reqmeta:: max_retry_times
@@ -844,6 +852,8 @@ The meta key is used set retry times per request. When set, the
 verbatim_url
 ------------
 
+.. versionadded:: 2.17.0
+
 Set this key to ``True`` to keep the request URL as passed to
 :class:`~scrapy.Request`, without URL percent-encoding.
 
@@ -854,7 +864,6 @@ characters that would otherwise be canonicalized get different fingerprints.
 In this mode, the ``keep_fragments`` parameter is ignored, and it is
 effectively true.
 
-
 .. _topics-stop-response-download:
 
 Stopping the download of a Response
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 11251a92c..a5148ee26 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -766,6 +766,9 @@ specific cipher that is not included in ``DEFAULT`` if a website requires it.
 Set this setting to ``None`` to use the default ciphers of the underlying TLS
 implementation.
 
+.. versionchanged:: 2.17.0
+   Added support for setting this to ``None``.
+
 .. _OpenSSL cipher list format: https://docs.openssl.org/master/man1/openssl-ciphers/#cipher-list-format
 
 .. note::
@@ -781,6 +784,8 @@ implementation.
 DOWNLOAD_TLS_MAX_VERSION
 ------------------------
 
+.. versionadded:: 2.17.0
+
 Default: ``None``
 
 Use this setting to change the maximum version of the TLS protocol allowed to
@@ -816,6 +821,8 @@ modern environments.
 DOWNLOAD_TLS_MIN_VERSION
 ------------------------
 
+.. versionadded:: 2.17.0
+
 Default: ``None``
 
 Use this setting to change the minimum version of the TLS protocol allowed to
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.py
index a0a0b60a2..e1dcd90d5 100644
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@@ -87,6 +87,9 @@ def get_retry_request(
     message logged when a request exceeds its retries. See
     :setting:`RETRY_GIVE_UP_LOG_LEVEL` for details.
 
+    .. versionadded:: 2.17.0
+       The *give_up_log_level* parameter.
+
     *stats_base_key* is a string to be used as the base key for the
     retry-related job stats
     """
diff --git a/scrapy/item.py b/scrapy/item.py
index 4d99ea79d..57d2272e2 100644
--- a/scrapy/item.py
+++ b/scrapy/item.py
@@ -100,7 +100,7 @@ class Item(MutableMapping[str, Any], object_ref, metaclass=ItemMeta):
     #: come first, followed by fields declared in subclasses, and a field
     #: redefined in a subclass keeps the position of its first definition.
     #:
-    #: .. versionchanged:: VERSION
+    #: .. versionchanged:: 2.17.0
     #:    Fields are now returned in definition order rather than alphabetical
     #:    order.
     fields: dict[str, Field]
diff --git a/scrapy/linkextractors/lxmlhtml.py b/scrapy/linkextractors/lxmlhtml.py
index 2efe7cd8e..3fb741d7a 100644
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@@ -235,7 +235,7 @@ class LxmlLinkExtractor:
         can be combined with ``tags='*'`` to consider every tag except a few.
         Defaults to ``()`` (no tag is excluded).
 
-        .. versionadded:: VERSION
+        .. versionadded:: 2.17.0
     :type deny_tags: str or list
 
     :param deny_attrs: an attribute or a list of attributes that should not be
@@ -244,7 +244,7 @@ class LxmlLinkExtractor:
         every attribute except a few. Defaults to ``()`` (no attribute is
         excluded).
 
-        .. versionadded:: VERSION
+        .. versionadded:: 2.17.0
     :type deny_attrs: str or list
 
     :param canonicalize: canonicalize each extracted url (using

From 9523e1ec8c41fde265a26d14563d178b6f1ad04b Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Tue, 7 Jul 2026 12:25:36 +0200
Subject: [PATCH 06/62] Merge commit from fork

---
 docs/news.rst                         | 20 ++++++++++++++++++++
 scrapy/core/downloader/handlers/s3.py |  2 +-
 tests/test_downloader_handlers.py     | 18 ++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)

diff --git a/docs/news.rst b/docs/news.rst
index 6966e7763..582c543ef 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -10,10 +10,30 @@ Scrapy 2.17.0 (unreleased)
 
 Highlights:
 
+-   Security bug fixes
+
 -   HTTP/2 and SOCKS proxy support for ``HttpxDownloadHandler``
 
 -   Improved settings for changing allowed TLS versions
 
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+-   ``s3://`` requests now use HTTPS by default, instead of plaintext HTTP.
+
+    Previously, :class:`~scrapy.core.downloader.handlers.s3.S3DownloadHandler`
+    sent signed S3 requests over plaintext HTTP unless
+    ``request.meta["is_secure"]`` was set to a true value, exposing the request
+    path, the AWS ``Authorization`` header, the ``X-Amz-Security-Token`` header
+    (when using temporary credentials), and the response contents to network
+    attackers, who could also tamper with responses. See the `76g3-c3x4-crvx`_
+    security advisory for details.
+
+    To restore the previous behavior for a given request, set
+    ``request.meta["is_secure"]`` to ``False``.
+
+    .. _76g3-c3x4-crvx: https://github.com/scrapy/scrapy/security/advisories/GHSA-76g3-c3x4-crvx
+
 Deprecations
 ~~~~~~~~~~~~
 
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index c6660ff48..9b7ceeb4c 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -49,7 +49,7 @@ class S3DownloadHandler(BaseDownloadHandler):
 
     async def download_request(self, request: Request) -> Response:
         p = urlparse_cached(request)
-        scheme = "https" if request.meta.get("is_secure") else "http"
+        scheme = "http" if request.meta.get("is_secure") is False else "https"
         bucket = p.hostname
         path = p.path + "?" + p.query if p.query else p.path
         url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index e685f607a..275199a76 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -170,6 +170,12 @@ class TestS3Anon:
         httpreq = await self.download_request(req)
         assert hasattr(self.s3reqh, "anon")
         assert self.s3reqh.anon
+        assert httpreq.url == "https://aws-publicdatasets.s3.amazonaws.com/"
+
+    @coroutine_test
+    async def test_anon_request_insecure(self):
+        req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
+        httpreq = await self.download_request(req)
         assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
 
 
@@ -200,6 +206,18 @@ class TestS3:
             mock_formatdate.return_value = date
             yield
 
+    @coroutine_test
+    async def test_secure_by_default(self):
+        req = Request("s3://johnsmith/photos/puppy.jpg")
+        httpreq = await self.download_request(req)
+        assert httpreq.url == "https://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
+
+    @coroutine_test
+    async def test_insecure_opt_out(self):
+        req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
+        httpreq = await self.download_request(req)
+        assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
+
     @coroutine_test
     async def test_request_signing1(self):
         # gets an object from the johnsmith bucket.

From feb692f552e3ed4533cf4e3af4809908d84cd763 Mon Sep 17 00:00:00 2001
From: Adrian Chaves <adrian@zyte.com>
Date: Tue, 7 Jul 2026 12:26:52 +0200
Subject: [PATCH 07/62] =?UTF-8?q?Bump=20version:=202.16.0=20=E2=86=92=202.?=
 =?UTF-8?q?17.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 SECURITY.md    | 4 ++--
 docs/news.rst  | 2 +-
 pyproject.toml | 2 +-
 scrapy/VERSION | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 49cafdf9e..c1a5482c6 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -4,8 +4,8 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 2.16.x     | :white_check_mark: |
-| < 2.16.x   | :x:                |
+| 2.17.x     | :white_check_mark: |
+| < 2.17.x   | :x:                |
 
 ## Reporting a Vulnerability
 
diff --git a/docs/news.rst b/docs/news.rst
index 582c543ef..7d2b07b09 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.17.0:
 
-Scrapy 2.17.0 (unreleased)
+Scrapy 2.17.0 (2026-07-07)
 --------------------------
 
 Highlights:
diff --git a/pyproject.toml b/pyproject.toml
index b8d9067f9..ed2e99bea 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -137,7 +137,7 @@ module = [
 ignore_missing_imports = true
 
 [tool.bumpversion]
-current_version = "2.16.0"
+current_version = "2.17.0"
 commit = true
 tag = true
 tag_name = "{new_version}"
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 752490696..d76bd2ba3 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.16.0
+2.17.0

From c5ec881f1de1c450920d6a6f3299f46a05915229 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Tue, 7 Jul 2026 14:27:26 +0200
Subject: [PATCH 08/62] Update tools (#7724)

---
 .pre-commit-config.yaml   |  4 ++--
 docs/faq.rst              |  1 -
 docs/topics/addons.rst    |  1 -
 docs/topics/selectors.rst | 13 +++++--------
 tox.ini                   | 20 ++++++++++----------
 5 files changed, 17 insertions(+), 22 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 754bd6f9c..311df7052 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -6,7 +6,7 @@ exclude: |
   )
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.15.2
+  rev: v0.15.20
   hooks:
     - id: ruff-check
       args: [ --fix ]
@@ -16,7 +16,7 @@ repos:
   hooks:
   - id: blacken-docs
     additional_dependencies:
-    - black==25.9.0
+    - black==26.5.1
 - repo: https://github.com/pre-commit/pre-commit-hooks
   rev: v6.0.0
   hooks:
diff --git a/docs/faq.rst b/docs/faq.rst
index df90122f5..6eeb81a31 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -409,7 +409,6 @@ How can I make a blank request?
 
     from scrapy import Request
 
-
     blank_request = Request("data:,")
 
 In this case, the URL is set to a data URI scheme. Data URLs allow you to include data
diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 01e4bcac1..78b1ab92b 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -168,7 +168,6 @@ Use a fallback component:
 
     from scrapy.utils.misc import build_from_crawler, load_object
 
-
     FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
 
 
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 35ea2a200..0b174341d 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -634,8 +634,7 @@ Example:
 .. code-block:: pycon
 
     >>> from scrapy import Selector
-    >>> sel = Selector(
-    ...     text="""
+    >>> sel = Selector(text="""
     ...     <ul class="list">
     ...         <li>1</li>
     ...         <li>2</li>
@@ -645,8 +644,8 @@ Example:
     ...         <li>4</li>
     ...         <li>5</li>
     ...         <li>6</li>
-    ...     </ul>"""
-    ... )
+    ...     </ul>""")
+    ...
     >>> xp = lambda x: sel.xpath(x).getall()
 
 This gets all first ``<li>``  elements under whatever it is its parent:
@@ -948,11 +947,9 @@ with groups of itemscopes and corresponding itemprops:
     >>> sel = Selector(text=doc, type="html")
     >>> for scope in sel.xpath("//div[@itemscope]"):
     ...     print("current scope:", scope.xpath("@itemtype").getall())
-    ...     props = scope.xpath(
-    ...         """
+    ...     props = scope.xpath("""
     ...                 set:difference(./descendant::*/@itemprop,
-    ...                                .//*[@itemscope]/*/@itemprop)"""
-    ...     )
+    ...                                .//*[@itemscope]/*/@itemprop)""")
     ...     print(f"    properties: {props.getall()}")
     ...     print("")
     ...
diff --git a/tox.ini b/tox.ini
index 5017d7636..7ace26e98 100644
--- a/tox.ini
+++ b/tox.ini
@@ -70,13 +70,13 @@ commands =
 basepython = python3.10
 deps =
     mypy==2.1.0
-    typing-extensions==4.15.0
-    Pillow==12.2.0
-    Protego==0.6.0
+    typing-extensions==4.16.0
+    Pillow==12.3.0
+    Protego==0.6.2
     Twisted==26.4.0
     attrs==26.1.0
-    boto3-stubs[s3]==1.43.9
-    botocore-stubs==1.42.41
+    boto3-stubs[s3]==1.43.41
+    botocore-stubs==1.43.14
     h2==4.3.0
     httpx==0.28.1
     itemadapter==0.13.1
@@ -84,12 +84,12 @@ deps =
     # newer ones require newer Python
     ipython==8.39.0
     pyOpenSSL==26.3.0
-    pytest==9.0.3
+    pytest==9.1.1
     socksio==1.0.0
-    types-Pygments==2.20.0.20260508
+    types-Pygments==2.20.0.20260518
     types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
-    types-pexpect==4.9.0.20260508
+    types-pexpect==4.9.0.20260518
     uvloop==0.22.1
     w3lib==2.4.1
     zstandard==0.25.0
@@ -116,7 +116,7 @@ commands =
 basepython = python3
 deps =
     {[testenv:extra-deps]deps}
-    pylint==4.0.2
+    pylint==4.0.6
     pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
 commands =
     pylint {posargs:conftest.py docs extras scrapy tests}
@@ -125,7 +125,7 @@ commands =
 basepython = python3
 deps =
     twine==6.2.0
-    build==1.3.0
+    build==1.5.0
 commands =
     python -m build --sdist
     twine check dist/*

From bdf3067935fd262927ffddfe6bc2ebc9d15d034e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 7 Jul 2026 18:39:34 +0500
Subject: [PATCH 09/62] Assorted docs fixes, part 1. (#7710)

---
 docs/README.rst                       |  68 ---------
 docs/contributing.rst                 |   7 +-
 docs/faq.rst                          |   8 +-
 docs/index.rst                        |   8 +-
 docs/intro/overview.rst               |   7 +-
 docs/news.rst                         |   8 +-
 docs/topics/asyncio.rst               |   8 +-
 docs/topics/commands.rst              |   4 +-
 docs/topics/coroutines.rst            |   2 +-
 docs/topics/downloader-middleware.rst |  13 +-
 docs/topics/dynamic-content.rst       |   2 +-
 docs/topics/exceptions.rst            |   2 +-
 docs/topics/exporters.rst             |  10 +-
 docs/topics/extensions.rst            |   8 +-
 docs/topics/feed-exports.rst          |   1 +
 docs/topics/items.rst                 |   4 +-
 docs/topics/leaks.rst                 |  10 +-
 docs/topics/loaders.rst               |  10 +-
 docs/topics/logging.rst               |   5 -
 docs/topics/media-pipeline.rst        |  11 +-
 docs/topics/request-response.rst      |   7 +-
 docs/topics/selectors.rst             |   2 +-
 docs/topics/settings.rst              | 202 ++++++++++++++------------
 docs/topics/signals.rst               |   2 +
 docs/topics/spider-middleware.rst     |   2 +-
 docs/topics/spiders.rst               |   9 +-
 docs/topics/telnetconsole.rst         |   6 +-
 docs/versioning.rst                   |   4 +-
 scrapy/settings/default_settings.py   |   2 +-
 29 files changed, 192 insertions(+), 240 deletions(-)
 delete mode 100644 docs/README.rst

diff --git a/docs/README.rst b/docs/README.rst
deleted file mode 100644
index 4d2236b53..000000000
--- a/docs/README.rst
+++ /dev/null
@@ -1,68 +0,0 @@
-:orphan:
-
-======================================
-Scrapy documentation quick start guide
-======================================
-
-This file provides a quick guide on how to compile the Scrapy documentation.
-
-
-Setup the environment
----------------------
-
-To compile the documentation you need Sphinx Python library. To install it
-and all its dependencies run the following command from this dir
-
-::
-
-    pip install -r requirements.txt
-
-
-Compile the documentation
--------------------------
-
-To compile the documentation (to classic HTML output) run the following command
-from this dir::
-
-    make html
-
-Documentation will be generated (in HTML format) inside the ``build/html`` dir.
-
-
-View the documentation
-----------------------
-
-To view the documentation run the following command::
-
-    make htmlview
-
-This command will fire up your default browser and open the main page of your
-(previously generated) HTML documentation.
-
-
-Start over
-----------
-
-To clean up all generated documentation files and start from scratch run::
-
-    make clean
-
-Keep in mind that this command won't touch any documentation source files.
-
-
-Recreating documentation on the fly
------------------------------------
-
-There is a way to recreate the doc automatically when you make changes, you
-need to install watchdog (``pip install watchdog``) and then use::
-
-    make watch
-
-Alternative method using tox
-----------------------------
-
-To compile the documentation to HTML run the following command::
-
-    tox -e docs
-
-Documentation will be generated inside the ``docs/_build/all`` dir.
diff --git a/docs/contributing.rst b/docs/contributing.rst
index c868a0ac4..b43d30796 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -323,9 +323,10 @@ deprecation removals are documented in the :ref:`release notes <news>`.
 Tests
 =====
 
-Tests are implemented using the :doc:`Twisted unit-testing framework
-<twisted:development/test-standard>`. Running tests requires
-:doc:`tox <tox:index>`.
+Tests are implemented using pytest_. Running tests requires :doc:`tox
+<tox:index>`.
+
+.. _pytest: https://pytest.org
 
 .. _running-tests:
 
diff --git a/docs/faq.rst b/docs/faq.rst
index 6eeb81a31..a396a8212 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -285,7 +285,8 @@ consume a lot of memory.
 In order to avoid parsing all the entire feed at once in memory, you can use
 the :func:`~scrapy.utils.iterators.xmliter_lxml` and
 :func:`~scrapy.utils.iterators.csviter` functions. In fact, this is what
-:class:`~scrapy.spiders.XMLFeedSpider` uses.
+:class:`~scrapy.spiders.XMLFeedSpider` and
+:class:`~scrapy.spiders.CSVFeedSpider` use.
 
 .. autofunction:: scrapy.utils.iterators.xmliter_lxml
 
@@ -360,10 +361,11 @@ method for this purpose. For example:
         def process_spider_output(self, response, result):
             for item_or_request in result:
                 if isinstance(item_or_request, Request):
+                    yield item_or_request
                     continue
-                adapter = ItemAdapter(item)
+                adapter = ItemAdapter(item_or_request)
                 for _ in range(adapter["multiply_by"]):
-                    yield deepcopy(item)
+                    yield deepcopy(item_or_request)
 
 Does Scrapy support IPv6 addresses?
 -----------------------------------
diff --git a/docs/index.rst b/docs/index.rst
index 8e8624a22..688cab81b 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -24,7 +24,7 @@ Having trouble? We'd like to help!
 * Ask or search questions in `StackOverflow using the scrapy tag`_.
 * Ask or search questions in the `Scrapy subreddit`_.
 * Search for questions on the archives of the `scrapy-users mailing list`_.
-* Ask a question in the `#scrapy IRC channel`_,
+* Ask a question in the `#scrapy IRC channel`_.
 * Report bugs with Scrapy in our `issue tracker`_.
 * Join the Discord community `Scrapy Discord`_.
 
@@ -91,15 +91,15 @@ Basic concepts
 :doc:`topics/selectors`
     Extract the data from web pages using XPath.
 
-:doc:`topics/shell`
-    Test your extraction code in an interactive environment.
-
 :doc:`topics/items`
     Define the data you want to scrape.
 
 :doc:`topics/loaders`
     Populate your items with the extracted data.
 
+:doc:`topics/shell`
+    Test your extraction code in an interactive environment.
+
 :doc:`topics/item-pipeline`
     Post-process and store your scraped data.
 
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index ee91ce7ca..488e1c2e3 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -90,9 +90,10 @@ to figure these settings out automatically.
 .. note::
 
     This is using :ref:`feed exports <topics-feed-exports>` to generate the
-    JSON file, you can easily change the export format (XML or CSV, for example) or the
-    storage backend (FTP or `Amazon S3`_, for example).  You can also write an
-    :ref:`item pipeline <topics-item-pipeline>` to store the items in a database.
+    JSON Lines file, you can easily change the export format (XML or CSV, for
+    example) or the storage backend (FTP or `Amazon S3`_, for example). You can
+    also write an :ref:`item pipeline <topics-item-pipeline>` to store the
+    items in a database.
 
 
 .. _topics-whatelse:
diff --git a/docs/news.rst b/docs/news.rst
index 7d2b07b09..63f0d69e4 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1035,7 +1035,7 @@ Deprecations
 New features
 ~~~~~~~~~~~~
 
--   Added a new setting, :setting:`REFERER_POLICIES`, to allow customizing
+-   Added a new setting, :setting:`REFERRER_POLICIES`, to allow customizing
     supported referrer policies.
 
 Bug fixes
@@ -6046,7 +6046,7 @@ The following changes may impact custom priority queue classes:
 
     *   A new keyword parameter has been added: ``key``. It is a string
         that is always an empty string for memory queues and indicates the
-        :setting:`JOB_DIR` value for disk queues.
+        :setting:`JOBDIR` value for disk queues.
 
     *   The parameter for disk queues that contains data from the previous
         crawl, ``startprios`` or ``slot_startprios``, is now passed as a
@@ -6600,7 +6600,7 @@ New features
     ``scrapy.pqueues.DownloaderAwarePriorityQueue``, may be
     :ref:`enabled <broad-crawls-scheduler-priority-queue>` for a significant
     scheduling improvement on crawls targeting multiple web domains, at the
-    cost of no :setting:`CONCURRENT_REQUESTS_PER_IP` support (:issue:`3520`)
+    cost of no ``CONCURRENT_REQUESTS_PER_IP`` support (:issue:`3520`)
 
 *   A new :attr:`.Request.cb_kwargs` attribute
     provides a cleaner way to pass keyword arguments to callback methods
@@ -9079,7 +9079,7 @@ New features and settings
 - In request errbacks, offending requests are now received in ``failure.request`` attribute (:rev:`2738`)
 - Big downloader refactoring to support per domain/ip concurrency limits (:rev:`2732`)
    - ``CONCURRENT_REQUESTS_PER_SPIDER`` setting has been deprecated and replaced by:
-      - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, :setting:`CONCURRENT_REQUESTS_PER_IP`
+      - :setting:`CONCURRENT_REQUESTS`, :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, ``CONCURRENT_REQUESTS_PER_IP``
    - check the documentation for more details
 - Added builtin caching DNS resolver (:rev:`2728`)
 - Moved Amazon AWS-related components/extensions (SQS spider queue, SimpleDB stats collector) to a separate project: [scaws](https://github.com/scrapinghub/scaws) (:rev:`2706`, :rev:`2714`)
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 1a151044f..17b322b5a 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -105,6 +105,9 @@ Scrapy API requires passing a Deferred to it) using the following helpers:
 
 .. autofunction:: scrapy.utils.defer.deferred_from_coro
 .. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
+
+The following function helps with a reverse wrapping:
+
 .. autofunction:: scrapy.utils.defer.ensure_awaitable
 
 
@@ -190,7 +193,7 @@ in future Scrapy versions. The following features are not available:
   :class:`~scrapy.crawler.CrawlerProcess`
   (:class:`~scrapy.crawler.AsyncCrawlerProcess` and
   :class:`~scrapy.crawler.AsyncCrawlerRunner` are available)
-* Twisted-specific DNS resolvers (the :setting:`DNS_RESOLVER` setting)
+* Twisted-specific DNS resolvers (the :setting:`TWISTED_DNS_RESOLVER` setting)
 * User and 3rd-party code that requires a reactor (see :ref:`below
   <asyncio-without-reactor-migrate>` for examples)
 
@@ -315,8 +318,7 @@ implementations, :class:`~asyncio.ProactorEventLoop` (default) and
 :class:`~asyncio.SelectorEventLoop` works with Twisted.
 
 Scrapy changes the event loop class to :class:`~asyncio.SelectorEventLoop`
-automatically when you change the :setting:`TWISTED_REACTOR` setting or call
-:func:`~scrapy.utils.reactor.install_reactor`.
+automatically when installing the asyncio reactor.
 
 .. note:: Other libraries you use may require
           :class:`~asyncio.ProactorEventLoop`, e.g. because it supports
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 8d1351eb9..270d9859d 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -199,6 +199,7 @@ Global commands:
 * :command:`fetch`
 * :command:`view`
 * :command:`version`
+* :command:`bench`
 
 Project-only commands:
 
@@ -207,7 +208,6 @@ Project-only commands:
 * :command:`list`
 * :command:`edit`
 * :command:`parse`
-* :command:`bench`
 
 .. command:: startproject
 
@@ -476,7 +476,7 @@ Supported options:
 
 * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
 
-* ``--a NAME=VALUE``: set spider argument (may be repeated)
+* ``-a NAME=VALUE``: set spider argument (may be repeated)
 
 * ``--callback`` or ``-c``: spider method to use as callback for parsing the
   response
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 116aac323..8aaf0aa31 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -268,5 +268,5 @@ You can also send multiple requests in parallel:
             yield {
                 "h1": response.css("h1::text").get(),
                 "price": responses[0].css(".price::text").get(),
-                "price2": responses[1].css(".color::text").get(),
+                "color": responses[1].css(".color::text").get(),
             }
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 054c1130c..383f331b5 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -505,7 +505,7 @@ Filesystem storage backend (default)
 
     *   ``response_body`` - the plain response body
 
-    *   ``response_headers`` - the request headers (in raw HTTP format)
+    *   ``response_headers`` - the response headers (in raw HTTP format)
 
     *   ``meta`` - some metadata of this cache resource in Python ``repr()``
         format (grep-friendly format)
@@ -547,7 +547,7 @@ defines the methods described below.
     .. method:: open_spider(spider)
 
       This method gets called after a spider has been opened for crawling. It handles
-      the :signal:`open_spider <spider_opened>` signal.
+      the :signal:`spider_opened` signal.
 
       :param spider: the spider which has been opened
       :type spider: :class:`~scrapy.Spider` object
@@ -555,7 +555,7 @@ defines the methods described below.
     .. method:: close_spider(spider)
 
       This method gets called after a spider has been closed. It handles
-      the :signal:`close_spider <spider_closed>` signal.
+      the :signal:`spider_closed` signal.
 
       :param spider: the spider which has been closed
       :type spider: :class:`~scrapy.Spider` object
@@ -814,7 +814,6 @@ HttpProxyMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 .. setting:: HTTPPROXY_ENABLED
-.. setting:: HTTPPROXY_AUTH_ENCODING
 
 HTTPPROXY_ENABLED
 ^^^^^^^^^^^^^^^^^
@@ -823,6 +822,8 @@ Default: ``True``
 
 Whether or not to enable the :class:`HttpProxyMiddleware`.
 
+.. setting:: HTTPPROXY_AUTH_ENCODING
+
 HTTPPROXY_AUTH_ENCODING
 ^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -984,7 +985,7 @@ Whether the Meta Refresh middleware will be enabled.
 METAREFRESH_IGNORE_TAGS
 ^^^^^^^^^^^^^^^^^^^^^^^
 
-Default: ``[]``
+Default: ``["noscript"]``
 
 Meta tags within these tags are ignored.
 
@@ -1091,7 +1092,7 @@ Default::
         'twisted.internet.error.ConnectionDone',
         'twisted.internet.error.ConnectError',
         'twisted.internet.error.ConnectionLost',
-        IOError,
+        OSError,
         'scrapy.core.downloader.handlers.http11.TunnelError',
     ]
 
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 5a399e094..30b6536c7 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -133,7 +133,7 @@ data from it depends on the type of response:
 
     .. code-block:: python
 
-        selector = Selector(data["html"])
+        selector = Selector(text=data["html"])
 
 -   If the response is JavaScript, or HTML with a ``<script/>`` element
     containing the desired data, see :ref:`topics-parsing-javascript`.
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index cc690d609..4c56a0f14 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -31,7 +31,7 @@ For example:
 .. code-block:: python
 
     def parse_page(self, response):
-        if "Bandwidth exceeded" in response.body:
+        if "Bandwidth exceeded" in response.text:
             raise CloseSpider("bandwidth_exceeded")
 
 DontCloseSpider
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index c4cd05683..f0b704b9d 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -153,7 +153,7 @@ output examples, which assume you're exporting these two items:
 BaseItemExporter
 ----------------
 
-.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0, dont_fail=False)
+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding=None, indent=None, dont_fail=False)
 
    This is the (abstract) base class for all Item Exporters. It provides
    support for common features used by all (concrete) Item Exporters, such as
@@ -239,7 +239,7 @@ BaseItemExporter
 
    .. attribute:: indent
 
-      Amount of spaces used to indent the output on each level. Defaults to ``0``.
+      Amount of spaces used to indent the output on each level. Defaults to ``None``.
 
       * ``indent=None`` selects the most compact representation,
         all items in the same line with no indentation
@@ -328,7 +328,7 @@ CsvItemExporter
 
    :param join_multivalued: The char (or chars) that will be used for joining
       multi-valued fields, if found.
-   :type include_headers_line: str
+   :type join_multivalued: str
 
    :param errors: The optional string that specifies how encoding and decoding
       errors are to be handled. For more information see
@@ -342,14 +342,14 @@ CsvItemExporter
 
    A typical output of this exporter would be::
 
-      product,price
+      name,price
       Color TV,1200
       DVD player,200
 
 PickleItemExporter
 ------------------
 
-.. class:: PickleItemExporter(file, protocol=0, **kwargs)
+.. class:: PickleItemExporter(file, protocol=4, **kwargs)
 
    Exports items in pickle format to the given file-like object.
 
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 5a05f67d9..5598ab983 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -341,9 +341,6 @@ closing the spider. If the spider generates more than that number of errors,
 it will be closed with the reason ``closespider_errorcount``. If zero (or non
 set), spiders won't be closed by number of errors.
 
-.. module:: scrapy.extensions.debug
-   :synopsis: Extensions for debugging Scrapy
-
 .. module:: scrapy.extensions.periodic_log
    :synopsis: Periodic stats logging
 
@@ -418,7 +415,7 @@ Example extension configuration:
     custom_settings = {
         "LOG_LEVEL": "INFO",
         "PERIODIC_LOG_STATS": {
-            "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count/"],
+            "include": ["downloader/", "scheduler/", "log_count/", "item_scraped_count"],
         },
         "PERIODIC_LOG_DELTA": {"include": ["downloader/"]},
         "PERIODIC_LOG_TIMING_ENABLED": True,
@@ -463,6 +460,9 @@ Default: ``False``
 Debugging extensions
 --------------------
 
+.. module:: scrapy.extensions.debug
+   :synopsis: Extensions for debugging Scrapy
+
 Stack trace dump extension
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9aa1b8d5f..db6dd127c 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -615,6 +615,7 @@ Default:
         "file": "scrapy.extensions.feedexport.FileFeedStorage",
         "stdout": "scrapy.extensions.feedexport.StdoutFeedStorage",
         "s3": "scrapy.extensions.feedexport.S3FeedStorage",
+        "gs": "scrapy.extensions.feedexport.GCSFeedStorage",
         "ftp": "scrapy.extensions.feedexport.FTPFeedStorage",
     }
 
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 7a1c1f907..b21d6b903 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -376,7 +376,9 @@ Creating dicts from items:
     >>> dict(product)  # create a dict from all populated values
     {'price': 1000, 'name': 'Desktop PC'}
 
-    Creating items from dicts:
+Creating items from dicts:
+
+.. code-block:: pycon
 
     >>> Product({"name": "Laptop PC", "price": 1500})
     Product(price=1500, name='Laptop PC')
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 0913a3310..70d563fdd 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -93,7 +93,7 @@ You can get the oldest object of each class using the
 Which objects are tracked?
 --------------------------
 
-The objects tracked by ``trackrefs`` are all from these classes (and all its
+The objects tracked by ``trackref`` are all from these classes (and all its
 subclasses):
 
 * :class:`scrapy.Request`
@@ -187,7 +187,7 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
     Inherit from this class if you want to track live
     instances with the ``trackref`` module.
 
-.. function:: print_live_refs(class_name, ignore=NoneType)
+.. function:: print_live_refs(ignore=NoneType)
 
     Print a report of live references, grouped by class name.
 
@@ -203,9 +203,9 @@ Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
 .. function:: iter_all(class_name)
 
-    Return an iterator over all objects alive with the given class name, or
-    ``None`` if none is found. Use :func:`print_live_refs` first to get a list
-    of all tracked live objects per class name.
+    Return an iterator over all objects alive with the given class name. Use
+    :func:`print_live_refs` first to get a list of all tracked live objects
+    per class name.
 
 .. skip: end
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 3ccc90bf9..801dd6583 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -76,7 +76,7 @@ data that will be assigned to the ``name`` field later.
 
 Afterwards, similar calls are used for ``price`` and ``stock`` fields
 (the latter using a CSS selector with the :meth:`~ItemLoader.add_css` method),
-and finally the ``last_update`` field is populated directly with a literal value
+and finally the ``last_updated`` field is populated directly with a literal value
 (``today``) using a different method: :meth:`~ItemLoader.add_value`.
 
 Finally, when all data is collected, the :meth:`ItemLoader.load_item` method is
@@ -273,8 +273,8 @@ The precedence order, for both input and output processors, is as follows:
 1. Item Loader field-specific attributes: ``field_in`` and ``field_out`` (most
    precedence)
 2. Field metadata (``input_processor`` and ``output_processor`` key)
-3. Item Loader defaults: :meth:`ItemLoader.default_input_processor` and
-   :meth:`ItemLoader.default_output_processor` (least precedence)
+3. Item Loader defaults: :attr:`ItemLoader.default_input_processor` and
+   :attr:`ItemLoader.default_output_processor` (least precedence)
 
 See also: :ref:`topics-loaders-extending`.
 
@@ -323,8 +323,8 @@ There are several ways to modify Item Loader context values:
       loader = ItemLoader(product, unit="cm")
 
 3. On Item Loader declaration, for those input/output processors that support
-   instantiating them with an Item Loader context. :class:`~processor.MapCompose` is one of
-   them:
+   instantiating them with an Item Loader context.
+   :class:`~itemloaders.processors.MapCompose` is one of them:
 
    .. code-block:: python
 
diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index bbb5d0458..b486eab63 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -4,11 +4,6 @@
 Logging
 =======
 
-.. note::
-    :mod:`scrapy.log` has been deprecated alongside its functions in favor of
-    explicit calls to the Python standard logging. Keep reading to learn more
-    about the new logging system.
-
 Scrapy uses :mod:`logging` for event logging. We'll
 provide some simple examples to get you started, but for more advanced
 use-cases it's strongly suggested to read thoroughly its documentation.
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index c67d2d627..04a744b0e 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -371,11 +371,12 @@ For the Images Pipeline, set :setting:`IMAGES_URLS_FIELD` and/or
 If you need something more complex and want to override the custom pipeline
 behaviour, see :ref:`topics-media-pipeline-override`.
 
-If you have multiple image pipelines inheriting from ImagePipeline and you want
-to have different settings in different pipelines you can set setting keys
-preceded with uppercase name of your pipeline class. E.g. if your pipeline is
-called MyPipeline and you want to have custom IMAGES_URLS_FIELD you define
-setting MYPIPELINE_IMAGES_URLS_FIELD and your custom settings will be used.
+If you have multiple image pipelines inheriting from :class:`ImagesPipeline`
+and you want to have different settings in different pipelines you can set
+setting keys preceded with uppercase name of your pipeline class. E.g. if your
+pipeline is called ``MyPipeline`` and you want to have custom
+:setting:`IMAGES_URLS_FIELD` you define setting
+``MYPIPELINE_IMAGES_URLS_FIELD`` and your custom settings will be used.
 
 
 Additional features
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 44bc09b7f..7796755af 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -247,7 +247,7 @@ Request objects
        Return a new Request which is a copy of this Request. See also:
        :ref:`topics-request-response-ref-request-callback-arguments`.
 
-    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs])
+    .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
 
        Return a Request object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
@@ -717,6 +717,7 @@ Those are:
 * :reqmeta:`download_fail_on_dataloss`
 * :reqmeta:`download_latency`
 * :reqmeta:`download_maxsize`
+* :reqmeta:`download_slot`
 * :reqmeta:`download_warnsize`
 * :reqmeta:`download_timeout`
 * ``ftp_password`` (See :setting:`FTP_PASSWORD` for more info)
@@ -1085,7 +1086,7 @@ Response objects
     .. attribute:: Response.flags
 
         A list that contains flags for this response. Flags are labels used for
-        tagging Responses. For example: ``'cached'``, ``'redirected``', etc. And
+        tagging Responses. For example: ``'cached'``, ``'redirected'``', etc. And
         they're shown on the string representation of the Response (``__str__()``
         method) which is used by the engine for logging.
 
@@ -1119,7 +1120,7 @@ Response objects
 
        Returns a new Response which is a copy of this Response.
 
-    .. method:: Response.replace([url, status, headers, body, request, flags, cls])
+    .. method:: Response.replace([url, status, headers, body, request, flags, certificate, ip_address, protocol, cls])
 
        Returns a Response object with the same members, except for those members
        given new values by whichever keyword arguments are specified. The
diff --git a/docs/topics/selectors.rst b/docs/topics/selectors.rst
index 0b174341d..6d4954c2d 100644
--- a/docs/topics/selectors.rst
+++ b/docs/topics/selectors.rst
@@ -308,7 +308,7 @@ Examples:
 
 * ``*::text`` selects all descendant text nodes of the current selector context:
 
-..skip: next
+.. skip: next
 .. code-block:: pycon
 
     >>> response.css("#images *::text").getall()
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a5148ee26..6084fdb5b 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -409,77 +409,6 @@ Default: ``{}``
 A dict containing paths to the add-ons enabled in your project and their
 priorities. For more information, see :ref:`topics-addons`.
 
-.. setting:: AWS_ACCESS_KEY_ID
-
-AWS_ACCESS_KEY_ID
------------------
-
-Default: ``None``
-
-The AWS access key used by code that requires access to `Amazon Web services`_,
-such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
-
-.. setting:: AWS_SECRET_ACCESS_KEY
-
-AWS_SECRET_ACCESS_KEY
----------------------
-
-Default: ``None``
-
-The AWS secret key used by code that requires access to `Amazon Web services`_,
-such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
-
-.. setting:: AWS_SESSION_TOKEN
-
-AWS_SESSION_TOKEN
------------------
-
-Default: ``None``
-
-The AWS security token used by code that requires access to `Amazon Web services`_,
-such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
-`temporary security credentials`_.
-
-.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
-
-.. setting:: AWS_ENDPOINT_URL
-
-AWS_ENDPOINT_URL
-----------------
-
-Default: ``None``
-
-Endpoint URL used for S3-like storage, for example Minio or s3.scality.
-
-.. setting:: AWS_USE_SSL
-
-AWS_USE_SSL
------------
-
-Default: ``None``
-
-Use this option if you want to disable SSL connection for communication with
-S3 or S3-like storage. By default SSL will be used.
-
-.. setting:: AWS_VERIFY
-
-AWS_VERIFY
-----------
-
-Default: ``None``
-
-Verify SSL connection between Scrapy and S3 or S3-like storage. By default
-SSL verification will occur.
-
-.. setting:: AWS_REGION_NAME
-
-AWS_REGION_NAME
----------------
-
-Default: ``None``
-
-The name of the region associated with the AWS client.
-
 .. setting:: ASYNCIO_EVENT_LOOP
 
 ASYNCIO_EVENT_LOOP
@@ -508,6 +437,77 @@ Note that the event loop class must inherit from :class:`asyncio.AbstractEventLo
 
 .. note:: This is a :ref:`reactor setting <reactor-settings>`.
 
+.. setting:: AWS_ACCESS_KEY_ID
+
+AWS_ACCESS_KEY_ID
+-----------------
+
+Default: ``None``
+
+The AWS access key used by code that requires access to `Amazon Web services`_,
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+
+.. setting:: AWS_ENDPOINT_URL
+
+AWS_ENDPOINT_URL
+----------------
+
+Default: ``None``
+
+Endpoint URL used for S3-like storage, for example Minio or s3.scality.
+
+.. setting:: AWS_REGION_NAME
+
+AWS_REGION_NAME
+---------------
+
+Default: ``None``
+
+The name of the region associated with the AWS client.
+
+.. setting:: AWS_SECRET_ACCESS_KEY
+
+AWS_SECRET_ACCESS_KEY
+---------------------
+
+Default: ``None``
+
+The AWS secret key used by code that requires access to `Amazon Web services`_,
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`.
+
+.. setting:: AWS_SESSION_TOKEN
+
+AWS_SESSION_TOKEN
+-----------------
+
+Default: ``None``
+
+The AWS security token used by code that requires access to `Amazon Web services`_,
+such as the :ref:`S3 feed storage backend <topics-feed-storage-s3>`, when using
+`temporary security credentials`_.
+
+.. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
+
+.. setting:: AWS_USE_SSL
+
+AWS_USE_SSL
+-----------
+
+Default: ``None``
+
+Use this option if you want to disable SSL connection for communication with
+S3 or S3-like storage. By default SSL will be used.
+
+.. setting:: AWS_VERIFY
+
+AWS_VERIFY
+----------
+
+Default: ``None``
+
+Verify SSL connection between Scrapy and S3 or S3-like storage. By default
+SSL verification will occur.
+
 .. setting:: BOT_NAME
 
 BOT_NAME
@@ -593,7 +593,7 @@ When writing an item pipeline, you can force a different log level by setting
 DEFAULT_ITEM_CLASS
 ------------------
 
-Default: ``'scrapy.Item'``
+Default: ``'scrapy.item.Item'``
 
 The default class that will be used for instantiating items in the :ref:`the
 Scrapy shell <topics-shell>`.
@@ -687,7 +687,7 @@ Whether to enable DNS in-memory cache.
     :class:`~scrapy.resolver.CachingThreadedResolver` and
     :class:`~scrapy.resolver.CachingHostnameResolver`. It has no effect when
     :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
-    either when :setting:`DNS_RESOLVER` is set to a different resolver.
+    either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
 
 .. note:: This is a :ref:`reactor setting <reactor-settings>`.
 
@@ -702,25 +702,6 @@ DNS in-memory cache size, see :setting:`DNSCACHE_ENABLED`.
 
 .. note:: This is a :ref:`reactor setting <reactor-settings>`.
 
-.. setting:: TWISTED_DNS_RESOLVER
-
-TWISTED_DNS_RESOLVER
---------------------
-
-Default: ``'scrapy.resolver.CachingThreadedResolver'``
-
-The class to be used by Twisted to resolve DNS names. The default
-``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
-DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
-addresses. Scrapy provides an alternative resolver,
-``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
-take the :setting:`DNS_TIMEOUT` setting into account.
-
-.. note::
-    This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
-
-.. note:: This is a :ref:`reactor setting <reactor-settings>`.
-
 .. setting:: DNS_TIMEOUT
 
 DNS_TIMEOUT
@@ -734,7 +715,7 @@ Timeout for processing of DNS queries in seconds. Float is supported.
     This setting is only used by
     :class:`~scrapy.resolver.CachingThreadedResolver`. It has no effect when
     :setting:`TWISTED_REACTOR_ENABLED` is ``False``, and may have no effect
-    either when :setting:`DNS_RESOLVER` is set to a different resolver.
+    either when :setting:`TWISTED_DNS_RESOLVER` is set to a different resolver.
 
 .. note:: This is a :ref:`reactor setting <reactor-settings>`.
 
@@ -1330,6 +1311,7 @@ Default:
 
     {
         "scrapy.extensions.corestats.CoreStats": 0,
+        "scrapy.extensions.logcount.LogCount": 0,
         "scrapy.extensions.telnet.TelnetConsole": 0,
         "scrapy.extensions.memusage.MemoryUsage": 0,
         "scrapy.extensions.memdebug.MemoryDebugger": 0,
@@ -1352,6 +1334,8 @@ and the :ref:`list of available extensions <topics-extensions-ref>`.
 FEED_TEMPDIR
 ------------
 
+Default: ``None``
+
 The Feed Temp dir allows you to set a custom folder to save crawler
 temporary files before uploading with :ref:`FTP feed storage <topics-feed-storage-ftp>` and
 :ref:`Amazon S3 <topics-feed-storage-s3>`.
@@ -1361,6 +1345,8 @@ temporary files before uploading with :ref:`FTP feed storage <topics-feed-storag
 FEED_STORAGE_GCS_ACL
 --------------------
 
+Default: ``""``
+
 The Access Control List (ACL) used when storing items to :ref:`Google Cloud Storage <topics-feed-storage-gcs>`.
 For more information on how to set this value, please refer to the column *JSON API* in `Google Cloud documentation <https://docs.cloud.google.com/storage/docs/access-control/lists>`_.
 
@@ -1629,6 +1615,8 @@ The following special items are also supported:
 
 -   ``Python``
 
+-   ``pyOpenSSL``
+
 .. setting:: LOGSTATS_INTERVAL
 
 LOGSTATS_INTERVAL
@@ -1756,7 +1744,7 @@ significant similarities in the time between their requests.
 
 The randomization policy is the same used by `wget`_ ``--random-wait`` option.
 
-If :setting:`DOWNLOAD_DELAY` is zero (default) this option has no effect.
+If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
 
 .. _wget: https://www.gnu.org/software/wget/manual/wget.html
 
@@ -1817,7 +1805,7 @@ The parser backend to use for parsing ``robots.txt`` files. For more information
 .. setting:: ROBOTSTXT_USER_AGENT
 
 ROBOTSTXT_USER_AGENT
-^^^^^^^^^^^^^^^^^^^^
+--------------------
 
 Default: ``None``
 
@@ -1972,6 +1960,8 @@ Default:
 
     {
         "scrapy.contracts.default.UrlContract": 1,
+        "scrapy.contracts.default.CallbackKeywordArgumentsContract": 1,
+        "scrapy.contracts.default.MetadataContract": 1,
         "scrapy.contracts.default.ReturnsContract": 2,
         "scrapy.contracts.default.ScrapesContract": 3,
     }
@@ -2036,6 +2026,7 @@ Default:
 .. code-block:: python
 
     {
+        "scrapy.spidermiddlewares.start.StartSpiderMiddleware": 25,
         "scrapy.spidermiddlewares.httperror.HttpErrorMiddleware": 50,
         "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
         "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
@@ -2111,6 +2102,25 @@ command.
 The project name must not conflict with the name of custom files or directories
 in the ``project`` subdirectory.
 
+.. setting:: TWISTED_DNS_RESOLVER
+
+TWISTED_DNS_RESOLVER
+--------------------
+
+Default: ``'scrapy.resolver.CachingThreadedResolver'``
+
+The class to be used by Twisted to resolve DNS names. The default
+``scrapy.resolver.CachingThreadedResolver`` supports specifying a timeout for
+DNS requests via the :setting:`DNS_TIMEOUT` setting, but works only with IPv4
+addresses. Scrapy provides an alternative resolver,
+``scrapy.resolver.CachingHostnameResolver``, which supports IPv4/IPv6 addresses but does not
+take the :setting:`DNS_TIMEOUT` setting into account.
+
+.. note::
+    This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
+
+.. note:: This is a :ref:`reactor setting <reactor-settings>`.
+
 .. setting:: TWISTED_REACTOR_ENABLED
 
 TWISTED_REACTOR_ENABLED
@@ -2252,7 +2262,7 @@ URLLENGTH_LIMIT
 
 Default: ``2083``
 
-Scope: ``spidermiddlewares.urllength``
+Scope: ``scrapy.spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs.
 
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index d13733623..bd14f5993 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -60,6 +60,8 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
 .. skip: next
 .. code-block:: python
 
+    import json
+
     import scrapy
     import treq
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index e2cd0f986..870adbc8c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -46,7 +46,7 @@ previous (or subsequent) middleware being applied.
 If you want to disable a builtin middleware (the ones defined in
 :setting:`SPIDER_MIDDLEWARES_BASE`, and enabled by default) you must define it
 in your project :setting:`SPIDER_MIDDLEWARES` setting and assign ``None`` as its
-value.  For example, if you want to disable the off-site middleware:
+value.  For example, if you want to disable the referer middleware:
 
 .. code-block:: python
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 506daf930..bb5952826 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -208,7 +208,7 @@ scrapy.Spider
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`
 
-   .. method:: log(message, [level, component])
+   .. method:: log(message, [level])
 
        Wrapper that sends a log message through the Spider's :attr:`logger`,
        kept for backward compatibility. For more information see
@@ -335,8 +335,8 @@ The above example can also be written as follows:
 
 If you are :ref:`running Scrapy from a script <run-from-script>`, you can
 specify spider arguments when calling
-:class:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
-:class:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
+:meth:`CrawlerProcess.crawl <scrapy.crawler.CrawlerProcess.crawl>` or
+:meth:`CrawlerRunner.crawl <scrapy.crawler.CrawlerRunner.crawl>`:
 
 .. skip: next
 .. code-block:: python
@@ -593,7 +593,7 @@ Let's now take a look at an example CrawlSpider with rules:
 This spider would start crawling example.com's home page, collecting category
 links, and item links, parsing the latter with the ``parse_item`` method. For
 each item response, some data will be extracted from the HTML using XPath, and
-an :class:`~scrapy.Item` will be filled with it.
+a dictionary will be filled with it.
 
 XMLFeedSpider
 -------------
@@ -953,6 +953,7 @@ Combine SitemapSpider with other sources of urls:
 
 .. code-block:: python
 
+    from scrapy import Request
     from scrapy.spiders import SitemapSpider
 
 
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index a30258112..72188c03f 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -36,9 +36,9 @@ disable it if you want. For more information about the extension itself see
 How to access the telnet console
 ================================
 
-The telnet console listens in the TCP port defined in the
-:setting:`TELNETCONSOLE_PORT` setting, which defaults to ``6023``. To access
-the console you need to type::
+The telnet console listens on the first available TCP port from the range
+defined in the :setting:`TELNETCONSOLE_PORT` setting, which defaults to
+``[6023, 6073]``. To access the console you need to type::
 
     telnet localhost 6023
     Trying localhost...
diff --git a/docs/versioning.rst b/docs/versioning.rst
index 0c6498244..e44496799 100644
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@@ -39,8 +39,8 @@ API stability
 
 API stability was one of the major goals for the *1.0* release.
 
-Methods or functions that start with a single dash (``_``) are private and
-should never be relied as stable.
+Methods or functions that start with a single underscore (``_``) are private
+and should never be relied upon as stable.
 
 Also, keep in mind that stable doesn't mean complete: stable APIs could grow
 new methods or functionality but the existing methods should keep working the
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 049921ba7..3a7dfd018 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -240,7 +240,7 @@ BOT_NAME = "scrapybot"
 CLOSESPIDER_ERRORCOUNT = 0
 CLOSESPIDER_ITEMCOUNT = 0
 CLOSESPIDER_PAGECOUNT = 0
-CLOSESPIDER_TIMEOUT = 0
+CLOSESPIDER_TIMEOUT = 0.0
 CLOSESPIDER_PAGECOUNT_NO_ITEM = 0
 CLOSESPIDER_TIMEOUT_NO_ITEM = 0
 

From 61f99f2df1d373d3387c5d2dffa33b38e14164fb Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 8 Jul 2026 13:05:45 +0500
Subject: [PATCH 10/62] More granular untyped defs checking in tests. (#7712)

---
 pyproject.toml                                | 77 ++++++++++++++++++-
 .../asyncio_deferred_signal.py                |  2 +-
 .../caching_hostname_resolver.py              |  1 +
 .../CrawlerProcess/asyncio_deferred_signal.py |  2 +-
 .../caching_hostname_resolver.py              |  1 +
 tests/CrawlerProcess/sleeping.py              |  2 +-
 tests/CrawlerRunner/ip_address.py             |  1 +
 tests/test_addons.py                          |  6 +-
 .../__init__.py                               | 31 ++++----
 tests/test_commands.py                        |  1 +
 tests/test_crawl.py                           | 10 ++-
 tests/test_crawler.py                         |  1 +
 tests/test_dependencies.py                    |  3 +
 tests/test_downloader_handlers.py             |  6 +-
 tests/test_downloadermiddleware.py            |  4 +-
 ...est_downloadermiddleware_defaultheaders.py | 47 +++++------
 ...st_downloadermiddleware_downloadtimeout.py | 72 +++++++++--------
 tests/test_downloadermiddleware_useragent.py  | 52 +++++++------
 tests/test_engine.py                          | 10 ++-
 tests/test_extension_periodic_log.py          |  8 +-
 tests/test_feedexport_batch.py                |  6 +-
 tests/test_http_cookies.py                    |  4 +-
 tests/test_responsetypes.py                   | 12 +--
 tests/test_scheduler.py                       |  3 +-
 tests/test_signals.py                         | 26 +++----
 tests/test_spider_start.py                    | 11 ++-
 tests/test_spidermiddleware.py                |  2 +-
 tests/test_spidermiddleware_httperror.py      |  3 +-
 tests/test_spidermiddleware_referer.py        |  9 ++-
 tests/test_spidermiddleware_start.py          |  9 ++-
 tests/test_utils_asyncio.py                   |  4 +-
 tests/test_utils_conf.py                      |  6 +-
 tests/test_utils_console.py                   |  3 +
 tests/test_utils_display.py                   | 20 ++++-
 tests/test_utils_iterators.py                 |  4 +-
 tests/test_utils_log.py                       |  2 +-
 tests/test_utils_misc/__init__.py             |  6 +-
 tests/test_utils_response.py                  |  4 +-
 tests/test_utils_signal.py                    | 12 ++-
 tests/test_utils_spider.py                    |  4 +-
 tests/test_utils_trackref.py                  |  6 +-
 41 files changed, 331 insertions(+), 162 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index ed2e99bea..ac9d37e78 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -94,7 +94,82 @@ untyped_calls_exclude = [
 [[tool.mypy.overrides]]
 module = "tests.*"
 allow_untyped_defs = true
-allow_incomplete_defs = true # 48 errors
+allow_incomplete_defs = true # 59 errors
+
+# TODO
+[[tool.mypy.overrides]]
+module = [
+    "tests.mockserver.*",
+    "tests.spiders",
+    "tests.test_closespider",
+    "tests.test_cmdline",
+    "tests.test_contracts",
+    "tests.test_core_downloader",
+    "tests.test_downloader_handler_twisted_ftp",
+    "tests.test_downloadermiddleware_cookies",
+    "tests.test_downloadermiddleware_httpauth",
+    "tests.test_downloadermiddleware_httpcache",
+    "tests.test_downloadermiddleware_httpcompression",
+    "tests.test_downloadermiddleware_httpproxy",
+    "tests.test_downloadermiddleware_offsite",
+    "tests.test_downloadermiddleware_redirect",
+    "tests.test_downloadermiddleware_redirect_base",
+    "tests.test_downloadermiddleware_redirect_metarefresh",
+    "tests.test_downloadermiddleware_retry",
+    "tests.test_downloadermiddleware_robotstxt",
+    "tests.test_downloadermiddleware_stats",
+    "tests.test_downloaderslotssettings",
+    "tests.test_dupefilters",
+    "tests.test_engine_loop",
+    "tests.test_exporters",
+    "tests.test_extension_statsmailer",
+    "tests.test_extension_throttle",
+    "tests.test_feedexport",
+    "tests.test_feedexport_postprocess",
+    "tests.test_feedexport_storages",
+    "tests.test_feedexport_uri_params",
+    "tests.test_http2_client_protocol",
+    "tests.test_http_headers",
+    "tests.test_http_request",
+    "tests.test_http_request_form",
+    "tests.test_http_response",
+    "tests.test_http_response_text",
+    "tests.test_item",
+    "tests.test_link",
+    "tests.test_linkextractors",
+    "tests.test_loader",
+    "tests.test_logformatter",
+    "tests.test_logstats",
+    "tests.test_mail",
+    "tests.test_pipeline_crawl",
+    "tests.test_pipeline_files",
+    "tests.test_pipeline_images",
+    "tests.test_pipeline_media",
+    "tests.test_pipelines",
+    "tests.test_pqueues",
+    "tests.test_request_attribute_binding",
+    "tests.test_request_cb_kwargs",
+    "tests.test_request_dict",
+    "tests.test_request_left",
+    "tests.test_robotstxt_interface",
+    "tests.test_scheduler_base",
+    "tests.test_settings",
+    "tests.test_spider",
+    "tests.test_spider_crawl",
+    "tests.test_spidermiddleware_output_chain",
+    "tests.test_spidermiddleware_process_start",
+    "tests.test_spider_sitemap",
+    "tests.test_squeues",
+    "tests.test_squeues_request",
+    "tests.test_stats",
+    "tests.test_utils_datatypes",
+    "tests.test_utils_decorators",
+    "tests.test_utils_defer",
+    "tests.test_utils_deprecate",
+    "tests.test_utils_misc.test_return_with_argument_inside_generator",
+    "tests.test_utils_python",
+    "tests.test_utils_request",
+]
 check_untyped_defs = false
 
 # Interface classes are hard to support
diff --git a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
index e000a23db..2f4f2d051 100644
--- a/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/AsyncCrawlerProcess/asyncio_deferred_signal.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
 
 
 class UppercasePipeline:
-    async def _open_spider(self, spider):
+    async def _open_spider(self, spider: Spider) -> None:
         spider.logger.info("async pipeline opened!")
         await asyncio.sleep(0.1)
 
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
index a6520ca22..7e3988d92 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver.py
@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
+    url: str
 
     async def start(self):
         yield scrapy.Request(self.url)
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index eb2e1105c..6e4db1990 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -9,7 +9,7 @@ from scrapy.utils.defer import deferred_from_coro
 
 
 class UppercasePipeline:
-    async def _open_spider(self, spider):
+    async def _open_spider(self, spider: Spider) -> None:
         spider.logger.info("async pipeline opened!")
         await asyncio.sleep(0.1)
 
diff --git a/tests/CrawlerProcess/caching_hostname_resolver.py b/tests/CrawlerProcess/caching_hostname_resolver.py
index 7f687e7d9..79ce9c707 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver.py
@@ -10,6 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
     """
 
     name = "caching_hostname_resolver_spider"
+    url: str
 
     async def start(self):
         yield scrapy.Request(self.url)
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
index a577b1909..9f7d14606 100644
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@@ -15,7 +15,7 @@ class SleepingSpider(scrapy.Spider):
     async def parse(self, response):
         from twisted.internet import reactor
 
-        d = Deferred()
+        d: Deferred[None] = Deferred()
         reactor.callLater(int(sys.argv[1]), d.callback, None)
         await maybe_deferred_to_future(d)
 
diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.py
index fae206610..bab4f65db 100644
--- a/tests/CrawlerRunner/ip_address.py
+++ b/tests/CrawlerRunner/ip_address.py
@@ -36,6 +36,7 @@ def createResolver(servers: list[tuple[str, int]]) -> ResolverBase:
 
 class LocalhostSpider(Spider):
     name = "localhost_spider"
+    url: str
 
     async def start(self):
         yield Request(self.url)
diff --git a/tests/test_addons.py b/tests/test_addons.py
index 14ebddda8..a041af6d6 100644
--- a/tests/test_addons.py
+++ b/tests/test_addons.py
@@ -107,11 +107,11 @@ class TestAddonManager:
         addonlist = []
         for i in range(3):
             addon = get_addon_cls({"KEY1": i})
-            addon.number = i
+            addon.number = i  # type: ignore[attr-defined]
             addonlist.append(addon)
         # Test for every possible ordering
         for ordered_addons in itertools.permutations(addonlist):
-            expected_order = [a.number for a in ordered_addons]
+            expected_order = [a.number for a in ordered_addons]  # type: ignore[attr-defined]
             settings = {"ADDONS": {a: i for i, a in enumerate(ordered_addons)}}
             crawler = get_crawler(settings_dict=settings)
             manager = crawler.addons
@@ -177,7 +177,7 @@ class TestAddonManager:
                     )
                 settings["SCHEDULER"] = "AddonScheduler"
 
-        settings_dict = {
+        settings_dict: dict[str, Any] = {
             "ADDONS": {AddonWithFallback: 1},
         }
         crawler = get_crawler(settings_dict=settings_dict)
diff --git a/tests/test_cmdline_crawl_with_pipeline/__init__.py b/tests/test_cmdline_crawl_with_pipeline/__init__.py
index f17543575..50b04de0b 100644
--- a/tests/test_cmdline_crawl_with_pipeline/__init__.py
+++ b/tests/test_cmdline_crawl_with_pipeline/__init__.py
@@ -1,21 +1,24 @@
+from __future__ import annotations
+
 import sys
 from pathlib import Path
 from subprocess import PIPE, Popen
 
 
-class TestCmdlineCrawlPipeline:
-    def _execute(self, spname):
-        args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
-        cwd = Path(__file__).resolve().parent
-        proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
-        _, stderr = proc.communicate()
-        return proc.returncode, stderr
+def _execute(spname: str) -> int:
+    args = (sys.executable, "-m", "scrapy.cmdline", "crawl", spname)
+    cwd = Path(__file__).resolve().parent
+    proc = Popen(args, stdout=PIPE, stderr=PIPE, cwd=cwd)
+    proc.communicate()
+    return proc.returncode
 
-    def test_open_spider_normally_in_pipeline(self):
-        returncode, _ = self._execute("normal")
-        assert returncode == 0
 
-    def test_exception_at_open_spider_in_pipeline(self):
-        returncode, _ = self._execute("exception")
-        # An exception in pipeline's open_spider should result in a non-zero exit code
-        assert returncode == 1
+def test_open_spider_normally_in_pipeline():
+    returncode = _execute("normal")
+    assert returncode == 0
+
+
+def test_exception_at_open_spider_in_pipeline():
+    returncode = _execute("exception")
+    # An exception in pipeline's open_spider should result in a non-zero exit code
+    assert returncode == 1
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 9f81e8602..bb4d96f00 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -89,6 +89,7 @@ class TestCommandSettings:
             args=["-s", f"FEEDS={feeds_json}", "spider.py"]
         )
         self.command.process_options(args, opts)
+        assert self.command.settings is not None
         assert isinstance(self.command.settings["FEEDS"], scrapy.settings.BaseSettings)
         assert dict(self.command.settings["FEEDS"]) == json.loads(feeds_json)
 
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index be6c80429..8c1716345 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -356,9 +356,10 @@ with multiples lines
 
     @coroutine_test
     async def test_engine_status(self, mockserver: MockServer) -> None:
-        est = []
+        est: list[list[tuple[str, Any]]] = []
 
         def cb(response):
+            assert crawler.engine
             est.append(get_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
@@ -373,9 +374,10 @@ with multiples lines
 
     @coroutine_test
     async def test_format_engine_status(self, mockserver: MockServer) -> None:
-        est = []
+        est: list[str] = []
 
         def cb(response):
+            assert crawler.engine
             est.append(format_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
@@ -386,8 +388,8 @@ with multiples lines
         assert len(est) == 1, est
         est = est[0].split("\n")[2:-2]  # remove header & footer
         # convert to dict
-        est = [x.split(":") for x in est]
-        est = [x for sublist in est for x in sublist]  # flatten
+        est_split = [x.split(":") for x in est]
+        est = [x for sublist in est_split for x in sublist]  # flatten
         est = [x.lstrip().rstrip() for x in est]
         it = iter(est)
         s = dict(zip(it, it, strict=True))
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index adac32df1..6a1eea027 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -523,6 +523,7 @@ class TestCrawlerLogging:
             }
 
             async def start(self):
+                assert crawler.stats
                 info_count_start = crawler.stats.get_value("log_count/INFO")
                 logging.debug("debug message")  # noqa: LOG015
                 logging.info("info message")  # noqa: LOG015
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 578e84359..41c7ce526 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import re
 from configparser import ConfigParser
@@ -23,6 +25,7 @@ class TestScrapyUtils:
         config_parser.read(tox_config_file_path)
         pattern = r"Twisted==([\d.]+)"
         match = re.search(pattern, config_parser["min"]["deps"])
+        assert match
         pinned_twisted_version_string = match[1]
 
         assert twisted_version.short() == pinned_twisted_version_string
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 275199a76..4af5ca0e8 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -16,7 +16,7 @@ from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
 from scrapy.core.downloader.handlers.file import FileDownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Request
+from scrapy.http import Request, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.misc import build_from_crawler
@@ -354,6 +354,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "A brief note"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "US-ASCII"
 
     @coroutine_test
@@ -362,6 +363,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "iso-8859-7"
 
     @coroutine_test
@@ -370,6 +372,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert response.body == b"\xbe\xd3\xbe"
+        assert isinstance(response, TextResponse)
         assert response.encoding == "iso-8859-7"
 
     @coroutine_test
@@ -382,6 +385,7 @@ class TestDataURI:
         response = await self.download_request(request)
         assert response.text == "\u038e\u03a3\u038e"
         assert type(response) is responsetypes.from_mimetype("text/plain")  # pylint: disable=unidiomatic-typecheck
+        assert isinstance(response, TextResponse)
         assert response.encoding == "utf-8"
 
     @coroutine_test
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index d321c9047..92eb18d33 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -233,7 +233,7 @@ class TestMiddlewareUsingDeferreds(TestManagerBase):
                 return result
 
             def process_request(self, request):
-                d = Deferred()
+                d: Deferred[Response] = Deferred()
                 d.addCallback(self.cb)
                 d.callback(resp)
                 return d
@@ -298,6 +298,8 @@ class TestDownloadDeprecated(TestManagerBase):
             return succeed(resp)
 
         async with self.get_mwman() as mwman:
+            assert mwman.crawler
+            assert mwman.crawler.spider
             with pytest.warns(
                 ScrapyDeprecationWarning,
                 match=r"DownloaderMiddlewareManager.download\(\) is deprecated, use download_async\(\) instead",
diff --git a/tests/test_downloadermiddleware_defaultheaders.py b/tests/test_downloadermiddleware_defaultheaders.py
index e697de9b7..8c89c3ffb 100644
--- a/tests/test_downloadermiddleware_defaultheaders.py
+++ b/tests/test_downloadermiddleware_defaultheaders.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
@@ -5,28 +7,29 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 
 
-class TestDefaultHeadersMiddleware:
-    def get_defaults_mw(self):
-        crawler = get_crawler(Spider)
-        defaults = {
-            to_bytes(k): [to_bytes(v)]
-            for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
-        }
-        return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
+def get_defaults_mw() -> tuple[dict[bytes, list[bytes]], DefaultHeadersMiddleware]:
+    crawler = get_crawler(Spider)
+    defaults = {
+        to_bytes(k): [to_bytes(v)]
+        for k, v in crawler.settings.get("DEFAULT_REQUEST_HEADERS").items()
+    }
+    return defaults, DefaultHeadersMiddleware.from_crawler(crawler)
 
-    def test_process_request(self):
-        defaults, mw = self.get_defaults_mw()
-        req = Request("http://www.scrapytest.org")
-        mw.process_request(req)
-        assert req.headers == defaults
 
-    def test_update_headers(self):
-        defaults, mw = self.get_defaults_mw()
-        headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
-        bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
-        req = Request("http://www.scrapytest.org", headers=headers)
-        assert req.headers == bytes_headers
+def test_process_request():
+    defaults, mw = get_defaults_mw()
+    req = Request("http://www.scrapytest.org")
+    mw.process_request(req)
+    assert req.headers == defaults
 
-        mw.process_request(req)
-        defaults.update(bytes_headers)
-        assert req.headers == defaults
+
+def test_update_headers():
+    defaults, mw = get_defaults_mw()
+    headers = {"Accept-Language": ["es"], "Test-Header": ["test"]}
+    bytes_headers = {b"Accept-Language": [b"es"], b"Test-Header": [b"test"]}
+    req = Request("http://www.scrapytest.org", headers=headers)
+    assert req.headers == bytes_headers
+
+    mw.process_request(req)
+    defaults.update(bytes_headers)
+    assert req.headers == defaults
diff --git a/tests/test_downloadermiddleware_downloadtimeout.py b/tests/test_downloadermiddleware_downloadtimeout.py
index e6b17960e..9b64cf349 100644
--- a/tests/test_downloadermiddleware_downloadtimeout.py
+++ b/tests/test_downloadermiddleware_downloadtimeout.py
@@ -1,43 +1,51 @@
+from __future__ import annotations
+
+from typing import Any
+
 from scrapy.downloadermiddlewares.downloadtimeout import DownloadTimeoutMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
-class TestDownloadTimeoutMiddleware:
-    def get_request_spider_mw(self, settings=None):
-        crawler = get_crawler(Spider, settings)
-        spider = crawler._create_spider("foo")
-        request = Request("http://scrapytest.org/")
-        return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
+def get_request_spider_mw(settings: dict[str, Any] | None = None):
+    crawler = get_crawler(Spider, settings)
+    spider = crawler._create_spider("foo")
+    request = Request("http://scrapytest.org/")
+    return request, spider, DownloadTimeoutMiddleware.from_crawler(crawler)
 
-    def test_default_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw()
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 180
 
-    def test_string_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 20.1
+def test_default_download_timeout():
+    req, spider, mw = get_request_spider_mw()
+    mw.spider_opened(spider)
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 180
 
-    def test_setting_has_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 2
 
-    def test_request_has_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
-        mw.spider_opened(spider)
-        req.meta["download_timeout"] = 1
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") == 1
+def test_string_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": "20.1"})
+    mw.spider_opened(spider)
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 20.1
 
-    def test_zero_download_timeout(self):
-        req, spider, mw = self.get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
-        mw.spider_opened(spider)
-        assert mw.process_request(req) is None
-        assert req.meta.get("download_timeout") is None
+
+def test_setting_has_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+    mw.spider_opened(spider)
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 2
+
+
+def test_request_has_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 2})
+    mw.spider_opened(spider)
+    req.meta["download_timeout"] = 1
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") == 1
+
+
+def test_zero_download_timeout():
+    req, spider, mw = get_request_spider_mw({"DOWNLOAD_TIMEOUT": 0})
+    mw.spider_opened(spider)
+    assert mw.process_request(req) is None
+    assert req.meta.get("download_timeout") is None
diff --git a/tests/test_downloadermiddleware_useragent.py b/tests/test_downloadermiddleware_useragent.py
index 539183cd6..0c925a0d1 100644
--- a/tests/test_downloadermiddleware_useragent.py
+++ b/tests/test_downloadermiddleware_useragent.py
@@ -1,33 +1,37 @@
+from __future__ import annotations
+
 from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 
 
-class TestUserAgentMiddleware:
-    def get_spider_and_mw(self, default_useragent):
-        crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
-        spider = crawler._create_spider("foo")
-        return spider, UserAgentMiddleware.from_crawler(crawler)
+def get_spider_and_mw(
+    default_useragent: str | None,
+) -> tuple[Spider, UserAgentMiddleware]:
+    crawler = get_crawler(Spider, {"USER_AGENT": default_useragent})
+    spider = crawler._create_spider("foo")
+    return spider, UserAgentMiddleware.from_crawler(crawler)
 
-    def test_default_agent(self):
-        _, mw = self.get_spider_and_mw("default_useragent")
-        req = Request("http://scrapytest.org/")
-        assert mw.process_request(req) is None
-        assert req.headers["User-Agent"] == b"default_useragent"
 
-    def test_header_agent(self):
-        spider, mw = self.get_spider_and_mw("default_useragent")
-        mw.spider_opened(spider)
-        req = Request(
-            "http://scrapytest.org/", headers={"User-Agent": "header_useragent"}
-        )
-        assert mw.process_request(req) is None
-        assert req.headers["User-Agent"] == b"header_useragent"
+def test_default_agent():
+    _, mw = get_spider_and_mw("default_useragent")
+    req = Request("http://scrapytest.org/")
+    assert mw.process_request(req) is None
+    assert req.headers["User-Agent"] == b"default_useragent"
 
-    def test_no_agent(self):
-        spider, mw = self.get_spider_and_mw(None)
-        mw.spider_opened(spider)
-        req = Request("http://scrapytest.org/")
-        assert mw.process_request(req) is None
-        assert "User-Agent" not in req.headers
+
+def test_header_agent():
+    spider, mw = get_spider_and_mw("default_useragent")
+    mw.spider_opened(spider)
+    req = Request("http://scrapytest.org/", headers={"User-Agent": "header_useragent"})
+    assert mw.process_request(req) is None
+    assert req.headers["User-Agent"] == b"header_useragent"
+
+
+def test_no_agent():
+    spider, mw = get_spider_and_mw(None)
+    mw.spider_opened(spider)
+    req = Request("http://scrapytest.org/")
+    assert mw.process_request(req) is None
+    assert "User-Agent" not in req.headers
diff --git a/tests/test_engine.py b/tests/test_engine.py
index e51eb4664..2b857d35a 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -38,6 +38,8 @@ from tests import get_testdata
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
     from twisted.python.failure import Failure
 
     from scrapy.core.scheduler import Scheduler
@@ -608,8 +610,8 @@ class TestEngineDownload(TestEngineDownloadAsync):
 @coroutine_test
 async def test_request_scheduled_signal():
     class TestScheduler(BaseScheduler):
-        def __init__(self):
-            self.enqueued = []
+        def __init__(self) -> None:
+            self.enqueued: list[Request] = []
 
         def enqueue_request(self, request: Request) -> bool:
             self.enqueued.append(request)
@@ -621,9 +623,9 @@ async def test_request_scheduled_signal():
 
     crawler = get_crawler(MySpider)
     engine = ExecutionEngine(crawler, lambda _: None)
-    scheduler = TestScheduler()
+    scheduler = TestScheduler()  # type: ignore[abstract]
 
-    async def start():
+    async def start() -> AsyncIterator[Any]:
         return
         yield
 
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 18782fb70..0fe2caca4 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -102,7 +102,9 @@ class TestPeriodicLog:
             ext.spider_closed(spider, reason="finished")
             return ext, a, b
 
-        def check(settings: dict[str, Any], condition: Callable) -> None:
+        def check(
+            settings: dict[str, Any], condition: Callable[[str, Any], bool]
+        ) -> None:
             ext, a, b = emulate(settings)
             assert list(a["delta"].keys()) == [
                 k for k, v in ext.stats._stats.items() if condition(k, v)
@@ -168,7 +170,9 @@ class TestPeriodicLog:
             ext.spider_closed(spider, reason="finished")
             return ext, a, b
 
-        def check(settings: dict[str, Any], condition: Callable) -> None:
+        def check(
+            settings: dict[str, Any], condition: Callable[[str, Any], bool]
+        ) -> None:
             ext, a, b = emulate(settings)
             assert list(a["stats"].keys()) == [
                 k for k, v in ext.stats._stats.items() if condition(k, v)
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 0a926479b..469985288 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -378,6 +378,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         }
         crawler = get_crawler(ItemSpider, settings)
         yield crawler.crawl(total=2, mockserver=self.mockserver)
+        assert crawler.stats
         assert "feedexport/success_count/FileFeedStorage" in crawler.stats.get_stats()
         assert crawler.stats.get_value("feedexport/success_count/FileFeedStorage") == 12
 
@@ -392,7 +393,9 @@ class TestBatchDeliveries(TestFeedExportBase):
         ]
 
         class CustomS3FeedStorage(S3FeedStorage):
-            stubs = []
+            from botocore.stub import Stubber  # noqa: PLC0415
+
+            stubs: list[Stubber] = []
 
             def open(self, *args, **kwargs):
                 from botocore import __version__ as botocore_version  # noqa: PLC0415
@@ -449,6 +452,7 @@ class TestBatchDeliveries(TestFeedExportBase):
         assert len(CustomS3FeedStorage.stubs) == len(items)
         for stub in CustomS3FeedStorage.stubs:
             stub.assert_no_pending_responses()
+        assert crawler.stats
         assert (
             "feedexport/success_count/CustomS3FeedStorage" in crawler.stats.get_stats()
         )
diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.py
index ce9296764..5d4d5a2bd 100644
--- a/tests/test_http_cookies.py
+++ b/tests/test_http_cookies.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from http.cookiejar import DefaultCookiePolicy
 
 from scrapy.http import Request, Response
@@ -46,7 +48,7 @@ class TestCookieJar:
     def test_set_policy(self):
         policy = DefaultCookiePolicy()
         self.jar.set_policy(policy)
-        assert self.jar.jar._policy is policy
+        assert self.jar.jar._policy is policy  # type: ignore[attr-defined]
 
     def test_check_expired_frequency(self):
         jar = CookieJar(check_expired_frequency=1)
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 42ab29267..7c9f717e4 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,3 +1,5 @@
+from typing import Any
+
 from scrapy.http import (
     Headers,
     HtmlResponse,
@@ -38,13 +40,13 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_content_disposition_no_filename(self):
         assert responsetypes.from_content_disposition(b"attachment") is Response
 
     def test_from_content_type(self):
-        mappings = [
+        mappings: list[tuple[str | bytes, type[Response]]] = [
             ("text/html; charset=UTF-8", HtmlResponse),
             ("text/xml; charset=UTF-8", XmlResponse),
             ("application/xhtml+xml; charset=UTF-8", HtmlResponse),
@@ -58,7 +60,7 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_type(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_body(self):
         mappings = [
@@ -71,7 +73,7 @@ class TestResponseTypes:
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_body(source)
-            assert retcls is cls, f"{source} ==> {retcls} != {cls}"
+            assert retcls is cls, f"{source!r} ==> {retcls} != {cls}"
 
     def test_from_headers(self):
         mappings = [
@@ -98,7 +100,7 @@ class TestResponseTypes:
 
     def test_from_args(self):
         # TODO: add more tests that check precedence between the different arguments
-        mappings = [
+        mappings: list[tuple[dict[str, Any], type[Response]]] = [
             ({"url": "http://www.example.com/data.csv"}, TextResponse),
             # headers takes precedence over url
             (
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 8637de41e..39ce87b65 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -379,13 +379,14 @@ class TestIntegrationWithDownloaderAwareInMemory:
             url = mockserver.url("/status?n=200", is_secure=False)
             start_urls = [url] * 6
             yield self.crawler.crawl(start_urls)
+            assert self.crawler.stats
             assert self.crawler.stats.get_value("downloader/response_count") == len(
                 start_urls
             )
 
 
 class TestIncompatibility:
-    def _incompatible(self):
+    def _incompatible(self) -> None:
         settings = {
             "SCHEDULER_PRIORITY_QUEUE": "scrapy.pqueues.DownloaderAwarePriorityQueue",
             "CONCURRENT_REQUESTS_PER_IP": 1,
diff --git a/tests/test_signals.py b/tests/test_signals.py
index f5f13b992..fc9520f49 100644
--- a/tests/test_signals.py
+++ b/tests/test_signals.py
@@ -1,13 +1,20 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 import pytest
 
 from scrapy import Request, Spider, signals
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
-from tests.mockserver.http import MockServer
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 
 class ItemSpider(Spider):
     name = "itemspider"
+    mockserver: MockServer
 
     async def start(self):
         for index in range(10):
@@ -34,15 +41,6 @@ class TestMain:
 
 
 class TestMockServer:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
     def setup_method(self):
         self.items = []
 
@@ -51,11 +49,11 @@ class TestMockServer:
         self.items.append(item)
 
     @pytest.mark.only_asyncio
-    @inline_callbacks_test
-    def test_simple_pipeline(self):
+    @coroutine_test
+    async def test_simple_pipeline(self, mockserver: MockServer) -> None:
         crawler = get_crawler(ItemSpider)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
         assert len(self.items) == 10
         for index in range(10):
             assert {"index": index} in self.items
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 9257e0232..7cef2fefa 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from asyncio import sleep
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
@@ -12,6 +12,9 @@ from scrapy.utils.test import get_crawler
 from .utils import twisted_sleep
 from .utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Callable
+
 SLEEP_SECONDS = 0.1
 
 ITEM_A = {"id": "a"}
@@ -67,7 +70,11 @@ class TestMain:
 
         await self._test_spider(TestSpider, [ITEM_A])
 
-    async def _test_start(self, start_, expected_items=None):
+    async def _test_start(
+        self,
+        start_: Callable[[Any], AsyncIterator[Any]],
+        expected_items: list[Any] | None = None,
+    ) -> None:
         class TestSpider(Spider):
             name = "test"
             start = start_
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index e5891474b..7824d5c20 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -186,7 +186,7 @@ class ProcessSpiderExceptionSimpleIterableMiddleware:
 class ProcessSpiderExceptionAsyncIteratorMiddleware:
     async def process_spider_exception(self, response, exception):
         yield {"foo": 1}
-        d = defer.Deferred()
+        d: defer.Deferred[None] = defer.Deferred()
         call_later(0, d.callback, None)
         await maybe_deferred_to_future(d)
         yield {"foo": 2}
diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index 6a054ded5..2ba083fd6 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -22,6 +22,7 @@ class _HttpErrorSpider(MockServerSpider):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
+        assert self.mockserver
         self.start_urls = [
             self.mockserver.url("/status?n=200"),
             self.mockserver.url("/status?n=404"),
@@ -36,7 +37,7 @@ class _HttpErrorSpider(MockServerSpider):
         for url in self.start_urls:
             yield Request(url, self.parse, errback=self.on_error)
 
-    def parse(self, response):
+    def parse(self, response: Response) -> None:
         self.parsed.add(response.url[-3:])
 
     def on_error(self, failure):
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 32e83f990..7591c0e7b 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import warnings
-from typing import Any
+from typing import TYPE_CHECKING, Any
 from urllib.parse import urlparse
 
 import pytest
@@ -35,6 +35,9 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
 
 class TestRefererMiddleware:
     req_meta: dict[str, Any] = {}
@@ -1031,7 +1034,7 @@ async def test_response_policy_only_supports_policy_names():
     crawler = get_crawler(settings_dict={"REFERRER_POLICY": "no-referrer"})
     mw = build_from_crawler(RefererMiddleware, crawler)
 
-    async def input_result():
+    async def input_result() -> AsyncIterator[Any]:
         yield Request("https://example.com/")
 
     response = Response(
@@ -1079,7 +1082,7 @@ async def test_referer_policies_setting():
     )
     mw = build_from_crawler(RefererMiddleware, crawler)
 
-    async def input_result():
+    async def input_result() -> AsyncIterator[Any]:
         yield Request("https://example.com/")
 
     # "no-referrer-when-downgrade": None,
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index def3a3df3..e879f3a32 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.start import StartSpiderMiddleware
 from scrapy.spiders import Spider
@@ -5,6 +9,9 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
 
 class TestMiddleware:
     @coroutine_test
@@ -12,7 +19,7 @@ class TestMiddleware:
         crawler = get_crawler(Spider)
         mw = build_from_crawler(StartSpiderMiddleware, crawler)
 
-        async def start():
+        async def start() -> AsyncIterator[Request]:
             yield Request("data:,1")
             yield Request("data:,2", meta={"is_start_request": True})
             yield Request("data:,2", meta={"is_start_request": False})
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index a198d1c09..7528dc51a 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -72,7 +72,7 @@ class TestParallelAsyncio:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable(length)
             await _parallel_asyncio(
                 ait,
@@ -91,7 +91,7 @@ class TestParallelAsyncio:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable_with_delays(length)
             await _parallel_asyncio(
                 ait,
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index b6a9d8b06..518cc3518 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+from typing import Any
+
 import pytest
 
 from scrapy.exceptions import UsageError
@@ -106,7 +110,7 @@ class TestFeedExportConfig:
             )
 
     def test_feed_complete_default_values_from_settings_empty(self):
-        feed = {}
+        feed: dict[str, Any] = {}
         settings = Settings(
             {
                 "FEED_EXPORT_ENCODING": "custom encoding",
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index dc1d96f66..05b272832 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.utils.console import get_shell_embed_func
@@ -38,4 +40,5 @@ def test_get_shell_embed_func_bpython():
 def test_get_shell_embed_func_ipython():
     # default shell should be 'ipython'
     shell = get_shell_embed_func()
+    assert shell is not None
     assert shell.__name__ == "_embed_ipython_shell"
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 8b02116b8..9f9e24957 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -1,9 +1,16 @@
+from __future__ import annotations
+
 import builtins
 from io import StringIO
+from typing import TYPE_CHECKING
 from unittest import mock
 
 from scrapy.utils.display import pformat, pprint
 
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+    from types import ModuleType
+
 value = {"a": 1}
 colorized_strings = {
     (
@@ -77,14 +84,19 @@ def test_pformat_no_pygments(isatty):
 
     real_import = builtins.__import__
 
-    def mock_import(name, globals_, locals_, fromlist, level):
+    def mock_import(
+        name: str,
+        globals_: Mapping[str, object] | None = None,
+        locals_: Mapping[str, object] | None = None,
+        fromlist: Sequence[str] | None = (),
+        level: int = 0,
+    ) -> ModuleType:
         if "pygments" in name:
             raise ImportError
         return real_import(name, globals_, locals_, fromlist, level)
 
-    builtins.__import__ = mock_import
-    assert pformat(value) == plain_string
-    builtins.__import__ = real_import
+    with mock.patch("builtins.__import__", mock_import):
+        assert pformat(value) == plain_string
 
 
 def test_pprint():
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 1276c1392..a59541edb 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -238,7 +238,7 @@ class TestXmliterBase(ABC):
             next(my_iter)
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")
+        i = self.xmliter(42, "product")  # type: ignore[arg-type]
         with pytest.raises(TypeError):
             next(i)
 
@@ -353,7 +353,7 @@ class TestLxmlXmliter(TestXmliterBase):
         assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")
+        i = self.xmliter(42, "product")  # type: ignore[arg-type]
         with pytest.raises(TypeError):
             next(i)
 
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index ee552df64..fe57492b6 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -38,7 +38,7 @@ class TestFailureToExcInfo:
         assert exc_info == failure_to_exc_info(failure)
 
     def test_non_failure(self):
-        assert failure_to_exc_info("test") is None
+        assert failure_to_exc_info("test") is None  # type: ignore[arg-type]
 
 
 class TestTopLevelFormatter:
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index c4c861404..ab6965ed5 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import sys
 from pathlib import Path
@@ -37,7 +39,7 @@ class TestUtilsMisc:
         with pytest.raises(NameError):
             load_object("scrapy.utils.misc.load_object999")
         with pytest.raises(TypeError):
-            load_object({})
+            load_object({})  # type: ignore[arg-type]
 
     def test_walk_modules(self):
         mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
@@ -114,7 +116,7 @@ class TestUtilsMisc:
         args = (True, 100.0)
         kwargs = {"key": "val"}
 
-        def _test_with_crawler(mock, crawler):
+        def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
             build_from_crawler(mock, crawler, *args, **kwargs)
             if hasattr(mock, "from_crawler"):
                 mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 4544cd29e..146cdb802 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -38,7 +38,7 @@ def test_open_in_browser():
 
     resp = Response(url, body=body)
     with pytest.raises(TypeError):
-        open_in_browser(resp, debug=True)  # pylint: disable=unexpected-keyword-arg
+        open_in_browser(resp, _openfunc=browser_open)  # type: ignore[arg-type]
 
 
 def test_get_meta_refresh():
@@ -320,4 +320,4 @@ def test_open_in_browser_text_response_uses_txt_extension():
 def test_open_in_browser_raises_for_unsupported_response_type():
     response = Response("http://www.example.com", body=b"binary")
     with pytest.raises(TypeError):
-        open_in_browser(response, _openfunc=lambda _: True)
+        open_in_browser(response, _openfunc=lambda _: True)  # type: ignore[arg-type]
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index 9ac0fd0c0..dc615eca6 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import asyncio
+from typing import TYPE_CHECKING
 
 import pytest
 from pydispatch import dispatcher
@@ -16,6 +19,9 @@ from scrapy.utils.signal import (
 from scrapy.utils.test import get_from_asyncio_queue
 from tests.utils.decorators import inline_callbacks_test
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures
@@ -24,7 +30,7 @@ class TestSendCatchLog:
     @inline_callbacks_test
     def test_send_catch_log(self):
         test_signal = object()
-        handlers_called = set()
+        handlers_called: set[Callable[..., None]] = set()
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
@@ -74,7 +80,7 @@ class TestSendCatchLogDeferred2(TestSendCatchLogDeferred):
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
         assert arg == "test"
-        d = defer.Deferred()
+        d: defer.Deferred[str] = defer.Deferred()
         call_later(0, d.callback, "OK")
         return d
 
@@ -108,7 +114,7 @@ class TestSendCatchLogAsync2(TestSendCatchLogAsync):
     def ok_handler(self, arg, handlers_called):
         handlers_called.add(self.ok_handler)
         assert arg == "test"
-        d = defer.Deferred()
+        d: defer.Deferred[str] = defer.Deferred()
         call_later(0, d.callback, "OK")
         return d
 
diff --git a/tests/test_utils_spider.py b/tests/test_utils_spider.py
index 05971f423..71f0f76ad 100644
--- a/tests/test_utils_spider.py
+++ b/tests/test_utils_spider.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from scrapy import Spider
 from scrapy.http import Request
 from scrapy.item import Item
@@ -17,7 +19,7 @@ def test_iterate_spider_output():
     r = Request("http://scrapytest.org")
     o = object()
 
-    assert list(iterate_spider_output(i)) == [i]
+    assert list(iterate_spider_output(i)) == [i]  # type: ignore[call-overload]
     assert list(iterate_spider_output(r)) == [r]
     assert list(iterate_spider_output(o)) == [o]
     assert list(iterate_spider_output([r, i, o])) == [r, i, o]
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 7a7a264d3..2334c76e9 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -79,9 +79,9 @@ def test_get_oldest():
     stale entries in `live_refs`, affecting results unless explicitly cleared.
     """
 
-    def _delete_o1():
+    def _delete_o1() -> None:
         """Delete `o1` and ensure it is actually collected on PyPy."""
-        nonlocal o1
+        nonlocal o1  # type: ignore[misc]
         del o1
 
         if _IS_PYPY:
@@ -89,7 +89,7 @@ def test_get_oldest():
             # still exist until the GC runs, so we force a collection cycle.
             garbage_collect()
 
-    def _do_asserts():
+    def _do_asserts() -> None:
         assert trackref.get_oldest("Foo") is o1
         assert trackref.get_oldest("Bar") is o2
         # Ensure the newer Foo is not incorrectly considered the oldest

From 9d9950df69477ee1916bb0b9102fb5385e77d027 Mon Sep 17 00:00:00 2001
From: Shaquon Kelley <shaquonkelley@gmail.com>
Date: Wed, 8 Jul 2026 08:49:29 -0400
Subject: [PATCH 11/62] Fix silent data loss in CsvItemExporter (closes #4002)
 (#7651)

Adds state tracking to CsvItemExporter to detect when auto-detected fields drop data on items. Injects a warning instructing users to configure FEED_EXPORT_FIELDS. Includes unit tests. Corrects the false positive logic in stalled PR #7613.
---
 scrapy/exporters.py     | 22 ++++++++++++++++++++++
 tests/test_exporters.py | 14 ++++++++++++++
 2 files changed, 36 insertions(+)

diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c3540d724..5a11df833 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats.
 from __future__ import annotations
 
 import csv
+import logging
 import marshal
 import pickle
 import pprint
@@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
 if TYPE_CHECKING:
     from json import JSONEncoder
 
+logger = logging.getLogger(__name__)
+
 __all__ = [
     "BaseItemExporter",
     "CsvItemExporter",
@@ -254,6 +257,8 @@ class CsvItemExporter(BaseItemExporter):
         self.csv_writer = csv.writer(self.stream, **self._kwargs)
         self._headers_not_written = True
         self._join_multivalued = join_multivalued
+        self._autodetected_fields = False
+        self._data_loss_warned = False
 
     def serialize_field(
         self, field: Mapping[str, Any] | Field, name: str, value: Any
@@ -274,6 +279,22 @@ class CsvItemExporter(BaseItemExporter):
             self._headers_not_written = False
             self._write_headers_and_set_fields_to_export(item)
 
+        if (
+            self._autodetected_fields
+            and self.fields_to_export is not None
+            and not self._data_loss_warned
+        ):
+            item_fields = ItemAdapter(item).field_names()
+            dropped_fields = set(item_fields) - set(self.fields_to_export)
+
+            if dropped_fields:
+                dropped_fields_display = sorted(dropped_fields)
+                logger.warning(
+                    f"CSVExporter dropped fields {dropped_fields_display}. "
+                    f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
+                    f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
+                )
+                self._data_loss_warned = True
         fields = self._get_serialized_fields(item, default_value="", include_empty=True)
         values = list(self._build_row(x for _, x in fields))
         self.csv_writer.writerow(values)
@@ -293,6 +314,7 @@ class CsvItemExporter(BaseItemExporter):
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
+                self._autodetected_fields = True
             fields: Iterable[str]
             if isinstance(self.fields_to_export, Mapping):
                 fields = self.fields_to_export.values()
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 877e0708b..3b997767e 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -384,6 +384,20 @@ class TestCsvItemExporter(TestBaseItemExporter):
             errors="xmlcharrefreplace",
         )
 
+    def test_csv_dropped_fields_warning(self, caplog):
+        out = BytesIO()
+        exporter = CsvItemExporter(out)
+        exporter.start_exporting()
+
+        exporter.export_item({"name": "Apple"})
+
+        with caplog.at_level("WARNING", logger="scrapy.exporters"):
+            exporter.export_item({"name": "Banana", "price": 2.00})
+
+        assert len(caplog.records) == 1
+        assert "CSVExporter dropped fields" in caplog.text
+        assert "price" in caplog.text
+
 
 class TestCsvItemExporterDataclass(TestCsvItemExporter):
     item_class = MyDataClass

From c9446931a80e63ea1d77e130ea5581b547e0f51b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 9 Jul 2026 20:02:05 +0500
Subject: [PATCH 12/62] Use external mitmproxy. (#7720)

---
 .github/workflows/tests-ubuntu.yml |   6 +-
 conftest.py                        |   6 +-
 docs/contributing.rst              |  18 +++++
 pyproject.toml                     |   2 +-
 tests/mockserver/mitm_proxy.py     | 104 ++++++++++++++++++++---------
 tests/mockserver/utils.py          |   8 +++
 tox.ini                            |  16 +----
 7 files changed, 109 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index f51dd9799..15f25d9b8 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -79,9 +79,6 @@ jobs:
         - python-version: "3.14"
           env:
             TOXENV: botocore
-        - python-version: "3.14"
-          env:
-            TOXENV: mitmproxy
 
     steps:
     - uses: actions/checkout@v6
@@ -97,6 +94,9 @@ jobs:
         sudo apt-get update
         sudo apt-get install libxml2-dev libxslt-dev
 
+    - name: Install mitmproxy
+      run: pipx install mitmproxy
+
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
diff --git a/conftest.py b/conftest.py
index 532f83f56..7403b07b2 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,7 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
 from scrapy.utils.reactorless import install_reactor_import_hook
 from tests.keys import generate_keys
 from tests.mockserver.http import MockServer
-from tests.mockserver.mitm_proxy import MitmProxy
+from tests.mockserver.mitm_proxy import MitmProxy, mitmdump_cmd
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -127,7 +127,6 @@ def pytest_runtest_setup(item):
         "uvloop",
         "botocore",
         "boto3",
-        "mitmproxy",
     ]
 
     for module in optional_deps:
@@ -137,6 +136,9 @@ def pytest_runtest_setup(item):
             except ImportError:
                 pytest.skip(f"{module} is not installed")
 
+    if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
+        pytest.skip("mitmdump is not available")
+
 
 # Generate localhost certificate files, needed by some tests
 generate_keys()
diff --git a/docs/contributing.rst b/docs/contributing.rst
index b43d30796..6d2e08fe8 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -372,6 +372,21 @@ To see coverage report install :doc:`coverage <coverage:index>`
 
 see output of ``coverage --help`` for more options like html or xml report.
 
+Some tests need a ``mitmdump`` executable (from mitmproxy_) to test against a
+fully featured proxy server; they are skipped when one cannot be found
+(``mitmproxy`` is intentionally not a test dependency that would be installed
+into test venvs, as that sometimes leads to various dependency conflicts).
+To run these tests, make ``mitmdump`` available in one of these ways:
+
+* install ``mitmproxy`` so that ``mitmdump`` is on your ``PATH``, e.g. with
+  pipx_ (``pipx install mitmproxy``) or uv_ (``uv tool install mitmproxy``);
+
+* have uv_ installed, in which case the tests will run
+  ``uvx --from mitmproxy mitmdump``;
+
+* set the ``MITMDUMP`` environment variable to the path of a ``mitmdump``
+  executable.
+
 Writing tests
 -------------
 
@@ -399,3 +414,6 @@ And their unit-tests are in::
 .. _pytest-xdist: https://github.com/pytest-dev/pytest-xdist
 .. _help wanted issues: https://github.com/scrapy/scrapy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22
 .. _test coverage: https://app.codecov.io/gh/scrapy/scrapy
+.. _mitmproxy: https://mitmproxy.org/
+.. _pipx: https://pipx.pypa.io/
+.. _uv: https://docs.astral.sh/uv/
diff --git a/pyproject.toml b/pyproject.toml
index ac9d37e78..b24397c02 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -344,7 +344,7 @@ markers = [
     "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
     "requires_botocore: marks tests that need botocore (but not boto3)",
     "requires_boto3: marks tests that need botocore and boto3",
-    "requires_mitmproxy: marks tests that need mitmproxy",
+    "requires_mitmproxy: marks tests that need a mitmdump executable",
     "requires_internet: marks tests that need real Internet access",
 ]
 filterwarnings = [
diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py
index 56620f84e..8a1025b37 100644
--- a/tests/mockserver/mitm_proxy.py
+++ b/tests/mockserver/mitm_proxy.py
@@ -1,11 +1,42 @@
 from __future__ import annotations
 
-import re
-import sys
+import contextlib
+import functools
+import os
+import shutil
+import signal
+import socket
+import time
 from pathlib import Path
-from subprocess import PIPE, Popen
+from subprocess import DEVNULL, Popen
 from urllib.parse import urlsplit, urlunsplit
 
+from .utils import _free_port
+
+
+@functools.cache
+def mitmdump_cmd() -> list[str] | None:
+    """Return the command prefix used to invoke ``mitmdump``, or ``None`` if it
+    cannot be resolved.
+
+    We don't want to install ``mitmproxy`` into the test env (it has a lot of
+    dependencies that can conflict with some of the Scrapy/test ones, and its
+    newer versions may not support older Python versions). So we expect it
+    installed externally. We look for the ``mitmdump`` binary in the following
+    sources:
+
+    1. the ``MITMDUMP`` environment variable;
+    2. a ``mitmdump`` binary on ``PATH``;
+    3. using ``uvx --from mitmproxy mitmdump`` if ``uvx`` is available.
+    """
+    if env := os.environ.get("MITMDUMP"):
+        return [env]
+    if path := shutil.which("mitmdump"):
+        return [path]
+    if uvx := shutil.which("uvx"):
+        return [uvx, "--from", "mitmproxy", "mitmdump"]
+    return None
+
 
 class MitmProxy:
     auth_user = "scrapy"
@@ -15,18 +46,22 @@ class MitmProxy:
         self.mode = mode
 
     def start(self) -> str:
-        script = """
-import sys
-from mitmproxy.tools.main import mitmdump
-sys.argv[0] = "mitmdump"
-sys.exit(mitmdump())
-        """
+        cmd = mitmdump_cmd()
+        if not cmd:
+            raise RuntimeError(
+                "mitmdump is not available. Please install mitmproxy or uv."
+            )
         cert_path = Path(__file__).parent.parent.resolve() / "keys"
+        # Choose a free port ourselves instead of reading the mitmdump output
+        # as there is no easy way to disable stdout buffering for all kinds of
+        # mitmdump installs that we support.
+        host = "127.0.0.1"
+        port = _free_port()
         args = [
             "--listen-host",
-            "127.0.0.1",
+            host,
             "--listen-port",
-            "0",
+            str(port),
             "--proxyauth",
             f"{self.auth_user}:{self.auth_pass}",
             "--set",
@@ -37,30 +72,39 @@ sys.exit(mitmdump())
         ]
         if self.mode:
             args += ["--mode", self.mode]
-        self.proc: Popen[str] = Popen(
-            [
-                sys.executable,
-                "-u",
-                "-c",
-                script,
-                *args,
-            ],
-            stdout=PIPE,
-            text=True,
+        self.proc: Popen[bytes] = Popen(
+            [*cmd, *args],
+            stdout=DEVNULL,
+            stderr=DEVNULL,
+            start_new_session=True,  # needed for killpg() to make sense
         )
-        assert self.proc.stdout is not None
         scheme = "socks5" if self.mode == "socks5" else "http"
-        line = ""
-        for line in self.proc.stdout:
-            m = re.search(r"listening at (?:\w+://)?([^:]+:\d+)", line)
-            if m:
-                host_port = m.group(1)
-                return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host_port}"
+        deadline = time.monotonic() + 60
+        while True:
+            if self.proc.poll() is not None:
+                raise RuntimeError(
+                    f"mitmdump exited with code {self.proc.returncode} before it "
+                    f"started listening"
+                )
+            try:
+                with socket.create_connection((host, port), timeout=1):
+                    return f"{scheme}://{self.auth_user}:{self.auth_pass}@{host}:{port}"
+            except OSError:
+                if time.monotonic() >= deadline:
+                    break
+                time.sleep(0.05)
         self.stop()
-        raise RuntimeError(f"Failed to parse mitmdump output: {line}")
+        raise RuntimeError(f"mitmdump did not start listening on {host}:{port} in time")
 
     def stop(self) -> None:
-        self.proc.kill()
+        if os.name == "posix":
+            # SIGKILL doesn't propagate to the actual process (child of uvx)
+            # https://github.com/astral-sh/uv/issues/11817#issuecomment-2688830077
+            # https://stackoverflow.com/a/61980200/113586
+            with contextlib.suppress(ProcessLookupError):
+                os.killpg(os.getpgid(self.proc.pid), signal.SIGKILL)
+        else:
+            self.proc.kill()
         self.proc.communicate()
 
 
diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py
index 5c4ca7457..33c3340b9 100644
--- a/tests/mockserver/utils.py
+++ b/tests/mockserver/utils.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import socket
 from pathlib import Path
 from typing import TYPE_CHECKING, cast
 
@@ -18,6 +19,13 @@ if TYPE_CHECKING:
     from twisted.internet.interfaces import IOpenSSLContextFactory
 
 
+def _free_port() -> int:
+    # racy but should be fine for tests
+    with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+        s.bind(("127.0.0.1", 0))
+        return cast("int", s.getsockname()[1])
+
+
 def ssl_context_factory(
     keyfile: str = "keys/localhost.key",
     certfile: str = "keys/localhost.crt",
diff --git a/tox.ini b/tox.ini
index 7ace26e98..526533480 100644
--- a/tox.ini
+++ b/tox.ini
@@ -28,7 +28,6 @@ envlist =
     no-reactor
     no-reactor-extra-deps
     botocore
-    mitmproxy
     pypy3
     pypy3-extra-deps
 minversion = 1.7.0
@@ -55,6 +54,7 @@ deps =
 passenv =
     PYTHONTRACEMALLOC
     PYTEST_ADDOPTS
+    MITMDUMP
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
@@ -319,17 +319,3 @@ setenv =
     {[min]setenv}
 commands =
     pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
-
-
-# Run proxy tests that use mitmproxy in a separate env to avoid installing
-# numerous mitmproxy deps in other envs (even in extra-deps), as they can
-# conflict with other deps we want, or don't want, to have installed there.
-
-[testenv:mitmproxy]
-deps =
-    {[testenv]deps}
-    # mitmproxy does not support PyPy
-    mitmproxy; implementation_name != "pypy"
-    httpx[http2,socks]
-commands =
-    pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=mitmproxy.junit.xml -o junit_family=legacy} -m requires_mitmproxy

From b3670369b8620ae645a08bb27f553ec6bf830573 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Mon, 13 Jul 2026 19:46:41 +0200
Subject: [PATCH 13/62] Solve the feed Path issue (#7674)

* Solve the feed Path issue

* Address additional, related issues
---
 docs/topics/feed-exports.rst    |   5 ++
 scrapy/extensions/feedexport.py |  39 +++++++++--
 tests/test_feedexport.py        | 117 ++++++++++++++++++++++++++++++++
 3 files changed, 155 insertions(+), 6 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index db6dd127c..ef1abe747 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -143,6 +143,11 @@ Here are some examples to illustrate:
 .. note:: :ref:`Spider arguments <spiderargs>` become spider attributes, hence
           they can also be used as storage URI parameters.
 
+.. note:: Only ``%(...)s`` parameters are replaced. Any other percent
+          character is kept as-is, so percent-encoded URIs (e.g. ``%20`` for a
+          space or percent-encoded FTP credentials) and :class:`pathlib.Path`
+          keys containing ``%(...)s`` parameters both work as expected.
+
 
 .. _topics-feed-storage-backends:
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index fa708a2e1..0d9b016a3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -47,6 +47,33 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+
+# Printf-style placeholders (e.g. %(time)s) used to build feed URIs. Any other
+# percent character in a URI (e.g. percent-encoding such as %20 or %23) must be
+# treated as a literal rather than as the start of a placeholder.
+_FEED_URI_PLACEHOLDER_RE = re.compile(
+    r"%\([^)]+\)[-+ #0]*(?:\d+|\*)?(?:\.(?:\d+|\*))?[diouxXeEfFgGcrsa]"
+)
+
+
+def apply_uri_params(uri_template: str, uri_params: dict[str, Any]) -> str:
+    """Return *uri_template* with its ``%(...)s`` placeholders replaced using
+    *uri_params*, leaving any other percent character untouched.
+
+    This allows feed URIs to contain percent-encoded characters (e.g. ``%20``
+    in a path with spaces or ``%23`` in FTP credentials) without them being
+    misinterpreted as printf-style formatting directives.
+    """
+    parts: list[str] = []
+    last = 0
+    for match in _FEED_URI_PLACEHOLDER_RE.finditer(uri_template):
+        parts.append(uri_template[last : match.start()].replace("%", "%%"))
+        parts.append(match.group(0))
+        last = match.end()
+    parts.append(uri_template[last:].replace("%", "%%"))
+    return "".join(parts) % uri_params
+
+
 UriParamsCallableT: TypeAlias = Callable[
     [dict[str, Any], Spider], dict[str, Any] | None
 ]
@@ -473,7 +500,7 @@ class FeedExporter:
             )
             uri = self.settings["FEED_URI"]
             # handle pathlib.Path objects
-            uri = str(uri) if not isinstance(uri, Path) else uri.absolute().as_uri()
+            uri = str(uri.absolute()) if isinstance(uri, Path) else str(uri)
             feed_options = {"format": self.settings["FEED_FORMAT"]}
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
@@ -485,9 +512,9 @@ class FeedExporter:
         for settings_uri, feed_options in self.settings.getdict("FEEDS").items():
             # handle pathlib.Path objects
             uri = (
-                str(settings_uri)
-                if not isinstance(settings_uri, Path)
-                else settings_uri.absolute().as_uri()
+                str(settings_uri.absolute())
+                if isinstance(settings_uri, Path)
+                else str(settings_uri)
             )
             self.feeds[uri] = feed_complete_default_values_from_settings(
                 feed_options, self.settings
@@ -514,7 +541,7 @@ class FeedExporter:
             self.slots.append(
                 self._start_new_batch(
                     batch_id=1,
-                    uri=uri % uri_params,
+                    uri=apply_uri_params(uri, uri_params),
                     feed_options=feed_options,
                     spider=spider,
                     uri_template=uri,
@@ -639,7 +666,7 @@ class FeedExporter:
                 slots.append(
                     self._start_new_batch(
                         batch_id=slot.batch_id + 1,
-                        uri=slot.uri_template % uri_params,
+                        uri=apply_uri_params(slot.uri_template, uri_params),
                         feed_options=self.feeds[slot.uri_template],
                         spider=spider,
                         uri_template=slot.uri_template,
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 7d751f188..5ab1a54c1 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -32,6 +32,7 @@ from scrapy.extensions.feedexport import (
     FeedSlot,
     FileFeedStorage,
     IFeedStorage,
+    apply_uri_params,
 )
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
@@ -526,6 +527,91 @@ class TestFeedExport(TestFeedExportBase):
         header = self.MyItem.fields.keys()
         await self.assertExported(items, header, rows)
 
+    @coroutine_test
+    async def test_pathlib_uri_with_placeholders(self):
+        feed_dir = Path(self.temp_dir, "pathlib_placeholders")
+        feed_dir.mkdir()
+        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                yield from items
+
+        TestSpider.start_urls = [self.mockserver.url("/")]
+        settings = {
+            "FEEDS": {
+                feed_dir / "%(time)s.json": {"format": "json"},
+            },
+        }
+        crawler = get_crawler(TestSpider, settings)
+        await crawler.crawl_async()
+
+        files = list(feed_dir.iterdir())
+        assert len(files) == 1
+        assert "%(time)s" not in files[0].name
+        assert files[0].suffix == ".json"
+
+    @coroutine_test
+    async def test_pathlib_uri_with_spaces_and_unicode(self):
+        # A pathlib.Path key with spaces and non-ASCII characters must be kept
+        # verbatim (not percent-encoded), while %()s placeholders are still
+        # substituted. %(name)s resolves to the spider name deterministically,
+        # so the resulting file name can be asserted exactly.
+        feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")
+        feed_dir.mkdir()
+        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                yield from items
+
+        TestSpider.start_urls = [self.mockserver.url("/")]
+        settings = {
+            "FEEDS": {
+                feed_dir / "out %(name)s ünïcode.json": {"format": "json"},
+            },
+        }
+        crawler = get_crawler(TestSpider, settings)
+        await crawler.crawl_async()
+
+        files = list(feed_dir.iterdir())
+        assert len(files) == 1
+        assert files[0].name == "out testspider ünïcode.json"
+
+    @coroutine_test
+    async def test_str_uri_with_percent_encoding_and_placeholder(self):
+        # A percent-encoded string URI (e.g. %20 for a space) must reach
+        # storage verbatim rather than being misinterpreted as a printf
+        # directive, while %()s placeholders are still substituted. See #6425
+        # and #5794.
+        feed_dir = Path(self.temp_dir, "dir with spaces")
+        feed_dir.mkdir()
+        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                yield from items
+
+        TestSpider.start_urls = [self.mockserver.url("/")]
+        settings = {
+            "FEEDS": {
+                f"{feed_dir.as_uri()}/%(time)s.json": {"format": "json"},
+            },
+        }
+        crawler = get_crawler(TestSpider, settings)
+        await crawler.crawl_async()
+
+        files = list(feed_dir.iterdir())
+        assert len(files) == 1
+        assert "%(time)s" not in files[0].name
+        assert files[0].suffix == ".json"
+
     @coroutine_test
     async def test_export_no_items_not_store_empty(self):
         for fmt in ("json", "jsonlines", "xml", "csv"):
@@ -1408,3 +1494,34 @@ class TestFeedExportInit:
         crawler = get_crawler(settings_dict=settings)
         exporter = FeedExporter.from_crawler(crawler)
         assert isinstance(exporter, FeedExporter)
+
+
+class TestApplyUriParams:
+    params = {
+        "name": "myspider",
+        "time": "2020-01-01T00-00-00",
+        "batch_id": 2,
+        "batch_time": "2020-01-01T00-00-00",
+    }
+
+    @pytest.mark.parametrize(
+        ("uri_template", "expected"),
+        [
+            # Placeholders are substituted, including width/flags.
+            ("/data/%(name)s/%(time)s.json", "/data/myspider/2020-01-01T00-00-00.json"),
+            ("/data/%(batch_id)05d.json", "/data/00002.json"),
+            # Percent-encoding is kept verbatim (#6425, #5794).
+            (
+                "file:///path%20with%20spaces/%(name)s.json",
+                "file:///path%20with%20spaces/myspider.json",
+            ),
+            (
+                "ftp://user:2%23um25%21M%23JZ@ftp.example.com/%(name)s.csv",
+                "ftp://user:2%23um25%21M%23JZ@ftp.example.com/myspider.csv",
+            ),
+            # A lone percent character next to a placeholder stays literal.
+            ("/100%/%(name)s.json", "/100%/myspider.json"),
+        ],
+    )
+    def test_apply_uri_params(self, uri_template, expected):
+        assert apply_uri_params(uri_template, self.params) == expected

From d8ba1571e714b069216e5f21804f3754cdc2a032 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 14 Jul 2026 00:49:44 +0500
Subject: [PATCH 14/62] Assorted docs fixes, part 2. (#7725)

* Assorted docs fixes, part 2.

* Second pass.

* Address feedback.
---
 docs/faq.rst                          |  5 +--
 docs/intro/install.rst                |  4 +--
 docs/intro/overview.rst               |  2 +-
 docs/topics/asyncio.rst               |  4 +--
 docs/topics/autothrottle.rst          |  2 +-
 docs/topics/commands.rst              | 51 +++++++++++++++++++--------
 docs/topics/coroutines.rst            | 10 +++---
 docs/topics/download-handlers.rst     |  2 +-
 docs/topics/downloader-middleware.rst | 21 +++--------
 docs/topics/exceptions.rst            |  6 ++--
 docs/topics/feed-exports.rst          |  6 ++--
 docs/topics/item-pipeline.rst         |  2 +-
 docs/topics/items.rst                 | 11 +++---
 docs/topics/jobs.rst                  |  4 +--
 docs/topics/leaks.rst                 |  8 ++---
 docs/topics/loaders.rst               |  2 +-
 docs/topics/media-pipeline.rst        | 24 ++++++-------
 docs/topics/practices.rst             |  6 ++--
 docs/topics/request-response.rst      |  4 +--
 docs/topics/settings.rst              | 26 ++++++++++----
 docs/topics/signals.rst               |  4 +--
 docs/topics/spiders.rst               |  2 +-
 docs/topics/stats.rst                 | 13 +++----
 docs/topics/telnetconsole.rst         |  4 +--
 scrapy/core/scheduler.py              |  6 ++--
 scrapy/settings/__init__.py           |  4 +--
 26 files changed, 129 insertions(+), 104 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index a396a8212..8f2013581 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -384,8 +384,9 @@ How to deal with ``<class 'ValueError'>: filedescriptor out of range in select()
 ----------------------------------------------------------------------------------------------
 
 This issue `has been reported`_ to appear when running broad crawls in macOS, where the default
-Twisted reactor is :class:`twisted.internet.selectreactor.SelectReactor`. Switching to a
-different reactor is possible by using the :setting:`TWISTED_REACTOR` setting.
+Twisted reactor was :class:`twisted.internet.selectreactor.SelectReactor` at that time.
+If you have switched to this reactor using the :setting:`TWISTED_REACTOR` setting you can switch
+to a different one in the same way.
 
 
 .. _faq-stop-response-download:
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 8cef04ff1..67a84e538 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -230,8 +230,8 @@ Installing Scrapy with PyPy on Windows is not tested.
 You can check that Scrapy is installed correctly by running ``scrapy bench``.
 If this command gives errors such as
 ``TypeError: ... got 2 unexpected keyword arguments``, this means
-that setuptools was unable to pick up one PyPy-specific dependency.
-To fix this issue, run ``pip install 'PyPyDispatcher>=2.1.0'``.
+that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run
+``pip install 'PyPyDispatcher>=2.1.0'``.
 
 
 .. _intro-install-troubleshooting:
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index 488e1c2e3..5937c5860 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -83,7 +83,7 @@ While this enables you to do very fast crawls (sending multiple concurrent
 requests at the same time, in a fault-tolerant way) Scrapy also gives you
 control over the politeness of the crawl through :ref:`a few settings
 <topics-settings-ref>`. You can do things like setting a download delay between
-each request, limiting the amount of concurrent requests per domain or per IP, and
+each request, limiting the amount of concurrent requests per domain, and
 even :ref:`using an auto-throttling extension <topics-autothrottle>` that tries
 to figure these settings out automatically.
 
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 17b322b5a..3d4ebe088 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -5,7 +5,7 @@ asyncio
 =======
 
 Scrapy supports :mod:`asyncio` natively. New projects created with
-:command:`scrapy startproject` have asyncio enabled by default, and you can use
+:command:`startproject` have asyncio enabled by default, and you can use
 :mod:`asyncio` and :mod:`asyncio`-powered libraries in any :doc:`coroutine
 <coroutines>`.
 
@@ -18,7 +18,7 @@ no additional setup is needed.
 Configuring the asyncio reactor
 ===============================
 
-New projects generated with :command:`scrapy startproject` have the asyncio
+New projects generated with :command:`startproject` have the asyncio
 reactor configured by default. No manual setup is needed.
 
 The :setting:`TWISTED_REACTOR` setting controls which Twisted reactor Scrapy
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 787ef842a..4f28019da 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -75,7 +75,7 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
 .. _download-latency:
 
 In Scrapy, the download latency is measured as the time elapsed between
-establishing the TCP connection and receiving the HTTP headers.
+sending the request and receiving the HTTP headers.
 
 Note that these latencies are very hard to measure accurately in a cooperative
 multitasking environment because Scrapy may be busy processing a spider
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 270d9859d..ee2c3a3cd 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -309,11 +309,25 @@ Usage examples::
       * parse_item
 
     $ scrapy check
-    [FAILED] first_spider:parse_item
-    >>> 'RetailPricex' field is missing
+    F.F.
+    ======================================================================
+    FAIL: [first_spider] parse (@returns post-hook)
+    ----------------------------------------------------------------------
+    Traceback (most recent call last):
+      ...
+    scrapy.exceptions.ContractFail: Returned 92 requests, expected 0..4
 
-    [FAILED] first_spider:parse
-    >>> Returned 92 requests, expected 0..4
+    ======================================================================
+    FAIL: [first_spider] parse_item (@scrapes post-hook)
+    ----------------------------------------------------------------------
+    Traceback (most recent call last):
+      ...
+    scrapy.exceptions.ContractFail: Missing fields: RetailPricex
+
+    ----------------------------------------------------------------------
+    Ran 4 contracts in 0.174s
+
+    FAILED (failures=2)
 
 .. skip: end
 
@@ -377,7 +391,7 @@ Supported options:
 
 * ``--spider=SPIDER``: bypass spider autodetection and force use of specific spider
 
-* ``--headers``: print the response's HTTP headers instead of the response's body
+* ``--headers``: print the request's and response's HTTP headers instead of the response's body
 
 * ``--no-redirect``: do not follow HTTP 3xx redirects (default is to follow them)
 
@@ -387,15 +401,19 @@ Usage examples::
     [ ... html content here ... ]
 
     $ scrapy fetch --nolog --headers http://www.example.com/
-    {'Accept-Ranges': ['bytes'],
-     'Age': ['1263   '],
-     'Connection': ['close     '],
-     'Content-Length': ['596'],
-     'Content-Type': ['text/html; charset=UTF-8'],
-     'Date': ['Wed, 18 Aug 2010 23:59:46 GMT'],
-     'Etag': ['"573c1-254-48c9c87349680"'],
-     'Last-Modified': ['Fri, 30 Jul 2010 15:30:18 GMT'],
-     'Server': ['Apache/2.2.3 (CentOS)']}
+    > Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
+    > Accept-Language: en
+    > User-Agent: Scrapy/2.16.0 (+https://scrapy.org)
+    > Accept-Encoding: gzip, deflate, br
+    >
+    < Date: Wed, 08 Jul 2026 06:15:01 GMT
+    < Content-Type: text/html
+    < Server: cloudflare
+    < Last-Modified: Wed, 01 Jul 2026 17:50:18 GMT
+    < Allow: GET, HEAD
+    < Cf-Cache-Status: HIT
+    < Age: 8184
+    < Cf-Ray: a17cf3b80eddf141-DME
 
 .. command:: view
 
@@ -605,7 +623,10 @@ shouldn't matter to the user running the command, but when the user :ref:`needs
 a non-default Twisted reactor <disable-asyncio>`, it may be important.
 
 Scrapy decides which of these two classes to use based on the value of the
-:setting:`TWISTED_REACTOR` setting. If the setting value is the default one
+:setting:`TWISTED_REACTOR` and :setting:`TWISTED_REACTOR_ENABLED` settings.
+With :setting:`TWISTED_REACTOR_ENABLED` set to ``False`` it will use
+:class:`~scrapy.crawler.AsyncCrawlerProcess`. Otherwise, if the
+:setting:`TWISTED_REACTOR` value is the default one
 (``'twisted.internet.asyncioreactor.AsyncioSelectorReactor'``),
 :class:`~scrapy.crawler.AsyncCrawlerProcess` will be used, otherwise
 :class:`~scrapy.crawler.CrawlerProcess` will be used. The :ref:`spider settings
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 8aaf0aa31..9dcd9d69c 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -16,7 +16,7 @@ Supported callables
 The following callables may be defined as coroutines using ``async def``, and
 hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
--   The :meth:`~scrapy.spiders.Spider.start` spider method, which *must* be
+-   The :meth:`~scrapy.Spider.start` spider method, which *must* be
     defined as an :term:`asynchronous generator`.
 
     .. versionadded:: 2.13
@@ -204,13 +204,15 @@ This means you can use many useful Python libraries providing such code:
 Common use cases for asynchronous code include:
 
 * requesting data from websites, databases and other services (in
-  :meth:`~scrapy.spiders.Spider.start`, callbacks, pipelines and
+  :meth:`~scrapy.Spider.start`, callbacks, pipelines and
   middlewares);
 * storing data in databases (in pipelines and middlewares);
 * delaying the spider initialization until some external event (in the
   :signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
-  (see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+* calling asynchronous Scrapy methods like
+  :meth:`ExecutionEngine.download_async()
+  <scrapy.core.engine.ExecutionEngine.download_async>` (see :ref:`the
+  screenshot pipeline example <ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
 
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 9bbfee9e2..5fbc5df06 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -89,7 +89,7 @@ the following API:
         If ``True``, the handler will only be instantiated when the first
         request handled by it needs to be downloaded.
 
-    .. method:: download_request(request: Request) -> Response:
+    .. method:: download_request(request: Request) -> Response
         :async:
 
         Download the given request and return a response.
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 383f331b5..096f87edd 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -591,8 +591,8 @@ In order to use your storage backend, set:
 HTTPCache middleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-The :class:`HttpCacheMiddleware` can be configured through the following
-settings:
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware` can be
+configured through the following settings:
 
 .. setting:: HTTPCACHE_ENABLED
 
@@ -868,9 +868,9 @@ OffsiteMiddleware
    .. reqmeta:: allow_offsite
 
    If the request has the :attr:`~scrapy.Request.dont_filter` attribute set to
-   ``True`` or :attr:`Request.meta` has ``allow_offsite`` set to ``True``, then
-   the OffsiteMiddleware will allow the request even if its domain is not listed
-   in allowed domains.
+   ``True`` or :attr:`Request.meta <scrapy.Request.meta>` has ``allow_offsite``
+   set to ``True``, then the OffsiteMiddleware will allow the request even if
+   its domain is not listed in allowed domains.
 
 RedirectMiddleware
 ------------------
@@ -1015,17 +1015,6 @@ RetryMiddleware
    A middleware to retry failed requests that are potentially caused by
    temporary problems such as a connection timeout or HTTP 500 error.
 
-Failed pages are collected on the scraping process and rescheduled at the
-end, once the spider has finished crawling all regular (non failed) pages.
-
-The :class:`RetryMiddleware` can be configured through the following
-settings (see the settings documentation for more info):
-
-* :setting:`RETRY_ENABLED`
-* :setting:`RETRY_TIMES`
-* :setting:`RETRY_HTTP_CODES`
-* :setting:`RETRY_EXCEPTIONS`
-
 .. reqmeta:: dont_retry
 
 If :attr:`Request.meta <scrapy.Request.meta>` has ``dont_retry`` key
diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 4c56a0f14..45acb1763 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -12,7 +12,8 @@ Exceptions
 Built-in Exceptions reference
 =============================
 
-Here's a list of all exceptions included in Scrapy and their usage.
+Here's a list of all exceptions included in Scrapy and their usage, except for
+the :ref:`download handler exceptions <download-handlers-exceptions>`.
 
 
 CloseSpider
@@ -71,7 +72,8 @@ remain disabled. Those components include:
 -   Downloader middlewares
 -   Spider middlewares
 
-The exception must be raised in the component's ``__init__`` method.
+The exception must be raised in the component's ``__init__()`` or
+``from_crawler()``  method.
 
 NotSupported
 ------------
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index ef1abe747..1bab3bfc2 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -166,7 +166,7 @@ The feeds are stored in the local filesystem.
 -   Required external libraries: none
 
 Note that for the local filesystem storage (only) you can omit the scheme if
-you specify an absolute path like ``/tmp/export.csv`` (Unix systems only).
+you specify a path (e.g. ``/tmp/export.csv``).
 Alternatively you can also use a :class:`pathlib.Path` object.
 
 .. _topics-feed-storage-ftp:
@@ -762,8 +762,8 @@ The function signature should be as follows:
    :param spider: source spider of the feed items
    :type spider: scrapy.Spider
 
-   .. caution:: The function should return a new dictionary, modifying
-                the received ``params`` in-place is deprecated.
+   .. caution:: The function must return a new dictionary instead of modifying
+                the received ``params`` in-place.
 
 For example, to include the :attr:`name <scrapy.Spider.name>` of the
 source spider in the feed URI:
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index f1fa463d8..951c0f485 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -121,7 +121,7 @@ Write items to MongoDB
 
 In this example we'll write items to MongoDB_ using pymongo_.
 MongoDB address and database name are specified in Scrapy settings;
-MongoDB collection is named after item class.
+MongoDB collection is specified in a class attribute.
 
 The main point of this example is to show how to :ref:`get the crawler
 <from-crawler>` and how to clean up the resources properly.
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index b21d6b903..0892dd839 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -23,7 +23,8 @@ Item Types
 
 Scrapy supports the following types of items, via the `itemadapter`_ library:
 :ref:`dictionaries <dict-items>`, :ref:`Item objects <item-objects>`,
-:ref:`dataclass objects <dataclass-items>`, and :ref:`attrs objects <attrs-items>`.
+:ref:`dataclass objects <dataclass-items>`, :ref:`attrs objects <attrs-items>`
+and :ref:`Pydantic models <pydantic-items>`.
 
 .. _itemadapter: https://github.com/scrapy/itemadapter
 
@@ -61,8 +62,8 @@ its ``__init__`` method.
 :class:`Item` also allows the defining of field metadata, which can be used to
 :ref:`customize serialization <topics-exporters-field-serialization>`.
 
-:mod:`trackref` tracks :class:`Item` objects to help find memory leaks
-(see :ref:`topics-leaks-trackrefs`).
+:mod:`scrapy.utils.trackref` tracks :class:`Item` objects to help find memory
+leaks (see :ref:`topics-leaks-trackrefs`).
 
 Example:
 
@@ -262,7 +263,7 @@ Creating items
 
     >>> product = Product(name="Desktop PC", price=1000)
     >>> print(product)
-    Product(name='Desktop PC', price=1000)
+    {'name': 'Desktop PC', 'price': 1000}
 
 
 Getting field values
@@ -381,7 +382,7 @@ Creating items from dicts:
 .. code-block:: pycon
 
     >>> Product({"name": "Laptop PC", "price": 1500})
-    Product(price=1500, name='Laptop PC')
+    {'name': 'Laptop PC', 'price': 1500}
 
     >>> Product({"name": "Laptop PC", "lala": 1500})  # warning: unknown field in dict
     Traceback (most recent call last):
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index 8d6b2458b..c9916110d 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -151,8 +151,8 @@ Where:
 -   :class:`~scrapy.pqueues.ScrapyPriorityQueue` creates the ``{priority}{s?}``
     directories.
 
--   :class:`scrapy.squeues.PickleLifoDiskQueue`, a subclass of
-    :class:`queuelib.LifoDiskQueue` that uses :mod:`pickle` to serialize
+-   :class:`scrapy.squeues.PickleFifoDiskQueue`, a subclass of
+    :class:`queuelib.FifoDiskQueue` that uses :mod:`pickle` to serialize
     :class:`dict` representations of :class:`scrapy.Request` objects, creates
     the ``info.json`` and ``q{00000}`` files.
 
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 70d563fdd..3337cc5d0 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -62,9 +62,9 @@ Debugging memory leaks with ``trackref``
 
 .. skip: start
 
-:mod:`trackref` is a module provided by Scrapy to debug the most common cases of
-memory leaks. It basically tracks the references to all live Request,
-Response, Item, Spider and Selector objects.
+:mod:`scrapy.utils.trackref` is a module provided by Scrapy to debug the most
+common cases of memory leaks. It basically tracks the references to all live
+Request, Response, Item, Spider and Selector objects.
 
 You can enter the telnet console and inspect how many objects (of the classes
 mentioned above) are currently alive using the ``prefs()`` function which is an
@@ -164,7 +164,7 @@ Too many spiders?
 -----------------
 
 If your project has too many spiders executed in parallel,
-the output of :func:`prefs` can be difficult to read.
+the output of ``prefs()`` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
 ignore a particular class (and all its subclasses). For
 example, this won't show any live references to spiders:
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 801dd6583..765e7f6d7 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -264,7 +264,7 @@ metadata. Here is an example:
     >>> il.add_value("name", ["Welcome to my", "<strong>website</strong>"])
     >>> il.add_value("price", ["&euro;", "<span>1000</span>"])
     >>> il.load_item()
-    {'name': 'Welcome to my website', 'price': '1000'}
+    Product(name='Welcome to my website', price='1000')
 
 .. skip: end
 
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 04a744b0e..b7a4f561e 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -41,11 +41,10 @@ this:
 2. The item is returned from the spider and goes to the item pipeline.
 
 3. When the item reaches the :class:`FilesPipeline`, the URLs in the
-   ``file_urls`` field are scheduled for download using the standard
-   Scrapy scheduler and downloader (which means the scheduler and downloader
-   middlewares are reused), but with a higher priority, processing them before other
-   pages are scraped. The item remains "locked" at that particular pipeline stage
-   until the files have finish downloading (or fail for some reason).
+   ``file_urls`` field are downloaded using the standard Scrapy downloader
+   (which means the downloader middlewares are used, but the spider middlewares
+   aren't). The item remains "locked" at that particular pipeline stage until
+   the files have finished downloading (or failed for some reason).
 
 4. When the files are downloaded, another field (``files``) will be populated
    with the results. This field will contain a list of dicts with information
@@ -548,10 +547,9 @@ See here the methods that you can override in your custom Files Pipeline:
 
    .. method:: FilesPipeline.get_media_requests(item, info)
 
-      As seen on the workflow, the pipeline will get the URLs of the images to
-      download from the item. In order to do this, you can override the
-      :meth:`~get_media_requests` method and return a Request for each
-      file URL:
+      As seen on the workflow, the pipeline will get the requests for the files
+      to download from the item by calling this method. You can override it to
+      change what requests are returned:
 
       .. code-block:: python
 
@@ -591,8 +589,9 @@ See here the methods that you can override in your custom Files Pipeline:
           * ``downloaded`` - file was downloaded.
           * ``uptodate`` - file was not downloaded, as it was downloaded recently,
             according to the file expiration policy.
-          * ``cached`` - file was already scheduled for download, by another item
-            sharing the same file.
+          * ``cached`` - file was taken from a cache (the response has a
+            ``"cached"`` flag, e.g. from
+            :class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
 
       The list of tuples received by :meth:`~item_completed` is
       guaranteed to retain the same order of the requests returned from the
@@ -619,9 +618,6 @@ See here the methods that you can override in your custom Files Pipeline:
               (False, Failure(...)),
           ]
 
-      By default the :meth:`get_media_requests` method returns ``None`` which
-      means there are no files to download for the item.
-
    .. method:: FilesPipeline.item_completed(results, item, info)
 
       The :meth:`FilesPipeline.item_completed` method called when all file
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index e7faf48c4..216b7e64f 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -17,8 +17,10 @@ Run Scrapy from a script
 You can use the :ref:`API <topics-api>` to run Scrapy from a script, instead of
 the typical way of running Scrapy via ``scrapy crawl``.
 
-Remember that Scrapy is built on top of the Twisted
-asynchronous networking library, so you need to run it inside the Twisted reactor.
+Remember that Scrapy requires a Twisted reactor or (with
+:setting:`TWISTED_REACTOR_ENABLED` set to ``False``) an asyncio event loop, so
+you need to run one of those in your script for it to work (helpers described
+below can do it for you).
 
 The first utility you can use to run your spiders is
 :class:`scrapy.crawler.AsyncCrawlerProcess` or
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 7796755af..a5f799444 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -1101,8 +1101,8 @@ Response objects
 
         The IP address of the server from which the Response originated.
 
-        This attribute is currently only populated by the HTTP 1.1 download
-        handler, i.e. for ``http(s)`` responses. For other handlers,
+        This attribute is currently only populated by the HTTP download
+        handlers, i.e. for ``http(s)`` responses. For other handlers,
         :attr:`ip_address` is always ``None``.
 
     .. attribute:: Response.protocol
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 6084fdb5b..a592f8bc5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -418,7 +418,9 @@ Default: ``None``
 
 Import path of a given ``asyncio`` event loop class.
 
-If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) this setting can be used to specify the
+If the asyncio reactor is enabled (see :setting:`TWISTED_REACTOR`) or when
+:ref:`running Scrapy without a reactor <asyncio-without-reactor>` this setting
+can be used to specify the
 asyncio event loop to be used with it. Set the setting to the import path of the
 desired asyncio event loop class. If the setting is set to ``None`` the default asyncio
 event loop will be used.
@@ -554,6 +556,8 @@ performed to any single domain.
 See also: :ref:`topics-autothrottle` and its
 :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.
 
+It is possible to change this setting per domain by using
+:setting:`DOWNLOAD_SLOTS`.
 
 .. setting:: DEFAULT_DROPITEM_LOG_LEVEL
 
@@ -917,9 +921,8 @@ desired.
 
     This delay can be set per spider using :attr:`download_delay` spider attribute.
 
-It is also possible to change this setting per domain, although it requires
-non-trivial code. See the implementation of the :ref:`AutoThrottle
-<topics-autothrottle>` extension for an example.
+It is possible to change this setting per domain by using
+:setting:`DOWNLOAD_SLOTS`.
 
 .. setting:: DOWNLOAD_BIND_ADDRESS
 
@@ -1358,14 +1361,19 @@ FORCE_CRAWLER_PROCESS
 Default: ``False``
 
 If ``False``, :ref:`Scrapy commands that need a CrawlerProcess
-<topics-commands-crawlerprocess>` will decide between using
+<topics-commands-crawlerprocess>`, when :setting:`TWISTED_REACTOR_ENABLED`
+is set to ``True``, will decide between using
 :class:`scrapy.crawler.AsyncCrawlerProcess` and
 :class:`scrapy.crawler.CrawlerProcess` based on the value of the
 :setting:`TWISTED_REACTOR` setting, but ignoring its value in :ref:`per-spider
 settings <spider-settings>`.
 
 If ``True``, these commands will always use
-:class:`~scrapy.crawler.CrawlerProcess`.
+:class:`~scrapy.crawler.CrawlerProcess` when :setting:`TWISTED_REACTOR_ENABLED`
+is set to ``True``.
+
+When :setting:`TWISTED_REACTOR_ENABLED` is set to ``False``,
+:class:`~scrapy.crawler.AsyncCrawlerProcess` will be used in all cases.
 
 Set this to ``True`` if you want to set :setting:`TWISTED_REACTOR` to a
 non-default value in :ref:`per-spider settings <spider-settings>`.
@@ -1746,6 +1754,9 @@ The randomization policy is the same used by `wget`_ ``--random-wait`` option.
 
 If :setting:`DOWNLOAD_DELAY` is zero this option has no effect.
 
+It is possible to change this setting per domain by using
+:setting:`DOWNLOAD_SLOTS`.
+
 .. _wget: https://www.gnu.org/software/wget/manual/wget.html
 
 .. setting:: REACTOR_THREADPOOL_MAXSIZE
@@ -2159,6 +2170,9 @@ Default: ``"twisted.internet.asyncioreactor.AsyncioSelectorReactor"``
 
 Import path of a given :mod:`~twisted.internet.reactor`.
 
+.. note::
+    This setting has no effect when :setting:`TWISTED_REACTOR_ENABLED` is ``False``.
+
 Scrapy will install this reactor if no other reactor is installed yet, such as
 when the ``scrapy`` CLI program is invoked or when using the
 :class:`~scrapy.crawler.AsyncCrawlerProcess` class or the
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index bd14f5993..531232149 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -454,7 +454,7 @@ bytes_received
 .. signal:: bytes_received
 .. function:: bytes_received(data, request, spider)
 
-    Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is
+    Sent by some download handlers when a group of bytes is
     received for a specific request. This signal might be fired multiple
     times for the same request, with partial data each time. For instance,
     a possible scenario for a 25 kb response would be two signals fired
@@ -482,7 +482,7 @@ headers_received
 .. signal:: headers_received
 .. function:: headers_received(headers, body_length, request, spider)
 
-    Sent by the HTTP 1.1 and S3 download handlers when the response headers are
+    Sent by some download handlers when the response headers are
     available for a given request, before downloading any additional content.
 
     Handlers for this signal can stop the download of a response while it
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index bb5952826..887da0a15 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -614,7 +614,7 @@ XMLFeedSpider
 
         A string which defines the iterator to use. It can be either:
 
-           - ``'iternodes'`` - a fast iterator based on regular expressions
+           - ``'iternodes'`` - a fast iterator based on ``lxml``
 
            - ``'html'`` - an iterator which uses :class:`~scrapy.Selector`.
              Keep in mind this uses DOM parsing and must load all DOM in memory
diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index ac714eacb..028fb009d 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -10,8 +10,8 @@ Collector, and can be accessed through the :attr:`~scrapy.crawler.Crawler.stats`
 attribute of the :ref:`topics-api-crawler`, as illustrated by the examples in
 the :ref:`topics-stats-usecases` section below.
 
-However, the Stats Collector is always available, so you can always import it
-in your module and use its API (to increment or set new stat keys), regardless
+The Stats Collector API is always available, so you can always use it (to
+increment or set new stat keys), regardless
 of whether the stats collection is enabled or not. If it's disabled, the API
 will still work but it won't collect anything. This is aimed at simplifying the
 stats collector usage: you should spend no more than one line of code for
@@ -21,9 +21,6 @@ using the Stats Collector from.
 Another feature of the Stats Collector is that it's very efficient (when
 enabled) and extremely efficient (almost unnoticeable) when disabled.
 
-The Stats Collector keeps a stats table per open spider which is automatically
-opened when the spider is opened, and closed when the spider is closed.
-
 .. _topics-stats-usecases:
 
 Common Stats Collector uses
@@ -87,13 +84,13 @@ Get all stats:
 Available Stats Collectors
 ==========================
 
+.. currentmodule:: scrapy.statscollectors
+
 Besides the basic :class:`StatsCollector` there are other Stats Collectors
 available in Scrapy which extend the basic Stats Collector. You can select
 which Stats Collector to use through the :setting:`STATS_CLASS` setting. The
 default Stats Collector used is the :class:`MemoryStatsCollector`.
 
-.. currentmodule:: scrapy.statscollectors
-
 MemoryStatsCollector
 --------------------
 
@@ -102,7 +99,7 @@ MemoryStatsCollector
     A simple stats collector that keeps the stats of the last scraping run (for
     each spider) in memory, after they're closed. The stats can be accessed
     through the :attr:`spider_stats` attribute, which is a dict keyed by spider
-    domain name.
+    name.
 
     This is the default Stats Collector used in Scrapy.
 
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index 72188c03f..54379b4b9 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -107,8 +107,8 @@ Here are some example tasks you can do with the telnet console:
 View engine status
 ------------------
 
-You can use the ``est()`` method of the Scrapy engine to quickly show its state
-using the telnet console::
+You can use the ``est()`` method provided by the console to quickly show the
+engine status::
 
     telnet localhost 6023
     >>> est()
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 82e31b90b..42c517222 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -289,11 +289,11 @@ class Scheduler(BaseScheduler):
 
         :param dqclass: A class to be used as persistent request queue.
                         The value for the :setting:`SCHEDULER_DISK_QUEUE` setting is used by default.
-        :type dqclass: class
+        :type dqclass: type
 
         :param mqclass: A class to be used as non-persistent request queue.
                         The value for the :setting:`SCHEDULER_MEMORY_QUEUE` setting is used by default.
-        :type mqclass: class
+        :type mqclass: type
 
         :param logunser: A boolean that indicates whether or not unserializable requests should be logged.
                         The value for the :setting:`SCHEDULER_DEBUG` setting is used by default.
@@ -306,7 +306,7 @@ class Scheduler(BaseScheduler):
 
         :param pqclass: A class to be used as priority queue for requests.
                         The value for the :setting:`SCHEDULER_PRIORITY_QUEUE` setting is used by default.
-        :type pqclass: class
+        :type pqclass: type
 
         :param crawler: The crawler object corresponding to the current crawl.
         :type crawler: :class:`scrapy.crawler.Crawler`
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 932463ba9..27b0f1187 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -288,10 +288,10 @@ class BaseSettings(MutableMapping[str, Any]):
         -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
 
         :param name: the setting name
-        :type name: string
+        :type name: str
 
         :param default: the value to return if no setting is found
-        :type default: any
+        :type default: object
         """
         value = self.get(name, default)
         if value is None:

From 11d7a05a6f0700a7bbae440bfa9da9ea4b917c7d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 20 Jul 2026 17:59:11 +0500
Subject: [PATCH 15/62] Deprecate is_secure=False for s3://. (#7738)

---
 scrapy/core/downloader/handlers/s3.py | 14 ++++++++++++--
 tests/test_downloader_handlers.py     | 12 ++++++++++--
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 9b7ceeb4c..1fba7f019 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
@@ -49,7 +50,16 @@ class S3DownloadHandler(BaseDownloadHandler):
 
     async def download_request(self, request: Request) -> Response:
         p = urlparse_cached(request)
-        scheme = "http" if request.meta.get("is_secure") is False else "https"
+        if request.meta.get("is_secure") is False:
+            warnings.warn(
+                "Passing is_secure=False for s3:// requests is deprecated."
+                " In future Scrapy releases this flag will be ignored.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            scheme = "http"
+        else:
+            scheme = "https"
         bucket = p.hostname
         path = p.path + "?" + p.query if p.query else p.path
         url = f"{scheme}://{bucket}.s3.amazonaws.com{path}"
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 4af5ca0e8..8cb9308ff 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -175,7 +175,11 @@ class TestS3Anon:
     @coroutine_test
     async def test_anon_request_insecure(self):
         req = Request("s3://aws-publicdatasets/", meta={"is_secure": False})
-        httpreq = await self.download_request(req)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing is_secure=False for s3:// requests is deprecated",
+        ):
+            httpreq = await self.download_request(req)
         assert httpreq.url == "http://aws-publicdatasets.s3.amazonaws.com/"
 
 
@@ -215,7 +219,11 @@ class TestS3:
     @coroutine_test
     async def test_insecure_opt_out(self):
         req = Request("s3://johnsmith/photos/puppy.jpg", meta={"is_secure": False})
-        httpreq = await self.download_request(req)
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match="Passing is_secure=False for s3:// requests is deprecated",
+        ):
+            httpreq = await self.download_request(req)
         assert httpreq.url == "http://johnsmith.s3.amazonaws.com/photos/puppy.jpg"
 
     @coroutine_test

From a591d15c0458a80738670902e9dac24fee087da0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 20 Jul 2026 17:59:55 +0500
Subject: [PATCH 16/62] Deprecate Spider.log(). (#7739)

---
 docs/topics/spiders.rst    | 6 ------
 scrapy/spiders/__init__.py | 7 +++++++
 tests/test_spider.py       | 8 +++++++-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 887da0a15..770640adf 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -208,12 +208,6 @@ scrapy.Spider
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`
 
-   .. method:: log(message, [level])
-
-       Wrapper that sends a log message through the Spider's :attr:`logger`,
-       kept for backward compatibility. For more information see
-       :ref:`topics-logging-from-spiders`.
-
    .. method:: closed(reason)
 
        Called when the spider closes. This method provides a shortcut to
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index de527d04f..02dfa2ac6 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -7,9 +7,11 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import logging
+import warnings
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
@@ -66,6 +68,11 @@ class Spider(object_ref):
         can use it directly (e.g. Spider.logger.info('msg')) or use any other
         Python logger too.
         """
+        warnings.warn(
+            "Spider.log() is deprecated, use methods of Spider.logger instead.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.logger.log(level, message, **kw)
 
     @classmethod
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 526cc8f23..fba1ef984 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -8,6 +8,7 @@ from testfixtures import LogCapture
 
 from scrapy import signals
 from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, XmlResponse
 from scrapy.settings import Settings
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
@@ -116,7 +117,12 @@ class TestSpider:
 
     def test_log(self):
         spider = self.spider_class("example.com")
-        with mock.patch("scrapy.spiders.Spider.logger") as mock_logger:
+        with (
+            mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
+            pytest.warns(
+                ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
+            ),
+        ):
             spider.log("test log msg", "INFO")
         mock_logger.log.assert_called_once_with("INFO", "test log msg")
 

From 7faf20c6b5cbf226101720f8e77ee92a1c1b6182 Mon Sep 17 00:00:00 2001
From: Madan kumar <winklemad@outlook.com>
Date: Mon, 20 Jul 2026 18:48:55 +0530
Subject: [PATCH 17/62] Merge repeated curl -d/--data options in
 Request.from_curl() (#7728)

curl merges repeated -d/--data/--data-raw options into a single request
body joined with "&" (e.g. `curl -d a=1 -d b=2` sends `a=1&b=2`). scrapy's
DataAction kept only the last value, silently dropping the earlier ones, so
`Request.from_curl("curl -d a=1 -d b=2 ...")` produced a body of just `b=2`.
Accumulate the values with "&" to match curl. -H/--header and -b/--cookie
already accumulate via action="append"; -d was the only repeatable data flag
that didn't.
---
 scrapy/utils/curl.py     |  5 +++++
 tests/test_utils_curl.py | 33 +++++++++++++++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/scrapy/utils/curl.py b/scrapy/utils/curl.py
index 646335fb1..c82440dc7 100644
--- a/scrapy/utils/curl.py
+++ b/scrapy/utils/curl.py
@@ -23,6 +23,11 @@ class DataAction(argparse.Action):
     ) -> None:
         value = str(values)
         value = value.removeprefix("$")
+        # curl merges repeated -d/--data/--data-raw options into a single body
+        # joined with "&"; mirror that instead of keeping only the last one.
+        previous = getattr(namespace, self.dest, None)
+        if previous is not None:
+            value = f"{previous}&{value}"
         setattr(namespace, self.dest, value)
 
 
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index 0627a4b93..fce9fc984 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -163,6 +163,39 @@ class TestCurlToRequestKwargs:
         }
         self._test_command(curl_command, expected_result)
 
+    def test_post_data_multiple(self):
+        # curl merges repeated -d/--data/--data-raw options into a single body
+        # joined with "&"; scrapy must do the same, not keep only the last one.
+        curl_command = "curl 'https://www.example.org/' -d 'a=1' -d 'b=2' -d 'c=3'"
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": "a=1&b=2&c=3",
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_post_data_multiple_mixed_flag_names(self):
+        curl_command = (
+            "curl 'https://www.example.org/' --data 'a=1' --data-raw 'b=2' -d 'c=3'"
+        )
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": "a=1&b=2&c=3",
+        }
+        self._test_command(curl_command, expected_result)
+
+    def test_post_data_multiple_with_string_prefix(self):
+        # The leading "$" left by bash $'...' quoting is stripped per option,
+        # before the values are merged.
+        curl_command = "curl 'https://www.example.org/' -d $'a=1' -d $'b=2'"
+        expected_result = {
+            "method": "POST",
+            "url": "https://www.example.org/",
+            "body": "a=1&b=2",
+        }
+        self._test_command(curl_command, expected_result)
+
     def test_explicit_get_with_data(self):
         curl_command = "curl httpbin.org/anything -X GET --data asdf"
         expected_result = {

From e80f94fe8a07ce5828d81fe195f654c4798ebb2f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 20 Jul 2026 18:21:28 +0500
Subject: [PATCH 18/62] Tests for extensions.debug and extensions.memdebug.
 (#7749)

* Tests for extensions.debug and extensions.memdebug.

* Use a unique class in test_crawl_sets_stats.

* Don't assume a single log message.
---
 tests/test_extension_debug.py    | 135 +++++++++++++++++++++++++++++++
 tests/test_extension_memdebug.py |  51 ++++++++++++
 2 files changed, 186 insertions(+)
 create mode 100644 tests/test_extension_debug.py
 create mode 100644 tests/test_extension_memdebug.py

diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py
new file mode 100644
index 000000000..a54807c1c
--- /dev/null
+++ b/tests/test_extension_debug.py
@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import logging
+import os
+import signal
+import sys
+import threading
+from typing import TYPE_CHECKING
+from unittest import mock
+
+import pytest
+
+from scrapy.extensions.debug import Debugger, StackTraceDump
+from scrapy.spiders import Spider
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+
+@pytest.fixture(autouse=True)
+def preserve_signal_handlers() -> Generator[None]:
+    """Restore the signal handlers that the extensions replace."""
+    signums = [
+        getattr(signal, name)
+        for name in ("SIGUSR2", "SIGQUIT")
+        if hasattr(signal, name)
+    ]
+    handlers = {signum: signal.getsignal(signum) for signum in signums}
+    yield
+    for signum, handler in handlers.items():
+        signal.signal(signum, handler)
+
+
+class SignalSpider(Spider):
+    name = "signal_spider"
+    start_urls = ["data:,"]
+
+    def parse(self, response):
+        os.kill(os.getpid(), signal.SIGUSR2)
+        return []
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32", reason="SIGUSR2 and SIGQUIT are POSIX-only"
+)
+def test_stacktracedump_installs_signal_handlers() -> None:
+    crawler = get_crawler()
+    ext = StackTraceDump.from_crawler(crawler)
+    assert signal.getsignal(signal.SIGUSR2) == ext.dump_stacktrace  # pylint: disable=comparison-with-callable
+    assert signal.getsignal(signal.SIGQUIT) == ext.dump_stacktrace  # pylint: disable=comparison-with-callable
+
+
+def test_stacktracedump_works_without_signal_support(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    # simulate win32 platforms, which don't support SIGUSR signals
+    monkeypatch.delattr(signal, "SIGUSR2", raising=False)
+    ext = StackTraceDump.from_crawler(get_crawler())
+    assert isinstance(ext, StackTraceDump)
+
+
+def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> None:
+    crawler = get_crawler()
+    crawler.engine = mock.Mock()
+    ext = StackTraceDump.from_crawler(crawler)
+    spider = DefaultSpider()
+    with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"):
+        ext.dump_stacktrace(0, None)
+    assert len(caplog.records) == 1
+    message = caplog.records[0].getMessage()
+    assert "Dumping stack trace and engine status" in message
+    assert "Execution engine status" in message
+    assert "Live References" in message
+    assert type(spider).__name__ in message
+    assert "# Thread: MainThread" in message
+    assert getattr(caplog.records[0], "crawler", None) is crawler
+
+
+def test_stacktracedump_thread_stacks() -> None:
+    ext = StackTraceDump.from_crawler(get_crawler())
+    stop = threading.Event()
+    thread = threading.Thread(target=stop.wait, name="dump-test-thread")
+    thread.start()
+    try:
+        stacks = ext._thread_stacks()
+    finally:
+        stop.set()
+        thread.join()
+    assert "# Thread: MainThread" in stacks
+    assert "# Thread: dump-test-thread" in stacks
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="SIGUSR2 is POSIX-only")
+@coroutine_test
+async def test_stacktracedump_dumps_on_signal(caplog: pytest.LogCaptureFixture) -> None:
+    settings = {
+        "EXTENSIONS": {"scrapy.extensions.debug.StackTraceDump": 0},
+        "LOG_LEVEL": "INFO",
+    }
+    crawler = get_crawler(spidercls=SignalSpider, settings_dict=settings)
+    with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"):
+        await crawler.crawl_async()
+    for r in caplog.records:
+        message = r.getMessage()
+        if "Dumping stack trace and engine status" in message:
+            assert "Dumping stack trace and engine status" in message
+            assert "engine.spider.name" in message
+            assert "signal_spider" in message
+            return
+    raise AssertionError("No stack trace dump log message found")
+
+
+@pytest.mark.skipif(sys.platform == "win32", reason="SIGUSR2 is POSIX-only")
+def test_debugger_installs_signal_handler() -> None:
+    ext = Debugger()
+    assert signal.getsignal(signal.SIGUSR2) == ext._enter_debugger  # pylint: disable=comparison-with-callable
+
+
+def test_debugger_works_without_signal_support(monkeypatch: pytest.MonkeyPatch) -> None:
+    # simulate win32 platforms, which don't support SIGUSR signals
+    monkeypatch.delattr(signal, "SIGUSR2", raising=False)
+    ext = Debugger()
+    assert isinstance(ext, Debugger)
+
+
+def test_debugger_enter_debugger(monkeypatch: pytest.MonkeyPatch) -> None:
+    pdb_cls = mock.Mock()
+    monkeypatch.setattr("scrapy.extensions.debug.Pdb", pdb_cls)
+    ext = Debugger()
+    frame = sys._getframe()
+    ext._enter_debugger(0, frame)
+    pdb_cls.return_value.set_trace.assert_called_once_with(frame.f_back)
diff --git a/tests/test_extension_memdebug.py b/tests/test_extension_memdebug.py
new file mode 100644
index 000000000..e32989aec
--- /dev/null
+++ b/tests/test_extension_memdebug.py
@@ -0,0 +1,51 @@
+from __future__ import annotations
+
+import gc
+
+import pytest
+
+from scrapy.exceptions import NotConfigured
+from scrapy.extensions.memdebug import MemoryDebugger
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from scrapy.utils.trackref import object_ref
+from tests.utils.decorators import coroutine_test
+
+
+def test_disabled_by_default() -> None:
+    with pytest.raises(NotConfigured):
+        MemoryDebugger.from_crawler(get_crawler())
+
+
+def test_spider_closed_sets_stats() -> None:
+    crawler = get_crawler(settings_dict={"MEMDEBUG_ENABLED": True})
+    ext = MemoryDebugger.from_crawler(crawler)
+
+    class TrackedObject(object_ref):
+        pass
+
+    class CollectedObject(object_ref):
+        pass
+
+    tracked = [TrackedObject(), TrackedObject()]
+    CollectedObject()
+
+    ext.spider_closed(DefaultSpider(), "finished")
+
+    assert crawler.stats
+    assert crawler.stats.get_value("memdebug/gc_garbage_count") == len(gc.garbage)
+    assert crawler.stats.get_value("memdebug/live_refs/TrackedObject") == len(tracked)
+    assert crawler.stats.get_value("memdebug/live_refs/CollectedObject") is None
+
+
+@coroutine_test
+async def test_crawl_sets_stats() -> None:
+    # unique class so that other tests don't pollute live_refs
+    class MemDebugSpider(DefaultSpider):
+        pass
+
+    crawler = get_crawler(MemDebugSpider, settings_dict={"MEMDEBUG_ENABLED": True})
+    await crawler.crawl_async()
+    assert crawler.stats
+    assert crawler.stats.get_value("memdebug/gc_garbage_count") is not None
+    assert crawler.stats.get_value("memdebug/live_refs/MemDebugSpider") == 1

From 4d4a04f3189fe2f36574a593814dc17319c4a66a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 13:17:56 +0500
Subject: [PATCH 19/62] Improve a flaky test_stacktracedump_dump_stacktrace().
 (#7753)

---
 tests/test_extension_debug.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/tests/test_extension_debug.py b/tests/test_extension_debug.py
index a54807c1c..27554116b 100644
--- a/tests/test_extension_debug.py
+++ b/tests/test_extension_debug.py
@@ -69,14 +69,16 @@ def test_stacktracedump_dump_stacktrace(caplog: pytest.LogCaptureFixture) -> Non
     spider = DefaultSpider()
     with caplog.at_level(logging.INFO, logger="scrapy.extensions.debug"):
         ext.dump_stacktrace(0, None)
-    assert len(caplog.records) == 1
-    message = caplog.records[0].getMessage()
-    assert "Dumping stack trace and engine status" in message
-    assert "Execution engine status" in message
-    assert "Live References" in message
-    assert type(spider).__name__ in message
-    assert "# Thread: MainThread" in message
-    assert getattr(caplog.records[0], "crawler", None) is crawler
+    for r in caplog.records:
+        message = r.getMessage()
+        if "Dumping stack trace and engine status" in message:
+            assert "Execution engine status" in message
+            assert "Live References" in message
+            assert type(spider).__name__ in message
+            assert "# Thread: MainThread" in message
+            assert getattr(r, "crawler", None) is crawler
+            return
+    raise AssertionError("No stack trace dump log message found")
 
 
 def test_stacktracedump_thread_stacks() -> None:

From 1c5404dce59e9936f7efbad6cade07c21358f7b5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 14:23:52 +0500
Subject: [PATCH 20/62] Add docs about running Scrapy from apps and notebooks.
 (#7751)

---
 docs/topics/asyncio.rst   |   6 +++
 docs/topics/practices.rst | 104 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 110 insertions(+)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 3d4ebe088..d0b99c38e 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -150,6 +150,12 @@ Using Scrapy without a Twisted reactor
 .. warning::
     This is currently experimental and may not be suitable for production use.
 
+.. note:: As the Twisted download handlers cannot be used without a reactor,
+    the default download handler in this mode is
+    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
+    will need to additionally install the ``httpx`` library to use it, unless
+    you switch to some different handler.
+
 It's possible to use Scrapy without installing a Twisted reactor at all, by
 setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
 mode Scrapy will use the asyncio event loop directly, and most of the Scrapy
diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 216b7e64f..23738c98c 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -247,6 +247,110 @@ Using :func:`asyncio.run` with :class:`~scrapy.crawler.AsyncCrawlerRunner`:
 
     asyncio.run(main())
 
+.. _run-spiders-in-apps:
+
+Running spiders inside existing applications
+============================================
+
+You may want to run Scrapy spiders inside an existing application. In simple
+cases (e.g. task queues that spawn a process for every task, or applications
+that can execute tasks synchronously in the same process) you can use the same
+approach as for standalone scripts (see :ref:`run-from-script`). More complex
+cases, e.g. asynchronous web applications, have additional caveats and
+limitations.
+
+If the application runs its own Twisted reactor, you can use
+:class:`~scrapy.crawler.AsyncCrawlerRunner` or
+:class:`~scrapy.crawler.CrawlerRunner` to run spiders using this reactor, see
+:ref:`run-from-script` for examples.
+
+If the application doesn't run a Twisted reactor or an asyncio event loop (for
+example, a Django web app deployed with a WSGI server such as uWSGI), you can
+use :class:`~scrapy.crawler.AsyncCrawlerProcess` with
+:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy starts and
+stops an asyncio event loop for every spider run:
+
+.. code-block:: python
+
+    import scrapy
+    from django.http import HttpResponse
+    from scrapy.crawler import AsyncCrawlerProcess
+
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+
+    def crawl_view(request):
+        process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+        process.crawl(MySpider)
+        process.start()  # returns when the spider finishes
+        return HttpResponse("Crawling finished")
+
+If the application runs its own asyncio event loop (for example, a Django web
+app deployed with an ASGI server such as uvicorn), you can use
+:class:`~scrapy.crawler.AsyncCrawlerRunner` with
+:setting:`TWISTED_REACTOR_ENABLED` set to ``False``, so that Scrapy uses the
+existing event loop:
+
+.. code-block:: python
+
+    import scrapy
+    from django.http import HttpResponse
+    from scrapy.crawler import AsyncCrawlerRunner
+
+
+    class MySpider(scrapy.Spider):
+        # Your spider definition
+        ...
+
+
+    async def crawl_view(request):
+        runner = AsyncCrawlerRunner(settings={"TWISTED_REACTOR_ENABLED": False})
+        await runner.crawl(MySpider)  # completes when the spider finishes
+        return HttpResponse("Crawling finished")
+
+.. note:: Running Scrapy without a Twisted reactor is experimental and has
+    some limitations, described in :ref:`asyncio-without-reactor`.
+
+.. _run-in-notebook:
+
+Running spiders in Jupyter notebooks
+====================================
+
+You can run Scrapy spiders in Jupyter notebooks. You need to use
+:class:`~scrapy.crawler.AsyncCrawlerRunner` with
+:setting:`TWISTED_REACTOR_ENABLED` set to ``False`` for this, so that Scrapy
+uses the event loop provided by the notebook kernel. As
+:class:`~scrapy.crawler.AsyncCrawlerRunner` doesn't configure logging, and you
+most likely want to see the spider log in the notebook, you should call
+:func:`scrapy.utils.log.configure_logging`. Here is a full example, which
+supports rerunning both as a single cell and as separate cells:
+
+.. code-block:: python
+
+    from scrapy import Spider
+    from scrapy.crawler import AsyncCrawlerRunner
+    from scrapy.utils.log import configure_logging
+
+    configure_logging()
+
+
+    class BooksSpider(Spider):
+        name = "books"
+        start_urls = ["https://books.toscrape.com"]
+
+        def parse(self, response):
+            for book in response.css("h3"):
+                yield {"title": book.css("a::attr(title)").get()}
+
+
+    runner = AsyncCrawlerRunner({"TWISTED_REACTOR_ENABLED": False})
+    await runner.crawl(BooksSpider)
+
+.. note:: Running Scrapy without a Twisted reactor is experimental and has
+    some limitations, described in :ref:`asyncio-without-reactor`.
 
 .. _run-multiple-spiders:
 

From b1b9efb4731040eb47bb12c213f63b2ea1a2a962 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 14:43:36 +0500
Subject: [PATCH 21/62] Rewrite GCSFilesStore tests to use mocking. (#7727)

---
 scrapy/pipelines/files.py         |   4 +-
 tests/test_feedexport_storages.py |  18 +--
 tests/test_pipeline_files.py      | 216 ++++++++++++++++++++----------
 tests/utils/cloud.py              |  22 +++
 tox.ini                           |   3 -
 5 files changed, 173 insertions(+), 90 deletions(-)
 create mode 100644 tests/utils/cloud.py

diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 44c422430..70dc36e35 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -300,13 +300,13 @@ class GCSFilesStore:
         )
         if "storage.objects.get" not in permissions:
             logger.warning(
-                "No 'storage.objects.get' permission for GSC bucket %(bucket)s. "
+                "No 'storage.objects.get' permission for GCS bucket %(bucket)s. "
                 "Checking if files are up to date will be impossible. Files will be downloaded every time.",
                 {"bucket": bucket},
             )
         if "storage.objects.create" not in permissions:
             logger.error(
-                "No 'storage.objects.create' permission for GSC bucket %(bucket)s. Saving files will be impossible!",
+                "No 'storage.objects.create' permission for GCS bucket %(bucket)s. Saving files will be impossible!",
                 {"bucket": bucket},
             )
 
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 11e79775f..b7009be62 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -27,26 +27,10 @@ from scrapy.extensions.feedexport import (
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
+from tests.utils.cloud import mock_google_cloud_storage
 from tests.utils.decorators import coroutine_test
 
 
-def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
-    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
-    classes and set their proper return values.
-    """
-    from google.cloud.storage import Blob, Bucket, Client  # noqa: PLC0415
-
-    client_mock = mock.create_autospec(Client)
-
-    bucket_mock = mock.create_autospec(Bucket)
-    client_mock.get_bucket.return_value = bucket_mock
-
-    blob_mock = mock.create_autospec(Blob)
-    bucket_mock.blob.return_value = blob_mock
-
-    return (client_mock, bucket_mock, blob_mock)
-
-
 class TestFileFeedStorage:
     def test_store_file_uri(self, tmp_path):
         path = tmp_path / "file.txt"
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 3829a35f5..0d9aac830 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -1,5 +1,6 @@
+import base64
 import dataclasses
-import os
+import logging
 import random
 import time
 from abc import ABC, abstractmethod
@@ -13,7 +14,6 @@ from tempfile import mkdtemp
 from typing import Any
 from unittest import mock
 from unittest.mock import MagicMock
-from urllib.parse import urlparse
 
 import attr
 import pytest
@@ -30,28 +30,21 @@ from scrapy.pipelines.files import (
     GCSFilesStore,
     S3FilesStore,
 )
+from scrapy.pipelines.media import MediaPipeline
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 from .test_pipeline_media import _mocked_download_func
+from .utils.cloud import mock_google_cloud_storage
 
-
-def get_gcs_content_and_delete(
-    bucket: Any, path: str
-) -> tuple[bytes, list[dict[str, str]], Any]:
-    from google.cloud import storage  # noqa: PLC0415
-
-    client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
-    bucket = client.get_bucket(bucket)
-    blob = bucket.get_blob(path)
-    content = blob.download_as_string()
-    acl = list(blob.acl)  # loads acl before it will be deleted
-    bucket.delete_blob(path)
-    return content, acl, blob
+# required by persist_file() and stat_file(), but as some stores don't use the argument
+# we can pass this singleton to keep type hints correct
+DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
 
 
 def get_ftp_content_and_delete(
@@ -625,7 +618,7 @@ class TestS3FilesStore:
             yield store.persist_file(
                 path,
                 buffer,
-                info=None,
+                info=DUMMY_SPIDER_INFO,
                 meta=meta,
                 headers={"Content-Type": content_type},
             )
@@ -658,7 +651,7 @@ class TestS3FilesStore:
                 },
             )
 
-            file_stats = yield store.stat_file("", info=None)
+            file_stats = yield store.stat_file("", info=DUMMY_SPIDER_INFO)
             assert file_stats == {
                 "checksum": checksum,
                 "last_modified": last_modified.timestamp(),
@@ -667,59 +660,144 @@ class TestS3FilesStore:
             stub.assert_no_pending_responses()
 
 
-@pytest.mark.skipif(
-    "GCS_PROJECT_ID" not in os.environ, reason="GCS_PROJECT_ID not found"
-)
 class TestGCSFilesStore:
-    @inline_callbacks_test
-    def test_persist(self):
-        uri = os.environ.get("GCS_TEST_FILE_URI")
-        if not uri:
-            pytest.skip("No GCS URI available for testing")
-        data = b"TestGCSFilesStore: \xe2\x98\x83"
-        buf = BytesIO(data)
-        meta = {"foo": "bar"}
-        path = "full/filename"
-        store = GCSFilesStore(uri)
-        store.POLICY = "authenticatedRead"
-        expected_policy = {"role": "READER", "entity": "allAuthenticatedUsers"}
-        yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
-        s = yield store.stat_file(path, info=None)
-        assert "last_modified" in s
-        assert "checksum" in s
-        assert s["checksum"] == "cdcda85605e46d0af6110752770dce3c"
-        u = urlparse(uri)
-        content, acl, blob = get_gcs_content_and_delete(u.hostname, u.path[1:] + path)
-        assert content == data
-        assert blob.metadata == {"foo": "bar"}
-        assert blob.cache_control == GCSFilesStore.CACHE_CONTROL
-        assert blob.content_type == "application/octet-stream"
-        assert expected_policy in acl
+    @staticmethod
+    def build_gcs_files_store(
+        *,
+        permissions: tuple[str, ...] = (
+            "storage.objects.get",
+            "storage.objects.create",
+        ),
+    ) -> tuple[GCSFilesStore, Any, Any]:
+        """Build a :class:`GCSFilesStore` mock.
 
-    @inline_callbacks_test
-    def test_blob_path_consistency(self):
+        Returns ``(store, bucket_mock, blob_mock)``. Skips the test if
+        google-cloud-storage is not installed. ``permissions`` is what
+        ``Bucket.test_iam_permissions`` will return.
+        """
+        pytest.importorskip("google.cloud.storage")
+
+        client_mock, bucket_mock, blob_mock = mock_google_cloud_storage()
+        bucket_mock.test_iam_permissions.return_value = list(permissions)
+
+        with mock.patch("google.cloud.storage.Client", return_value=client_mock):
+            store = GCSFilesStore("gs://my_bucket/my_prefix/")
+        return store, bucket_mock, blob_mock
+
+    def test_init(self, caplog: pytest.LogCaptureFixture) -> None:
+        with caplog.at_level(logging.WARNING):
+            self.build_gcs_files_store()
+        assert not caplog.records
+
+    def test_get_perm_missing(self, caplog: pytest.LogCaptureFixture) -> None:
+        self.build_gcs_files_store(permissions=("storage.objects.create",))
+        assert (
+            "No 'storage.objects.get' permission for GCS bucket my_bucket"
+            in caplog.text
+        )
+
+    def test_create_perm_missing(self, caplog: pytest.LogCaptureFixture) -> None:
+        self.build_gcs_files_store(permissions=("storage.objects.get",))
+        assert (
+            "No 'storage.objects.create' permission for GCS bucket my_bucket"
+            in caplog.text
+        )
+
+    def test_update_stores(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setattr(GCSFilesStore, "GCS_PROJECT_ID", None)
+        monkeypatch.setattr(GCSFilesStore, "POLICY", None)
+
+        settings = Settings(
+            {"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": "publicRead"}
+        )
+        FilesPipeline._update_stores(settings)
+        assert GCSFilesStore.GCS_PROJECT_ID == "my-project"
+        assert GCSFilesStore.POLICY == "publicRead"
+
+        # An empty FILES_STORE_GCS_ACL is normalised to None.
+        settings = Settings({"GCS_PROJECT_ID": "my-project", "FILES_STORE_GCS_ACL": ""})
+        FilesPipeline._update_stores(settings)
+        assert GCSFilesStore.POLICY is None
+
+    @coroutine_test
+    async def test_persist(self) -> None:
+        store, bucket, blob = self.build_gcs_files_store()
+        await maybe_deferred_to_future(
+            store.persist_file(
+                "full/filename",
+                BytesIO(b"hello"),
+                info=DUMMY_SPIDER_INFO,
+                meta={"foo": 1},
+                headers={"Content-Type": "image/png"},
+            )
+        )
+        bucket.blob.assert_called_once_with("my_prefix/full/filename")
+        assert blob.cache_control == GCSFilesStore.CACHE_CONTROL
+        assert blob.metadata == {"foo": "1"}
+        blob.upload_from_string.assert_called_once_with(
+            data=b"hello",
+            content_type="image/png",
+            predefined_acl=store.POLICY,
+        )
+
+    @coroutine_test
+    async def test_persist_defaults(self) -> None:
+        store, _, blob = self.build_gcs_files_store()
+        await maybe_deferred_to_future(
+            store.persist_file(
+                "full/filename",
+                BytesIO(b"hello"),
+                info=DUMMY_SPIDER_INFO,
+            )
+        )
+        blob.upload_from_string.assert_called_once_with(
+            data=b"hello",
+            content_type="application/octet-stream",
+            predefined_acl=store.POLICY,
+        )
+        assert blob.metadata == {}
+
+    @coroutine_test
+    async def test_stat(self) -> None:
+        store, bucket, blob = self.build_gcs_files_store()
+        checksum = "cdcda85605e46d0af6110752770dce3c"
+        blob.md5_hash = base64.b64encode(bytes.fromhex(checksum)).decode()
+        updated = datetime(2019, 12, 1)
+        blob.updated = updated
+        bucket.get_blob.return_value = blob
+        stat = await maybe_deferred_to_future(
+            store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)
+        )
+        bucket.get_blob.assert_called_once_with("my_prefix/full/filename")
+        assert stat == {
+            "checksum": checksum,
+            "last_modified": time.mktime(updated.timetuple()),
+        }
+
+    @coroutine_test
+    async def test_stat_missing_blob(self) -> None:
+        store, bucket, _ = self.build_gcs_files_store()
+        bucket.get_blob.return_value = None
+        stat = await maybe_deferred_to_future(
+            store.stat_file("full/filename", info=DUMMY_SPIDER_INFO)
+        )
+        assert stat == {}
+
+    @coroutine_test
+    async def test_blob_path_consistency(self) -> None:
         """Test to make sure that paths used to store files is the same as the one used to get
         already uploaded files.
         """
-        try:
-            import google.cloud.storage  # noqa: F401,PLC0415
-        except ModuleNotFoundError:
-            pytest.skip("google-cloud-storage is not installed")
-        with (
-            mock.patch("google.cloud.storage"),
-            mock.patch("scrapy.pipelines.files.time"),
-        ):
-            uri = "gs://my_bucket/my_prefix/"
-            store = GCSFilesStore(uri)
-            store.bucket = mock.Mock()
-            path = "full/my_data.txt"
-            yield store.persist_file(
-                path, mock.Mock(), info=None, meta=None, headers=None
-            )
-            yield store.stat_file(path, info=None)
-            expected_blob_path = store.prefix + path
-            store.bucket.blob.assert_called_with(expected_blob_path)
-            store.bucket.get_blob.assert_called_with(expected_blob_path)
+        store, bucket, _ = self.build_gcs_files_store()
+        bucket.get_blob.return_value = None
+        path = "full/my_data.txt"
+        await maybe_deferred_to_future(
+            store.persist_file(path, BytesIO(b""), info=DUMMY_SPIDER_INFO)
+        )
+        await maybe_deferred_to_future(store.stat_file(path, info=DUMMY_SPIDER_INFO))
+        expected_blob_path = store.prefix + path
+        bucket.blob.assert_called_with(expected_blob_path)
+        bucket.get_blob.assert_called_with(expected_blob_path)
 
 
 class TestFTPFileStore:
@@ -735,10 +813,12 @@ class TestFTPFileStore:
             FTPFilesStore.FTP_PASSWORD = "guest"
 
             store = FTPFilesStore(ftp_server.url("/"))
-            empty_dict = yield store.stat_file(path, info=None)
+            empty_dict = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
             assert empty_dict == {}
-            yield store.persist_file(path, buf, info=None, meta=meta, headers=None)
-            stat = yield store.stat_file(path, info=None)
+            yield store.persist_file(
+                path, buf, info=DUMMY_SPIDER_INFO, meta=meta, headers=None
+            )
+            stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
             assert "last_modified" in stat
             assert "checksum" in stat
             assert stat["checksum"] == "d113d66b2ec7258724a268bd88eef6b6"
diff --git a/tests/utils/cloud.py b/tests/utils/cloud.py
new file mode 100644
index 000000000..662e0b3ee
--- /dev/null
+++ b/tests/utils/cloud.py
@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+from typing import Any
+from unittest import mock
+
+
+def mock_google_cloud_storage() -> tuple[Any, Any, Any]:
+    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
+    classes and set their proper return values.
+    """
+    from google.cloud.storage import Blob, Bucket, Client  # noqa: PLC0415
+
+    client_mock = mock.create_autospec(Client)
+
+    bucket_mock = mock.create_autospec(Bucket)
+    client_mock.bucket.return_value = bucket_mock
+    client_mock.get_bucket.return_value = bucket_mock
+
+    blob_mock = mock.create_autospec(Blob)
+    bucket_mock.blob.return_value = blob_mock
+
+    return (client_mock, bucket_mock, blob_mock)
diff --git a/tox.ini b/tox.ini
index 526533480..ec7273b42 100644
--- a/tox.ini
+++ b/tox.ini
@@ -55,12 +55,9 @@ passenv =
     PYTHONTRACEMALLOC
     PYTEST_ADDOPTS
     MITMDUMP
-    S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
     AWS_SECRET_ACCESS_KEY
     AWS_SESSION_TOKEN
-    GCS_TEST_FILE_URI
-    GCS_PROJECT_ID
 #allow tox virtualenv to upgrade pip/wheel/setuptools
 download = true
 commands =

From 36bf1185e58e1249bf2c05ffb064851a5306a20d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 15:22:04 +0500
Subject: [PATCH 22/62] Modernize code examples, drop docs for nonexistent
 `MEMDEBUG_NOTIFY`  (#7737)

---
 docs/topics/addons.rst              | 12 ++++++--
 docs/topics/developer-tools.rst     |  3 +-
 docs/topics/exporters.rst           | 44 ++++++++++++++++++--------
 docs/topics/feed-exports.rst        | 48 ++++++++++++++++-------------
 docs/topics/leaks.rst               | 11 +++++--
 docs/topics/link-extractors.rst     |  4 ++-
 docs/topics/loaders.rst             |  4 ++-
 docs/topics/media-pipeline.rst      |  8 +++--
 docs/topics/request-response.rst    | 38 +++++++++++++++++------
 docs/topics/settings.rst            | 33 ++++++++------------
 docs/topics/shell.rst               |  4 ++-
 docs/topics/signals.rst             |  4 +--
 docs/topics/spiders.rst             | 33 ++++++++++++++------
 scrapy/settings/default_settings.py |  2 --
 14 files changed, 155 insertions(+), 93 deletions(-)

diff --git a/docs/topics/addons.rst b/docs/topics/addons.rst
index 78b1ab92b..75f15b3ae 100644
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@@ -21,10 +21,14 @@ The ``ADDONS`` setting is a dict in which every key is an add-on class or its
 import path and the value is its priority.
 
 This is an example where two add-ons are enabled in a project's
-``settings.py``::
+``settings.py``:
+
+.. skip: next
+
+.. code-block:: python
 
     ADDONS = {
-        'path.to.someaddon': 0,
+        "path.to.someaddon": 0,
         SomeAddonClass: 1,
     }
 
@@ -56,7 +60,9 @@ the following methods:
     :type settings: :class:`~scrapy.settings.BaseSettings`
 
 The settings set by the add-on should use the ``addon`` priority (see
-:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`)::
+:ref:`populating-settings` and :func:`scrapy.settings.BaseSettings.set`):
+
+.. code-block:: python
 
     class MyAddon:
         def update_settings(self, settings):
diff --git a/docs/topics/developer-tools.rst b/docs/topics/developer-tools.rst
index 05dffcdda..a5ffe00f1 100644
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@@ -246,7 +246,6 @@ also request each page to get every quote on the site:
 .. code-block:: python
 
     import scrapy
-    import json
 
 
     class QuoteSpider(scrapy.Spider):
@@ -256,7 +255,7 @@ also request each page to get every quote on the site:
         start_urls = ["https://quotes.toscrape.com/api/quotes?page=1"]
 
         def parse(self, response):
-            data = json.loads(response.text)
+            data = response.json()
             for quote in data["quotes"]:
                 yield {"quote": quote["text"]}
             if data["has_next"]:
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f0b704b9d..ecd154122 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -211,13 +211,17 @@ BaseItemExporter
 
       -   ``None`` (all fields [2]_, default)
 
-      -   A list of fields::
+      -   A list of fields:
 
-              ['field1', 'field2']
+          .. code-block:: python
 
-      -   A dict where keys are fields and values are output names::
+              ["field1", "field2"]
 
-              {'field1': 'Field 1', 'field2': 'Field 2'}
+      -   A dict where keys are fields and values are output names:
+
+          .. code-block:: python
+
+              {"field1": "Field 1", "field2": "Field 2"}
 
       .. [1] Not all exporters respect the specified field order.
       .. [2] When using :ref:`item objects <item-types>` that do not expose
@@ -273,7 +277,9 @@ XmlItemExporter
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method.
 
-   A typical output of this exporter would be::
+   A typical output of this exporter would be:
+
+   .. code-block:: xml
 
        <?xml version="1.0" encoding="utf-8"?>
        <items>
@@ -291,11 +297,17 @@ XmlItemExporter
    exported by serializing each value inside a ``<value>`` element. This is for
    convenience, as multi-valued fields are very common.
 
-   For example, the item::
+   For example, the item:
 
-        Item(name=['John', 'Doe'], age='23')
+   .. skip: next
 
-   Would be serialized as::
+   .. code-block:: python
+
+        Item(name=["John", "Doe"], age="23")
+
+   Would be serialized as:
+
+   .. code-block:: xml
 
        <?xml version="1.0" encoding="utf-8"?>
        <items>
@@ -379,10 +391,12 @@ PprintItemExporter
    The additional keyword arguments of this ``__init__`` method are passed to the
    :class:`BaseItemExporter` ``__init__`` method.
 
-   A typical output of this exporter would be::
+   A typical output of this exporter would be:
 
-        {'name': 'Color TV', 'price': '1200'}
-        {'name': 'DVD player', 'price': '200'}
+   .. code-block:: python
+
+        {"name": "Color TV", "price": "1200"}
+        {"name": "DVD player", "price": "200"}
 
    Longer lines (when present) are pretty-formatted.
 
@@ -400,7 +414,9 @@ JsonItemExporter
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
-   A typical output of this exporter would be::
+   A typical output of this exporter would be:
+
+   .. code-block:: json
 
         [{"name": "Color TV", "price": "1200"},
         {"name": "DVD player", "price": "200"}]
@@ -429,7 +445,9 @@ JsonLinesItemExporter
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
 
-   A typical output of this exporter would be::
+   A typical output of this exporter would be:
+
+   .. code-block:: json
 
         {"name": "Color TV", "price": "1200"}
         {"name": "DVD player", "price": "200"}
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 1bab3bfc2..51e142b04 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -434,33 +434,37 @@ This setting is required for enabling the feed export feature.
 
 See :ref:`topics-feed-storage-backends` for supported URI schemes.
 
-For instance::
+For instance:
+
+.. skip: next
+
+.. code-block:: python
 
     {
-        'items.json': {
-            'format': 'json',
-            'encoding': 'utf8',
-            'store_empty': False,
-            'item_classes': [MyItemClass1, 'myproject.items.MyItemClass2'],
-            'fields': None,
-            'indent': 4,
-            'item_export_kwargs': {
-               'export_empty_fields': True,
+        "items.json": {
+            "format": "json",
+            "encoding": "utf8",
+            "store_empty": False,
+            "item_classes": [MyItemClass1, "myproject.items.MyItemClass2"],
+            "fields": None,
+            "indent": 4,
+            "item_export_kwargs": {
+                "export_empty_fields": True,
             },
         },
-        '/home/user/documents/items.xml': {
-            'format': 'xml',
-            'fields': ['name', 'price'],
-            'item_filter': MyCustomFilter1,
-            'encoding': 'latin1',
-            'indent': 8,
+        "/home/user/documents/items.xml": {
+            "format": "xml",
+            "fields": ["name", "price"],
+            "item_filter": MyCustomFilter1,
+            "encoding": "latin1",
+            "indent": 8,
         },
-        pathlib.Path('items.csv.gz'): {
-            'format': 'csv',
-            'fields': ['price', 'name'],
-            'item_filter': 'myproject.filters.MyCustomFilter2',
-            'postprocessing': [MyPlugin1, 'scrapy.extensions.postprocessing.GzipPlugin'],
-            'gzip_compresslevel': 5,
+        pathlib.Path("items.csv.gz"): {
+            "format": "csv",
+            "fields": ["price", "name"],
+            "item_filter": "myproject.filters.MyCustomFilter2",
+            "postprocessing": [MyPlugin1, "scrapy.extensions.postprocessing.GzipPlugin"],
+            "gzip_compresslevel": 5,
         },
     }
 
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 3337cc5d0..d4577601d 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -106,10 +106,15 @@ A real example
 --------------
 
 Let's see a concrete example of a hypothetical case of memory leaks.
-Suppose we have some spider with a line similar to this one::
+Suppose we have some spider with a line similar to this one:
 
-    return Request(f"http://www.somenastyspider.com/product.php?pid={product_id}",
-                   callback=self.parse, cb_kwargs={'referer': response})
+.. code-block:: python
+
+    return Request(
+        f"http://www.somenastyspider.com/product.php?pid={product_id}",
+        callback=self.parse,
+        cb_kwargs={"referer": response},
+    )
 
 That line is passing a response reference inside a request which effectively
 ties the response lifetime to the requests' one, and that would definitely
diff --git a/docs/topics/link-extractors.rst b/docs/topics/link-extractors.rst
index 3fc896507..c1d872dd0 100644
--- a/docs/topics/link-extractors.rst
+++ b/docs/topics/link-extractors.rst
@@ -36,7 +36,9 @@ Link extractor reference
 
 The link extractor class is
 :class:`scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor`. For convenience it
-can also be imported as ``scrapy.linkextractors.LinkExtractor``::
+can also be imported as ``scrapy.linkextractors.LinkExtractor``:
+
+.. code-block:: python
 
     from scrapy.linkextractors import LinkExtractor
 
diff --git a/docs/topics/loaders.rst b/docs/topics/loaders.rst
index 765e7f6d7..314e2c1ae 100644
--- a/docs/topics/loaders.rst
+++ b/docs/topics/loaders.rst
@@ -350,7 +350,9 @@ When parsing related values from a subsection of a document, it can be
 useful to create nested loaders.  Imagine you're extracting details from
 a footer of a page that looks something like:
 
-Example::
+Example:
+
+.. code-block:: html
 
     <footer>
         <a class="social" href="https://facebook.com/whatever">Like Us</a>
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index b7a4f561e..56b53d18d 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -470,7 +470,9 @@ When using the Images Pipeline, you can drop images which are too small, by
 specifying the minimum allowed size in the :setting:`IMAGES_MIN_HEIGHT` and
 :setting:`IMAGES_MIN_WIDTH` settings.
 
-For example::
+For example:
+
+.. code-block:: python
 
    IMAGES_MIN_HEIGHT = 110
    IMAGES_MIN_WIDTH = 110
@@ -493,7 +495,9 @@ Allowing redirections
 By default media pipelines ignore redirects, i.e. an HTTP redirection
 to a media file URL request will mean the media download is considered failed.
 
-To handle media redirections, set this setting to ``True``::
+To handle media redirections, set this setting to ``True``:
+
+.. code-block:: python
 
     MEDIA_ALLOW_REDIRECTS = True
 
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index a5f799444..e56be0a22 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -436,7 +436,7 @@ errors if needed:
                 )
 
         def parse_httpbin(self, response):
-            self.logger.info("Got successful response from {}".format(response.url))
+            self.logger.info(f"Got successful response from {response.url}")
             # do something useful here...
 
         def errback_httpbin(self, failure):
@@ -1028,9 +1028,13 @@ Response objects
         :meth:`~scrapy.http.headers.Headers.get` to return the last header value with
         the specified name or :meth:`~scrapy.http.headers.Headers.getlist` to return
         all header values with the specified name. For example, this call will give you
-        all cookies in the headers::
+        all cookies in the headers:
 
-            response.headers.getlist('Set-Cookie')
+        .. skip: next
+
+        .. code-block:: python
+
+            response.headers.getlist("Set-Cookie")
 
     .. attribute:: Response.body
 
@@ -1132,7 +1136,11 @@ Response objects
         a possible relative url.
 
         This is a wrapper over :func:`~urllib.parse.urljoin`, it's merely an alias for
-        making this call::
+        making this call:
+
+        .. skip: next
+
+        .. code-block:: python
 
             urllib.parse.urljoin(response.url, url)
 
@@ -1221,21 +1229,31 @@ TextResponse objects
 
     .. method:: TextResponse.jmespath(query)
 
-        A shortcut to ``TextResponse.selector.jmespath(query)``::
+        .. skip: start
 
-            response.jmespath('object.[*]')
+        A shortcut to ``TextResponse.selector.jmespath(query)``:
+
+        .. code-block:: python
+
+            response.jmespath("object.[*]")
 
     .. method:: TextResponse.xpath(query)
 
-        A shortcut to ``TextResponse.selector.xpath(query)``::
+        A shortcut to ``TextResponse.selector.xpath(query)``:
 
-            response.xpath('//p')
+        .. code-block:: python
+
+            response.xpath("//p")
 
     .. method:: TextResponse.css(query)
 
-        A shortcut to ``TextResponse.selector.css(query)``::
+        A shortcut to ``TextResponse.selector.css(query)``:
 
-            response.css('p')
+        .. code-block:: python
+
+            response.css("p")
+
+        .. skip: end
 
     .. automethod:: TextResponse.follow
 
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a592f8bc5..8068824e3 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -899,7 +899,9 @@ Use :setting:`DOWNLOAD_DELAY` to throttle your crawling speed, to avoid hitting
 servers too hard.
 
 Decimal numbers are supported. For example, to send a maximum of 4 requests
-every 10 seconds::
+every 10 seconds:
+
+.. code-block:: python
 
     DOWNLOAD_DELAY = 2.5
 
@@ -1231,7 +1233,9 @@ the ``dont_filter`` parameter to ``True`` on the ``__init__`` method of a
 specific :class:`~scrapy.Request` object that should not be filtered out.
 
 A class assigned to :setting:`DUPEFILTER_CLASS` must implement the following
-interface::
+interface:
+
+.. code-block:: python
 
     class MyDupeFilter:
 
@@ -1644,21 +1648,6 @@ Default: ``False``
 
 Whether to enable memory debugging.
 
-.. setting:: MEMDEBUG_NOTIFY
-
-MEMDEBUG_NOTIFY
----------------
-
-Default: ``[]``
-
-When memory debugging is enabled a memory report will be sent to the specified
-addresses if this setting is not empty, otherwise the report will be written to
-the log.
-
-Example::
-
-    MEMDEBUG_NOTIFY = ['user@example.com']
-
 .. setting:: MEMUSAGE_ENABLED
 
 MEMUSAGE_ENABLED
@@ -1732,9 +1721,11 @@ Default: ``"<project name>.spiders"`` (:ref:`fallback <default-settings>`: ``""`
 
 Module where to create new spiders using the :command:`genspider` command.
 
-Example::
+Example:
 
-    NEWSPIDER_MODULE = 'mybot.spiders_dev'
+.. code-block:: python
+
+    NEWSPIDER_MODULE = "mybot.spiders_dev"
 
 .. setting:: RANDOMIZE_DOWNLOAD_DELAY
 
@@ -2209,7 +2200,7 @@ In order to use the reactor installed by Scrapy:
 
         def __init__(self, *args, **kwargs):
             self.timeout = int(kwargs.pop("timeout", "60"))
-            super(QuotesSpider, self).__init__(*args, **kwargs)
+            super().__init__(*args, **kwargs)
 
         async def start(self):
             reactor.callLater(self.timeout, self.stop)
@@ -2238,7 +2229,7 @@ which raises an exception, becomes:
 
         def __init__(self, *args, **kwargs):
             self.timeout = int(kwargs.pop("timeout", "60"))
-            super(QuotesSpider, self).__init__(*args, **kwargs)
+            super().__init__(*args, **kwargs)
 
         async def start(self):
             from twisted.internet import reactor
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index 5b826ed18..ea97a22aa 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -34,7 +34,9 @@ is unavailable.
 Through Scrapy's settings you can configure it to use any one of
 ``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
 are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
-variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`::
+variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:
+
+.. code-block:: ini
 
     [settings]
     shell = bpython
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 531232149..03996bee6 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -34,7 +34,7 @@ Here is a simple example showing how you can catch signals and perform some acti
 
         @classmethod
         def from_crawler(cls, crawler, *args, **kwargs):
-            spider = super(DmozSpider, cls).from_crawler(crawler, *args, **kwargs)
+            spider = super().from_crawler(crawler, *args, **kwargs)
             crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
             return spider
 
@@ -72,7 +72,7 @@ Let's take an example using :ref:`coroutines <topics-coroutines>`:
 
         @classmethod
         def from_crawler(cls, crawler, *args, **kwargs):
-            spider = super(SignalSpider, cls).from_crawler(crawler, *args, **kwargs)
+            spider = super().from_crawler(crawler, *args, **kwargs)
             crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
             return spider
 
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 770640adf..69ff08fa1 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -308,7 +308,7 @@ Spiders can access arguments in their `__init__` methods:
         name = "myspider"
 
         def __init__(self, category=None, *args, **kwargs):
-            super(MySpider, self).__init__(*args, **kwargs)
+            super().__init__(*args, **kwargs)
             self.start_urls = [f"http://www.example.com/categories/{category}"]
             # ...
 
@@ -622,9 +622,11 @@ XMLFeedSpider
 
     .. attribute:: itertag
 
-        A string with the name of the node (or element) to iterate in. Example::
+        A string with the name of the node (or element) to iterate in. Example:
 
-            itertag = 'product'
+        .. code-block:: python
+
+            itertag = "product"
 
     .. attribute:: namespaces
 
@@ -637,12 +639,17 @@ XMLFeedSpider
         You can then specify nodes with namespaces in the :attr:`itertag`
         attribute.
 
-        Example::
+        Example:
+
+        .. code-block:: python
+
+            from scrapy.spiders import XMLFeedSpider
+
 
             class YourSpider(XMLFeedSpider):
 
-                namespaces = [('n', 'http://www.sitemaps.org/schemas/sitemap/0.9')]
-                itertag = 'n:url'
+                namespaces = [("n", "http://www.sitemaps.org/schemas/sitemap/0.9")]
+                itertag = "n:url"
                 # ...
 
     Apart from these new attributes, this spider has the following overridable
@@ -802,9 +809,11 @@ SitemapSpider
           the regular expression. ``callback`` can be a string (indicating the
           name of a spider method) or a callable.
 
-        For example::
+        For example:
 
-            sitemap_rules = [('/product/', 'parse_product')]
+        .. code-block:: python
+
+            sitemap_rules = [("/product/", "parse_product")]
 
         Rules are applied in order, and only the first one that matches will be
         used.
@@ -826,7 +835,9 @@ SitemapSpider
         are links for the same website in another language passed within
         the same ``url`` block.
 
-        For example::
+        For example:
+
+        .. code-block:: xml
 
             <url>
                 <loc>http://example.com/</loc>
@@ -844,7 +855,9 @@ SitemapSpider
         This is a filter function that could be overridden to select sitemap entries
         based on their attributes.
 
-        For example::
+        For example:
+
+        .. code-block:: xml
 
             <url>
                 <loc>http://example.com/</loc>
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 3a7dfd018..d283336ab 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,7 +154,6 @@ __all__ = [
     "MAIL_TLS",
     "MAIL_USER",
     "MEMDEBUG_ENABLED",
-    "MEMDEBUG_NOTIFY",
     "MEMUSAGE_CHECK_INTERVAL_SECONDS",
     "MEMUSAGE_ENABLED",
     "MEMUSAGE_LIMIT_MB",
@@ -470,7 +469,6 @@ MAIL_SSL = False
 MAIL_TLS = False
 
 MEMDEBUG_ENABLED = False  # enable memory debugging
-MEMDEBUG_NOTIFY = []  # send memory debugging report by mail at engine shutdown
 
 MEMUSAGE_ENABLED = True
 MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0

From 4ee3676464d927b0eaeead015bc95f2be5639283 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 15:44:56 +0500
Subject: [PATCH 23/62] Test restructuring (#7736)

---
 tests/spiders.py                              |  16 +
 tests/test_command_check.py                   |   2 +-
 tests/test_command_crawl.py                   |   2 +-
 tests/test_command_genspider.py               |  11 +-
 tests/test_command_parse.py                   |   2 +-
 tests/test_command_runspider.py               |   7 +-
 tests/test_commands.py                        |  31 +-
 tests/test_crawl.py                           |  49 +--
 tests/test_crawler.py                         | 254 +---------------
 tests/test_crawler_runners.py                 | 284 ++++++++++++++++++
 tests/test_downloadermiddleware_redirect.py   |   7 +-
 ...test_downloadermiddleware_redirect_base.py |  39 ---
 ...wnloadermiddleware_redirect_metarefresh.py |   4 +-
 tests/test_downloadermiddleware_robotstxt.py  |   2 +-
 tests/test_engine.py                          | 228 +-------------
 tests/test_engine_close_spider.py             | 153 ++++++++++
 tests/test_engine_download.py                 | 106 +++++++
 tests/test_engine_loop.py                     |  36 ++-
 tests/test_feedexport.py                      |  64 ++--
 tests/test_feedexport_batch.py                |  21 +-
 tests/test_feedexport_postprocess.py          |   3 +-
 tests/test_robotstxt_interface.py             |  12 +-
 tests/test_scheduler.py                       |  35 +--
 tests/utils/__init__.py                       |  13 +-
 tests/utils/base_commands.py                  |  34 +++
 tests/utils/cmdline.py                        |  12 +-
 tests/utils/feedexport.py                     |  29 ++
 tests/utils/redirect.py                       |  40 +++
 tests/utils/robotstxt.py                      |  12 +
 29 files changed, 796 insertions(+), 712 deletions(-)
 create mode 100644 tests/test_crawler_runners.py
 create mode 100644 tests/test_engine_close_spider.py
 create mode 100644 tests/test_engine_download.py
 create mode 100644 tests/utils/base_commands.py
 create mode 100644 tests/utils/feedexport.py
 create mode 100644 tests/utils/redirect.py
 create mode 100644 tests/utils/robotstxt.py

diff --git a/tests/spiders.py b/tests/spiders.py
index 612dc11c9..da14fdbe3 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -571,3 +571,19 @@ class HeadersReceivedErrbackSpider(HeadersReceivedCallbackSpider):
     def headers_received(self, headers, body_length, request, spider):
         self.meta["headers_received"] = headers
         raise StopDownload(fail=True)
+
+
+class ExceptionSpider(Spider):
+    name = "exception"
+
+    @classmethod
+    def from_crawler(cls, crawler, *args, **kwargs):
+        raise ValueError("Exception in from_crawler method")
+
+
+class NoRequestsSpider(Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 815f87026..794c6d400 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -7,7 +7,7 @@ from unittest import TestCase
 from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
 
 from scrapy.commands.check import Command, TextTestResult
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py
index dd69d36ba..6293e973e 100644
--- a/tests/test_command_crawl.py
+++ b/tests/test_command_crawl.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index 94db14c30..f465d0b30 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -6,15 +6,8 @@ from pathlib import Path
 
 import pytest
 
-from tests.test_commands import TestProjectBase
-from tests.utils.cmdline import call, proc
-
-
-def write_recording_editor(editor: Path) -> None:
-    """Create an executable editor script that writes the path it is asked to
-    open (its last argument) into the file given as its first argument."""
-    editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
-    editor.chmod(0o755)
+from tests.utils.base_commands import TestProjectBase
+from tests.utils.cmdline import call, proc, write_recording_editor
 
 
 def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index c210a06e8..a3581d764 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -8,7 +8,7 @@ import pytest
 
 from scrapy.commands import parse
 from scrapy.settings import Settings
-from tests.test_commands import TestProjectBase
+from tests.utils.base_commands import TestProjectBase
 from tests.utils.cmdline import call, proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_command_runspider.py b/tests/test_command_runspider.py
index 9dabbc942..2b410b5c6 100644
--- a/tests/test_command_runspider.py
+++ b/tests/test_command_runspider.py
@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING
 
 import pytest
 
-from tests.test_crawler import ExceptionSpider, NoRequestsSpider
+from tests.spiders import ExceptionSpider, NoRequestsSpider
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
@@ -65,13 +65,14 @@ class BadSpider(scrapy.Spider):
 
     def test_run_fail_spider(self, tmp_path: Path) -> None:
         ret, _, _ = self.runspider(
-            tmp_path, "import scrapy\n" + inspect.getsource(ExceptionSpider)
+            tmp_path, "from scrapy import Spider\n" + inspect.getsource(ExceptionSpider)
         )
         assert ret != 0
 
     def test_run_good_spider(self, tmp_path: Path) -> None:
         ret, _, _ = self.runspider(
-            tmp_path, "import scrapy\n" + inspect.getsource(NoRequestsSpider)
+            tmp_path,
+            "from scrapy import Spider\n" + inspect.getsource(NoRequestsSpider),
         )
         assert ret == 0
 
diff --git a/tests/test_commands.py b/tests/test_commands.py
index bb4d96f00..9e7d4d5a1 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -4,7 +4,6 @@ import argparse
 import json
 import sys
 from io import StringIO
-from shutil import copytree
 from typing import TYPE_CHECKING
 from unittest import mock
 
@@ -16,7 +15,8 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.reactor import _asyncio_reactor_path
-from tests.utils.cmdline import call, proc
+from tests.utils.base_commands import TestProjectBase
+from tests.utils.cmdline import call, proc, write_recording_editor
 
 if TYPE_CHECKING:
     from pathlib import Path
@@ -109,29 +109,6 @@ class TestCommandSettings:
         )
 
 
-class TestProjectBase:
-    """A base class for tests that may need a Scrapy project."""
-
-    project_name = "testproject"
-
-    @pytest.fixture(scope="session")
-    def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
-        """Create a Scrapy project in a temporary directory and return its path.
-
-        Used as a cache for ``proj_path``.
-        """
-        tmp_path = tmp_path_factory.mktemp("proj")
-        call("startproject", self.project_name, cwd=tmp_path)
-        return tmp_path / self.project_name
-
-    @pytest.fixture
-    def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
-        """Copy a pre-generated Scrapy project into a temporary directory and return its path."""
-        proj_path = tmp_path / self.project_name
-        copytree(_proj_path_cached, proj_path)
-        return proj_path
-
-
 class TestCommandCrawlerProcess(TestProjectBase):
     """Test that the command uses the expected kind of *CrawlerProcess
     and produces expected errors when needed."""
@@ -430,9 +407,7 @@ class TestEditCommand(TestProjectBase):
         spider = proj_path / self.project_name / "spiders" / "example.py"
         edited = proj_path / "edited.txt"
         editor = proj_path / "fake-editor.sh"
-        # Records the file it is asked to open ($2) into the file given as $1.
-        editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
-        editor.chmod(0o755)
+        write_recording_editor(editor)
         monkeypatch.setenv("EDITOR", f"{editor} {edited}")
 
         assert call("genspider", "example", "example.com", cwd=proj_path) == 0
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 8c1716345..9c479068a 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -14,14 +14,13 @@ from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
 from scrapy import Spider, signals
-from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
 from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.engine import format_engine_status, get_engine_status
 from scrapy.utils.python import to_unicode
-from scrapy.utils.test import get_crawler, get_reactor_settings
+from scrapy.utils.test import get_crawler
 from tests import NON_EXISTING_RESOLVABLE
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
@@ -430,50 +429,6 @@ with multiples lines
             )
         assert not crawler.crawling
 
-    @coroutine_test
-    async def test_crawlerrunner_accepts_crawler(
-        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
-    ) -> None:
-        crawler = Crawler(SimpleSpider, get_reactor_settings())
-        runner = CrawlerRunner()
-        with caplog.at_level(logging.DEBUG):
-            await maybe_deferred_to_future(
-                runner.crawl(
-                    crawler,
-                    mockserver.url("/status?n=200"),
-                    mockserver=mockserver,
-                )
-            )
-        assert "Got response 200" in caplog.text
-
-    @coroutine_test
-    async def test_crawl_multiple(
-        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
-    ) -> None:
-        settings_dict = get_reactor_settings()
-        runner_cls = (
-            CrawlerRunner
-            if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
-            else AsyncCrawlerRunner
-        )
-        runner = runner_cls(settings_dict)
-        runner.crawl(
-            SimpleSpider,
-            mockserver.url("/status?n=200"),
-            mockserver=mockserver,
-        )
-        runner.crawl(
-            SimpleSpider,
-            mockserver.url("/status?n=503"),
-            mockserver=mockserver,
-        )
-
-        with caplog.at_level(logging.DEBUG):
-            await ensure_awaitable(runner.join())
-
-        self._assert_retried(caplog.text)
-        assert "Got response 200" in caplog.text
-
     @coroutine_test
     async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(SimpleSpider)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 6a1eea027..b4f906e25 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -10,22 +10,14 @@ from typing import TYPE_CHECKING, Any, ClassVar
 from unittest.mock import MagicMock
 
 import pytest
-from zope.interface.exceptions import MultipleInvalid
 
 import scrapy
 from scrapy import Spider
-from scrapy.crawler import (
-    AsyncCrawlerProcess,
-    AsyncCrawlerRunner,
-    Crawler,
-    CrawlerProcess,
-    CrawlerRunner,
-    CrawlerRunnerBase,
-)
+from scrapy.crawler import AsyncCrawlerProcess, Crawler, CrawlerProcess
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
-from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.log import (
     _uninstall_scrapy_root_handler,
     configure_logging,
@@ -33,6 +25,8 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.spiders import NoRequestsSpider
+from tests.utils import assert_option_is_default
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -52,14 +46,7 @@ def get_raw_crawler(
     return Crawler(spidercls or DefaultSpider, settings)
 
 
-class TestBaseCrawler:
-    @staticmethod
-    def assertOptionIsDefault(settings: Settings, key: str) -> None:
-        assert isinstance(settings, Settings)
-        assert settings[key] == getattr(default_settings, key)
-
-
-class TestCrawler(TestBaseCrawler):
+class TestCrawler:
     def test_populate_spidercls_settings(self) -> None:
         spider_settings: dict[str, Any] = {
             "TEST1": "spider",
@@ -90,11 +77,11 @@ class TestCrawler(TestBaseCrawler):
     def test_crawler_accepts_dict(self) -> None:
         crawler = get_crawler(DefaultSpider, {"foo": "bar"})
         assert crawler.settings["foo"] == "bar"
-        self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
+        assert_option_is_default(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_accepts_None(self) -> None:
         crawler = Crawler(DefaultSpider)
-        self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
+        assert_option_is_default(crawler.settings, "RETRY_ENABLED")
 
     def test_crawler_rejects_spider_objects(self) -> None:
         with pytest.raises(ValueError, match="spidercls argument must be a class"):
@@ -586,78 +573,6 @@ class TestCrawlerLogging:
         assert "debug message" in logged
 
 
-class SpiderLoaderWithWrongInterface:
-    def unneeded_method(self) -> None:
-        pass
-
-
-class TestCrawlerRunner(TestBaseCrawler):
-    def test_spider_manager_verify_interface(self) -> None:
-        settings = Settings(
-            {
-                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
-            }
-        )
-        with pytest.raises(MultipleInvalid):
-            CrawlerRunner(settings)
-
-    def test_crawler_runner_accepts_dict(self) -> None:
-        runner = CrawlerRunner({"foo": "bar"})
-        assert runner.settings["foo"] == "bar"
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-    def test_crawler_runner_accepts_None(self) -> None:
-        runner = CrawlerRunner()
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-class TestAsyncCrawlerRunner(TestBaseCrawler):
-    def test_spider_manager_verify_interface(self) -> None:
-        settings = Settings(
-            {
-                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
-            }
-        )
-        with pytest.raises(MultipleInvalid):
-            AsyncCrawlerRunner(settings)
-
-    def test_crawler_runner_accepts_dict(self) -> None:
-        runner = AsyncCrawlerRunner({"foo": "bar"})
-        assert runner.settings["foo"] == "bar"
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-    def test_crawler_runner_accepts_None(self) -> None:
-        runner = AsyncCrawlerRunner()
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-class TestCrawlerProcess(TestBaseCrawler):
-    def test_crawler_process_accepts_dict(self) -> None:
-        runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
-        assert runner.settings["foo"] == "bar"
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-    def test_crawler_process_accepts_None(self) -> None:
-        runner = CrawlerProcess(install_root_handler=False)
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
-@pytest.mark.only_asyncio
-class TestAsyncCrawlerProcess(TestBaseCrawler):
-    def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
-        runner = AsyncCrawlerProcess(
-            {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
-            install_root_handler=False,
-        )
-        assert runner.settings["foo"] == "bar"
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-    @pytest.mark.requires_reactor  # can't pass TWISTED_REACTOR_ENABLED=False
-    def test_crawler_process_accepts_None(self) -> None:
-        runner = AsyncCrawlerProcess(install_root_handler=False)
-        self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
-
-
 class TestAsyncCrawlerProcessReactorlessHelpers:
     """Unit tests for the reactorless shutdown helpers of AsyncCrawlerProcess.
 
@@ -797,161 +712,6 @@ class TestAsyncCrawlerProcessReactorlessHelpers:
         )
 
 
-@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_runner_settings_applied_to_crawler_instance(
-    runner_cls: type[CrawlerRunnerBase],
-) -> None:
-    runner = runner_cls({"FOO": "runner"})
-    crawler = Crawler(DefaultSpider)
-    result = runner.create_crawler(crawler)
-    assert result is crawler
-    assert result.settings["FOO"] == "runner"
-
-
-@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_spider_custom_settings_override_runner(
-    runner_cls: type[CrawlerRunnerBase],
-) -> None:
-    class MySpider(DefaultSpider):
-        custom_settings = {"FOO": "spider"}
-
-    runner = runner_cls({"FOO": "runner"})
-    crawler = Crawler(MySpider)
-    runner.create_crawler(crawler)
-    assert crawler.settings["FOO"] == "spider"
-
-
-def test_create_crawler_instance_consistent_with_spider_class() -> None:
-    runner = AsyncCrawlerRunner({"FOO": "runner"})
-
-    crawler_from_class = runner.create_crawler(DefaultSpider)
-
-    pre_built = Crawler(DefaultSpider)
-    runner.create_crawler(pre_built)
-
-    assert crawler_from_class.settings["FOO"] == "runner"
-    assert pre_built.settings["FOO"] == "runner"
-
-
-@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_create_crawler_rejects_spider_object(
-    runner_cls: type[CrawlerRunnerBase],
-) -> None:
-    runner = runner_cls()
-    with pytest.raises(ValueError, match="cannot be a spider object"):
-        runner.create_crawler(DefaultSpider())  # type: ignore[arg-type]
-
-
-@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
-def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
-    runner = runner_cls()
-    with pytest.raises(ValueError, match="cannot be a spider object"):
-        runner.crawl(DefaultSpider())  # type: ignore[arg-type]
-
-
-class ExceptionSpider(scrapy.Spider):
-    name = "exception"
-
-    @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
-        raise ValueError("Exception in from_crawler method")
-
-
-class NoRequestsSpider(scrapy.Spider):
-    name = "no_request"
-
-    async def start(self):
-        return
-        yield
-
-
-@pytest.mark.requires_reactor  # CrawlerRunner requires a reactor
-class TestCrawlerRunnerHasSpider:
-    @pytest.fixture
-    def runner(self) -> CrawlerRunnerBase:
-        return CrawlerRunner(get_reactor_settings())
-
-    @staticmethod
-    async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
-        await ensure_awaitable(runner.crawl(spider))
-
-    @coroutine_test
-    async def test_crawler_runner_bootstrap_successful(
-        self, runner: CrawlerRunnerBase
-    ) -> None:
-        await self._crawl(runner, NoRequestsSpider)
-        assert not runner.bootstrap_failed
-
-    @coroutine_test
-    async def test_crawler_runner_bootstrap_successful_for_several(
-        self, runner: CrawlerRunnerBase
-    ) -> None:
-        await self._crawl(runner, NoRequestsSpider)
-        await self._crawl(runner, NoRequestsSpider)
-        assert not runner.bootstrap_failed
-
-    @coroutine_test
-    async def test_crawler_runner_bootstrap_failed(
-        self, runner: CrawlerRunnerBase
-    ) -> None:
-        try:
-            await self._crawl(runner, ExceptionSpider)
-        except ValueError:
-            pass
-        else:
-            pytest.fail("Exception should be raised from spider")
-
-        assert runner.bootstrap_failed
-
-    @coroutine_test
-    async def test_crawler_runner_bootstrap_failed_for_several(
-        self, runner: CrawlerRunnerBase
-    ) -> None:
-        try:
-            await self._crawl(runner, ExceptionSpider)
-        except ValueError:
-            pass
-        else:
-            pytest.fail("Exception should be raised from spider")
-
-        await self._crawl(runner, NoRequestsSpider)
-
-        assert runner.bootstrap_failed
-
-    @coroutine_test
-    async def test_crawler_runner_asyncio_enabled_true(
-        self, reactor_pytest: str
-    ) -> None:
-        if reactor_pytest != "asyncio":
-            runner = CrawlerRunner(
-                settings={
-                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                }
-            )
-            with pytest.raises(
-                Exception,
-                match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
-            ):
-                await self._crawl(runner, NoRequestsSpider)
-        else:
-            runner = CrawlerRunner(
-                settings={
-                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
-                }
-            )
-            await self._crawl(runner, NoRequestsSpider)
-
-
-@pytest.mark.only_asyncio
-class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
-    @pytest.fixture
-    def runner(self) -> CrawlerRunnerBase:
-        return AsyncCrawlerRunner(get_reactor_settings())
-
-    def test_crawler_runner_asyncio_enabled_true(self) -> None:  # type: ignore[override]
-        pytest.skip("This test is only for CrawlerRunner")
-
-
 @pytest.mark.parametrize(
     ("settings", "items"),
     [
diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py
new file mode 100644
index 000000000..2f3649b43
--- /dev/null
+++ b/tests/test_crawler_runners.py
@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+import pytest
+from zope.interface.exceptions import MultipleInvalid
+
+from scrapy.crawler import (
+    AsyncCrawlerProcess,
+    AsyncCrawlerRunner,
+    Crawler,
+    CrawlerProcess,
+    CrawlerRunner,
+    CrawlerRunnerBase,
+)
+from scrapy.settings import Settings
+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_reactor_settings
+from tests.spiders import ExceptionSpider, NoRequestsSpider, SimpleSpider
+from tests.utils import assert_option_is_default
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from scrapy import Spider
+    from tests.mockserver.http import MockServer
+
+
+class SpiderLoaderWithWrongInterface:
+    def unneeded_method(self) -> None:
+        pass
+
+
+class TestCrawlerRunner:
+    def test_spider_manager_verify_interface(self) -> None:
+        settings = Settings(
+            {
+                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
+            }
+        )
+        with pytest.raises(MultipleInvalid):
+            CrawlerRunner(settings)
+
+    def test_crawler_runner_accepts_dict(self) -> None:
+        runner = CrawlerRunner({"foo": "bar"})
+        assert runner.settings["foo"] == "bar"
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+    def test_crawler_runner_accepts_None(self) -> None:
+        runner = CrawlerRunner()
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+class TestAsyncCrawlerRunner:
+    def test_spider_manager_verify_interface(self) -> None:
+        settings = Settings(
+            {
+                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
+            }
+        )
+        with pytest.raises(MultipleInvalid):
+            AsyncCrawlerRunner(settings)
+
+    def test_crawler_runner_accepts_dict(self) -> None:
+        runner = AsyncCrawlerRunner({"foo": "bar"})
+        assert runner.settings["foo"] == "bar"
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+    def test_crawler_runner_accepts_None(self) -> None:
+        runner = AsyncCrawlerRunner()
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+class TestCrawlerProcess:
+    def test_crawler_process_accepts_dict(self) -> None:
+        runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
+        assert runner.settings["foo"] == "bar"
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+    def test_crawler_process_accepts_None(self) -> None:
+        runner = CrawlerProcess(install_root_handler=False)
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+@pytest.mark.only_asyncio
+class TestAsyncCrawlerProcess:
+    def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None:
+        runner = AsyncCrawlerProcess(
+            {"foo": "bar", "TWISTED_REACTOR_ENABLED": reactor_pytest != "none"},
+            install_root_handler=False,
+        )
+        assert runner.settings["foo"] == "bar"
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+    @pytest.mark.requires_reactor  # can't pass TWISTED_REACTOR_ENABLED=False
+    def test_crawler_process_accepts_None(self) -> None:
+        runner = AsyncCrawlerProcess(install_root_handler=False)
+        assert_option_is_default(runner.settings, "RETRY_ENABLED")
+
+
+@pytest.mark.requires_reactor  # CrawlerRunner requires a reactor
+class TestCrawlerRunnerHasSpider:
+    @pytest.fixture
+    def runner(self) -> CrawlerRunnerBase:
+        return CrawlerRunner(get_reactor_settings())
+
+    @staticmethod
+    async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
+        await ensure_awaitable(runner.crawl(spider))
+
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_successful(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        await self._crawl(runner, NoRequestsSpider)
+        assert not runner.bootstrap_failed
+
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_successful_for_several(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        await self._crawl(runner, NoRequestsSpider)
+        await self._crawl(runner, NoRequestsSpider)
+        assert not runner.bootstrap_failed
+
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_failed(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        try:
+            await self._crawl(runner, ExceptionSpider)
+        except ValueError:
+            pass
+        else:
+            pytest.fail("Exception should be raised from spider")
+
+        assert runner.bootstrap_failed
+
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_failed_for_several(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        try:
+            await self._crawl(runner, ExceptionSpider)
+        except ValueError:
+            pass
+        else:
+            pytest.fail("Exception should be raised from spider")
+
+        await self._crawl(runner, NoRequestsSpider)
+
+        assert runner.bootstrap_failed
+
+    @coroutine_test
+    async def test_crawler_runner_asyncio_enabled_true(
+        self, reactor_pytest: str
+    ) -> None:
+        if reactor_pytest != "asyncio":
+            runner = CrawlerRunner(
+                settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                }
+            )
+            with pytest.raises(
+                Exception,
+                match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
+            ):
+                await self._crawl(runner, NoRequestsSpider)
+        else:
+            runner = CrawlerRunner(
+                settings={
+                    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                }
+            )
+            await self._crawl(runner, NoRequestsSpider)
+
+
+@pytest.mark.only_asyncio
+class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
+    @pytest.fixture
+    def runner(self) -> CrawlerRunnerBase:
+        return AsyncCrawlerRunner(get_reactor_settings())
+
+    def test_crawler_runner_asyncio_enabled_true(self) -> None:  # type: ignore[override]
+        pytest.skip("This test is only for CrawlerRunner")
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_runner_settings_applied_to_crawler_instance(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    runner = runner_cls({"FOO": "runner"})
+    crawler = Crawler(DefaultSpider)
+    result = runner.create_crawler(crawler)
+    assert result is crawler
+    assert result.settings["FOO"] == "runner"
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_spider_custom_settings_override_runner(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    class MySpider(DefaultSpider):
+        custom_settings = {"FOO": "spider"}
+
+    runner = runner_cls({"FOO": "runner"})
+    crawler = Crawler(MySpider)
+    runner.create_crawler(crawler)
+    assert crawler.settings["FOO"] == "spider"
+
+
+def test_create_crawler_instance_consistent_with_spider_class() -> None:
+    runner = AsyncCrawlerRunner({"FOO": "runner"})
+
+    crawler_from_class = runner.create_crawler(DefaultSpider)
+
+    pre_built = Crawler(DefaultSpider)
+    runner.create_crawler(pre_built)
+
+    assert crawler_from_class.settings["FOO"] == "runner"
+    assert pre_built.settings["FOO"] == "runner"
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_create_crawler_rejects_spider_object(
+    runner_cls: type[CrawlerRunnerBase],
+) -> None:
+    runner = runner_cls()
+    with pytest.raises(ValueError, match="cannot be a spider object"):
+        runner.create_crawler(DefaultSpider())  # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize("runner_cls", [AsyncCrawlerRunner, CrawlerRunner])
+def test_crawl_rejects_spider_object(runner_cls: type[CrawlerRunnerBase]) -> None:
+    runner = runner_cls()
+    with pytest.raises(ValueError, match="cannot be a spider object"):
+        runner.crawl(DefaultSpider())  # type: ignore[arg-type]
+
+
+@coroutine_test
+async def test_crawlerrunner_accepts_crawler(
+    caplog: pytest.LogCaptureFixture, mockserver: MockServer
+) -> None:
+    crawler = Crawler(SimpleSpider, get_reactor_settings())
+    runner = CrawlerRunner()
+    with caplog.at_level(logging.DEBUG):
+        await maybe_deferred_to_future(
+            runner.crawl(
+                crawler,
+                mockserver.url("/status?n=200"),
+                mockserver=mockserver,
+            )
+        )
+    assert "Got response 200" in caplog.text
+
+
+@coroutine_test
+async def test_crawl_multiple(
+    caplog: pytest.LogCaptureFixture, mockserver: MockServer
+) -> None:
+    settings_dict = get_reactor_settings()
+    runner_cls = (
+        CrawlerRunner
+        if settings_dict.get("TWISTED_REACTOR_ENABLED", True)
+        else AsyncCrawlerRunner
+    )
+    runner = runner_cls(settings_dict)
+    runner.crawl(
+        SimpleSpider,
+        mockserver.url("/status?n=200"),
+        mockserver=mockserver,
+    )
+    runner.crawl(
+        SimpleSpider,
+        mockserver.url("/status?n=503"),
+        mockserver=mockserver,
+    )
+
+    with caplog.at_level(logging.DEBUG):
+        await ensure_awaitable(runner.join())
+
+    assert "Got response 200" in caplog.text
+    assert "Gave up retrying" in caplog.text
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 1da7bbf3e..0824ab531 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -16,11 +16,8 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
-    REDIRECT_SCHEME_CASES,
-    SCHEME_PARAMS,
-    Base,
-)
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
 
 
 class TestRedirectMiddleware(Base.Test):
diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py
index 32935769f..bd8bc796c 100644
--- a/tests/test_downloadermiddleware_redirect_base.py
+++ b/tests/test_downloadermiddleware_redirect_base.py
@@ -1,7 +1,5 @@
 from __future__ import annotations
 
-from itertools import product
-
 import pytest
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
@@ -10,43 +8,6 @@ from scrapy.http import Request, Response
 from scrapy.utils.misc import set_environ
 from scrapy.utils.test import get_crawler
 
-SCHEME_PARAMS = ("url", "location", "target")
-HTTP_SCHEMES = ("http", "https")
-NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
-REDIRECT_SCHEME_CASES = (
-    # http/https → http/https redirects
-    *(
-        (
-            f"{input_scheme}://example.com/a",
-            f"{output_scheme}://example.com/b",
-            f"{output_scheme}://example.com/b",
-        )
-        for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
-    ),
-    # http/https → data/file/ftp/s3/foo does not redirect
-    *(
-        (
-            f"{input_scheme}://example.com/a",
-            f"{output_scheme}://example.com/b",
-            None,
-        )
-        for input_scheme in HTTP_SCHEMES
-        for output_scheme in NON_HTTP_SCHEMES
-    ),
-    # http/https → relative redirects
-    *(
-        (
-            f"{scheme}://example.com/a",
-            location,
-            f"{scheme}://example.com/b",
-        )
-        for scheme in HTTP_SCHEMES
-        for location in ("//example.com/b", "/b")
-    ),
-    # Note: We do not test data/file/ftp/s3 schemes for the initial URL
-    # because their download handlers cannot return a status code of 3xx.
-)
-
 
 class Base:
     class Test:
diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py
index d849cc8fb..b5d39080c 100644
--- a/tests/test_downloadermiddleware_redirect_metarefresh.py
+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py
@@ -12,12 +12,12 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import (
+from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.redirect import (
     HTTP_SCHEMES,
     NON_HTTP_SCHEMES,
     REDIRECT_SCHEME_CASES,
     SCHEME_PARAMS,
-    Base,
 )
 
 
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 082fc743e..f82041a62 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -15,8 +15,8 @@ from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
-from tests.test_robotstxt_interface import rerp_available
 from tests.utils.decorators import coroutine_test
+from tests.utils.robotstxt import rerp_available
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2b857d35a..3ab8f6d1b 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -6,8 +6,8 @@ import subprocess
 import sys
 from collections import defaultdict
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any, cast
-from unittest.mock import Mock, call
+from typing import TYPE_CHECKING, Any
+from unittest.mock import Mock
 from urllib.parse import urlparse
 
 import attr
@@ -25,7 +25,6 @@ from scrapy.http import Headers, Request, Response
 from scrapy.item import Field, Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
-from scrapy.statscollectors import MemoryStatsCollector
 from scrapy.utils.defer import (
     _schedule_coro,
     deferred_from_coro,
@@ -42,8 +41,6 @@ if TYPE_CHECKING:
 
     from twisted.python.failure import Failure
 
-    from scrapy.core.scheduler import Scheduler
-    from scrapy.crawler import Crawler
     from tests.mockserver.http import MockServer
 
 
@@ -515,98 +512,6 @@ class TestEngine(TestEngineBase):
         assert "AssertionError" not in stderr_str, stderr_str
 
 
-class TestEngineDownloadAsync:
-    """Test cases for ExecutionEngine.download_async()."""
-
-    @pytest.fixture
-    def engine(self) -> ExecutionEngine:
-        crawler = get_crawler(MySpider)
-        engine = ExecutionEngine(crawler, lambda _: None)
-        engine.downloader.close()
-        engine.downloader = Mock()
-        engine._slot = Mock()
-        engine._slot.inprogress = set()
-        return engine
-
-    @staticmethod
-    async def _download(engine: ExecutionEngine, request: Request) -> Response:
-        return await engine.download_async(request)
-
-    @coroutine_test
-    async def test_download_async_success(self, engine):
-        """Test basic successful async download of a request."""
-        request = Request("http://example.com")
-        response = Response("http://example.com", body=b"test body")
-        engine.spider = Mock()
-        engine.downloader.fetch.return_value = defer.succeed(response)
-        engine._slot.add_request = Mock()
-        engine._slot.remove_request = Mock()
-
-        result = await self._download(engine, request)
-        assert result == response
-        engine._slot.add_request.assert_called_once_with(request)
-        engine._slot.remove_request.assert_called_once_with(request)
-        engine.downloader.fetch.assert_called_once_with(request)
-
-    @coroutine_test
-    async def test_download_async_redirect(self, engine):
-        """Test async download with a redirect request."""
-        original_request = Request("http://example.com")
-        redirect_request = Request("http://example.com/redirect")
-        final_response = Response("http://example.com/redirect", body=b"redirected")
-
-        # First call returns redirect request, second call returns final response
-        engine.downloader.fetch.side_effect = [
-            defer.succeed(redirect_request),
-            defer.succeed(final_response),
-        ]
-        engine.spider = Mock()
-        engine._slot.add_request = Mock()
-        engine._slot.remove_request = Mock()
-
-        result = await self._download(engine, original_request)
-        assert result == final_response
-        assert engine.downloader.fetch.call_count == 2
-        engine._slot.add_request.assert_has_calls(
-            [call(original_request), call(redirect_request)]
-        )
-        engine._slot.remove_request.assert_has_calls(
-            [call(original_request), call(redirect_request)]
-        )
-
-    @coroutine_test
-    async def test_download_async_no_spider(self, engine):
-        """Test async download attempt when no spider is available."""
-        request = Request("http://example.com")
-        engine.spider = None
-        with pytest.raises(RuntimeError, match="No open spider to crawl:"):
-            await self._download(engine, request)
-
-    @coroutine_test
-    async def test_download_async_failure(self, engine):
-        """Test async download when the downloader raises an exception."""
-        request = Request("http://example.com")
-        error = RuntimeError("Download failed")
-        engine.spider = Mock()
-        engine.downloader.fetch.return_value = defer.fail(error)
-        engine._slot.add_request = Mock()
-        engine._slot.remove_request = Mock()
-
-        with pytest.raises(RuntimeError, match="Download failed"):
-            await self._download(engine, request)
-        engine._slot.add_request.assert_called_once_with(request)
-        engine._slot.remove_request.assert_called_once_with(request)
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestEngineDownload(TestEngineDownloadAsync):
-    """Test cases for ExecutionEngine.download()."""
-
-    @staticmethod
-    async def _download(engine: ExecutionEngine, request: Request) -> Response:
-        return await maybe_deferred_to_future(engine.download(request))
-
-
 @coroutine_test
 async def test_request_scheduled_signal():
     class TestScheduler(BaseScheduler):
@@ -640,132 +545,3 @@ async def test_request_scheduled_signal():
         f"{scheduler.enqueued!r} != [{keep_request!r}]"
     )
     crawler.signals.disconnect(signal_handler, signals.request_scheduled)
-
-
-class TestEngineCloseSpider:
-    """Tests for exception handling coverage during close_spider_async()."""
-
-    @pytest.fixture
-    def crawler(self) -> Crawler:
-        crawler = get_crawler(DefaultSpider)
-        crawler.spider = crawler._create_spider()
-        return crawler
-
-    @coroutine_test
-    async def test_no_slot(self, crawler: Crawler) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        slot = engine._slot
-        engine._slot = None
-        with pytest.raises(RuntimeError, match="Engine slot not assigned"):
-            await engine.close_spider_async()
-        # close it correctly
-        engine._slot = slot
-        await engine.close_spider_async()
-
-    @coroutine_test
-    async def test_no_spider(self, crawler: Crawler) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        with pytest.raises(RuntimeError, match="Spider not opened"):
-            await engine.close_spider_async()
-        engine.downloader.close()  # cleanup
-
-    @coroutine_test
-    async def test_exception_slot(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        assert engine._slot
-        del engine._slot.heartbeat
-        await engine.close_spider_async()
-        assert "Slot close failure" in caplog.text
-
-    @coroutine_test
-    async def test_exception_downloader(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        del engine.downloader.slots
-        await engine.close_spider_async()
-        assert "Downloader close failure" in caplog.text
-
-    @coroutine_test
-    async def test_exception_scraper(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        engine.scraper.slot = None
-        await engine.close_spider_async()
-        assert "Scraper close failure" in caplog.text
-
-    @coroutine_test
-    async def test_exception_scheduler(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        assert engine._slot
-        del cast("Scheduler", engine._slot.scheduler).dqs
-        await engine.close_spider_async()
-        assert "Scheduler close failure" in caplog.text
-
-    @coroutine_test
-    async def test_exception_signal(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        signal_manager = engine.signals
-        del engine.signals
-        await engine.close_spider_async()
-        assert "Error while sending spider_close signal" in caplog.text
-        # send the spider_closed signal to close various components
-        await signal_manager.send_catch_log_async(
-            signal=signals.spider_closed,
-            spider=engine.spider,
-            reason="cancelled",
-        )
-
-    @coroutine_test
-    async def test_exception_stats(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: None)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        assert isinstance(crawler.stats, MemoryStatsCollector)
-        del crawler.stats.spider_stats
-        await engine.close_spider_async()
-        assert "Stats close failure" in caplog.text
-
-    @coroutine_test
-    async def test_exception_callback(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
-        crawler.engine = engine
-        await engine.open_spider_async()
-        await engine.close_spider_async()
-        assert "Error running spider_closed_callback" in caplog.text
-
-    @coroutine_test
-    async def test_exception_async_callback(
-        self, crawler: Crawler, caplog: pytest.LogCaptureFixture
-    ) -> None:
-        async def cb(_):
-            raise ValueError
-
-        engine = ExecutionEngine(crawler, cb)
-        crawler.engine = engine
-        await engine.open_spider_async()
-        await engine.close_spider_async()
-        assert "Error running spider_closed_callback" in caplog.text
diff --git a/tests/test_engine_close_spider.py b/tests/test_engine_close_spider.py
new file mode 100644
index 000000000..8e9c9bd43
--- /dev/null
+++ b/tests/test_engine_close_spider.py
@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, cast
+
+import pytest
+from twisted.internet import defer
+
+from scrapy import signals
+from scrapy.core.engine import ExecutionEngine
+from scrapy.statscollectors import MemoryStatsCollector
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from scrapy.core.scheduler import Scheduler
+    from scrapy.crawler import Crawler
+
+
+@pytest.fixture
+def crawler() -> Crawler:
+    crawler = get_crawler(DefaultSpider)
+    crawler.spider = crawler._create_spider()
+    return crawler
+
+
+@coroutine_test
+async def test_no_slot(crawler: Crawler) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    slot = engine._slot
+    engine._slot = None
+    with pytest.raises(RuntimeError, match="Engine slot not assigned"):
+        await engine.close_spider_async()
+    # close it correctly
+    engine._slot = slot
+    await engine.close_spider_async()
+
+
+@coroutine_test
+async def test_no_spider(crawler: Crawler) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    with pytest.raises(RuntimeError, match="Spider not opened"):
+        await engine.close_spider_async()
+    engine.downloader.close()  # cleanup
+
+
+@coroutine_test
+async def test_exception_slot(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    assert engine._slot
+    del engine._slot.heartbeat
+    await engine.close_spider_async()
+    assert "Slot close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_downloader(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    del engine.downloader.slots
+    await engine.close_spider_async()
+    assert "Downloader close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_scraper(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    engine.scraper.slot = None
+    await engine.close_spider_async()
+    assert "Scraper close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_scheduler(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    assert engine._slot
+    del cast("Scheduler", engine._slot.scheduler).dqs
+    await engine.close_spider_async()
+    assert "Scheduler close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_signal(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    signal_manager = engine.signals
+    del engine.signals
+    await engine.close_spider_async()
+    assert "Error while sending spider_close signal" in caplog.text
+    # send the spider_closed signal to close various components
+    await signal_manager.send_catch_log_async(
+        signal=signals.spider_closed,
+        spider=engine.spider,
+        reason="cancelled",
+    )
+
+
+@coroutine_test
+async def test_exception_stats(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: None)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    assert isinstance(crawler.stats, MemoryStatsCollector)
+    del crawler.stats.spider_stats
+    await engine.close_spider_async()
+    assert "Stats close failure" in caplog.text
+
+
+@coroutine_test
+async def test_exception_callback(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
+    crawler.engine = engine
+    await engine.open_spider_async()
+    await engine.close_spider_async()
+    assert "Error running spider_closed_callback" in caplog.text
+
+
+@coroutine_test
+async def test_exception_async_callback(
+    crawler: Crawler, caplog: pytest.LogCaptureFixture
+) -> None:
+    async def cb(_):
+        raise ValueError
+
+    engine = ExecutionEngine(crawler, cb)
+    crawler.engine = engine
+    await engine.open_spider_async()
+    await engine.close_spider_async()
+    assert "Error running spider_closed_callback" in caplog.text
diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py
new file mode 100644
index 000000000..f15bfd5e2
--- /dev/null
+++ b/tests/test_engine_download.py
@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+from unittest.mock import Mock, call
+
+import pytest
+from twisted.internet import defer
+
+from scrapy.core.engine import ExecutionEngine
+from scrapy.http import Request, Response
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.spider import DefaultSpider
+from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+
+
+@pytest.fixture
+def engine() -> ExecutionEngine:
+    crawler = get_crawler(DefaultSpider)
+    engine = ExecutionEngine(crawler, lambda _: None)
+    engine.downloader.close()
+    engine.downloader = Mock()
+    engine._slot = Mock()
+    engine._slot.inprogress = set()
+    return engine
+
+
+class TestEngineDownloadAsync:
+    """Test cases for ExecutionEngine.download_async()."""
+
+    @staticmethod
+    async def _download(engine: ExecutionEngine, request: Request) -> Response:
+        return await engine.download_async(request)
+
+    @coroutine_test
+    async def test_download_async_success(self, engine):
+        """Test basic successful async download of a request."""
+        request = Request("http://example.com")
+        response = Response("http://example.com", body=b"test body")
+        engine.spider = Mock()
+        engine.downloader.fetch.return_value = defer.succeed(response)
+        engine._slot.add_request = Mock()
+        engine._slot.remove_request = Mock()
+
+        result = await self._download(engine, request)
+        assert result == response
+        engine._slot.add_request.assert_called_once_with(request)
+        engine._slot.remove_request.assert_called_once_with(request)
+        engine.downloader.fetch.assert_called_once_with(request)
+
+    @coroutine_test
+    async def test_download_async_redirect(self, engine):
+        """Test async download with a redirect request."""
+        original_request = Request("http://example.com")
+        redirect_request = Request("http://example.com/redirect")
+        final_response = Response("http://example.com/redirect", body=b"redirected")
+
+        # First call returns redirect request, second call returns final response
+        engine.downloader.fetch.side_effect = [
+            defer.succeed(redirect_request),
+            defer.succeed(final_response),
+        ]
+        engine.spider = Mock()
+        engine._slot.add_request = Mock()
+        engine._slot.remove_request = Mock()
+
+        result = await self._download(engine, original_request)
+        assert result == final_response
+        assert engine.downloader.fetch.call_count == 2
+        engine._slot.add_request.assert_has_calls(
+            [call(original_request), call(redirect_request)]
+        )
+        engine._slot.remove_request.assert_has_calls(
+            [call(original_request), call(redirect_request)]
+        )
+
+    @coroutine_test
+    async def test_download_async_no_spider(self, engine):
+        """Test async download attempt when no spider is available."""
+        request = Request("http://example.com")
+        engine.spider = None
+        with pytest.raises(RuntimeError, match="No open spider to crawl:"):
+            await self._download(engine, request)
+
+    @coroutine_test
+    async def test_download_async_failure(self, engine):
+        """Test async download when the downloader raises an exception."""
+        request = Request("http://example.com")
+        error = RuntimeError("Download failed")
+        engine.spider = Mock()
+        engine.downloader.fetch.return_value = defer.fail(error)
+        engine._slot.add_request = Mock()
+        engine._slot.remove_request = Mock()
+
+        with pytest.raises(RuntimeError, match="Download failed"):
+            await self._download(engine, request)
+        engine._slot.add_request.assert_called_once_with(request)
+        engine._slot.remove_request.assert_called_once_with(request)
+
+
+@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
+class TestEngineDownload(TestEngineDownloadAsync):
+    """Test cases for ExecutionEngine.download()."""
+
+    @staticmethod
+    async def _download(engine: ExecutionEngine, request: Request) -> Response:
+        return await maybe_deferred_to_future(engine.download(request))
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 6cc8c0650..ca52961f4 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -2,13 +2,13 @@ from __future__ import annotations
 
 from collections import deque
 from logging import ERROR
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
+from scrapy.core.scheduler import BaseScheduler
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
-from tests.test_scheduler import MemoryScheduler
 from tests.utils import async_sleep
 from tests.utils.decorators import coroutine_test
 
@@ -18,6 +18,38 @@ if TYPE_CHECKING:
     from scrapy.http import Response
 
 
+class MemoryScheduler(BaseScheduler):
+    paused = False
+
+    def __init__(self, *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.queue: deque[Request] = deque(
+            Request(value) if isinstance(value, str) else value
+            for value in getattr(self, "queue", [])
+        )
+
+    def enqueue_request(self, request: Request) -> bool:
+        self.queue.append(request)
+        return True
+
+    def has_pending_requests(self) -> bool:
+        return self.paused or bool(self.queue)
+
+    def next_request(self) -> Request | None:
+        if self.paused:
+            return None
+        try:
+            return self.queue.pop()
+        except IndexError:
+            return None
+
+    def pause(self) -> None:
+        self.paused = True
+
+    def unpause(self) -> None:
+        self.paused = False
+
+
 class TestMain:
     @coroutine_test
     async def test_sleep(self):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 5ab1a54c1..853243b97 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -13,8 +13,6 @@ from pathlib import Path
 from string import ascii_letters, digits
 from typing import IO, TYPE_CHECKING, Any
 from unittest import mock
-from urllib.parse import urljoin
-from urllib.request import pathname2url
 
 import lxml.etree
 import pytest
@@ -39,19 +37,12 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Callable, Iterable
 
 
-def path_to_url(path: str | Path) -> str:
-    return urljoin("file:", pathname2url(str(path)))
-
-
-def printf_escape(s: str) -> str:
-    return s.replace("%", "%%")
-
-
 class FromCrawlerMixin:
     init_with_crawler = False
 
@@ -114,15 +105,6 @@ class LogOnStoreFileStorage:
 class TestFeedExportBase(ABC):
     mockserver: MockServer
 
-    class MyItem(scrapy.Item):
-        foo = scrapy.Field()
-        egg = scrapy.Field()
-        baz = scrapy.Field()
-
-    class MyItem2(scrapy.Item):
-        foo = scrapy.Field()
-        hello = scrapy.Field()
-
     def _random_temp_filename(self, inter_dir="") -> Path:
         chars = [random.choice(ascii_letters + digits) for _ in range(15)]
         filename = "".join(chars)
@@ -517,21 +499,21 @@ class TestFeedExport(TestFeedExportBase):
     async def test_export_items(self):
         # feed exporters use field names from Item
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
         ]
         rows = [
             {"egg": "spam1", "foo": "bar1", "baz": ""},
             {"egg": "spam2", "foo": "bar2", "baz": "quux2"},
         ]
-        header = self.MyItem.fields.keys()
+        header = MyItem.fields.keys()
         await self.assertExported(items, header, rows)
 
     @coroutine_test
     async def test_pathlib_uri_with_placeholders(self):
         feed_dir = Path(self.temp_dir, "pathlib_placeholders")
         feed_dir.mkdir()
-        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+        items = [MyItem({"foo": "bar1", "egg": "spam1"})]
 
         class TestSpider(scrapy.Spider):
             name = "testspider"
@@ -561,7 +543,7 @@ class TestFeedExport(TestFeedExportBase):
         # so the resulting file name can be asserted exactly.
         feed_dir = Path(self.temp_dir, "pathlib_spaces_unicode")
         feed_dir.mkdir()
-        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+        items = [MyItem({"foo": "bar1", "egg": "spam1"})]
 
         class TestSpider(scrapy.Spider):
             name = "testspider"
@@ -590,7 +572,7 @@ class TestFeedExport(TestFeedExportBase):
         # and #5794.
         feed_dir = Path(self.temp_dir, "dir with spaces")
         feed_dir.mkdir()
-        items = [self.MyItem({"foo": "bar1", "egg": "spam1"})]
+        items = [MyItem({"foo": "bar1", "egg": "spam1"})]
 
         class TestSpider(scrapy.Spider):
             name = "testspider"
@@ -627,7 +609,7 @@ class TestFeedExport(TestFeedExportBase):
     @coroutine_test
     async def test_start_finish_exporting_items(self):
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
         ]
         settings = {
             "FEEDS": {
@@ -665,7 +647,7 @@ class TestFeedExport(TestFeedExportBase):
     @coroutine_test
     async def test_start_finish_exporting_items_exception(self):
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
         ]
         settings = {
             "FEEDS": {
@@ -743,15 +725,15 @@ class TestFeedExport(TestFeedExportBase):
     @coroutine_test
     async def test_export_multiple_item_classes(self):
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem2({"hello": "world2", "foo": "bar2"}),
-            self.MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem2({"hello": "world2", "foo": "bar2"}),
+            MyItem({"foo": "bar3", "egg": "spam3", "baz": "quux3"}),
             {"hello": "world4", "egg": "spam4"},
         ]
 
         # by default, Scrapy uses fields of the first Item for CSV and
         # all fields for JSON Lines
-        header = self.MyItem.fields.keys()
+        header = MyItem.fields.keys()
         rows_csv = [
             {"egg": "spam1", "foo": "bar1", "baz": ""},
             {"egg": "", "foo": "bar2", "baz": ""},
@@ -826,8 +808,8 @@ class TestFeedExport(TestFeedExportBase):
     @coroutine_test
     async def test_export_based_on_item_classes(self):
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem2({"hello": "world2", "foo": "bar2"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem2({"hello": "world2", "foo": "bar2"}),
             {"hello": "world3", "egg": "spam3"},
         ]
 
@@ -849,15 +831,15 @@ class TestFeedExport(TestFeedExportBase):
             "FEEDS": {
                 self._random_temp_filename(): {
                     "format": "csv",
-                    "item_classes": [self.MyItem],
+                    "item_classes": [MyItem],
                 },
                 self._random_temp_filename(): {
                     "format": "json",
-                    "item_classes": [self.MyItem2],
+                    "item_classes": [MyItem2],
                 },
                 self._random_temp_filename(): {
                     "format": "jsonlines",
-                    "item_classes": [self.MyItem, self.MyItem2],
+                    "item_classes": [MyItem, MyItem2],
                 },
                 self._random_temp_filename(): {
                     "format": "xml",
@@ -872,13 +854,11 @@ class TestFeedExport(TestFeedExportBase):
     @coroutine_test
     async def test_export_based_on_custom_filters(self):
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem2({"hello": "world2", "foo": "bar2"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem2({"hello": "world2", "foo": "bar2"}),
             {"hello": "world3", "egg": "spam3"},
         ]
 
-        MyItem = self.MyItem
-
         class CustomFilter1:
             def __init__(self, feed_options):
                 pass
@@ -918,7 +898,7 @@ class TestFeedExport(TestFeedExportBase):
                 },
                 self._random_temp_filename(): {
                     "format": "jsonlines",
-                    "item_classes": [self.MyItem, self.MyItem2],
+                    "item_classes": [MyItem, MyItem2],
                     "item_filter": CustomFilter3,
                 },
             },
@@ -957,7 +937,7 @@ class TestFeedExport(TestFeedExportBase):
         # FEED_EXPORT_FIELDS option allows to order export fields
         # and to select a subset of fields to export, both for Items and dicts.
 
-        for item_cls in [self.MyItem, dict]:
+        for item_cls in [MyItem, dict]:
             items = [
                 item_cls({"foo": "bar1", "egg": "spam1"}),
                 item_cls({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 469985288..6e0536110 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -24,6 +24,7 @@ from scrapy.utils.test import get_crawler
 from tests.spiders import ItemSpider
 from tests.test_feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.feedexport import MyItem
 
 if TYPE_CHECKING:
     from os import PathLike
@@ -197,9 +198,9 @@ class TestBatchDeliveries(TestFeedExportBase):
     async def test_export_items(self):
         """Test partial deliveries in all supported formats"""
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
-            self.MyItem({"foo": "bar3", "baz": "quux3"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
         rows = [
             {"egg": "spam1", "foo": "bar1", "baz": ""},
@@ -207,7 +208,7 @@ class TestBatchDeliveries(TestFeedExportBase):
             {"foo": "bar3", "baz": "quux3", "egg": ""},
         ]
         settings = {"FEED_EXPORT_BATCH_ITEM_COUNT": 2}
-        header = self.MyItem.fields.keys()
+        header = MyItem.fields.keys()
         await self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
@@ -349,9 +350,9 @@ class TestBatchDeliveries(TestFeedExportBase):
         So %(batch_id)d replaced with the current id.
         """
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
-            self.MyItem({"foo": "bar3", "baz": "quux3"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
         settings = {
             "FEEDS": {
@@ -387,9 +388,9 @@ class TestBatchDeliveries(TestFeedExportBase):
     def test_s3_export(self):
         bucket = "mybucket"
         items = [
-            self.MyItem({"foo": "bar1", "egg": "spam1"}),
-            self.MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
-            self.MyItem({"foo": "bar3", "baz": "quux3"}),
+            MyItem({"foo": "bar1", "egg": "spam1"}),
+            MyItem({"foo": "bar2", "egg": "spam2", "baz": "quux2"}),
+            MyItem({"foo": "bar3", "baz": "quux3"}),
         ]
 
         class CustomS3FeedStorage(S3FeedStorage):
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index 6ebcab152..83d976e94 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -13,8 +13,9 @@ from typing import TYPE_CHECKING, Any
 import pytest
 
 from scrapy.utils.test import get_crawler
-from tests.test_feedexport import TestFeedExportBase, path_to_url, printf_escape
+from tests.test_feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test
+from tests.utils.feedexport import path_to_url, printf_escape
 
 if TYPE_CHECKING:
     from scrapy import Spider
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 5249736f2..da94a4e95 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -7,17 +7,7 @@ from scrapy.robotstxt import (
     decode_robotstxt,
 )
 from scrapy.utils._deps_compat import STDLIB_IMPROVED_ROBOTFILEPARSER
-
-
-def rerp_available() -> bool:
-    # check if robotexclusionrulesparser is installed
-    try:
-        from robotexclusionrulesparser import (  # noqa: PLC0415
-            RobotExclusionRulesParser,  # noqa: F401
-        )
-    except ImportError:
-        return False
-    return True
+from tests.utils.robotstxt import rerp_available
 
 
 class BaseRobotParserTest:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 39ce87b65..4fb594444 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import warnings
 from abc import ABC, abstractmethod
-from collections import deque
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
 from typing import TYPE_CHECKING, Any, NamedTuple, cast
 from unittest.mock import Mock
@@ -10,7 +9,7 @@ from unittest.mock import Mock
 import pytest
 
 from scrapy.core.downloader import Downloader
-from scrapy.core.scheduler import BaseScheduler, Scheduler
+from scrapy.core.scheduler import Scheduler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
@@ -27,38 +26,6 @@ if TYPE_CHECKING:
     from pathlib import Path
 
 
-class MemoryScheduler(BaseScheduler):
-    paused = False
-
-    def __init__(self, *args: Any, **kwargs: Any):
-        super().__init__(*args, **kwargs)
-        self.queue: deque[Request] = deque(
-            Request(value) if isinstance(value, str) else value
-            for value in getattr(self, "queue", [])
-        )
-
-    def enqueue_request(self, request: Request) -> bool:
-        self.queue.append(request)
-        return True
-
-    def has_pending_requests(self) -> bool:
-        return self.paused or bool(self.queue)
-
-    def next_request(self) -> Request | None:
-        if self.paused:
-            return None
-        try:
-            return self.queue.pop()
-        except IndexError:
-            return None
-
-    def pause(self) -> None:
-        self.paused = True
-
-    def unpause(self) -> None:
-        self.paused = False
-
-
 class MockSlot(NamedTuple):
     active: list[Any]
 
diff --git a/tests/utils/__init__.py b/tests/utils/__init__.py
index 73c0e3776..b0632a7ea 100644
--- a/tests/utils/__init__.py
+++ b/tests/utils/__init__.py
@@ -1,13 +1,19 @@
+from __future__ import annotations
+
 import asyncio
 import os
-from collections.abc import Callable
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from twisted.internet.defer import Deferred
 
+from scrapy.settings import Settings, default_settings
 from scrapy.utils.asyncio import is_asyncio_available
 from scrapy.utils.defer import maybe_deferred_to_future
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
 
 def twisted_sleep(seconds: float):
     from twisted.internet import reactor
@@ -48,3 +54,8 @@ class OneShotLoop:
 
     def stop(self) -> None:
         self.running = False
+
+
+def assert_option_is_default(settings: Settings, key: str) -> None:
+    assert isinstance(settings, Settings)
+    assert settings[key] == getattr(default_settings, key)
diff --git a/tests/utils/base_commands.py b/tests/utils/base_commands.py
new file mode 100644
index 000000000..594544c83
--- /dev/null
+++ b/tests/utils/base_commands.py
@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from shutil import copytree
+from typing import TYPE_CHECKING
+
+import pytest
+
+from tests.utils.cmdline import call
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+class TestProjectBase:
+    """A base class for tests that may need a Scrapy project."""
+
+    project_name = "testproject"
+
+    @pytest.fixture(scope="session")
+    def _proj_path_cached(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
+        """Create a Scrapy project in a temporary directory and return its path.
+
+        Used as a cache for ``proj_path``.
+        """
+        tmp_path = tmp_path_factory.mktemp("proj")
+        call("startproject", self.project_name, cwd=tmp_path)
+        return tmp_path / self.project_name
+
+    @pytest.fixture
+    def proj_path(self, tmp_path: Path, _proj_path_cached: Path) -> Path:
+        """Copy a pre-generated Scrapy project into a temporary directory and return its path."""
+        proj_path = tmp_path / self.project_name
+        copytree(_proj_path_cached, proj_path)
+        return proj_path
diff --git a/tests/utils/cmdline.py b/tests/utils/cmdline.py
index 122e0236d..62dff3d4c 100644
--- a/tests/utils/cmdline.py
+++ b/tests/utils/cmdline.py
@@ -2,12 +2,15 @@ from __future__ import annotations
 
 import subprocess
 import sys
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 import pytest
 
 from scrapy.utils.test import get_testenv
 
+if TYPE_CHECKING:
+    from pathlib import Path
+
 
 def call(*args: str, **popen_kwargs: Any) -> int:
     args = (sys.executable, "-m", "scrapy.cmdline", *args)
@@ -36,3 +39,10 @@ def proc(*args: str, **popen_kwargs: Any) -> tuple[int, str, str]:
         pytest.fail("Command took too much time to complete")
 
     return p.returncode, p.stdout, p.stderr
+
+
+def write_recording_editor(editor: Path) -> None:
+    """Create an executable editor script that writes the path it is asked to
+    open (its last argument) into the file given as its first argument."""
+    editor.write_text('#!/bin/sh\nprintf "%s" "$2" > "$1"\n', encoding="utf-8")
+    editor.chmod(0o755)
diff --git a/tests/utils/feedexport.py b/tests/utils/feedexport.py
new file mode 100644
index 000000000..515082217
--- /dev/null
+++ b/tests/utils/feedexport.py
@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from urllib.parse import urljoin
+from urllib.request import pathname2url
+
+from scrapy import Field, Item
+
+if TYPE_CHECKING:
+    from pathlib import Path
+
+
+def path_to_url(path: str | Path) -> str:
+    return urljoin("file:", pathname2url(str(path)))
+
+
+def printf_escape(s: str) -> str:
+    return s.replace("%", "%%")
+
+
+class MyItem(Item):
+    foo = Field()
+    egg = Field()
+    baz = Field()
+
+
+class MyItem2(Item):
+    foo = Field()
+    hello = Field()
diff --git a/tests/utils/redirect.py b/tests/utils/redirect.py
new file mode 100644
index 000000000..a3a572988
--- /dev/null
+++ b/tests/utils/redirect.py
@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from itertools import product
+
+SCHEME_PARAMS = ("url", "location", "target")
+HTTP_SCHEMES = ("http", "https")
+NON_HTTP_SCHEMES = ("data", "file", "ftp", "s3", "foo")
+REDIRECT_SCHEME_CASES = (
+    # http/https → http/https redirects
+    *(
+        (
+            f"{input_scheme}://example.com/a",
+            f"{output_scheme}://example.com/b",
+            f"{output_scheme}://example.com/b",
+        )
+        for input_scheme, output_scheme in product(HTTP_SCHEMES, repeat=2)
+    ),
+    # http/https → data/file/ftp/s3/foo does not redirect
+    *(
+        (
+            f"{input_scheme}://example.com/a",
+            f"{output_scheme}://example.com/b",
+            None,
+        )
+        for input_scheme in HTTP_SCHEMES
+        for output_scheme in NON_HTTP_SCHEMES
+    ),
+    # http/https → relative redirects
+    *(
+        (
+            f"{scheme}://example.com/a",
+            location,
+            f"{scheme}://example.com/b",
+        )
+        for scheme in HTTP_SCHEMES
+        for location in ("//example.com/b", "/b")
+    ),
+    # Note: We do not test data/file/ftp/s3 schemes for the initial URL
+    # because their download handlers cannot return a status code of 3xx.
+)
diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py
new file mode 100644
index 000000000..c1b2f28c6
--- /dev/null
+++ b/tests/utils/robotstxt.py
@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+
+def rerp_available() -> bool:
+    # check if robotexclusionrulesparser is installed
+    try:
+        from robotexclusionrulesparser import (  # noqa: PLC0415
+            RobotExclusionRulesParser,  # noqa: F401
+        )
+    except ImportError:
+        return False
+    return True

From e3229052552c51df18f88b5c106e0de694d6be3e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 15:54:43 +0500
Subject: [PATCH 24/62] Assorted test fixes, part 3. (#7741)

* Assorted test fixes, part 3.

* Fix asyncio_enabled_reactor_different_loop.py on 3.14.
---
 .../asyncio_enabled_reactor_different_loop.py    |  2 +-
 tests/test_crawler_subprocess.py                 | 10 +++++-----
 tests/test_downloader_handlers_http_base.py      | 14 +++++++-------
 tests/test_downloadermiddleware_redirect.py      |  2 +-
 tests/test_downloadermiddleware_retry.py         |  9 +++++----
 tests/test_engine_loop.py                        |  2 +-
 tests/test_extension_periodic_log.py             |  1 -
 tests/test_feedexport.py                         |  2 +-
 tests/test_http_response_text.py                 | 16 ----------------
 tests/test_linkextractors.py                     | 13 -------------
 tests/test_pipeline_media.py                     |  2 +-
 tests/test_spider.py                             |  2 +-
 tests/test_spiderloader/__init__.py              |  1 -
 tests/test_spidermiddleware_referer.py           | 10 +++++-----
 tests/test_stats.py                              |  6 +++++-
 tests/test_utils_iterators.py                    |  5 -----
 tests/test_utils_url.py                          | 10 ----------
 17 files changed, 33 insertions(+), 74 deletions(-)

diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
index 4257bc0ac..13a0850f9 100644
--- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
+++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -8,7 +8,7 @@ from scrapy.crawler import AsyncCrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncioreactor.install(asyncio.get_event_loop())
+asyncioreactor.install()
 
 
 class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 018a2b31b..6e43615e4 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -352,7 +352,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         ) in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_same_loop(self) -> None:
+    def test_asyncio_custom_loop_custom_settings_same(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -362,7 +362,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_different_loop(self) -> None:
+    def test_asyncio_custom_loop_custom_settings_different(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -437,14 +437,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             in log
         )
 
-    def test_shutdown_graceful(self) -> None:
+    def test_reactorless_shutdown_graceful(self) -> None:
         self._test_shutdown_graceful("reactorless_sleeping.py")
 
     @coroutine_test
-    async def test_shutdown_forced(self) -> None:
+    async def test_reactorless_shutdown_forced(self) -> None:
         await self._test_shutdown_forced("reactorless_sleeping.py")
 
-    def test_shutdown_graceful_reactorless_no_stop(self) -> None:
+    def test_reactorless_shutdown_graceful_no_stop(self) -> None:
         self._test_shutdown_graceful("reactorless_sleeping.py", "--no-stop")
 
     def test_asyncio_enabled_reactor_same_loop_default(self) -> None:
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index c4a8193c8..29e1d9757 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -538,13 +538,6 @@ class TestHttpBase(ABC):
         else:
             assert latency > 0
 
-    @coroutine_test
-    async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
-        request = Request(mockserver.url("/text", is_secure=self.is_secure))
-        async with self.get_dh() as download_handler:
-            response = await download_handler.download_request(request)
-        assert response.body == b"Works"
-
     @coroutine_test
     async def test_response_class_choosing_request(
         self, mockserver: MockServer
@@ -560,6 +553,13 @@ class TestHttpBase(ABC):
             response = await download_handler.download_request(request)
         assert type(response) is TextResponse  # pylint: disable=unidiomatic-typecheck
 
+    @coroutine_test
+    async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
+        request = Request(mockserver.url("/text", is_secure=self.is_secure))
+        async with self.get_dh({"DOWNLOAD_MAXSIZE": 0}) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.body == b"Works"
+
     @coroutine_test
     async def test_download_with_maxsize(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 0824ab531..28cf8ba34 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -33,7 +33,7 @@ class TestRedirectMiddleware(Base.Test):
         headers = {"Location": location}
         return Response(request.url, status=status, headers=headers)
 
-    def test_redirect_3xx_permanent(self):
+    def test_redirect_307_308_preserve_method(self):
         def _test(method, status: int):
             url = f"http://www.example.com/{status}"
             url2 = "http://www.example.com/redirected"
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 56d21a4d2..825efce2f 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -42,17 +42,18 @@ class TestRetry:
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
         rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
-        # first retry
+        # no retry
         r = self.mw.process_response(req, rsp)
         assert r is rsp
 
         # Test retry when dont_retry set to False
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": False})
-        rsp = Response("http://www.scrapytest.org/503")
+        rsp = Response("http://www.scrapytest.org/503", body=b"", status=503)
 
         # first retry
-        r = self.mw.process_response(req, rsp)
-        assert r is rsp
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        assert req.meta["retry_times"] == 1
 
     def test_dont_retry_exc(self):
         req = Request("http://www.scrapytest.org/503", meta={"dont_retry": True})
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index ca52961f4..c15c396d3 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -152,7 +152,7 @@ class TestRequestSendOrder:
 
     @classmethod
     def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)  # increase if flaky
+        cls.mockserver.__exit__(None, None, None)
 
     def request(self, num, response_seconds, download_slots, priority=0):
         url = self.mockserver.url(f"/delay?n={response_seconds}&{num}")
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index 0fe2caca4..ffe7a0dc7 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -73,7 +73,6 @@ def extension(settings: dict[str, Any] | None = None) -> CustomPeriodicLog:
 class TestPeriodicLog:
     def test_extension_enabled(self):
         # Expected that settings for this extension loaded successfully
-        # And on certain conditions - extension raising NotConfigured
 
         # "PERIODIC_LOG_STATS": True -> set to {"enabled": True}
         # due to TypeError exception from settings.getdict
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 853243b97..88070513f 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -706,7 +706,7 @@ class TestFeedExport(TestFeedExportBase):
 
     @coroutine_test
     async def test_export_no_items_multiple_feeds(self):
-        """Make sure that `storage.store` is called for every feed."""
+        """Make sure that `storage.store` is not called."""
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 5ef89fe4a..daf6e66e5 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -405,22 +405,6 @@ class TestTextResponse(TestResponse):
             encoding="cp1251",
         )
 
-    def test_follow_flags(self):
-        res = self.response_class("http://example.com/")
-        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
-        assert fol.flags == ["cached", "allowed"]
-
-    def test_follow_all_flags(self):
-        re = self.response_class("http://www.example.com/")
-        urls = [
-            "http://www.example.com/",
-            "http://www.example.com/2",
-            "http://www.example.com/foo",
-        ]
-        fol = re.follow_all(urls, flags=["cached", "allowed"])
-        for req in fol:
-            assert req.flags == ["cached", "allowed"]
-
     def test_follow_all_css(self):
         expected = [
             "http://example.com/sample3.html",
diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py
index db23e9aef..95a6aee54 100644
--- a/tests/test_linkextractors.py
+++ b/tests/test_linkextractors.py
@@ -798,19 +798,6 @@ class Base:
 class TestLxmlLinkExtractor(Base.TestLinkExtractorBase):
     extractor_cls = LxmlLinkExtractor
 
-    def test_link_wrong_href(self):
-        html = b"""
-        <a href="http://example.org/item1.html">Item 1</a>
-        <a href="http://[example.org/item2.html">Item 2</a>
-        <a href="http://example.org/item3.html">Item 3</a>
-        """
-        response = HtmlResponse("http://example.org/index.html", body=html)
-        lx = self.extractor_cls()
-        assert list(lx.extract_links(response)) == [
-            Link(url="http://example.org/item1.html", text="Item 1", nofollow=False),
-            Link(url="http://example.org/item3.html", text="Item 3", nofollow=False),
-        ]
-
     def test_link_restrict_text(self):
         html = b"""
         <a href="http://example.org/item1.html">Pic of a cat</a>
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 8ac949da7..6862c34eb 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -394,7 +394,7 @@ class TestMediaPipelineAllowRedirectSettings:
             # the downloader to handle itself
             (301, False),
             (302, False),
-            (302, False),
+            (303, False),
             (307, False),
             (308, False),
             # we still want to get 4xx and 5xx
diff --git a/tests/test_spider.py b/tests/test_spider.py
index fba1ef984..7b2727370 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -31,7 +31,7 @@ class TestSpider:
         assert spider.foo == "bar"
 
     def test_spider_without_name(self):
-        """``__init__`` method arguments are assigned to spider attributes"""
+        """``__init__`` raises when the name is not provided."""
         msg = "must have a name"
         with pytest.raises(ValueError, match=msg):
             self.spider_class()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index d85942f90..81e49bb75 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -192,7 +192,6 @@ class TestDuplicateSpiderNameLoader:
     def test_multiple_dupename_warning(self, spider_loader_env):
         settings, spiders_dir = spider_loader_env
         # copy 2 spider modules so as to have duplicate spider name
-        # This should issue 2 warning, 1 for each duplicate spider name
         shutil.copyfile(spiders_dir / "spider1.py", spiders_dir / "spider1dupe.py")
         shutil.copyfile(spiders_dir / "spider2.py", spiders_dir / "spider2dupe.py")
 
diff --git a/tests/test_spidermiddleware_referer.py b/tests/test_spidermiddleware_referer.py
index 7591c0e7b..5ca27d15c 100644
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@@ -239,7 +239,7 @@ class MixinSameOrigin:
         ("https://example.com/page.html", "http://not.example.com/", None),
         ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
         ("ftp://example.com/urls.zip", "http://example.com/not-page.html", None),
-        ("ftps://example.com/urls.zip", "https://example.com/not-page.html", None),
+        ("ftps://example.com/urls.zip", "http://example.com/not-page.html", None),
         # test for user/password stripping
         (
             "https://user:password@example.com/page.html",
@@ -395,7 +395,7 @@ class MixinOriginWhenCrossOrigin:
         ),
         (
             "ftps://example4.com/urls.zip",
-            "https://example4.com/not-page.html",
+            "http://example4.com/not-page.html",
             b"ftps://example4.com/",
         ),
         # test for user/password stripping
@@ -506,9 +506,9 @@ class MixinStrictOriginWhenCrossOrigin:
             b"ftps://example4.com/",
         ),
         (
-            "ftps://example4.com/urls.zip",
-            "https://example4.com/not-page.html",
-            b"ftps://example4.com/",
+            "ftp://example4.com/urls.zip",
+            "http://example4.com/not-page.html",
+            b"ftp://example4.com/",
         ),
         # test for user/password stripping
         (
diff --git a/tests/test_stats.py b/tests/test_stats.py
index 05f609fda..c33a00090 100644
--- a/tests/test_stats.py
+++ b/tests/test_stats.py
@@ -32,7 +32,11 @@ class TestCoreStatsExtension:
     @mock.patch("scrapy.extensions.corestats.monotonic", return_value=0)
     @mock.patch("scrapy.extensions.corestats.datetime")
     def test_core_stats_default_stats_collector(
-        self, mock_datetime: mock.Mock, crawler: Crawler, spider: Spider
+        self,
+        mock_datetime: mock.Mock,
+        mock_monotonic: mock.Mock,
+        crawler: Crawler,
+        spider: Spider,
     ) -> None:
         fixed_datetime = datetime(2019, 12, 1, 11, 38)
         mock_datetime.now = mock.Mock(return_value=fixed_datetime)
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index a59541edb..4fb18305e 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -352,11 +352,6 @@ class TestLxmlXmliter(TestXmliterBase):
         node = next(my_iter)
         assert node.xpath("f:name/text()").getall() == ["African Coffee Table"]
 
-    def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")  # type: ignore[arg-type]
-        with pytest.raises(TypeError):
-            next(i)
-
 
 class TestUtilsCsv:
     def test_csviter_defaults(self):
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index a74b9a41d..5b98131a1 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -48,16 +48,6 @@ def test_url_is_from_spider():
     assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
 
 
-def test_url_is_from_spider_class_attributes():
-    class MySpider(Spider):
-        name = "example.com"
-
-    assert url_is_from_spider("http://www.example.com/some/page.html", MySpider)
-    assert url_is_from_spider("http://sub.example.com/some/page.html", MySpider)
-    assert not url_is_from_spider("http://www.example.org/some/page.html", MySpider)
-    assert not url_is_from_spider("http://www.example.net/some/page.html", MySpider)
-
-
 def test_url_is_from_spider_with_allowed_domains():
     class MySpider(Spider):
         name = "example.com"

From 394c2797f34094299dfcfd621d7315947cb8dd05 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 16:30:21 +0500
Subject: [PATCH 25/62] Remove non-Twisted uses of zope.interface (#7731)

---
 docs/news.rst                            | 24 ++++++++++++-
 docs/topics/api.rst                      | 45 ++++--------------------
 scrapy/core/downloader/contextfactory.py |  2 --
 scrapy/extensions/feedexport.py          | 33 ++++++++---------
 scrapy/interfaces.py                     | 24 +++++++------
 scrapy/spiderloader.py                   | 44 +++++++++++++----------
 tests/test_crawler_runners.py            | 20 -----------
 tests/test_feedexport.py                 |  5 ---
 tests/test_feedexport_batch.py           |  6 +---
 tests/test_feedexport_storages.py        |  9 -----
 tests/test_spiderloader/__init__.py      |  5 ---
 11 files changed, 87 insertions(+), 130 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 63f0d69e4..4ebab3c6a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,28 @@
 Release notes
 =============
 
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-   The following runtime usage of zope.interface_ interfaces is removed:
+
+    - :class:`~scrapy.spiderloader.SpiderLoader` and
+      :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
+      as implementing the ``ISpiderLoader`` interface.
+
+    - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
+      configured spider loader implements the ``ISpiderLoader`` interface.
+
+    - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
+      :class:`~scrapy.extensions.feedexport.FileFeedStorage` and
+      :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
+      marked as implementing the ``IFeedStorage`` interface.
+
+    (:issue:`6585`, :issue:`7731`)
+
 .. _release-2.17.0:
 
 Scrapy 2.17.0 (2026-07-07)
@@ -2643,7 +2665,7 @@ Deprecation removals
     (:issue:`6109`, :issue:`6116`)
 
 -   A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
-    does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
+    does not implement the ``ISpiderLoader`` interface
     will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
     run time. Non-compliant classes have been triggering a deprecation warning
     since Scrapy 1.0.0.
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 19082d9d7..7ff8d3464 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -172,46 +172,15 @@ SpiderLoader API
 .. module:: scrapy.spiderloader
    :synopsis: The spider loader
 
-.. class:: SpiderLoader
+Custom spider loaders can be employed by specifying their path in the
+:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
+:class:`SpiderLoaderProtocol`.
 
-    This class is in charge of retrieving and handling the spider classes
-    defined across the project.
+.. autoclass:: SpiderLoaderProtocol
+    :members:
 
-    Custom spider loaders can be employed by specifying their path in the
-    :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
-    the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
-    errorless execution.
-
-    .. method:: from_settings(settings)
-
-       This class method is used by Scrapy to create an instance of the class.
-       It's called with the current project settings, and it loads the spiders
-       found recursively in the modules of the :setting:`SPIDER_MODULES`
-       setting.
-
-       :param settings: project settings
-       :type settings: :class:`~scrapy.settings.Settings` instance
-
-    .. method:: load(spider_name)
-
-       Get the Spider class with the given name. It'll look into the previously
-       loaded spiders for a spider class with name ``spider_name`` and will raise
-       a KeyError if not found.
-
-       :param spider_name: spider class name
-       :type spider_name: str
-
-    .. method:: list()
-
-       Get the names of the available spiders in the project.
-
-    .. method:: find_by_request(request)
-
-       List the spiders' names that can handle the given request. Will try to
-       match the request's url against the domains of the spiders.
-
-       :param request: queried request
-       :type request: :class:`~scrapy.Request` instance
+.. autoclass:: SpiderLoader
+    :members:
 
 .. autoclass:: DummySpiderLoader
 
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index a934cbbc7..ef948997d 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -13,7 +13,6 @@ from twisted.internet.ssl import (
 from twisted.web.client import BrowserLikePolicyForHTTPS
 from twisted.web.iweb import IPolicyForHTTPS
 from zope.interface.declarations import implementer
-from zope.interface.verify import verifyObject
 
 from scrapy.core.downloader.tls import (
     _TWISTED_VERSION_MAP,
@@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory:
     # all of this with _ScrapyClientContextFactory.acceptableProtocols.
 
     def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
-        verifyObject(IPolicyForHTTPS, context_factory)
         self._wrapped_context_factory: Any = context_factory
         self._acceptable_protocols: list[bytes] = acceptable_protocols
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0d9b016a3..678a29e2e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse
 
 from twisted.internet.defer import Deferred, DeferredList
 from w3lib.url import file_uri_to_path
-from zope.interface import Interface, implementer
+from zope.interface import Interface
 
 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
@@ -115,25 +115,18 @@ class ItemFilter:
         return True  # accept all items by default
 
 
-class IFeedStorage(Interface):  # type: ignore[misc]
-    """Interface that all Feed Storages must implement"""
-
+class _IFeedStorage(Interface):  # type: ignore[misc]  # pragma: no cover
     # pylint: disable=no-self-argument
 
-    def __init__(uri, *, feed_options=None):  # type: ignore[no-untyped-def]  # pylint: disable=super-init-not-called
-        """Initialize the storage with the parameters given in the URI and the
-        feed-specific options (see :setting:`FEEDS`)"""
+    def __init__(uri, *, feed_options=None): ...  # type: ignore[no-untyped-def]  # pylint: disable=super-init-not-called
 
-    def open(spider):  # type: ignore[no-untyped-def]
-        """Open the storage for the given spider. It must return a file-like
-        object that will be used for the exporters"""
+    def open(spider): ...  # type: ignore[no-untyped-def]
 
-    def store(file):  # type: ignore[no-untyped-def]
-        """Store the given file stream"""
+    def store(file): ...  # type: ignore[no-untyped-def]
 
 
 class FeedStorageProtocol(Protocol):
-    """Reimplementation of ``IFeedStorage`` that can be used in type hints."""
+    """Protocol that all Feed Storages must follow."""
 
     def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
         """Initialize the storage with the parameters given in the URI and the
@@ -147,7 +140,6 @@ class FeedStorageProtocol(Protocol):
         """Store the given file stream"""
 
 
-@implementer(IFeedStorage)
 class BlockingFeedStorage(ABC):
     def open(self, spider: Spider) -> IO[bytes]:
         path = spider.crawler.settings["FEED_TEMPDIR"]
@@ -164,7 +156,6 @@ class BlockingFeedStorage(ABC):
         raise NotImplementedError
 
 
-@implementer(IFeedStorage)
 class StdoutFeedStorage:
     def __init__(
         self,
@@ -191,7 +182,6 @@ class StdoutFeedStorage:
         pass
 
 
-@implementer(IFeedStorage)
 class FileFeedStorage:
     def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
         self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
@@ -758,3 +748,14 @@ class FeedExporter:
             feed_options.get("item_filter", ItemFilter)
         )
         return item_filter_class(feed_options)
+
+
+def __getattr__(name: str) -> Any:  # pragma: no cover
+    if name == "IFeedStorage":
+        warnings.warn(
+            "scrapy.extensions.feedexport.IFeedStorage is deprecated.",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return _IFeedStorage
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index b4f1d9394..34d7bdd82 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -1,19 +1,23 @@
+# pragma: no file cover
 # pylint: disable=no-method-argument,no-self-argument
+import warnings
 
 from zope.interface import Interface
 
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+warnings.warn(
+    "The scrapy.interfaces module is deprecated.",
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
+
 
 class ISpiderLoader(Interface):
-    def from_settings(settings):
-        """Return an instance of the class for the given settings"""
+    def from_settings(settings): ...
 
-    def load(spider_name):
-        """Return the Spider class for the given spider name. If the spider
-        name is not found, it must raise a KeyError."""
+    def load(spider_name): ...
 
-    def list():
-        """Return a list with the names of all spiders available in the
-        project"""
+    def list(): ...
 
-    def find_by_request(request):
-        """Return the list of spiders names that can handle the given request"""
+    def find_by_request(request): ...
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 8c980fd46..244b7338f 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -5,10 +5,8 @@ import warnings
 from collections import defaultdict
 from typing import TYPE_CHECKING, Protocol, cast
 
-from zope.interface import implementer
-from zope.interface.verify import verifyClass
-
-from scrapy.interfaces import ISpiderLoader
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from scrapy import Request, Spider  # noqa: TC001
 from scrapy.utils.misc import load_object, walk_modules_iter
 from scrapy.utils.spider import iter_spider_classes
 
@@ -18,7 +16,6 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
-    from scrapy import Request, Spider
     from scrapy.settings import BaseSettings
 
 
@@ -26,28 +23,31 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
     """Get SpiderLoader instance from settings"""
     cls_path = settings.get("SPIDER_LOADER_CLASS")
     loader_cls = load_object(cls_path)
-    verifyClass(ISpiderLoader, loader_cls)
     return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
 
 
 class SpiderLoaderProtocol(Protocol):
+    """Protocol for spider loader implementations.
+
+    See :setting:`SPIDER_LOADER_CLASS`.
+    """
+
     @classmethod
     def from_settings(cls, settings: BaseSettings) -> Self:
-        """Return an instance of the class for the given settings"""
+        """Return an instance of the class for the given settings."""
 
     def load(self, spider_name: str) -> type[Spider]:
-        """Return the Spider class for the given spider name. If the spider
-        name is not found, it must raise a KeyError."""
+        """Return the spider class for the given spider name. If the spider
+        name is not found, it must raise a :exc:`KeyError`."""
 
     def list(self) -> list[str]:
         """Return a list with the names of all spiders available in the
-        project"""
+        project."""
 
     def find_by_request(self, request: Request) -> __builtins__.list[str]:
-        """Return the list of spiders names that can handle the given request"""
+        """Return the list of spiders names that can handle the given request."""
 
 
-@implementer(ISpiderLoader)
 class SpiderLoader:
     """
     SpiderLoader is a class which locates and loads spiders
@@ -106,12 +106,18 @@ class SpiderLoader:
 
     @classmethod
     def from_settings(cls, settings: BaseSettings) -> Self:
+        """Create an instance of the class.
+
+        It's called with the current project settings, and it loads the spiders
+        found recursively in the modules of the :setting:`SPIDER_MODULES`
+        setting.
+        """
         return cls(settings)
 
     def load(self, spider_name: str) -> type[Spider]:
-        """
-        Return the Spider class for the given spider name. If the spider
-        name is not found, raise a KeyError.
+        """Return the spider class for the given spider name.
+
+        If the spider name is not found, raise a :exc:`KeyError`.
         """
         try:
             return self._spiders[spider_name]
@@ -121,19 +127,19 @@ class SpiderLoader:
     def find_by_request(self, request: Request) -> list[str]:
         """
         Return the list of spider names that can handle the given request.
+
+        It will try to match the request's url against the domains of
+        the spiders.
         """
         return [
             name for name, cls in self._spiders.items() if cls.handles_request(request)
         ]
 
     def list(self) -> list[str]:
-        """
-        Return a list with the names of all spiders available in the project.
-        """
+        """Return a list with the names of all spiders available in the project."""
         return list(self._spiders.keys())
 
 
-@implementer(ISpiderLoader)
 class DummySpiderLoader:
     """A dummy spider loader that does not load any spiders."""
 
diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py
index 2f3649b43..9e35a38e2 100644
--- a/tests/test_crawler_runners.py
+++ b/tests/test_crawler_runners.py
@@ -4,7 +4,6 @@ import logging
 from typing import TYPE_CHECKING
 
 import pytest
-from zope.interface.exceptions import MultipleInvalid
 
 from scrapy.crawler import (
     AsyncCrawlerProcess,
@@ -14,7 +13,6 @@ from scrapy.crawler import (
     CrawlerRunner,
     CrawlerRunnerBase,
 )
-from scrapy.settings import Settings
 from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_reactor_settings
@@ -33,15 +31,6 @@ class SpiderLoaderWithWrongInterface:
 
 
 class TestCrawlerRunner:
-    def test_spider_manager_verify_interface(self) -> None:
-        settings = Settings(
-            {
-                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
-            }
-        )
-        with pytest.raises(MultipleInvalid):
-            CrawlerRunner(settings)
-
     def test_crawler_runner_accepts_dict(self) -> None:
         runner = CrawlerRunner({"foo": "bar"})
         assert runner.settings["foo"] == "bar"
@@ -53,15 +42,6 @@ class TestCrawlerRunner:
 
 
 class TestAsyncCrawlerRunner:
-    def test_spider_manager_verify_interface(self) -> None:
-        settings = Settings(
-            {
-                "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
-            }
-        )
-        with pytest.raises(MultipleInvalid):
-            AsyncCrawlerRunner(settings)
-
     def test_crawler_runner_accepts_dict(self) -> None:
         runner = AsyncCrawlerRunner({"foo": "bar"})
         assert runner.settings["foo"] == "bar"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 88070513f..3f659ee0b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,7 +18,6 @@ import lxml.etree
 import pytest
 from testfixtures import LogCapture
 from w3lib.url import file_uri_to_path
-from zope.interface import implementer
 
 import scrapy
 from scrapy import Spider, signals
@@ -29,7 +28,6 @@ from scrapy.extensions.feedexport import (
     FeedExporter,
     FeedSlot,
     FileFeedStorage,
-    IFeedStorage,
     apply_uri_params,
 )
 from scrapy.utils.python import to_unicode
@@ -83,7 +81,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
         raise OSError("Cannot store")
 
 
-@implementer(IFeedStorage)
 class LogOnStoreFileStorage:
     """
     This storage logs inside `store` method.
@@ -1300,7 +1297,6 @@ class TestFeedExport(TestFeedExportBase):
 
     @coroutine_test
     async def test_storage_file_no_postprocessing(self):
-        @implementer(IFeedStorage)
         class Storage:
             def __init__(self, uri, *, feed_options=None):
                 pass
@@ -1322,7 +1318,6 @@ class TestFeedExport(TestFeedExportBase):
 
     @coroutine_test
     async def test_storage_file_postprocessing(self):
-        @implementer(IFeedStorage)
         class Storage:
             def __init__(self, uri, *, feed_options=None):
                 pass
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 6e0536110..3b50cd492 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -12,12 +12,11 @@ from urllib.parse import urljoin
 import lxml.etree
 import pytest
 from packaging.version import Version
-from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy import Spider
 from scrapy.exceptions import NotConfigured
-from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage
+from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
@@ -436,9 +435,6 @@ class TestBatchDeliveries(TestFeedExportBase):
                 },
             },
         }
-        crawler = get_crawler(settings_dict=settings)
-        storage = S3FeedStorage.from_crawler(crawler, uri)
-        verifyObject(IFeedStorage, storage)
 
         class TestSpider(scrapy.Spider):
             name = "testspider"
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index b7009be62..82e2597a5 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -12,7 +12,6 @@ from urllib.parse import quote
 import pytest
 from testfixtures import LogCapture
 from w3lib.url import path_to_file_uri
-from zope.interface.verify import verifyObject
 
 import scrapy
 from scrapy.extensions.feedexport import (
@@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import (
     FileFeedStorage,
     FTPFeedStorage,
     GCSFeedStorage,
-    IFeedStorage,
     S3FeedStorage,
     StdoutFeedStorage,
 )
@@ -55,11 +53,6 @@ class TestFileFeedStorage:
         finally:
             os.chdir(old_cwd)
 
-    def test_interface(self, tmp_path):
-        path = tmp_path / "file.txt"
-        st = FileFeedStorage(str(path))
-        verifyObject(IFeedStorage, st)
-
     @staticmethod
     def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
         storage = FileFeedStorage(str(path), feed_options=feed_options)
@@ -115,7 +108,6 @@ class TestFTPFeedStorage:
             uri,
             feed_options=feed_options,
         )
-        verifyObject(IFeedStorage, storage)
         spider = self.get_test_spider()
         file = storage.open(spider)
         file.write(content)
@@ -259,7 +251,6 @@ class TestS3FeedStorage:
         bucket = "mybucket"
         key = "export.csv"
         storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
-        verifyObject(IFeedStorage, storage)
 
         file = mock.MagicMock()
 
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 81e49bb75..81840f7e0 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -5,14 +5,12 @@ from pathlib import Path
 from unittest import mock
 
 import pytest
-from zope.interface.verify import verifyObject
 
 # ugly hack to avoid cyclic imports of scrapy.spiders when running this test
 # alone
 import scrapy
 from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
-from scrapy.interfaces import ISpiderLoader
 from scrapy.settings import Settings
 from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
 
@@ -45,9 +43,6 @@ def spider_loader(spider_loader_env):
 
 
 class TestSpiderLoader:
-    def test_interface(self, spider_loader):
-        verifyObject(ISpiderLoader, spider_loader)
-
     def test_list(self, spider_loader):
         assert set(spider_loader.list()) == {
             "spider1",

From 1ddf024b89441a603b775e401786eee1854f8ec6 Mon Sep 17 00:00:00 2001
From: Mridankan Mandal <xerontitan90@gmail.com>
Date: Tue, 21 Jul 2026 18:45:30 +0530
Subject: [PATCH 26/62] Log ignored responses exceeding DOWNLOAD_MAXSIZE
 (#7742)

* Log ignored responses exceeding DOWNLOAD_MAXSIZE

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>

* Use caplog in compression warning test

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>

* Make compression warning test robust

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>

---------

Signed-off-by: Mridankan Mandal <xerontitan90@gmail.com>
---
 .../downloadermiddlewares/httpcompression.py  |  6 +++--
 ...st_downloadermiddleware_httpcompression.py | 24 +++++++++++++++++++
 2 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 6ca04a50e..13a432476 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -123,12 +123,14 @@ class HttpCompressionMiddleware:
                     response.body, content_encoding, max_size
                 )
             except _DecompressionMaxSizeExceeded as e:
-                raise IgnoreRequest(
+                msg = (
                     f"Ignored response {response} because its body "
                     f"({len(response.body)} B compressed, "
                     f"{e.decompressed_size} B decompressed so far) exceeded "
                     f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
-                ) from e
+                )
+                logger.warning(msg)
+                raise IgnoreRequest(msg) from e
             if len(response.body) < warn_size <= len(decoded_body):
                 logger.warning(
                     f"{response} body size after decompression "
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 5c4085657..311aff7df 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -558,6 +558,30 @@ class TestHttpCompression:
 
         self._test_compression_bomb_setting("zstd")
 
+    def test_compression_bomb_setting_logs_warning(self, caplog):
+        settings = {"DOWNLOAD_MAXSIZE": 1_000_000}
+        crawler = get_crawler(Spider, settings_dict=settings)
+        spider = crawler._create_spider("scrapytest.org")
+        mw = HttpCompressionMiddleware.from_crawler(crawler)
+        mw.open_spider(spider)
+
+        response = self._getresponse("bomb-gzip")  # 11_511_612 B
+        caplog.clear()
+        with (
+            caplog.at_level(
+                WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+            ),
+            pytest.raises(IgnoreRequest) as exc_info,
+        ):
+            mw.process_response(response.request, response)
+        assert caplog.record_tuples == [
+            (
+                "scrapy.downloadermiddlewares.httpcompression",
+                WARNING,
+                str(exc_info.value),
+            )
+        ]
+
     def _test_compression_bomb_spider_attr(self, compression_id):
         class DownloadMaxSizeSpider(Spider):
             download_maxsize = 1_000_000

From ca21306df769754ab4ad01596898a63f2fc90de6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 21 Jul 2026 19:58:40 +0500
Subject: [PATCH 27/62] Reactorless import hook improvements. (#7747)

* Reactorless import hook improvements.

* Replace prints with logging.

* More tests.

* Uninstall the import hook earlier.

* Add pragma: no branch.
---
 docs/topics/asyncio.rst                       |  3 +-
 scrapy/crawler.py                             | 13 ++++-
 scrapy/utils/reactorless.py                   | 19 +++++--
 .../reactorless_import_hook_multiple.py       | 29 +++++++++++
 .../reactorless_import_hook_uninstall.py      | 29 +++++++++++
 .../reactorless_then_reactor.py               | 19 +++++++
 tests/test_crawler_subprocess.py              | 32 ++++++++++++
 tests/test_utils_reactorless.py               | 50 +++++++++++++++++++
 8 files changed, 188 insertions(+), 6 deletions(-)
 create mode 100644 tests/AsyncCrawlerProcess/reactorless_import_hook_multiple.py
 create mode 100644 tests/AsyncCrawlerProcess/reactorless_import_hook_uninstall.py
 create mode 100644 tests/AsyncCrawlerProcess/reactorless_then_reactor.py
 create mode 100644 tests/test_utils_reactorless.py

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index d0b99c38e..83e40656b 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -227,7 +227,8 @@ for its differences and limitations compared to
 
 Additionally, :class:`~scrapy.crawler.AsyncCrawlerProcess` will install a
 :term:`meta path finder` that prevents :mod:`twisted.internet.reactor` from
-being imported.
+being imported. It will be uninstalled when :meth:`AsyncCrawlerProcess.start()
+<scrapy.crawler.AsyncCrawlerProcess.start>` exits.
 
 .. _asyncio-without-reactor-migrate:
 
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index c72752915..c8d74fba7 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -39,7 +39,11 @@ from scrapy.utils.reactor import (
     verify_installed_asyncio_event_loop,
     verify_installed_reactor,
 )
-from scrapy.utils.reactorless import install_reactor_import_hook
+from scrapy.utils.reactorless import (
+    ReactorImportHook,
+    install_reactor_import_hook,
+    uninstall_reactor_import_hook,
+)
 
 if TYPE_CHECKING:
     from collections.abc import Awaitable, Generator, Iterable
@@ -837,6 +841,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
         super().__init__(settings, install_root_handler)
         logger.debug("Using AsyncCrawlerProcess")
         self._reactorless_loop: asyncio.AbstractEventLoop | None = None
+        self._reactor_import_hook: ReactorImportHook | None = None
         # We want the asyncio event loop to be installed early, so that it's
         # always the correct one. And as we do that, we can also install the
         # reactor here.
@@ -849,7 +854,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
                     "TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed."
                 )
             self._reactorless_loop = set_asyncio_event_loop(loop_path)
-            install_reactor_import_hook()
+            self._reactor_import_hook = install_reactor_import_hook()
         elif is_reactor_installed():
             # The user could install a reactor before this class is instantiated.
             # We need to make sure the reactor is the correct one and the loop
@@ -979,6 +984,10 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
             loop.run_until_complete(loop.shutdown_asyncgens())
             loop.run_until_complete(loop.shutdown_default_executor())
         finally:
+            # loop.close() can raise, so we uninstall the hook first
+            if self._reactor_import_hook:  # pragma: no branch
+                uninstall_reactor_import_hook(self._reactor_import_hook)
+                self._reactor_import_hook = None
             self._reactorless_main_task = None
             asyncio.set_event_loop(None)
             loop.close()
diff --git a/scrapy/utils/reactorless.py b/scrapy/utils/reactorless.py
index aca76951b..243d80821 100644
--- a/scrapy/utils/reactorless.py
+++ b/scrapy/utils/reactorless.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import sys
 from importlib.abc import MetaPathFinder
 from typing import TYPE_CHECKING
@@ -47,7 +48,19 @@ class ReactorImportHook(MetaPathFinder):
         return None
 
 
-def install_reactor_import_hook() -> None:
-    """Prevent importing :mod:`twisted.internet.reactor`."""
+def install_reactor_import_hook() -> ReactorImportHook:
+    """Prevent importing :mod:`twisted.internet.reactor`.
 
-    sys.meta_path.insert(0, ReactorImportHook())
+    The hook is returned and can later be uninstalled with
+    :func:`uninstall_reactor_import_hook()`.
+    """
+
+    hook = ReactorImportHook()
+    sys.meta_path.insert(0, hook)
+    return hook
+
+
+def uninstall_reactor_import_hook(hook: ReactorImportHook) -> None:
+    """Uninstall the hook installed with :func:`install_reactor_import_hook()`."""
+    with contextlib.suppress(ValueError):
+        sys.meta_path.remove(hook)
diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook_multiple.py b/tests/AsyncCrawlerProcess/reactorless_import_hook_multiple.py
new file mode 100644
index 000000000..960bcbccb
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/reactorless_import_hook_multiple.py
@@ -0,0 +1,29 @@
+import logging
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.utils.reactorless import ReactorImportHook
+
+logger = logging.getLogger(__name__)
+
+
+def hook_count() -> int:
+    return sum(1 for finder in sys.meta_path if isinstance(finder, ReactorImportHook))
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        self.logger.info(f"Hooks during run: {hook_count()}")
+        return
+        yield
+
+
+for _ in range(2):
+    process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+    process.crawl(NoRequestsSpider)
+    process.start()
+
+logger.info(f"Hooks after runs: {hook_count()}")
diff --git a/tests/AsyncCrawlerProcess/reactorless_import_hook_uninstall.py b/tests/AsyncCrawlerProcess/reactorless_import_hook_uninstall.py
new file mode 100644
index 000000000..e1be1b490
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/reactorless_import_hook_uninstall.py
@@ -0,0 +1,29 @@
+import logging
+import sys
+
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+from scrapy.utils.reactorless import ReactorImportHook
+
+logger = logging.getLogger(__name__)
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+
+process.crawl(NoRequestsSpider)
+process.start()
+
+hook_count = sum(1 for finder in sys.meta_path if isinstance(finder, ReactorImportHook))
+logger.info(f"Hooks in sys.meta_path after start(): {hook_count}")
+
+import twisted.internet.reactor  # noqa: E402,F401,TID253
+
+logger.info("Reactor imported after start()")
diff --git a/tests/AsyncCrawlerProcess/reactorless_then_reactor.py b/tests/AsyncCrawlerProcess/reactorless_then_reactor.py
new file mode 100644
index 000000000..2566c1c9d
--- /dev/null
+++ b/tests/AsyncCrawlerProcess/reactorless_then_reactor.py
@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import AsyncCrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = "no_request"
+
+    async def start(self):
+        return
+        yield
+
+
+process = AsyncCrawlerProcess(settings={"TWISTED_REACTOR_ENABLED": False})
+process.crawl(NoRequestsSpider)
+process.start()
+
+process2 = AsyncCrawlerProcess()
+process2.crawl(NoRequestsSpider)
+process2.start()
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index 6e43615e4..fe2f83161 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -407,6 +407,38 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "Spider closed (finished)" in log
         assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
 
+    def test_reactorless_import_hook_uninstall(self) -> None:
+        """The import hook is removed when start() returns, so importing
+        twisted.internet.reactor becomes possible again."""
+        log = self.run_script("reactorless_import_hook_uninstall.py")
+        assert "Not using a Twisted reactor" in log
+        assert "Spider closed (finished)" in log
+        assert "Hooks in sys.meta_path after start(): 0" in log
+        assert "Reactor imported after start()" in log
+        assert "ImportError" not in log
+
+    def test_reactorless_import_hook_multiple(self) -> None:
+        """Sequential AsyncCrawlerProcess instances don't accumulate import
+        hooks: there is exactly one during each run and none afterwards."""
+        log = self.run_script("reactorless_import_hook_multiple.py")
+        assert log.count("Spider closed (finished)") == 2
+        assert log.count("Hooks during run: 1") == 2
+        assert "Hooks after runs: 0" in log
+        assert "ERROR: " not in log
+
+    def test_reactorless_then_reactor(self) -> None:
+        """After a reactorless run finishes, a reactor-based run is possible
+        in the same process."""
+        log = self.run_script("reactorless_then_reactor.py")
+        assert log.count("Spider closed (finished)") == 2
+        assert "Not using a Twisted reactor" in log
+        assert (
+            "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+            in log
+        )
+        assert "ImportError" not in log
+        assert "ERROR: " not in log
+
     def test_reactorless_telnetconsole_default(self) -> None:
         """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
         log = self.run_script("reactorless_simple.py")  # no need for a separate script
diff --git a/tests/test_utils_reactorless.py b/tests/test_utils_reactorless.py
new file mode 100644
index 000000000..2a575a8b3
--- /dev/null
+++ b/tests/test_utils_reactorless.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+import sys
+
+import pytest
+
+from scrapy.utils.reactorless import (
+    ReactorImportHook,
+    install_reactor_import_hook,
+    uninstall_reactor_import_hook,
+)
+
+
+class TestReactorImportHook:
+    def test_install_and_uninstall(self) -> None:
+        hook = install_reactor_import_hook()
+        try:
+            assert isinstance(hook, ReactorImportHook)
+            assert sys.meta_path[0] is hook
+        finally:
+            uninstall_reactor_import_hook(hook)
+        assert hook not in sys.meta_path
+
+    def test_uninstall_twice(self) -> None:
+        hook = install_reactor_import_hook()
+        uninstall_reactor_import_hook(hook)
+        uninstall_reactor_import_hook(hook)
+        assert hook not in sys.meta_path
+
+    def test_uninstall_not_installed(self) -> None:
+        uninstall_reactor_import_hook(ReactorImportHook())
+
+    def test_uninstall_only_removes_the_given_hook(self) -> None:
+        hook1 = install_reactor_import_hook()
+        hook2 = install_reactor_import_hook()
+        try:
+            uninstall_reactor_import_hook(hook1)
+            assert hook1 not in sys.meta_path
+            assert hook2 in sys.meta_path
+        finally:
+            uninstall_reactor_import_hook(hook1)
+            uninstall_reactor_import_hook(hook2)
+
+    def test_find_spec(self) -> None:
+        hook = ReactorImportHook()
+        with pytest.raises(
+            ImportError, match=r"Import of twisted\.internet\.reactor is forbidden"
+        ):
+            hook.find_spec("twisted.internet.reactor", None)
+        assert hook.find_spec("twisted.internet.defer", None) is None

From 64358f547bfa562ca9a4a6d20ab1e7dec0ec4d00 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 22 Jul 2026 11:58:05 +0500
Subject: [PATCH 28/62] Replace import tests with find_spec() or
 importorskip(). (#7755)

---
 conftest.py                                   | 13 +++-------
 .../downloadermiddlewares/httpcompression.py  |  7 ++----
 scrapy/utils/boto.py                          |  9 +++----
 ...st_downloadermiddleware_httpcompression.py | 25 ++++---------------
 tests/test_feedexport_storages.py             | 20 +++------------
 tests/utils/robotstxt.py                      | 11 +++-----
 6 files changed, 21 insertions(+), 64 deletions(-)

diff --git a/conftest.py b/conftest.py
index 7403b07b2..1bfb002cf 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-import importlib
+from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
 
@@ -50,9 +50,7 @@ if not H2_ENABLED:
         )
     )
 
-try:
-    import httpx  # noqa: F401
-except ImportError:
+if not find_spec("httpx"):
     collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
 
 
@@ -130,11 +128,8 @@ def pytest_runtest_setup(item):
     ]
 
     for module in optional_deps:
-        if item.get_closest_marker(f"requires_{module}"):
-            try:
-                importlib.import_module(module)
-            except ImportError:
-                pytest.skip(f"{module} is not installed")
+        if item.get_closest_marker(f"requires_{module}") and find_spec(module) is None:
+            pytest.skip(f"{module} is not installed")
 
     if item.get_closest_marker("requires_mitmproxy") and mitmdump_cmd() is None:
         pytest.skip("mitmdump is not available")
diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
index 13a432476..b99c323c5 100644
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from importlib.util import find_spec
 from itertools import chain
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
@@ -51,11 +52,7 @@ else:
     else:
         ACCEPTED_ENCODINGS.append(b"br")
 
-try:
-    import zstandard  # noqa: F401
-except ImportError:
-    pass
-else:
+if find_spec("zstandard") is not None:
     ACCEPTED_ENCODINGS.append(b"zstd")
 
 
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 73f86bc71..2a77ee2ac 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,10 +1,7 @@
 """Boto/botocore helpers"""
 
+from importlib.util import find_spec
+
 
 def is_botocore_available() -> bool:
-    try:
-        import botocore  # noqa: F401,PLC0415
-
-        return True
-    except ImportError:
-        return False
+    return find_spec("botocore") is not None
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index 311aff7df..ca18c8038 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -1,4 +1,5 @@
 from gzip import GzipFile
+from importlib.util import find_spec
 from io import BytesIO
 from logging import WARNING
 from pathlib import Path
@@ -65,10 +66,7 @@ def _skip_if_no_br() -> None:
 
 
 def _skip_if_no_zstd() -> None:
-    try:
-        import zstandard  # noqa: F401,PLC0415
-    except ImportError:
-        pytest.skip("no zstd support (zstandard)")
+    pytest.importorskip("zstandard")
 
 
 class TestHttpCompression:
@@ -172,17 +170,8 @@ class TestHttpCompression:
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
     def test_process_response_br_unsupported(self):
-        try:
-            try:
-                import brotli  # noqa: F401,PLC0415
-
-                pytest.skip("Requires not having brotli support")
-            except ImportError:
-                import brotlicffi  # noqa: F401,PLC0415
-
-                pytest.skip("Requires not having brotli support")
-        except ImportError:
-            pass
+        if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
+            pytest.skip("Requires not having brotli support")
         response = self._getresponse("br")
         request = response.request
         assert response.headers["Content-Encoding"] == b"br"
@@ -226,12 +215,8 @@ class TestHttpCompression:
             assert "Content-Encoding" not in newresponse.headers
 
     def test_process_response_zstd_unsupported(self):
-        try:
-            import zstandard  # noqa: F401,PLC0415
-
+        if find_spec("zstandard") is not None:
             pytest.skip("Requires not having zstandard support")
-        except ImportError:
-            pass
         response = self._getresponse("zstd-static-content-size")
         request = response.request
         assert response.headers["Content-Encoding"] == b"zstd"
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 82e2597a5..bdd0d7614 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -436,10 +436,7 @@ class TestS3FeedStorage:
 
 class TestGCSFeedStorage:
     def test_parse_settings(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": "publicRead"}
         crawler = get_crawler(settings_dict=settings)
@@ -450,10 +447,7 @@ class TestGCSFeedStorage:
         assert storage.blob_name == "export.csv"
 
     def test_parse_empty_acl(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         settings = {"GCS_PROJECT_ID": "123", "FEED_STORAGE_GCS_ACL": ""}
         crawler = get_crawler(settings_dict=settings)
@@ -467,10 +461,7 @@ class TestGCSFeedStorage:
 
     @coroutine_test
     async def test_store(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         uri = "gs://mybucket/export.csv"
         project_id = "myproject-123"
@@ -492,10 +483,7 @@ class TestGCSFeedStorage:
 
     @coroutine_test
     async def test_store_closes_file_on_upload_error(self):
-        try:
-            from google.cloud.storage import Client  # noqa: F401,PLC0415
-        except ImportError:
-            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+        pytest.importorskip("google.cloud.storage")
 
         uri = "gs://mybucket/export.csv"
         project_id = "myproject-123"
diff --git a/tests/utils/robotstxt.py b/tests/utils/robotstxt.py
index c1b2f28c6..38eaebf41 100644
--- a/tests/utils/robotstxt.py
+++ b/tests/utils/robotstxt.py
@@ -1,12 +1,7 @@
 from __future__ import annotations
 
+from importlib.util import find_spec
+
 
 def rerp_available() -> bool:
-    # check if robotexclusionrulesparser is installed
-    try:
-        from robotexclusionrulesparser import (  # noqa: PLC0415
-            RobotExclusionRulesParser,  # noqa: F401
-        )
-    except ImportError:
-        return False
-    return True
+    return find_spec("robotexclusionrulesparser") is not None

From a54c438da192da7df6b0c6fe6583aecf91839159 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Wed, 22 Jul 2026 10:44:26 +0200
Subject: [PATCH 29/62] Provide extras for optional dependencies (#7596)

---
 docs/conf.py                          |  1 +
 docs/intro/install.rst                | 50 ++++++++++++++++++++++++
 docs/topics/download-handlers.rst     | 28 +++++++++-----
 docs/topics/downloader-middleware.rst | 13 +++----
 docs/topics/feed-exports.rst          | 18 ++++-----
 docs/topics/media-pipeline.rst        | 24 +++++-------
 docs/topics/shell.rst                 | 27 +++++++------
 pyproject.toml                        | 19 +++++++++
 scrapy/utils/console.py               |  2 +-
 tests/test_utils_console.py           | 55 ++++++++++++++++++---------
 tox.ini                               |  4 +-
 11 files changed, 167 insertions(+), 74 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index b950c4ee2..de722baac 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -158,6 +158,7 @@ scrapy_intersphinx_enable = [
     "itemloaders",
     "parsel",
     "pytest",
+    "pypug",
     "scrapy-lint",
     "sphinx",
     "tox",
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 67a84e538..cba8c15a1 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -89,6 +89,56 @@ just like any other Python package.
 (See :ref:`platform-specific guides <intro-install-platform-notes>`
 below for non-Python dependencies that you may need to install beforehand).
 
+.. _extras:
+
+Optional extras
+===============
+
+Scrapy provides optional :ref:`extras <pypug:dependency-specifiers-extras>`
+that install additional dependencies to enable specific features. To install
+Scrapy with one or more extras, list them in square brackets:
+
+.. code-block:: console
+
+    pip install scrapy[s3,images]
+
+The following extras are available:
+
+.. list-table::
+   :header-rows: 1
+
+   * - Extra
+     - Provides
+   * - ``bpython``
+     - :ref:`bpython shell <shell-config>`
+   * - ``brotli``
+     - :ref:`Brotli response decompression <http-compression>`
+   * - ``gcs``
+     - :ref:`Google Cloud Storage <topics-feed-storage-gcs>` for
+       :ref:`feed exports <topics-feed-exports>` and
+       :ref:`media pipelines <media-pipeline-gcs>`
+   * - ``httpx``
+     - :ref:`httpx-handler`, including its HTTP/2 and SOCKS proxy support
+   * - ``images``
+     - :ref:`Images pipeline <images-pipeline>`
+   * - ``ipython``
+     - :ref:`IPython shell <shell-config>`
+   * - ``ptpython``
+     - :ref:`ptpython shell <shell-config>`
+   * - ``robotparser``
+     - :ref:`Robotexclusionrulesparser robots.txt parsing <rerp-parser>`
+   * - ``s3``
+     - :ref:`Amazon S3 <topics-feed-storage-s3>` storage for
+       :ref:`feed exports <topics-feed-exports>`,
+       :ref:`media pipelines <media-pipelines-s3>`, and
+       :ref:`S3 downloads <s3-handler>`
+   * - ``twisted-http2``
+     - :ref:`twisted-http2-handler`
+   * - ``uvloop``
+     - `uvloop <https://github.com/MagicStack/uvloop>`_ event loop
+   * - ``zstd``
+     - :ref:`Zstandard response decompression <http-compression>`
+
 
 .. _intro-install-platform-notes:
 
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 5fbc5df06..33cbf185e 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -173,6 +173,8 @@ of this package for more information.
 H2DownloadHandler
 -----------------
 
+.. note:: Requires the :ref:`twisted-http2 <extras>` extra.
+
 .. autoclass:: scrapy.core.downloader.handlers.http2.H2DownloadHandler
 
 | Supported scheme: ``https``.
@@ -185,9 +187,6 @@ for them.
 
 It's implemented using :mod:`twisted.web.client` and the ``h2`` library.
 
-For this handler to work you need to install the ``Twisted[http2]`` extra
-dependency.
-
 If you want to use this handler you need to replace the default one for the
 ``https`` scheme:
 
@@ -273,9 +272,13 @@ Other limitations:
 
 -   HTTPS proxies to HTTPS destinations are not supported.
 
+.. _httpx-handler:
+
 HttpxDownloadHandler
 --------------------
 
+.. note:: Requires the :ref:`httpx <extras>` extra.
+
 .. versionadded:: 2.15.0
 
 .. autoclass:: scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler
@@ -288,7 +291,9 @@ HttpxDownloadHandler
 This handler supports ``http://host/path`` and ``https://host/path`` URLs and
 uses the HTTP/1.1 or HTTP/2 protocol for them.
 
-It's implemented using the ``httpx`` library and needs it to be installed.
+It's implemented using the httpx_ library.
+
+.. _httpx: https://www.python-httpx.org/
 
 If you want to use this handler you need to replace the default ones for the
 ``http`` and ``https`` schemes:
@@ -311,8 +316,8 @@ Features and limitations
 
 =========================== =======================================
 HTTP proxies                Yes
-SOCKS proxies               Yes (SOCKS5; requires ``httpx[socks]``)
-HTTP/2                      Yes (requires ``httpx[http2]``)
+SOCKS proxies               Yes (SOCKS5)
+HTTP/2                      Yes
 ``response.certificate``    DER bytes
 Per-request ``bindaddress`` No (not supported by the library)
 TLS implementation          Standard library ``ssl``
@@ -333,8 +338,7 @@ HTTPX_HTTP2_ENABLED
 
 Default: ``False``
 
-Whether to enable HTTP/2 support in this handler. The ``httpx[http2]`` extra
-needs to be installed if you want to enable this setting.
+Whether to enable HTTP/2 support in this handler.
 
 Built-in non-HTTP download handlers reference
 =============================================
@@ -378,9 +382,13 @@ This handler supports ``ftp://host/path`` FTP URIs.
 
 It's implemented using :mod:`twisted.protocols.ftp`.
 
+.. _s3-handler:
+
 S3DownloadHandler
 -----------------
 
+.. note:: Requires the :ref:`s3 <extras>` extra.
+
 .. autoclass:: scrapy.core.downloader.handlers.s3.S3DownloadHandler
 
 | Supported scheme: ``s3``.
@@ -390,4 +398,6 @@ S3DownloadHandler
 
 This handler supports ``s3://bucket/path`` S3 URIs.
 
-It's implemented using the ``botocore`` library and needs it to be installed.
+It's implemented using the botocore_ library.
+
+.. _botocore: https://github.com/boto/botocore
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 096f87edd..934eb19ac 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -727,6 +727,8 @@ We assume that the spider will not issue Cache-Control directives
 in requests unless it actually needs them, so directives in requests are
 not filtered.
 
+.. _http-compression:
+
 HttpCompressionMiddleware
 -------------------------
 
@@ -738,14 +740,12 @@ HttpCompressionMiddleware
    This middleware allows compressed (gzip, deflate) traffic to be
    sent/received from web sites.
 
-   This middleware also supports decoding `brotli-compressed`_ as well as
-   `zstd-compressed`_ responses, provided that `brotli`_ or `zstandard`_ is
-   installed, respectively.
+   This middleware also supports decoding `brotli-compressed`_ responses with
+   the :ref:`brotli <extras>` extra, and `zstd-compressed`_
+   responses with the :ref:`zstd <extras>` extra.
 
 .. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
-.. _brotli: https://pypi.org/project/Brotli/
 .. _zstd-compressed: https://www.ietf.org/rfc/rfc8478.txt
-.. _zstandard: https://pypi.org/project/zstandard/
 
 
 HttpCompressionMiddleware Settings
@@ -1239,8 +1239,7 @@ Based on `Robotexclusionrulesparser <https://pypi.org/project/robotexclusionrule
 
 In order to use this parser:
 
-* Install ``Robotexclusionrulesparser`` by running
-  ``pip install robotexclusionrulesparser``
+* Install the :ref:`robotparser <extras>` extra.
 
 * Set :setting:`ROBOTSTXT_PARSER` setting to
   ``scrapy.robotstxt.RerpRobotParser``
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 51e142b04..66768c97b 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -92,7 +92,6 @@ Marshal
 -   Value for the ``format`` key in the :setting:`FEEDS` setting: ``marshal``
 -   Exporter used: :class:`~scrapy.exporters.MarshalItemExporter`
 
-
 .. _topics-feed-storage:
 
 Storages
@@ -106,14 +105,13 @@ The storages backends supported out of the box are:
 
 -   :ref:`topics-feed-storage-fs`
 -   :ref:`topics-feed-storage-ftp`
--   :ref:`topics-feed-storage-s3` (requires boto3_)
--   :ref:`topics-feed-storage-gcs` (requires `google-cloud-storage`_)
+-   :ref:`topics-feed-storage-s3` (requires the :ref:`s3 <extras>` extra)
+-   :ref:`topics-feed-storage-gcs` (requires the :ref:`gcs <extras>` extra)
 -   :ref:`topics-feed-storage-stdout`
 
-Some storage backends may be unavailable if the required external libraries are
-not available. For example, the S3 backend is only available if the boto3_
-library is installed.
-
+Some storage backends may be unavailable if the required :ref:`extras <extras>`
+are not installed. For example, the S3 backend requires the :ref:`s3 <extras>`
+extra.
 
 .. _topics-feed-uri-params:
 
@@ -209,7 +207,7 @@ The feeds are stored on `Amazon S3`_.
 
     -   ``s3://aws_key:aws_secret@mybucket/path/to/export.csv``
 
--   Required external libraries: `boto3`_ >= 1.20.0
+-   Required extras: :ref:`s3 <extras>`
 
 The AWS credentials can be passed as user/password in the URI, or they can be
 passed through the following settings:
@@ -249,7 +247,7 @@ The feeds are stored on `Google Cloud Storage`_.
 
     -   ``gs://mybucket/path/to/export.csv``
 
--   Required external libraries: `google-cloud-storage`_.
+-   Required extras: :ref:`gcs <extras>`
 
 For more information about authentication, please refer to `Google Cloud documentation <https://docs.cloud.google.com/docs/authentication>`_.
 
@@ -266,7 +264,6 @@ storage backend is: ``True``.
 
 This storage backend uses :ref:`delayed file delivery <delayed-file-delivery>`.
 
-.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
 
 
 .. _topics-feed-storage-stdout:
@@ -794,6 +791,5 @@ source spider in the feed URI:
 
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
 .. _Amazon S3: https://aws.amazon.com/s3/
-.. _boto3: https://github.com/boto/boto3
 .. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
 .. _Google Cloud Storage: https://cloud.google.com/storage/
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 56b53d18d..4ceb4732a 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -60,6 +60,8 @@ this:
 Using the Images Pipeline
 =========================
 
+.. note:: Requires the :ref:`images <extras>` extra.
+
 Using the :class:`ImagesPipeline` is a lot like using the :class:`FilesPipeline`,
 except the default field names used are different: you use ``image_urls`` for
 the image URLs of an item and it will populate an ``images`` field for the information
@@ -69,12 +71,6 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for
-thumbnailing and normalizing images to JPEG/RGB format.
-
-.. _Pillow: https://github.com/python-pillow/Pillow
-
-
 .. _topics-media-pipeline-enabling:
 
 Enabling your Media Pipeline
@@ -231,12 +227,13 @@ set the :setting:`FEED_STORAGE_FTP_ACTIVE` setting to ``True``.
 Amazon S3 storage
 -----------------
 
+.. note:: Requires the :ref:`s3 <extras>` extra.
+
 .. setting:: FILES_STORE_S3_ACL
 .. setting:: IMAGES_STORE_S3_ACL
 
-If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and
-:setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
-automatically upload the files to the bucket.
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent an Amazon S3
+bucket. Scrapy will automatically upload the files to the bucket.
 
 For example, this is a valid :setting:`IMAGES_STORE` value:
 
@@ -271,7 +268,6 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
     AWS_USE_SSL = False  # or True (None by default)
     AWS_VERIFY = False  # or True (None by default)
 
-.. _botocore: https://github.com/boto/botocore
 .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
 .. _Minio: https://github.com/minio/minio
 .. _Zenko CloudServer: https://www.zenko.io/cloudserver/
@@ -282,13 +278,13 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
 Google Cloud Storage
 ---------------------
 
+.. note:: Requires the :ref:`gcs <extras>` extra.
+
 .. setting:: FILES_STORE_GCS_ACL
 .. setting:: IMAGES_STORE_GCS_ACL
 
-:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud Storage
-bucket. Scrapy will automatically upload the files to the bucket. (requires `google-cloud-storage`_ )
-
-.. _google-cloud-storage: https://docs.cloud.google.com/storage/docs/reference/libraries#client-libraries-install-python
+:setting:`FILES_STORE` and :setting:`IMAGES_STORE` can represent a Google Cloud
+Storage bucket. Scrapy will automatically upload the files to the bucket.
 
 For example, these are valid :setting:`IMAGES_STORE` and :setting:`GCS_PROJECT_ID` settings:
 
diff --git a/docs/topics/shell.rst b/docs/topics/shell.rst
index ea97a22aa..6f7e67cf9 100644
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@@ -17,32 +17,35 @@ spider, without having to run the spider to test every change.
 Once you get familiarized with the Scrapy shell, you'll see that it's an
 invaluable tool for developing and debugging your spiders.
 
+.. _shell-config:
+
 Configuring the shell
 =====================
 
-If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
-standard Python console). The `IPython`_ console is much more powerful and
-provides smart auto-completion and colorized output, among other things.
+With the :ref:`ptpython <extras>` extra, the Scrapy shell will use ptpython_
+instead of the :term:`REPL`. ptpython provides syntax highlighting, smart
+auto-completion, and more.
 
-We highly recommend you install `IPython`_, especially if you're working on
-Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
-for more info.
+Failing that, with the :ref:`ipython <extras>` extra, the Scrapy shell will
+use IPython_ instead. IPython provides smart auto-completion, colorized
+output, and more.
 
-Scrapy also has support for `bpython`_, and will try to use it where `IPython`_
-is unavailable.
+Scrapy also has support for `bpython`_ via the :ref:`bpython <extras>` extra,
+and will try to use it where neither ptpython nor IPython is available.
 
 Through Scrapy's settings you can configure it to use any one of
-``ipython``, ``bpython`` or the standard ``python`` shell, regardless of which
-are installed. This is done by setting the ``SCRAPY_PYTHON_SHELL`` environment
-variable; or by defining it in your :ref:`scrapy.cfg <topics-config-settings>`:
+``ptpython``, ``ipython``, ``bpython`` or the standard ``python`` shell,
+regardless of which are installed. This is done by setting the
+``SCRAPY_PYTHON_SHELL`` environment variable; or by defining it in your
+:ref:`scrapy.cfg <topics-config-settings>`:
 
 .. code-block:: ini
 
     [settings]
     shell = bpython
 
+.. _ptpython: https://github.com/prompt-toolkit/ptpython
 .. _IPython: https://ipython.org/
-.. _IPython installation guide: https://ipython.org/install/
 .. _bpython: https://bpython-interpreter.org/
 
 Launch the shell
diff --git a/pyproject.toml b/pyproject.toml
index b24397c02..81d697c12 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -60,6 +60,25 @@ Source = "https://github.com/scrapy/scrapy"
 Tracker = "https://github.com/scrapy/scrapy/issues"
 "Release notes" = "https://docs.scrapy.org/en/latest/news.html"
 
+[project.optional-dependencies]
+bpython = ["bpython>=0.7.1"]
+brotli = [
+    "brotli>=1.2.0; implementation_name != 'pypy'",
+    "brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
+]
+gcs = ["google-cloud-storage>=1.29.0"]
+httpx = ["httpx[http2,socks]>=0.27.1"]
+images = ["Pillow>=8.3.2"]
+ipython = ["ipython>=7.1.0"]
+ptpython = ["ptpython>=2.0.1"]
+robotparser = ["robotexclusionrulesparser>=1.6.2"]
+s3 = ["boto3>=1.20.0"]
+twisted-http2 = ["Twisted[http2]>=21.7.0"]
+uvloop = [
+    "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'",
+]
+zstd = ["zstandard>=0.16.0; implementation_name != 'pypy'"]
+
 [project.scripts]
 scrapy = "scrapy.cmdline:execute"
 
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 93fbb9033..31a4bb32f 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -60,7 +60,7 @@ def _embed_ptpython_shell(
     namespace: dict[str, Any] | None = None, banner: str = ""
 ) -> EmbedFuncT:
     """Start a ptpython shell"""
-    import ptpython.repl  # noqa: PLC0415  # pylint: disable=import-error
+    import ptpython.repl  # noqa: PLC0415
 
     @wraps(_embed_ptpython_shell)
     def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index 05b272832..ad9aa3dff 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,24 +1,11 @@
 from __future__ import annotations
 
+from importlib.util import find_spec
+
 import pytest
 
 from scrapy.utils.console import get_shell_embed_func
 
-try:
-    import bpython
-
-    bpy = True
-    del bpython
-except ImportError:
-    bpy = False
-try:
-    import IPython
-
-    ipy = True
-    del IPython
-except ImportError:
-    ipy = False
-
 
 def test_get_shell_embed_func():
     shell = get_shell_embed_func(["invalid"])
@@ -29,16 +16,46 @@ def test_get_shell_embed_func():
     assert shell.__name__ == "_embed_standard_shell"
 
 
-@pytest.mark.skipif(not bpy, reason="bpython not available in testenv")
+def test_get_shell_embed_func_python():
+    # the standard shell is always available
+    shell = get_shell_embed_func(["python"])
+    assert callable(shell)
+    assert shell.__name__ == "_embed_standard_shell"
+
+
 def test_get_shell_embed_func_bpython():
+    pytest.importorskip("bpython")
     shell = get_shell_embed_func(["bpython"])
     assert callable(shell)
     assert shell.__name__ == "_embed_bpython_shell"
 
 
-@pytest.mark.skipif(not ipy, reason="IPython not available in testenv")
 def test_get_shell_embed_func_ipython():
-    # default shell should be 'ipython'
-    shell = get_shell_embed_func()
+    pytest.importorskip("IPython")
+    shell = get_shell_embed_func(["ipython"])
     assert shell is not None
     assert shell.__name__ == "_embed_ipython_shell"
+
+
+def test_get_shell_embed_func_ptpython():
+    pytest.importorskip("ptpython")
+    shell = get_shell_embed_func(["ptpython"])
+    assert shell is not None
+    assert shell.__name__ == "_embed_ptpython_shell"
+
+
+def test_get_shell_embed_func_default():
+    # with no shells given, the first available shell in preference order
+    # (ptpython, ipython, bpython, python) is returned; the standard shell
+    # is always available, so the default is never None
+    shell = get_shell_embed_func()
+    assert shell is not None
+    if find_spec("ptpython") is not None:
+        expected = "_embed_ptpython_shell"
+    elif find_spec("IPython") is not None:
+        expected = "_embed_ipython_shell"
+    elif find_spec("bpython") is not None:
+        expected = "_embed_bpython_shell"
+    else:
+        expected = "_embed_standard_shell"
+    assert shell.__name__ == expected
diff --git a/tox.ini b/tox.ini
index ec7273b42..b2f3b0db1 100644
--- a/tox.ini
+++ b/tox.ini
@@ -173,6 +173,7 @@ deps =
     google-cloud-storage
     httpx[http2,socks]
     ipython
+    ptpython  # optional for shell wrapper tests
     robotexclusionrulesparser
     uvloop; platform_system != "Windows" and implementation_name != "pypy"
     zstandard; implementation_name != "pypy"  # optional for HTTP compress downloader middleware tests
@@ -188,8 +189,9 @@ deps =
     brotli==1.2.0; implementation_name != "pypy"
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
-    httpx==0.27.1
+    httpx[http2,socks]==0.27.1
     ipython==7.1.0
+    ptpython==2.0.1
     robotexclusionrulesparser==1.6.2
     uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
     zstandard==0.16.0; implementation_name != "pypy"

From 1157b3e235c957f89e5d598ad2cc2db7779dae9c Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Wed, 22 Jul 2026 13:27:31 +0200
Subject: [PATCH 30/62] Add a middleware to detect response.meta copies (#7588)

---
 docs/topics/request-response.rst        |   3 +
 docs/topics/spider-middleware.rst       |  29 ++++++
 scrapy/settings/default_settings.py     |   1 +
 scrapy/spidermiddlewares/base.py        |   7 +-
 scrapy/spidermiddlewares/depth.py       |  13 ++-
 scrapy/spidermiddlewares/metacopy.py    |  65 +++++++++++++
 tests/test_spidermiddleware_depth.py    |  27 ++++++
 tests/test_spidermiddleware_metacopy.py | 121 ++++++++++++++++++++++++
 8 files changed, 260 insertions(+), 6 deletions(-)
 create mode 100644 scrapy/spidermiddlewares/metacopy.py
 create mode 100644 tests/test_spidermiddleware_metacopy.py

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index e56be0a22..7468f1b64 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -238,6 +238,9 @@ Request objects
         Also mind that the :meth:`copy` and :meth:`replace` request methods
         :doc:`shallow-copy <library/copy>` request metadata.
 
+        .. seealso:: :class:`~scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware`
+            for a built-in middleware that warns about this issue at run time.
+
     .. autoattribute:: dont_filter
 
     .. autoattribute:: Request.attributes
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 870adbc8c..aa14f6801 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -316,6 +316,35 @@ Default: ``False``
 Pass all responses, regardless of its status code.
 
 
+MetaCopyDetectionMiddleware
+---------------------------
+
+.. module:: scrapy.spidermiddlewares.metacopy
+   :synopsis: Meta Copy Detection Spider Middleware
+
+.. class:: MetaCopyDetectionMiddleware
+
+   Warns when a spider yields a request that contains internal meta keys which
+   should not be copied from :attr:`response.meta <scrapy.http.Response.meta>`
+   into new requests. See :attr:`~scrapy.http.Request.meta` to learn why.
+
+   Only 1 warning is emitted per crawl.
+
+   MetaCopyDetectionMiddleware settings
+   ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+   .. setting:: META_COPY_WARN_SKIP_KEYS
+
+   META_COPY_WARN_SKIP_KEYS
+   ^^^^^^^^^^^^^^^^^^^^^^^^
+
+   Default: ``[]``
+
+   A list of internal meta key names to exclude from the internal-keys check.
+   Use this when you intentionally copy one of the monitored keys and want to
+   suppress the resulting warning without disabling the middleware entirely.
+
+
 RefererMiddleware
 -----------------
 
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index d283336ab..993f2436d 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -554,6 +554,7 @@ SPIDER_MIDDLEWARES_BASE = {
     "scrapy.spidermiddlewares.referer.RefererMiddleware": 700,
     "scrapy.spidermiddlewares.urllength.UrlLengthMiddleware": 800,
     "scrapy.spidermiddlewares.depth.DepthMiddleware": 900,
+    "scrapy.spidermiddlewares.metacopy.MetaCopyDetectionMiddleware": 1000,
     # Spider side
 }
 
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index 6c62dccb8..3b14d2b0a 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -48,7 +48,10 @@ class BaseSpiderMiddleware:
 
     @_warn_spider_arg
     def process_spider_output(
-        self, response: Response, result: Iterable[Any], spider: Spider | None = None
+        self,
+        response: Response | None,
+        result: Iterable[Any],
+        spider: Spider | None = None,
     ) -> Iterable[Any]:
         for o in result:
             if (o := self._get_processed(o, response)) is not None:
@@ -57,7 +60,7 @@ class BaseSpiderMiddleware:
     @_warn_spider_arg
     async def process_spider_output_async(
         self,
-        response: Response,
+        response: Response | None,
         result: AsyncIterator[Any],
         spider: Spider | None = None,
     ) -> AsyncIterator[Any]:
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 9c387530c..054804119 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -55,19 +55,24 @@ class DepthMiddleware(BaseSpiderMiddleware):
 
     @_warn_spider_arg
     def process_spider_output(
-        self, response: Response, result: Iterable[Any], spider: Spider | None = None
+        self,
+        response: Response | None,
+        result: Iterable[Any],
+        spider: Spider | None = None,
     ) -> Iterable[Any]:
-        self._init_depth(response)
+        if response is not None:
+            self._init_depth(response)
         yield from super().process_spider_output(response, result)
 
     @_warn_spider_arg
     async def process_spider_output_async(
         self,
-        response: Response,
+        response: Response | None,
         result: AsyncIterator[Any],
         spider: Spider | None = None,
     ) -> AsyncIterator[Any]:
-        self._init_depth(response)
+        if response is not None:
+            self._init_depth(response)
         async for o in super().process_spider_output_async(response, result):
             yield o
 
diff --git a/scrapy/spidermiddlewares/metacopy.py b/scrapy/spidermiddlewares/metacopy.py
new file mode 100644
index 000000000..aa5a120c6
--- /dev/null
+++ b/scrapy/spidermiddlewares/metacopy.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+    from scrapy.http import Request, Response
+
+
+logger = logging.getLogger(__name__)
+
+
+class MetaCopyDetectionMiddleware(BaseSpiderMiddleware):
+    """Warn when a spider yields a request with internal meta keys that should
+    not be copied from response.meta, or when two requests share the same meta
+    dict object.
+
+    Each warning is emitted at most once per crawl.
+    """
+
+    _INTERNAL_KEYS: frozenset[str] = frozenset(
+        {
+            "_auth_proxy",
+            "_dont_cache",
+            "_scheme_proxy",
+            "download_latency",
+            "redirect_reasons",
+            "redirect_times",
+            "redirect_ttl",
+            "redirect_urls",
+            "retry_times",
+        }
+    )
+
+    def __init__(self, crawler: Crawler) -> None:
+        super().__init__(crawler)
+        skip = frozenset(crawler.settings.getlist("META_COPY_WARN_SKIP_KEYS", []))
+        self._keys: frozenset[str] = self._INTERNAL_KEYS - skip
+        self._warned: bool = False
+
+    def get_processed_request(
+        self, request: Request, response: Response | None
+    ) -> Request | None:
+        if response is None:
+            return request
+
+        if not self._warned:
+            found = self._keys & request.meta.keys()
+            if found:
+                spider_name = type(self.crawler.spider).__name__
+                logger.warning(
+                    f"{spider_name} yielded a request containing internal "
+                    f"meta keys that were likely copied from response.meta "
+                    f"and should not be forwarded to new requests: "
+                    f"{sorted(found)}. See the MetaCopyDetectionMiddleware "
+                    f"documentation for more information. Source response: "
+                    f"{response}, target request: {request}",
+                    extra={"spider": self.crawler.spider},
+                )
+                self._warned = True
+
+        return request
diff --git a/tests/test_spidermiddleware_depth.py b/tests/test_spidermiddleware_depth.py
index 307ed2440..32a2ea8f2 100644
--- a/tests/test_spidermiddleware_depth.py
+++ b/tests/test_spidermiddleware_depth.py
@@ -7,8 +7,10 @@ import pytest
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.depth import DepthMiddleware
 from scrapy.spiders import Spider
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -58,6 +60,31 @@ def test_process_spider_output(mw: DepthMiddleware, stats: StatsCollector) -> No
     assert rdm == 1
 
 
+def test_process_spider_output_no_response(
+    mw: DepthMiddleware, stats: StatsCollector
+) -> None:
+    result = [Request("https://example.com")]
+
+    out = list(mw.process_spider_output(None, result))
+    assert out == result
+    assert "depth" not in out[0].meta
+    assert stats.get_value("request_depth_count/0") is None
+
+
+@coroutine_test
+async def test_process_spider_output_async_no_response(
+    mw: DepthMiddleware, stats: StatsCollector
+) -> None:
+    result = [Request("https://example.com")]
+
+    out = await collect_asyncgen(
+        mw.process_spider_output_async(None, as_async_generator(result))
+    )
+    assert out == result
+    assert "depth" not in out[0].meta
+    assert stats.get_value("request_depth_count/0") is None
+
+
 def test_priority_and_non_verbose_stats() -> None:
     crawler = get_crawler(
         Spider,
diff --git a/tests/test_spidermiddleware_metacopy.py b/tests/test_spidermiddleware_metacopy.py
new file mode 100644
index 000000000..bfa27ac68
--- /dev/null
+++ b/tests/test_spidermiddleware_metacopy.py
@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from logging import WARNING
+from typing import TYPE_CHECKING
+
+import pytest
+
+from scrapy.http import Request, Response
+from scrapy.spidermiddlewares.metacopy import MetaCopyDetectionMiddleware
+from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
+
+def make_response(url: str = "https://example.com") -> Response:
+    response = Response(url)
+    response.request = Request(url)
+    return response
+
+
+@pytest.fixture
+def crawler() -> Crawler:
+    return get_crawler(Spider)
+
+
+@pytest.fixture
+def mw(crawler: Crawler) -> MetaCopyDetectionMiddleware:
+    return MetaCopyDetectionMiddleware.from_crawler(crawler)
+
+
+def process(
+    mw: MetaCopyDetectionMiddleware,
+    requests: list[Request],
+    response: Response | None = None,
+) -> list[Request]:
+    if response is None:
+        response = make_response()
+    return list(mw.process_spider_output(response, requests))
+
+
+class TestInternalKeysCheck:
+    def test_no_warning_for_clean_request(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        req = Request("https://example.com/1", meta={"my_key": "value"})
+        with caplog.at_level(WARNING):
+            process(mw, [req])
+        assert not caplog.records
+
+    def test_warns_on_internal_key(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        req = Request("https://example.com/1", meta={"retry_times": 1})
+        with caplog.at_level(WARNING):
+            process(mw, [req])
+        assert len(caplog.records) == 1
+        assert "retry_times" in caplog.text
+        assert "https://example.com/1" in caplog.text
+
+    def test_warns_once_across_multiple_requests(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        reqs = [
+            Request("https://example.com/1", meta={"retry_times": 1}),
+            Request("https://example.com/2", meta={"retry_times": 2}),
+        ]
+        with caplog.at_level(WARNING):
+            process(mw, reqs)
+        assert len(caplog.records) == 1
+
+    def test_reports_all_found_keys(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        req = Request(
+            "https://example.com/1",
+            meta={"retry_times": 1, "redirect_times": 2},
+        )
+        with caplog.at_level(WARNING):
+            process(mw, [req])
+        assert "retry_times" in caplog.text
+        assert "redirect_times" in caplog.text
+
+    def test_includes_source_response_in_message(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        response = make_response("https://source.example.com")
+        req = Request("https://example.com/1", meta={"retry_times": 1})
+        with caplog.at_level(WARNING):
+            process(mw, [req], response=response)
+        assert "https://source.example.com" in caplog.text
+
+    def test_skip_keys_setting(self, caplog: pytest.LogCaptureFixture) -> None:
+        with caplog.at_level(WARNING):
+            crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
+            mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
+            req = Request("https://example.com/1", meta={"retry_times": 1})
+            process(mw, [req])
+        assert not caplog.records
+
+    def test_skip_keys_setting_partial(self, caplog: pytest.LogCaptureFixture) -> None:
+        with caplog.at_level(WARNING):
+            crawler = get_crawler(Spider, {"META_COPY_WARN_SKIP_KEYS": ["retry_times"]})
+            mw = MetaCopyDetectionMiddleware.from_crawler(crawler)
+            req = Request(
+                "https://example.com/1",
+                meta={"retry_times": 1, "redirect_times": 2},
+            )
+            process(mw, [req])
+        assert len(caplog.records) == 1
+        assert "retry_times" not in caplog.text
+        assert "redirect_times" in caplog.text
+
+    def test_no_warning_for_start_requests(
+        self, mw: MetaCopyDetectionMiddleware, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        req = Request("https://example.com/1", meta={"retry_times": 1})
+        with caplog.at_level(WARNING):
+            list(mw.process_spider_output(None, [req]))
+        assert not caplog.records

From 56dee203e97aafb92e25e724d7bbbfbf4f940d3e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Jul 2026 13:29:22 +0500
Subject: [PATCH 31/62] Use httpx2 as a preferred alternative to httpx. (#7762)

---
 conftest.py                               |  2 +-
 docs/topics/asyncio.rst                   |  4 ++--
 docs/topics/download-handlers.rst         |  4 ++--
 pyproject.toml                            |  2 +-
 scrapy/core/downloader/handlers/_httpx.py | 23 +++++++++++++----------
 tests/test_downloader_handler_httpx.py    |  4 +++-
 tox.ini                                   | 10 +++++-----
 7 files changed, 27 insertions(+), 22 deletions(-)

diff --git a/conftest.py b/conftest.py
index 1bfb002cf..b8a9dc19e 100644
--- a/conftest.py
+++ b/conftest.py
@@ -50,7 +50,7 @@ if not H2_ENABLED:
         )
     )
 
-if not find_spec("httpx"):
+if find_spec("httpx2") is None and find_spec("httpx") is None:
     collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
 
 
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 83e40656b..63c217e93 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -153,8 +153,8 @@ Using Scrapy without a Twisted reactor
 .. note:: As the Twisted download handlers cannot be used without a reactor,
     the default download handler in this mode is
     :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`. You
-    will need to additionally install the ``httpx`` library to use it, unless
-    you switch to some different handler.
+    will need to additionally install the :ref:`httpx <extras>` extra to use
+    it, unless you switch to some different handler.
 
 It's possible to use Scrapy without installing a Twisted reactor at all, by
 setting the :setting:`TWISTED_REACTOR_ENABLED` setting to ``False``. In this
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 33cbf185e..34ab4f105 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -291,9 +291,9 @@ HttpxDownloadHandler
 This handler supports ``http://host/path`` and ``https://host/path`` URLs and
 uses the HTTP/1.1 or HTTP/2 protocol for them.
 
-It's implemented using the httpx_ library.
+It's implemented using the httpx2_ library.
 
-.. _httpx: https://www.python-httpx.org/
+.. _httpx2: https://httpx2.pydantic.dev/
 
 If you want to use this handler you need to replace the default ones for the
 ``http`` and ``https`` schemes:
diff --git a/pyproject.toml b/pyproject.toml
index 81d697c12..4220b12ed 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -67,7 +67,7 @@ brotli = [
     "brotlicffi>=1.2.0.0; implementation_name == 'pypy'",
 ]
 gcs = ["google-cloud-storage>=1.29.0"]
-httpx = ["httpx[http2,socks]>=0.27.1"]
+httpx = ["httpx2[http2,socks]>=2.0.0"]
 images = ["Pillow>=8.3.2"]
 ipython = ["ipython>=7.1.0"]
 ptpython = ["ptpython>=2.0.1"]
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 9b54b44d8..d5a4e9fcd 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -30,7 +30,7 @@ from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
-    from httpcore import AsyncNetworkStream
+    from httpcore2 import AsyncNetworkStream
 
     from scrapy import Request
     from scrapy.crawler import Crawler
@@ -39,8 +39,11 @@ if TYPE_CHECKING:
 HAS_SOCKS = HAS_HTTP2 = False
 
 try:
-    import httpx
-except ImportError:
+    try:
+        import httpx2 as httpx
+    except ImportError:  # pragma: no cover
+        import httpx  # type: ignore[import-not-found,no-redef]
+except ImportError:  # pragma: no cover
     httpx = None  # type: ignore[assignment]
 else:
     # a small hack to avoid importing these optional extras unconditionally
@@ -84,7 +87,7 @@ class HttpxDownloadHandler(_Base):
         self._enable_h2: bool = crawler.settings.getbool("HTTPX_HTTP2_ENABLED")
         if self._enable_h2 and not HAS_HTTP2:  # pragma: no cover
             raise NotConfigured(
-                f"HTTP/2 support in {type(self).__name__} requires the 'httpx[http2]' extra to be installed."
+                f"HTTP/2 support in {type(self).__name__} requires the 'httpx2[http2]' extra to be installed."
             )
         self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
         self._bind_host: str | None = self._get_bind_address_host()
@@ -96,7 +99,7 @@ class HttpxDownloadHandler(_Base):
         )
 
         self._default_client: httpx.AsyncClient = self._make_client()
-        # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,
+        # httpx2 doesn't support per-request proxies: https://github.com/pydantic/httpx2/issues/818,
         # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
         self._proxy_clients: dict[str, httpx.AsyncClient] = {}
 
@@ -104,7 +107,7 @@ class HttpxDownloadHandler(_Base):
     def _check_deps_installed() -> None:
         if httpx is None:  # pragma: no cover
             raise NotConfigured(
-                "HttpxDownloadHandler requires the httpx library to be installed."
+                "HttpxDownloadHandler requires the httpx2 library to be installed."
             )
 
     def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
@@ -128,7 +131,7 @@ class HttpxDownloadHandler(_Base):
                 proxy=proxy,
             ),
         )
-        # https://github.com/encode/httpx/discussions/1566
+        # https://github.com/pydantic/httpx2/issues/368
         for header_name in ("accept", "accept-encoding", "user-agent"):
             client.headers.pop(header_name, None)
         return client
@@ -149,7 +152,7 @@ class HttpxDownloadHandler(_Base):
         proxy = self._extract_proxy_url_with_creds(request)
         if proxy and proxy.startswith("socks") and not HAS_SOCKS:  # pragma: no cover
             raise ValueError(
-                f"SOCKS proxy support in {type(self).__name__} requires the 'httpx[socks]' extra to be installed."
+                f"SOCKS proxy support in {type(self).__name__} requires the 'httpx2[socks]' extra to be installed."
             )
         client = self._get_client(proxy)
         headers = self._request_headers(request).to_tuple_list()
@@ -175,7 +178,7 @@ class HttpxDownloadHandler(_Base):
             raise DownloadConnectionRefusedError(str(e)) from e
         except httpx.ProxyError as e:
             raise DownloadConnectionRefusedError(str(e)) from e
-        except DOWNLOAD_FAILED_EXCEPTIONS as e:
+        except DOWNLOAD_FAILED_EXCEPTIONS as e:  # pylint: disable=catching-non-exception
             raise DownloadFailedError(str(e)) from e
 
     @staticmethod
@@ -218,7 +221,7 @@ class HttpxDownloadHandler(_Base):
     def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
         network_stream: AsyncNetworkStream = response.extensions["network_stream"]
         extra_ssl_object = network_stream.get_extra_info("ssl_object")
-        if isinstance(extra_ssl_object, ssl.SSLObject):
+        if isinstance(extra_ssl_object, ssl.SSLObject):  # pragma: no branch
             _log_sslobj_debug_info(extra_ssl_object)
 
     async def close(self) -> None:
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index fdacad963..fd50248a7 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 import sys
+from importlib.util import find_spec
 from typing import TYPE_CHECKING, Any, ClassVar
 
 import pytest
@@ -37,7 +38,8 @@ if TYPE_CHECKING:
 
 pytestmark = pytest.mark.only_asyncio
 
-pytest.importorskip("httpx")
+if find_spec("httpx2") is None and find_spec("httpx") is None:
+    pytest.skip("Neither httpx2 nor httpx are installed", allow_module_level=True)
 
 
 class HttpxDownloadHandlerMixin:
diff --git a/tox.ini b/tox.ini
index b2f3b0db1..d35fe91db 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,7 +36,7 @@ minversion = 1.7.0
 deps =
     attrs
     coverage >= 7.10.6
-    httpx
+    httpx2
     pexpect >= 4.8.0
     pyftpdlib >= 2.0.1
     pygments
@@ -75,7 +75,7 @@ deps =
     boto3-stubs[s3]==1.43.41
     botocore-stubs==1.43.14
     h2==4.3.0
-    httpx==0.28.1
+    httpx2==2.7.0
     itemadapter==0.13.1
     ptpython==3.0.32
     # newer ones require newer Python
@@ -136,7 +136,7 @@ deps =
     Twisted==21.7.0
     cryptography==37.0.0
     cssselect==0.9.1
-    httpx==0.27.1
+    httpx2==2.0.0
     itemadapter==0.1.0
     lxml==4.6.4
     parsel==1.5.0
@@ -171,7 +171,7 @@ deps =
     brotli >= 1.2.0; implementation_name != "pypy"  # optional for HTTP compress downloader middleware tests
     brotlicffi >= 1.2.0.0; implementation_name == "pypy"  # optional for HTTP compress downloader middleware tests
     google-cloud-storage
-    httpx[http2,socks]
+    httpx2[http2,socks]
     ipython
     ptpython  # optional for shell wrapper tests
     robotexclusionrulesparser
@@ -189,7 +189,7 @@ deps =
     brotli==1.2.0; implementation_name != "pypy"
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
-    httpx[http2,socks]==0.27.1
+    httpx2[http2,socks]==2.0.0
     ipython==7.1.0
     ptpython==2.0.1
     robotexclusionrulesparser==1.6.2

From 628a3afbbda919ef1fbe7a11772842564135123a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 23 Jul 2026 13:38:56 +0500
Subject: [PATCH 32/62] Remove deprecated xmliter(), deprecate re_rsearch()
 (#7765)

---
 docs/news.rst                 |  2 +-
 scrapy/utils/iterators.py     | 60 ----------------------------
 scrapy/utils/python.py        |  8 +++-
 tests/test_utils_iterators.py | 75 ++++++++---------------------------
 4 files changed, 24 insertions(+), 121 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 4ebab3c6a..8f8477eaa 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5155,7 +5155,7 @@ Bug fixes
 *   The system file mode creation mask no longer affects the permissions of
     files generated using the :command:`startproject` command (:issue:`4722`)
 
-*   :func:`scrapy.utils.iterators.xmliter` now supports namespaced node names
+*   ``scrapy.utils.iterators.xmliter`` now supports namespaced node names
     (:issue:`861`, :issue:`4746`)
 
 *   :class:`~scrapy.Request` objects can now have ``about:`` URLs, which can
diff --git a/scrapy/utils/iterators.py b/scrapy/utils/iterators.py
index 9bb64c021..540fe407c 100644
--- a/scrapy/utils/iterators.py
+++ b/scrapy/utils/iterators.py
@@ -2,7 +2,6 @@ from __future__ import annotations
 
 import csv
 import logging
-import re
 from io import StringIO
 from typing import TYPE_CHECKING, Any, Literal, cast, overload
 from warnings import warn
@@ -12,7 +11,6 @@ from lxml import etree
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
-from scrapy.utils.python import re_rsearch
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterator
@@ -20,64 +18,6 @@ if TYPE_CHECKING:
 logger = logging.getLogger(__name__)
 
 
-def xmliter(obj: Response | str | bytes, nodename: str) -> Iterator[Selector]:
-    """Return a iterator of Selector's over all nodes of a XML document,
-       given the name of the node to iterate. Useful for parsing XML feeds.
-
-    obj can be:
-    - a Response object
-    - a unicode string
-    - a string encoded as utf-8
-    """
-    warn(
-        (
-            "xmliter is deprecated and its use strongly discouraged because "
-            "it is vulnerable to ReDoS attacks. Use xmliter_lxml instead. See "
-            "https://github.com/scrapy/scrapy/security/advisories/GHSA-cc65-xxvf-f7r9"
-        ),
-        ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-
-    nodename_patt = re.escape(nodename)
-
-    DOCUMENT_HEADER_RE = re.compile(r"<\?xml[^>]+>\s*", re.DOTALL)
-    HEADER_END_RE = re.compile(rf"<\s*/{nodename_patt}\s*>", re.DOTALL)
-    END_TAG_RE = re.compile(r"<\s*/([^\s>]+)\s*>", re.DOTALL)
-    NAMESPACE_RE = re.compile(r"((xmlns[:A-Za-z]*)=[^>\s]+)", re.DOTALL)
-    text = _body_or_str(obj)
-
-    document_header_match = re.search(DOCUMENT_HEADER_RE, text)
-    document_header = (
-        document_header_match.group().strip() if document_header_match else ""
-    )
-    header_end_idx = re_rsearch(HEADER_END_RE, text)
-    header_end = text[header_end_idx[1] :].strip() if header_end_idx else ""
-    namespaces: dict[str, str] = {}
-    if header_end:
-        for tagname in reversed(re.findall(END_TAG_RE, header_end)):
-            assert header_end_idx
-            tag = re.search(
-                rf"<\s*{tagname}.*?xmlns[:=][^>]*>",
-                text[: header_end_idx[1]],
-                re.DOTALL,
-            )
-            if tag:
-                for x in re.findall(NAMESPACE_RE, tag.group()):
-                    namespaces[x[1]] = x[0]
-
-    r = re.compile(rf"<{nodename_patt}[\s>].*?</{nodename_patt}>", re.DOTALL)
-    for match in r.finditer(text):
-        nodetext = (
-            document_header
-            + match.group().replace(
-                nodename, f"{nodename} {' '.join(namespaces.values())}", 1
-            )
-            + header_end
-        )
-        yield Selector(text=nodetext, type="xml")
-
-
 def xmliter_lxml(
     obj: Response | str | bytes,
     nodename: str,
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 00cf818b7..8a7517c1d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -113,7 +113,7 @@ def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
 
 def re_rsearch(
     pattern: str | Pattern[str], text: str, chunk_size: int = 1024
-) -> tuple[int, int] | None:
+) -> tuple[int, int] | None:  # pragma: no cover
     """
     This function does a reverse search in a text using a regular expression
     given in the attribute 'pattern'.
@@ -127,6 +127,12 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
+    warnings.warn(
+        "re_rsearch() is deprecated and will be removed in a future Scrapy version.",
+        category=ScrapyDeprecationWarning,
+        stacklevel=2,
+    )
+
     if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
diff --git a/tests/test_utils_iterators.py b/tests/test_utils_iterators.py
index 4fb18305e..d431b8017 100644
--- a/tests/test_utils_iterators.py
+++ b/tests/test_utils_iterators.py
@@ -1,28 +1,13 @@
 from __future__ import annotations
 
-from abc import ABC, abstractmethod
-from typing import TYPE_CHECKING, Any
-
 import pytest
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, XmlResponse
-from scrapy.utils.iterators import _body_or_str, csviter, xmliter, xmliter_lxml
+from scrapy.utils.iterators import _body_or_str, csviter, xmliter_lxml
 from tests import get_testdata
 
-if TYPE_CHECKING:
-    from collections.abc import Iterator
-
-    from scrapy import Selector
-
-
-class TestXmliterBase(ABC):
-    @abstractmethod
-    def xmliter(
-        self, obj: Response | str | bytes, nodename: str, *args: Any
-    ) -> Iterator[Selector]:
-        raise NotImplementedError
 
+class TestXmliter:
     def test_xmliter(self):
         body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
@@ -46,7 +31,7 @@ class TestXmliterBase(ABC):
                 x.xpath("name/text()").getall(),
                 x.xpath("./type/text()").getall(),
             )
-            for x in self.xmliter(response, "product")
+            for x in xmliter_lxml(response, "product")
         ]
 
         assert attrs == [
@@ -63,7 +48,7 @@ class TestXmliterBase(ABC):
         """
         response = XmlResponse(url="http://example.com", body=body)
         nodenames = [
-            e.xpath("name()").getall() for e in self.xmliter(response, "matchme...")
+            e.xpath("name()").getall() for e in xmliter_lxml(response, "matchme...")
         ]
         assert nodenames == [["matchme..."]]
 
@@ -117,7 +102,7 @@ class TestXmliterBase(ABC):
                     x.xpath("./skammstafanir/stuttskammstöfun/text()").getall(),
                     x.xpath("./tímabil/fyrstaþing/text()").getall(),
                 )
-                for x in self.xmliter(r, "þingflokkur")
+                for x in xmliter_lxml(r, "þingflokkur")
             ]
 
             assert attrs == [
@@ -132,7 +117,7 @@ class TestXmliterBase(ABC):
             "<products><product>one</product><product>two</product></products>"
         )
 
-        assert [x.xpath("text()").getall() for x in self.xmliter(body, "product")] == [
+        assert [x.xpath("text()").getall() for x in xmliter_lxml(body, "product")] == [
             ["one"],
             ["two"],
         ]
@@ -157,7 +142,7 @@ class TestXmliterBase(ABC):
             </rss>
         """
         response = XmlResponse(url="http://mydummycompany.com", body=body)
-        my_iter = self.xmliter(response, "item")
+        my_iter = xmliter_lxml(response, "item")
         node = next(my_iter)
         node.register_namespace("g", "http://base.google.com/ns/1.0")
         assert node.xpath("title/text()").getall() == ["Item 1"]
@@ -194,7 +179,7 @@ class TestXmliterBase(ABC):
             </rss>
         """
         response = XmlResponse(url="http://mydummycompany.com", body=body)
-        my_iter = self.xmliter(response, "g:image_link")
+        my_iter = xmliter_lxml(response, "g:image_link")
         node = next(my_iter)
         node.register_namespace("g", "http://base.google.com/ns/1.0")
         assert node.xpath("text()").extract() == [
@@ -221,7 +206,7 @@ class TestXmliterBase(ABC):
             </rss>
         """
         response = XmlResponse(url="http://mydummycompany.com", body=body)
-        my_iter = self.xmliter(response, "g:link_image")
+        my_iter = xmliter_lxml(response, "g:link_image")
         with pytest.raises(StopIteration):
             next(my_iter)
 
@@ -231,14 +216,14 @@ class TestXmliterBase(ABC):
             "<products><product>one</product><product>two</product></products>"
         )
 
-        my_iter = self.xmliter(body, "product")
+        my_iter = xmliter_lxml(body, "product")
         next(my_iter)
         next(my_iter)
         with pytest.raises(StopIteration):
             next(my_iter)
 
     def test_xmliter_objtype_exception(self):
-        i = self.xmliter(42, "product")  # type: ignore[arg-type]
+        i = xmliter_lxml(42, "product")  # type: ignore[arg-type]
         with pytest.raises(TypeError):
             next(i)
 
@@ -251,38 +236,10 @@ class TestXmliterBase(ABC):
         )
         response = XmlResponse("http://www.example.com", body=body)
         assert (
-            next(self.xmliter(response, "item")).get()
+            next(xmliter_lxml(response, "item")).get()
             == "<item>Some Turkish Characters \xd6\xc7\u015e\u0130\u011e\xdc \xfc\u011f\u0131\u015f\xe7\xf6</item>"
         )
 
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestXmliter(TestXmliterBase):
-    def xmliter(
-        self, obj: Response | str | bytes, nodename: str, *args: Any
-    ) -> Iterator[Selector]:
-        return xmliter(obj, nodename)
-
-    def test_deprecation(self):
-        body = b"""
-            <?xml version="1.0" encoding="UTF-8"?>
-            <products>
-              <product></product>
-            </products>
-        """
-        with pytest.warns(
-            ScrapyDeprecationWarning,
-            match="xmliter",
-        ):
-            next(self.xmliter(body, "product"))
-
-
-class TestLxmlXmliter(TestXmliterBase):
-    def xmliter(
-        self, obj: Response | str | bytes, nodename: str, *args: Any
-    ) -> Iterator[Selector]:
-        return xmliter_lxml(obj, nodename, *args)
-
     def test_xmliter_iterate_namespace(self):
         body = b"""
             <?xml version="1.0" encoding="UTF-8"?>
@@ -303,10 +260,10 @@ class TestLxmlXmliter(TestXmliterBase):
         """
         response = XmlResponse(url="http://mydummycompany.com", body=body)
 
-        no_namespace_iter = self.xmliter(response, "image_link")
+        no_namespace_iter = xmliter_lxml(response, "image_link")
         assert len(list(no_namespace_iter)) == 0
 
-        namespace_iter = self.xmliter(
+        namespace_iter = xmliter_lxml(
             response, "image_link", "http://base.google.com/ns/1.0"
         )
         node = next(namespace_iter)
@@ -338,14 +295,14 @@ class TestLxmlXmliter(TestXmliterBase):
         </root>
         """
         response = XmlResponse(url="http://mydummycompany.com", body=body)
-        my_iter = self.xmliter(response, "table", "http://www.w3.org/TR/html4/", "h")
+        my_iter = xmliter_lxml(response, "table", "http://www.w3.org/TR/html4/", "h")
 
         node = next(my_iter)
         assert len(node.xpath("h:tr/h:td").getall()) == 2
         assert node.xpath("h:tr/h:td[1]/text()").getall() == ["Apples"]
         assert node.xpath("h:tr/h:td[2]/text()").getall() == ["Bananas"]
 
-        my_iter = self.xmliter(
+        my_iter = xmliter_lxml(
             response, "table", "http://www.w3schools.com/furniture", "f"
         )
 

From 8489b3dad82d2323883575b17253e197cbdc80d3 Mon Sep 17 00:00:00 2001
From: Mridankan Mandal <xerontitan90@gmail.com>
Date: Thu, 23 Jul 2026 16:07:21 +0530
Subject: [PATCH 33/62] Support iterable values in BaseSettings.update()
 (#7763)

---
 scrapy/settings/__init__.py     | 15 +++++++++++----
 tests/test_settings/__init__.py | 10 ++++------
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 27b0f1187..5c5edf101 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -25,7 +25,9 @@ if TYPE_CHECKING:
     # typing.Self requires Python 3.11
     from typing_extensions import Self
 
-    _SettingsInput: TypeAlias = SupportsItems[str, Any] | str | None
+    _SettingsInput: TypeAlias = (
+        SupportsItems[str, Any] | Iterable[tuple[str, Any]] | str | None
+    )
 
 
 SETTINGS_PRIORITIES: dict[str, int] = {
@@ -560,7 +562,7 @@ class BaseSettings(MutableMapping[str, Any]):
             if key.isupper():
                 self.set(key, getattr(module, key), priority)
 
-    # BaseSettings.update() doesn't support all inputs that MutableMapping.update() supports
+    # BaseSettings.update() doesn't support kwargs input like MutableMapping.update().
     def update(self, values: _SettingsInput, priority: int | str = "project") -> None:  # type: ignore[override]
         """
         Store key/value pairs with a given priority.
@@ -577,7 +579,7 @@ class BaseSettings(MutableMapping[str, Any]):
         command.
 
         :param values: the settings names and values
-        :type values: dict or string or :class:`~scrapy.settings.BaseSettings`
+        :type values: dict, iterable, string or :class:`~scrapy.settings.BaseSettings`
 
         :param priority: the priority of the settings. Should be a key of
             :attr:`~scrapy.settings.SETTINGS_PRIORITIES` or an integer
@@ -591,7 +593,12 @@ class BaseSettings(MutableMapping[str, Any]):
                 for name, value in values.items():
                     self.set(name, value, cast("int", values.getpriority(name)))
             else:
-                for name, value in values.items():
+                items: Iterable[tuple[str, Any]]
+                if hasattr(values, "items"):
+                    items = cast("SupportsItems[str, Any]", values).items()
+                else:
+                    items = values
+                for name, value in items:
                     self.set(name, value, priority)
 
     def delete(self, name: str, priority: int | str = "project") -> None:
diff --git a/tests/test_settings/__init__.py b/tests/test_settings/__init__.py
index 3282b0591..97c095ae0 100644
--- a/tests/test_settings/__init__.py
+++ b/tests/test_settings/__init__.py
@@ -221,13 +221,11 @@ class TestBaseSettings:
         settings = BaseSettings({"key": 0})
         settings.update(key=1)  # pylint: disable=unexpected-keyword-arg
 
-    @pytest.mark.xfail(
-        raises=AttributeError,
-        reason="BaseSettings.update doesn't support iterable input",
-    )
     def test_update_iterable(self):
-        settings = BaseSettings({"key": 0})
-        settings.update([("key", 1)])
+        settings = BaseSettings({"key": 0}, priority=0)
+        settings.update([("key", 1)], priority=10)
+        assert settings["key"] == 1
+        assert settings.getpriority("key") == 10
 
     def test_update_jsonstring(self):
         settings = BaseSettings({"number": 0, "dict": BaseSettings({"key": "val"})})

From 67e52826845f29dedf33a21ae9d83211e4867be6 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 23 Jul 2026 13:02:41 +0200
Subject: [PATCH 34/62] Use autodoc for exceptions and improve their docs
 (#7767)

---
 docs/topics/exceptions.rst | 110 +++----------------------------------
 scrapy/exceptions.py       |  75 +++++++++++++++++++++----
 2 files changed, 74 insertions(+), 111 deletions(-)

diff --git a/docs/topics/exceptions.rst b/docs/topics/exceptions.rst
index 45acb1763..0aab90a43 100644
--- a/docs/topics/exceptions.rst
+++ b/docs/topics/exceptions.rst
@@ -1,117 +1,25 @@
 .. _topics-exceptions:
+.. _topics-exceptions-ref:
 
 ==========
 Exceptions
 ==========
 
-.. module:: scrapy.exceptions
-   :synopsis: Scrapy exceptions
-
-.. _topics-exceptions-ref:
-
-Built-in Exceptions reference
-=============================
-
 Here's a list of all exceptions included in Scrapy and their usage, except for
 the :ref:`download handler exceptions <download-handlers-exceptions>`.
 
+.. module:: scrapy.exceptions
 
-CloseSpider
------------
+.. autoexception:: CloseSpider
 
-.. exception:: CloseSpider(reason='cancelled')
+.. autoexception:: DontCloseSpider
 
-    This exception can be raised from a spider callback to request the spider to be
-    closed/stopped. Supported arguments:
+.. autoexception:: DropItem
 
-    :param reason: the reason for closing
-    :type reason: str
+.. autoexception:: IgnoreRequest
 
-For example:
+.. autoexception:: NotConfigured
 
-.. code-block:: python
+.. autoexception:: NotSupported
 
-    def parse_page(self, response):
-        if "Bandwidth exceeded" in response.text:
-            raise CloseSpider("bandwidth_exceeded")
-
-DontCloseSpider
----------------
-
-.. exception:: DontCloseSpider
-
-This exception can be raised in a :signal:`spider_idle` signal handler to
-prevent the spider from being closed.
-
-DropItem
---------
-
-.. exception:: DropItem
-
-The exception that must be raised by item pipeline stages to stop processing an
-Item. For more information see :ref:`topics-item-pipeline`.
-
-IgnoreRequest
--------------
-
-.. exception:: IgnoreRequest
-
-This exception can be raised by the Scheduler or any downloader middleware to
-indicate that the request should be ignored.
-
-NotConfigured
--------------
-
-.. exception:: NotConfigured
-
-This exception can be raised by some components to indicate that they will
-remain disabled. Those components include:
-
--   Extensions
--   Item pipelines
--   Downloader middlewares
--   Spider middlewares
-
-The exception must be raised in the component's ``__init__()`` or
-``from_crawler()``  method.
-
-NotSupported
-------------
-
-.. exception:: NotSupported
-
-This exception is raised to indicate an unsupported feature.
-
-StopDownload
--------------
-
-.. exception:: StopDownload(fail=True)
-
-Raised from a :class:`~scrapy.signals.bytes_received` or :class:`~scrapy.signals.headers_received`
-signal handler to indicate that no further bytes should be downloaded for a response.
-
-The ``fail`` boolean parameter controls which method will handle the resulting
-response:
-
-* If ``fail=True`` (default), the request errback is called. The response object is
-  available as the ``response`` attribute of the ``StopDownload`` exception,
-  which is in turn stored as the ``value`` attribute of the received
-  :class:`~twisted.python.failure.Failure` object. This means that in an errback
-  defined as ``def errback(self, failure)``, the response can be accessed though
-  ``failure.value.response``.
-
-* If ``fail=False``, the request callback is called instead.
-
-In both cases, the response could have its body truncated: the body contains
-all bytes received up until the exception is raised, including the bytes
-received in the signal handler that raises the exception. Also, the response
-object is marked with ``"download_stopped"`` in its :attr:`~scrapy.http.Response.flags`
-attribute.
-
-.. note:: ``fail`` is a keyword-only parameter, i.e. raising
-    ``StopDownload(False)`` or ``StopDownload(True)`` will raise
-    a :class:`TypeError`.
-
-See the documentation for the :class:`~scrapy.signals.bytes_received` and
-:class:`~scrapy.signals.headers_received` signals
-and the :ref:`topics-stop-response-download` topic for additional information and examples.
+.. autoexception:: StopDownload
diff --git a/scrapy/exceptions.py b/scrapy/exceptions.py
index 5330eab48..ccaccddf2 100644
--- a/scrapy/exceptions.py
+++ b/scrapy/exceptions.py
@@ -16,7 +16,15 @@ if TYPE_CHECKING:
 
 
 class NotConfigured(Exception):
-    """Indicates a missing configuration situation"""
+    """Raised by a :ref:`component <topics-components>` from its ``__init__()``
+    or :meth:`from_crawler` method to indicate that it will remain disabled.
+
+    Only the following components can be disabled this way:
+
+    -   :ref:`Downloader middlewares <topics-downloader-middleware>`
+    -   :ref:`Extensions <topics-extensions>`
+    -   :ref:`Item pipelines <topics-item-pipeline>`
+    -   :ref:`Spider middlewares <topics-spider-middleware>`"""
 
 
 class _InvalidOutput(TypeError):
@@ -30,15 +38,37 @@ class _InvalidOutput(TypeError):
 
 
 class IgnoreRequest(Exception):
-    """Indicates a decision was made not to process a request"""
+    """Raised to indicate that a request should be ignored.
+
+    A :ref:`downloader middleware <topics-downloader-middleware>` can raise it
+    from its
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_request`
+    or
+    :meth:`~scrapy.downloadermiddlewares.DownloaderMiddleware.process_response`
+    method to drop a request, and a :signal:`request_scheduled` signal handler
+    can raise it to drop a request before it reaches the
+    :ref:`scheduler <topics-scheduler>`."""
 
 
 class DontCloseSpider(Exception):
-    """Request the spider not to be closed yet"""
+    """Raised in a :signal:`spider_idle` signal handler to prevent the spider
+    from being closed."""
 
 
 class CloseSpider(Exception):
-    """Raise this from callbacks to request the spider to be closed"""
+    """Raised from a :ref:`spider callback <topics-spiders>` to request the
+    spider to be closed/stopped.
+
+    *reason* is a string with the reason for closing.
+
+    For example:
+
+    .. code-block:: python
+
+        def parse_page(self, response):
+            if "Bandwidth exceeded" in response.text:
+                raise CloseSpider("bandwidth_exceeded")
+    """
 
     def __init__(self, reason: str = "cancelled"):
         super().__init__()
@@ -46,10 +76,27 @@ class CloseSpider(Exception):
 
 
 class StopDownload(Exception):
-    """
-    Stop the download of the body for a given response.
-    The 'fail' boolean parameter indicates whether or not the resulting partial response
-    should be handled by the request errback. Note that 'fail' is a keyword-only argument.
+    """Raised from a :class:`~scrapy.signals.bytes_received` or
+    :class:`~scrapy.signals.headers_received` signal handler to :ref:`stop the
+    download <topics-stop-response-download>` of the response body.
+
+    The ``fail`` boolean parameter controls which method will handle the
+    resulting response:
+
+    * If ``fail=True`` (default), the request errback is called. The response
+      object is available as the ``response`` attribute of the ``StopDownload``
+      exception, which is in turn stored as the ``value`` attribute of the
+      received :class:`~twisted.python.failure.Failure` object. This means that
+      in an errback defined as ``def errback(self, failure)``, the response can
+      be accessed though ``failure.value.response``.
+
+    * If ``fail=False``, the request callback is called instead.
+
+    In both cases, the response could have its body truncated: the body contains
+    all bytes received up until the exception is raised, including the bytes
+    received in the signal handler that raises the exception. Also, the response
+    object is marked with ``"download_stopped"`` in its
+    :attr:`~scrapy.http.Response.flags` attribute.
     """
 
     response: Response | None
@@ -91,7 +138,8 @@ class UnsupportedURLSchemeError(Exception):
 
 
 class DropItem(Exception):
-    """Drop item from the item pipeline"""
+    """Raised from the :meth:`process_item` method of an :ref:`item pipeline
+    <topics-item-pipeline>` to stop the processing of an item."""
 
     def __init__(self, message: str, log_level: str | None = None):
         super().__init__(message)
@@ -99,7 +147,14 @@ class DropItem(Exception):
 
 
 class NotSupported(Exception):
-    """Indicates a feature or method is not supported"""
+    """Raised to indicate that a requested feature is not supported.
+
+    For example, Scrapy raises it when text-parsing shortcuts such as
+    :meth:`response.css() <scrapy.http.TextResponse.css>` or
+    :meth:`response.xpath() <scrapy.http.TextResponse.xpath>` are used on a
+    :class:`~scrapy.http.Response` whose content is not text, or when sending a
+    request whose URL scheme has no matching :ref:`download handler
+    <topics-download-handlers>`."""
 
 
 # Commands

From abbc024bbe81a9562a25f4e46a2bc1f2bae5bd7e Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 23 Jul 2026 16:48:59 +0200
Subject: [PATCH 35/62] Use autodoc for stats.rst (#7769)

---
 docs/topics/stats.rst     | 23 +++--------------------
 scrapy/statscollectors.py | 21 ++++++++++++++++++++-
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/docs/topics/stats.rst b/docs/topics/stats.rst
index 028fb009d..0cf4a72cc 100644
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@@ -94,27 +94,10 @@ default Stats Collector used is the :class:`MemoryStatsCollector`.
 MemoryStatsCollector
 --------------------
 
-.. class:: MemoryStatsCollector
-
-    A simple stats collector that keeps the stats of the last scraping run (for
-    each spider) in memory, after they're closed. The stats can be accessed
-    through the :attr:`spider_stats` attribute, which is a dict keyed by spider
-    name.
-
-    This is the default Stats Collector used in Scrapy.
-
-    .. attribute:: spider_stats
-
-       A dict of dicts (keyed by spider name) containing the stats of the last
-       scraping run for each spider.
+.. autoclass:: MemoryStatsCollector
+   :members:
 
 DummyStatsCollector
 -------------------
 
-.. class:: DummyStatsCollector
-
-    A Stats collector which does nothing but is very efficient (because it does
-    nothing). This stats collector can be set via the :setting:`STATS_CLASS`
-    setting, to disable stats collect in order to improve performance. However,
-    the performance penalty of stats collection is usually marginal compared to
-    other Scrapy workload like parsing pages.
+.. autoclass:: DummyStatsCollector
diff --git a/scrapy/statscollectors.py b/scrapy/statscollectors.py
index 1f9894519..3143fed63 100644
--- a/scrapy/statscollectors.py
+++ b/scrapy/statscollectors.py
@@ -101,9 +101,21 @@ class StatsCollector:
 
 
 class MemoryStatsCollector(StatsCollector):
+    """A simple stats collector that keeps the stats of the last scraping run
+    (for each spider) in memory, after they're closed. The stats can be
+    accessed through the :attr:`spider_stats` attribute, which is a dict keyed
+    by spider name.
+
+    This is the default stats collector used in Scrapy.
+    """
+
+    spider_stats: dict[str, StatsT]
+    """A dict of dicts (keyed by spider name) containing the stats of the last
+    scraping run for each spider."""
+
     def __init__(self, crawler: Crawler):
         super().__init__(crawler)
-        self.spider_stats: dict[str, StatsT] = {}
+        self.spider_stats = {}
 
     def _persist_stats(self, stats: StatsT) -> None:
         if self._crawler.spider:
@@ -111,6 +123,13 @@ class MemoryStatsCollector(StatsCollector):
 
 
 class DummyStatsCollector(StatsCollector):
+    """A stats collector which does nothing but is very efficient (because it
+    does nothing). This stats collector can be set via the
+    :setting:`STATS_CLASS` setting, to disable stats collection in order to
+    improve performance. However, the performance penalty of stats collection
+    is usually marginal compared to other Scrapy workload like parsing pages.
+    """
+
     def get_value(
         self, key: str, default: Any = None, spider: Spider | None = None
     ) -> Any:

From 0b578c1cbfec68b050c9e088bf326ab44cba147d Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Fri, 24 Jul 2026 13:24:54 +0200
Subject: [PATCH 36/62] trackref: use autodoc (#7771)

---
 docs/topics/leaks.rst    | 25 ++++---------------------
 scrapy/utils/trackref.py | 18 ++++++++++++++----
 2 files changed, 18 insertions(+), 25 deletions(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index d4577601d..0a8b146d2 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -187,30 +187,13 @@ scrapy.utils.trackref module
 
 Here are the functions available in the :mod:`~scrapy.utils.trackref` module.
 
-.. class:: object_ref
+.. autoclass:: object_ref
 
-    Inherit from this class if you want to track live
-    instances with the ``trackref`` module.
+.. autofunction:: print_live_refs(ignore=NoneType)
 
-.. function:: print_live_refs(ignore=NoneType)
+.. autofunction:: get_oldest
 
-    Print a report of live references, grouped by class name.
-
-    :param ignore: if given, all objects from the specified class (or tuple of
-        classes) will be ignored.
-    :type ignore: type or tuple
-
-.. function:: get_oldest(class_name)
-
-    Return the oldest object alive with the given class name, or ``None`` if
-    none is found. Use :func:`print_live_refs` first to get a list of all
-    tracked live objects per class name.
-
-.. function:: iter_all(class_name)
-
-    Return an iterator over all objects alive with the given class name. Use
-    :func:`print_live_refs` first to get a list of all tracked live objects
-    per class name.
+.. autofunction:: iter_all
 
 .. skip: end
 
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 22f9eadd0..0bbe68def 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -34,7 +34,8 @@ live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
 
 
 class object_ref:
-    """Inherit from this class to a keep a record of live instances"""
+    """Inherit from this class if you want to track live instances with the
+    ``trackref`` module."""
 
     __slots__ = ()
 
@@ -60,12 +61,19 @@ def format_live_refs(ignore: Any = NoneType) -> str:
 
 
 def print_live_refs(*a: Any, **kw: Any) -> None:
-    """Print tracked objects"""
+    """Print a report of live references, grouped by class name.
+
+    :param ignore: if given, all objects from the specified class (or tuple of
+        classes) will be ignored.
+    :type ignore: type or tuple
+    """
     print(format_live_refs(*a, **kw))
 
 
 def get_oldest(class_name: str) -> Any:
-    """Get the oldest object for a specific class name"""
+    """Return the oldest object alive with the given class name, or ``None`` if
+    none is found. Use :func:`print_live_refs` first to get a list of all
+    tracked live objects per class name."""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
@@ -75,7 +83,9 @@ def get_oldest(class_name: str) -> Any:
 
 
 def iter_all(class_name: str) -> Iterable[Any]:
-    """Iterate over all objects of the same class by its class name"""
+    """Return an iterator over all objects alive with the given class name. Use
+    :func:`print_live_refs` first to get a list of all tracked live objects per
+    class name."""
     for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             return wdict.keys()

From 41bb09741a5d24ccae8687fed990c41219d2f0d5 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Fri, 24 Jul 2026 15:35:54 +0200
Subject: [PATCH 37/62] Use autodoc for contracts (#7775)

---
 docs/topics/contracts.rst    | 60 +++++--------------------------
 scrapy/contracts/__init__.py | 18 +++++++++-
 scrapy/contracts/default.py  | 68 ++++++++++++++++++++++++++----------
 3 files changed, 76 insertions(+), 70 deletions(-)

diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index 61aef4bbb..df67bee02 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -30,43 +30,15 @@ You can use the following contracts:
 
 .. module:: scrapy.contracts.default
 
-.. class:: UrlContract
+.. autoclass:: UrlContract
 
-    This contract (``@url``) sets the sample URL used when checking other
-    contract conditions for this spider. This contract is mandatory. All
-    callbacks lacking this contract are ignored when running the checks::
+.. autoclass:: CallbackKeywordArgumentsContract
 
-    @url url
+.. autoclass:: MetadataContract
 
-.. class:: CallbackKeywordArgumentsContract
+.. autoclass:: ReturnsContract
 
-    This contract (``@cb_kwargs``) sets the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
-    attribute for the sample request. It must be a valid JSON dictionary.
-    ::
-
-    @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: MetadataContract
-
-    This contract (``@meta``) sets the :attr:`meta <scrapy.Request.meta>`
-    attribute for the sample request. It must be a valid JSON dictionary.
-    ::
-
-    @meta {"arg1": "value1", "arg2": "value2", ...}
-
-.. class:: ReturnsContract
-
-    This contract (``@returns``) sets lower and upper bounds for the items and
-    requests returned by the spider. The upper bound is optional::
-
-    @returns item(s)|request(s) [min [max]]
-
-.. class:: ScrapesContract
-
-    This contract (``@scrapes``) checks that all the items returned by the
-    callback have the specified fields::
-
-    @scrapes field_1 field_2 ...
+.. autoclass:: ScrapesContract
 
 Use the :command:`check` command to run the contract checks.
 
@@ -89,30 +61,16 @@ override three methods:
 
 .. module:: scrapy.contracts
 
-.. class:: Contract(method, *args)
+.. autoclass:: Contract
 
-    :param method: callback function to which the contract is associated
-    :type method: collections.abc.Callable
+    .. automethod:: adjust_request_args
 
-    :param args: list of arguments passed into the docstring (whitespace
-        separated)
-    :type args: list
-
-    .. method:: Contract.adjust_request_args(args)
-
-        This receives a ``dict`` as an argument containing default arguments
-        for request object. :class:`~scrapy.Request` is used by default,
-        but this can be changed with the ``request_cls`` attribute.
-        If multiple contracts in chain have this attribute defined, the last one is used.
-
-        Must return the same or a modified version of it.
-
-    .. method:: Contract.pre_process(response)
+    .. method:: pre_process(response)
 
         This allows hooking in various checks on the response received from the
         sample request, before it's being passed to the callback.
 
-    .. method:: Contract.post_process(output)
+    .. method:: post_process(output)
 
         This allows processing the output of the callback. Iterators are
         converted to lists before being passed to this hook.
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index ebbdf1b98..dbdbbd456 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -22,7 +22,16 @@ if TYPE_CHECKING:
 
 
 class Contract:
-    """Abstract class for contracts"""
+    """Base class for :ref:`custom contracts <topics-contracts>`.
+
+    *method* is the callback function to which the contract is associated.
+
+    *args* is the list of arguments passed into the docstring, separated by
+    whitespace.
+
+    Subclasses may override :meth:`adjust_request_args`, and define a
+    ``pre_process`` method or a ``post_process`` method, or both.
+    """
 
     request_cls: type[Request] | None = None
     name: str
@@ -90,6 +99,13 @@ class Contract:
         return request
 
     def adjust_request_args(self, args: dict[str, Any]) -> dict[str, Any]:
+        """Receive a ``dict`` with the default arguments for the sample request
+        and return it, either unmodified or with changes.
+
+        :class:`~scrapy.Request` is used by default, but this can be changed
+        with the ``request_cls`` attribute. If multiple contracts in the chain
+        define this attribute, the last one is used.
+        """
         return args
 
 
diff --git a/scrapy/contracts/default.py b/scrapy/contracts/default.py
index 9b42ca36f..e2e27165a 100644
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@@ -15,8 +15,15 @@ if TYPE_CHECKING:
 
 # contracts
 class UrlContract(Contract):
-    """Contract to set the url of the request (mandatory)
-    @url http://scrapy.org
+    """Sets (``@url``) the sample URL used when checking the other contract
+    conditions of a callback.
+
+    This contract is mandatory: callbacks lacking it are ignored when running
+    the checks.
+
+    .. code-block:: none
+
+        @url url
     """
 
     name = "url"
@@ -27,10 +34,14 @@ class UrlContract(Contract):
 
 
 class CallbackKeywordArgumentsContract(Contract):
-    """Contract to set the keyword arguments for the request.
-    The value should be a JSON-encoded dictionary, e.g.:
+    """Sets (``@cb_kwargs``) the :attr:`cb_kwargs <scrapy.Request.cb_kwargs>`
+    attribute of the sample request.
 
-    @cb_kwargs {"arg1": "some value"}
+    Its value must be a valid JSON dictionary.
+
+    .. code-block:: none
+
+        @cb_kwargs {"arg1": "value1", "arg2": "value2", ...}
     """
 
     name = "cb_kwargs"
@@ -41,10 +52,14 @@ class CallbackKeywordArgumentsContract(Contract):
 
 
 class MetadataContract(Contract):
-    """Contract to set metadata arguments for the request.
-    The value should be JSON-encoded dictionary, e.g.:
+    """Sets (``@meta``) the :attr:`meta <scrapy.Request.meta>` attribute of the
+    sample request.
 
-    @meta {"arg1": "some value"}
+    Its value must be a valid JSON dictionary.
+
+    .. code-block:: none
+
+        @meta {"arg1": "value1", "arg2": "value2", ...}
     """
 
     name = "meta"
@@ -55,16 +70,29 @@ class MetadataContract(Contract):
 
 
 class ReturnsContract(Contract):
-    """Contract to check the output of a callback
+    """Sets (``@returns``) lower and upper bounds for the items and requests
+    returned by a callback.
 
-    general form:
-    @returns request(s)/item(s) [min=1 [max]]
+    The upper bound is optional:
 
-    e.g.:
-    @returns request
-    @returns request 2
-    @returns request 2 10
-    @returns request 0 10
+    .. code-block:: none
+
+        @returns item(s)|request(s) [min [max]]
+
+    For example:
+
+    .. code-block:: none
+
+        @returns request
+        @returns request 2
+        @returns request 2 10
+        @returns request 0 10
+
+    Set both bounds to the same value to require an exact number:
+
+    .. code-block:: none
+
+        @returns request 2 2
     """
 
     name = "returns"
@@ -115,8 +143,12 @@ class ReturnsContract(Contract):
 
 
 class ScrapesContract(Contract):
-    """Contract to check presence of fields in scraped items
-    @scrapes page_name page_body
+    """Checks (``@scrapes``) that all items returned by a callback have the
+    specified fields.
+
+    .. code-block:: none
+
+        @scrapes field_1 field_2 ...
     """
 
     name = "scrapes"

From 58ed9fdcccc9d8a3c278a0bd0af54ff77863b034 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Fri, 24 Jul 2026 17:08:58 +0200
Subject: [PATCH 38/62] Document urlparse_cached (#7777)

---
 docs/topics/request-response.rst |  2 ++
 scrapy/utils/httpobj.py          | 11 +++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 7468f1b64..b83a04032 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -342,6 +342,8 @@ Other functions related to requests
 
 .. autofunction:: scrapy.utils.request.request_from_dict
 
+.. autofunction:: scrapy.utils.httpobj.urlparse_cached
+
 
 .. _topics-request-response-ref-request-callback-arguments:
 
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index 58b4539bf..5965e1f21 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = (
 
 
 def urlparse_cached(request_or_response: Request | Response) -> ParseResult:
-    """Return urlparse.urlparse caching the result, where the argument can be a
-    Request or Response object
+    """Return the result of parsing the URL of *request_or_response*, a
+    :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with
+    :func:`urllib.parse.urlparse`.
+
+    The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on
+    *request_or_response*, so that the URL of a given object is parsed only
+    once. Prefer this function over calling :func:`urllib.parse.urlparse` on
+    ``request_or_response.url`` directly when the same URL may be parsed more
+    than once.
     """
     if request_or_response not in _urlparse_cache:
         _urlparse_cache[request_or_response] = urlparse(request_or_response.url)

From e4ae4aad52145ba3e525be0d1ed90b008ee6fa6d Mon Sep 17 00:00:00 2001
From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com>
Date: Sat, 25 Jul 2026 08:19:53 +0300
Subject: [PATCH 39/62] docs: fix missing 'of' in faq.rst (#7780)

'instead joining the strings' -> 'instead of joining the strings'.
---
 docs/faq.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 8f2013581..dc909cf56 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -136,7 +136,7 @@ middleware with a :ref:`custom downloader middleware
 <topics-downloader-middleware-custom>` that requires less memory. For example:
 
 -   If your domain names are similar enough, use your own regular expression
-    instead joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
+    instead of joining the strings in :attr:`~scrapy.Spider.allowed_domains` into
     a complex regular expression.
 
 -   If you can meet the installation requirements, use pyre2_ instead of

From 96195e4a61209070845df40c3ba046a00f1e5714 Mon Sep 17 00:00:00 2001
From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com>
Date: Sat, 25 Jul 2026 08:20:18 +0300
Subject: [PATCH 40/62] docs: fix a/an article before XML in faq.rst (#7779)

'a XML document' -> 'an XML document' (XML is pronounced with a vowel sound).
---
 docs/faq.rst | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index dc909cf56..0446a6868 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -332,8 +332,8 @@ section of the site (which varies each time). In that case, the credentials to
 log in would be settings, while the url of the section to scrape would be a
 spider argument.
 
-I'm scraping a XML document and my XPath selector doesn't return any items
---------------------------------------------------------------------------
+I'm scraping an XML document and my XPath selector doesn't return any items
+---------------------------------------------------------------------------
 
 You may need to remove namespaces. See :ref:`removing-namespaces`.
 

From e710b9c18e18f0a3fe104fbfc72d49c221dfe448 Mon Sep 17 00:00:00 2001
From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com>
Date: Sat, 25 Jul 2026 08:20:30 +0300
Subject: [PATCH 41/62] core: fix verb tense in http2 stream comment (#7778)

'needs to be send' -> 'needs to be sent'. Comment-only, no functional change.
---
 scrapy/core/http2/stream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 4d072c555..c6226bbca 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -315,7 +315,7 @@ class Stream:
             0, self.metadata["remaining_content_length"]
         )
 
-        # End the stream if no more data needs to be send
+        # End the stream if no more data needs to be sent
         if self.metadata["remaining_content_length"] == 0:
             self._protocol.conn.end_stream(self.stream_id)
 

From 13be37e4b1076643d2914bb891ba68dd84675d99 Mon Sep 17 00:00:00 2001
From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com>
Date: Mon, 27 Jul 2026 11:07:32 +0300
Subject: [PATCH 42/62] Add type hints to test_link.py and
 test_downloadermiddleware_stats.py (#7785)

---
 pyproject.toml                           |  3 ---
 tests/test_downloadermiddleware_stats.py | 25 +++++++++++++++---------
 tests/test_link.py                       | 16 ++++++++-------
 tests/test_logstats.py                   | 11 +++++++----
 4 files changed, 32 insertions(+), 23 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 4220b12ed..5ed07a9ce 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -136,7 +136,6 @@ module = [
     "tests.test_downloadermiddleware_redirect_metarefresh",
     "tests.test_downloadermiddleware_retry",
     "tests.test_downloadermiddleware_robotstxt",
-    "tests.test_downloadermiddleware_stats",
     "tests.test_downloaderslotssettings",
     "tests.test_dupefilters",
     "tests.test_engine_loop",
@@ -154,11 +153,9 @@ module = [
     "tests.test_http_response",
     "tests.test_http_response_text",
     "tests.test_item",
-    "tests.test_link",
     "tests.test_linkextractors",
     "tests.test_loader",
     "tests.test_logformatter",
-    "tests.test_logstats",
     "tests.test_mail",
     "tests.test_pipeline_crawl",
     "tests.test_pipeline_files",
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 5609360a7..b8fda25b4 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats, get_header_size
@@ -12,8 +14,9 @@ class MyException(Exception):
 
 
 class TestDownloaderStats:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.crawler = get_crawler(Spider)
+        assert self.crawler.stats is not None
         self.mw = DownloaderStats(self.crawler.stats)
 
         self.crawler.stats.open_spider()
@@ -21,20 +24,21 @@ class TestDownloaderStats:
         self.req = Request("http://scrapytest.org")
         self.res = Response("http://scrapytest.org", status=400)
 
-    def assertStatsEqual(self, key, value):
+    def assertStatsEqual(self, key: str, value: object) -> None:
+        assert self.crawler.stats is not None
         assert self.crawler.stats.get_value(key) == value, str(
             self.crawler.stats.get_stats()
         )
 
-    def test_process_request(self):
+    def test_process_request(self) -> None:
         self.mw.process_request(self.req)
         self.assertStatsEqual("downloader/request_count", 1)
 
-    def test_process_response(self):
+    def test_process_response(self) -> None:
         self.mw.process_response(self.req, self.res)
         self.assertStatsEqual("downloader/response_count", 1)
 
-    def test_process_exception(self):
+    def test_process_exception(self) -> None:
         self.mw.process_exception(self.req, MyException())
         self.assertStatsEqual("downloader/exception_count", 1)
         self.assertStatsEqual(
@@ -42,14 +46,17 @@ class TestDownloaderStats:
             1,
         )
 
-    def test_from_crawler_not_configured(self):
+    def test_from_crawler_not_configured(self) -> None:
         crawler = get_crawler(Spider, {"DOWNLOADER_STATS": False})
         with pytest.raises(NotConfigured):
             DownloaderStats.from_crawler(crawler)
 
-    def teardown_method(self):
+    def teardown_method(self) -> None:
+        assert self.crawler.stats is not None
         self.crawler.stats.close_spider()
 
 
-def test_get_header_size_non_list_value():
-    assert get_header_size({"Content-Type": "text/html"}) == 0
+def test_get_header_size_non_list_value() -> None:
+    # Deliberately passing a non-list/tuple header value to make sure
+    # get_header_size() degrades gracefully instead of raising.
+    assert get_header_size({"Content-Type": "text/html"}) == 0  # type: ignore[dict-item]
diff --git a/tests/test_link.py b/tests/test_link.py
index 0eeffe12b..40c53edd3 100644
--- a/tests/test_link.py
+++ b/tests/test_link.py
@@ -1,18 +1,20 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.link import Link
 
 
 class TestLink:
-    def _assert_same_links(self, link1, link2):
+    def _assert_same_links(self, link1: Link, link2: Link) -> None:
         assert link1 == link2
         assert hash(link1) == hash(link2)
 
-    def _assert_different_links(self, link1, link2):
+    def _assert_different_links(self, link1: Link, link2: Link) -> None:
         assert link1 != link2
         assert hash(link1) != hash(link2)
 
-    def test_eq_and_hash(self):
+    def test_eq_and_hash(self) -> None:
         l1 = Link("http://www.example.com")
         l2 = Link("http://www.example.com/other")
         l3 = Link("http://www.example.com")
@@ -45,17 +47,17 @@ class TestLink:
         self._assert_different_links(l7, l9)
         self._assert_different_links(l7, l10)
 
-    def test_repr(self):
+    def test_repr(self) -> None:
         l1 = Link(
             "http://www.example.com", text="test", fragment="something", nofollow=True
         )
         l2 = eval(repr(l1))
         self._assert_same_links(l1, l2)
 
-    def test_bytes_url(self):
+    def test_bytes_url(self) -> None:
         with pytest.raises(TypeError):
-            Link(b"http://www.example.com/\xc2\xa3")
+            Link(b"http://www.example.com/\xc2\xa3")  # type: ignore[arg-type]
 
-    def test_eq_non_link(self):
+    def test_eq_non_link(self) -> None:
         url = "http://example.com"
         assert Link(url) != url
diff --git a/tests/test_logstats.py b/tests/test_logstats.py
index 370728e6a..213681ad1 100644
--- a/tests/test_logstats.py
+++ b/tests/test_logstats.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from datetime import datetime
 
 import pytest
@@ -9,16 +11,17 @@ from tests.utils.decorators import coroutine_test
 
 
 class TestLogStats:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.crawler = get_crawler(SimpleSpider)
         self.spider = self.crawler._create_spider("spidey")
+        assert self.crawler.stats is not None
         self.stats = self.crawler.stats
 
         self.stats.set_value("response_received_count", 4802)
         self.stats.set_value("item_scraped_count", 3201)
 
     @coroutine_test
-    async def test_stats_calculations(self):
+    async def test_stats_calculations(self) -> None:
         logstats = LogStats.from_crawler(self.crawler)
 
         with pytest.raises(AttributeError):
@@ -56,7 +59,7 @@ class TestLogStats:
         assert self.stats.get_value("responses_per_minute") == 172.9
         assert self.stats.get_value("items_per_minute") == 116.4
 
-    def test_stats_calculations_no_time(self):
+    def test_stats_calculations_no_time(self) -> None:
         """The stat values should be None since the start and finish time are
         not available.
         """
@@ -65,7 +68,7 @@ class TestLogStats:
         assert self.stats.get_value("responses_per_minute") is None
         assert self.stats.get_value("items_per_minute") is None
 
-    def test_stats_calculation_no_elapsed_time(self):
+    def test_stats_calculation_no_elapsed_time(self) -> None:
         """The stat values should be None since the elapsed time is 0."""
         logstats = LogStats.from_crawler(self.crawler)
         self.stats.set_value("start_time", datetime.fromtimestamp(1655100172))

From cec86f216e02072b2e9ed5b032810e061143277c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 27 Jul 2026 15:19:12 +0500
Subject: [PATCH 43/62] Remove remaining cross-imports in test modules. (#7782)

---
 pyproject.toml                                |   3 +
 tests/mockserver/http_base.py                 |   8 +-
 tests/test_command_check.py                   |   2 +-
 tests/test_command_crawl.py                   |   2 +-
 tests/test_command_genspider.py               |   2 +-
 tests/test_command_parse.py                   |   2 +-
 tests/test_commands.py                        |   2 +-
 tests/test_downloader_handler_httpx.py        |   2 +-
 .../test_downloader_handler_twisted_http11.py |   2 +-
 .../test_downloader_handler_twisted_http2.py  |   2 +-
 tests/test_downloadermiddleware_redirect.py   |   4 +-
 ...test_downloadermiddleware_redirect_base.py | 984 -----------------
 ...wnloadermiddleware_redirect_metarefresh.py |   4 +-
 tests/test_engine.py                          | 337 +-----
 tests/test_engine_stop_download_bytes.py      |   6 +-
 tests/test_engine_stop_download_headers.py    |   6 +-
 tests/test_feedexport.py                      | 141 +--
 tests/test_feedexport_batch.py                |   2 +-
 tests/test_feedexport_postprocess.py          |   2 +-
 tests/test_http_request.py                    | 485 +--------
 tests/test_http_request_form.py               |   4 +-
 tests/test_http_request_json.py               |   4 +-
 tests/test_http_response.py                   | 411 +-------
 tests/test_http_response_text.py              |   4 +-
 tests/test_pipeline_files.py                  |   4 +-
 tests/test_pipeline_media.py                  |  14 +-
 tests/test_pqueues.py                         |   2 +-
 tests/test_scheduler.py                       |  30 +-
 tests/test_spider.py                          | 123 +--
 tests/test_spider_crawl.py                    |   4 +-
 tests/test_spider_sitemap.py                  |   4 +-
 tests/test_spidermiddleware_process_start.py  |   3 +-
 tests/utils/bases/__init__.py                 |   0
 .../{base_commands.py => bases/commands.py}   |   0
 .../bases/download_handlers_http.py}          |   2 -
 tests/utils/bases/engine.py                   | 160 +++
 tests/utils/bases/feedexport.py               | 151 +++
 tests/utils/bases/http_request.py             | 490 +++++++++
 tests/utils/bases/http_response.py            | 417 ++++++++
 tests/utils/bases/redirect.py                 | 988 ++++++++++++++++++
 tests/utils/bases/spider.py                   | 131 +++
 tests/utils/downloader.py                     |  35 +
 tests/utils/engine.py                         | 184 ++++
 tests/utils/media_pipelines.py                |  15 +
 44 files changed, 2642 insertions(+), 2536 deletions(-)
 delete mode 100644 tests/test_downloadermiddleware_redirect_base.py
 create mode 100644 tests/utils/bases/__init__.py
 rename tests/utils/{base_commands.py => bases/commands.py} (100%)
 rename tests/{test_downloader_handlers_http_base.py => utils/bases/download_handlers_http.py} (99%)
 create mode 100644 tests/utils/bases/engine.py
 create mode 100644 tests/utils/bases/feedexport.py
 create mode 100644 tests/utils/bases/http_request.py
 create mode 100644 tests/utils/bases/http_response.py
 create mode 100644 tests/utils/bases/redirect.py
 create mode 100644 tests/utils/bases/spider.py
 create mode 100644 tests/utils/downloader.py
 create mode 100644 tests/utils/engine.py
 create mode 100644 tests/utils/media_pipelines.py

diff --git a/pyproject.toml b/pyproject.toml
index 5ed07a9ce..ee7f4ecdf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -185,6 +185,9 @@ module = [
     "tests.test_utils_misc.test_return_with_argument_inside_generator",
     "tests.test_utils_python",
     "tests.test_utils_request",
+    "tests.utils.bases.http_request",
+    "tests.utils.bases.http_response",
+    "tests.utils.bases.spider",
 ]
 check_untyped_defs = false
 
diff --git a/tests/mockserver/http_base.py b/tests/mockserver/http_base.py
index 7b38409ff..343c79781 100644
--- a/tests/mockserver/http_base.py
+++ b/tests/mockserver/http_base.py
@@ -20,6 +20,9 @@ if TYPE_CHECKING:
 
     from twisted.web import resource
 
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
 
 class BaseMockServer(ABC):
     listen_http: bool = True
@@ -39,13 +42,14 @@ class BaseMockServer(ABC):
         self.http_port: int | None = None
         self.https_port: int | None = None
 
-    def __enter__(self):
+    def __enter__(self) -> Self:
         self.proc = Popen(
             [sys.executable, "-u", "-m", self.module_name, *self.get_additional_args()],
             stdout=PIPE,
             env=get_script_run_env(),
             text=True,
         )
+        assert self.proc.stdout is not None
         if self.listen_http:
             http_address = self.proc.stdout.readline().strip()
             http_parsed = urlparse(http_address)
@@ -56,7 +60,7 @@ class BaseMockServer(ABC):
             self.https_port = https_parsed.port
         return self
 
-    def __exit__(self, exc_type, exc_value, traceback):
+    def __exit__(self, exc_type, exc_value, traceback) -> None:
         if self.proc:
             self.proc.kill()
             self.proc.communicate()
diff --git a/tests/test_command_check.py b/tests/test_command_check.py
index 794c6d400..240f44584 100644
--- a/tests/test_command_check.py
+++ b/tests/test_command_check.py
@@ -7,7 +7,7 @@ from unittest import TestCase
 from unittest.mock import MagicMock, Mock, PropertyMock, call, patch
 
 from scrapy.commands.check import Command, TextTestResult
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_command_crawl.py b/tests/test_command_crawl.py
index 6293e973e..70c26e6d0 100644
--- a/tests/test_command_crawl.py
+++ b/tests/test_command_crawl.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index f465d0b30..8bb6a2332 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -6,7 +6,7 @@ from pathlib import Path
 
 import pytest
 
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc, write_recording_editor
 
 
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index a3581d764..9b7131c7a 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -8,7 +8,7 @@ import pytest
 
 from scrapy.commands import parse
 from scrapy.settings import Settings
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc
 
 if TYPE_CHECKING:
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 9e7d4d5a1..51f98db1b 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -15,7 +15,7 @@ from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
 from scrapy.utils.reactor import _asyncio_reactor_path
-from tests.utils.base_commands import TestProjectBase
+from tests.utils.bases.commands import TestProjectBase
 from tests.utils.cmdline import call, proc, write_recording_editor
 
 if TYPE_CHECKING:
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index fd50248a7..5ceb93382 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -15,7 +15,7 @@ from scrapy.core.downloader.handlers._httpx import (
     HttpxDownloadHandler,
 )
 from scrapy.exceptions import DownloadFailedError
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
     TestHttpsBase,
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 79750a136..32dfd7540 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -11,7 +11,7 @@ from scrapy import Spider
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpBase,
     TestHttpProxyBase,
     TestHttpsBase,
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index bea97642e..daa89df58 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -13,7 +13,7 @@ from scrapy import Spider
 from scrapy.crawler import Crawler
 from scrapy.exceptions import DownloadFailedError, NotConfigured
 from scrapy.http import Request
-from tests.test_downloader_handlers_http_base import (
+from tests.utils.bases.download_handlers_http import (
     TestHttpProxyBase,
     TestHttpsBase,
     TestHttpsCustomCiphersBase,
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 28cf8ba34..ef2774a93 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -16,11 +16,11 @@ from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.bases.redirect import TestRedirectBase
 from tests.utils.redirect import REDIRECT_SCHEME_CASES, SCHEME_PARAMS
 
 
-class TestRedirectMiddleware(Base.Test):
+class TestRedirectMiddleware(TestRedirectBase):
     mwcls = RedirectMiddleware
     reason = 302
 
diff --git a/tests/test_downloadermiddleware_redirect_base.py b/tests/test_downloadermiddleware_redirect_base.py
deleted file mode 100644
index bd8bc796c..000000000
--- a/tests/test_downloadermiddleware_redirect_base.py
+++ /dev/null
@@ -1,984 +0,0 @@
-from __future__ import annotations
-
-import pytest
-
-from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
-from scrapy.exceptions import IgnoreRequest
-from scrapy.http import Request, Response
-from scrapy.utils.misc import set_environ
-from scrapy.utils.test import get_crawler
-
-
-class Base:
-    class Test:
-        def test_priority_adjust(self):
-            req = Request("http://a.example")
-            rsp = self.get_response(req, "http://a.example/redirected")
-            req2 = self.mw.process_response(req, rsp)
-            assert req2.priority > req.priority
-
-        def test_dont_redirect(self):
-            url = "http://www.example.com/301"
-            url2 = "http://www.example.com/redirected"
-            req = Request(url, meta={"dont_redirect": True})
-            rsp = self.get_response(req, url2)
-
-            r = self.mw.process_response(req, rsp)
-            assert isinstance(r, Response)
-            assert r is rsp
-
-            # Test that it redirects when dont_redirect is False
-            req = Request(url, meta={"dont_redirect": False})
-            rsp = self.get_response(req, url2)
-
-            r = self.mw.process_response(req, rsp)
-            assert isinstance(r, Request)
-
-        def test_post(self):
-            url = "http://www.example.com/302"
-            url2 = "http://www.example.com/redirected2"
-            req = Request(
-                url,
-                method="POST",
-                body="test",
-                headers={"Content-Type": "text/plain", "Content-length": "4"},
-            )
-            rsp = self.get_response(req, url2)
-
-            req2 = self.mw.process_response(req, rsp)
-            assert isinstance(req2, Request)
-            assert req2.url == url2
-            assert req2.method == "GET"
-            assert "Content-Type" not in req2.headers, (
-                "Content-Type header must not be present in redirected request"
-            )
-            assert "Content-Length" not in req2.headers, (
-                "Content-Length header must not be present in redirected request"
-            )
-            assert not req2.body, f"Redirected body must be empty, not '{req2.body}'"
-
-        def test_max_redirect_times(self):
-            self.mw.max_redirect_times = 1
-            req = Request("http://a.example/302")
-            rsp = self.get_response(req, "/redirected")
-
-            req = self.mw.process_response(req, rsp)
-            assert isinstance(req, Request)
-            assert "redirect_times" in req.meta
-            assert req.meta["redirect_times"] == 1
-            with pytest.raises(IgnoreRequest):
-                self.mw.process_response(req, rsp)
-
-        def test_ttl(self):
-            self.mw.max_redirect_times = 100
-            req = Request("http://a.example/302", meta={"redirect_ttl": 1})
-            rsp = self.get_response(req, "/a")
-
-            req = self.mw.process_response(req, rsp)
-            assert isinstance(req, Request)
-            with pytest.raises(IgnoreRequest):
-                self.mw.process_response(req, rsp)
-
-        def test_redirect_urls(self):
-            req1 = Request("http://a.example/first")
-            rsp1 = self.get_response(req1, "/redirected")
-            req2 = self.mw.process_response(req1, rsp1)
-            rsp2 = self.get_response(req2, "/redirected2")
-            req3 = self.mw.process_response(req2, rsp2)
-
-            assert req2.url == "http://a.example/redirected"
-            assert req2.meta["redirect_urls"] == ["http://a.example/first"]
-            assert req3.url == "http://a.example/redirected2"
-            assert req3.meta["redirect_urls"] == [
-                "http://a.example/first",
-                "http://a.example/redirected",
-            ]
-
-        def test_redirect_reasons(self):
-            req1 = Request("http://a.example/first")
-            rsp1 = self.get_response(req1, "/redirected1")
-            req2 = self.mw.process_response(req1, rsp1)
-            rsp2 = self.get_response(req2, "/redirected2")
-            req3 = self.mw.process_response(req2, rsp2)
-            assert req2.meta["redirect_reasons"] == [self.reason]
-            assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
-
-        def test_cross_origin_header_dropping(self):
-            safe_headers = {"A": "B"}
-            cookie_header = {"Cookie": "a=b"}
-            authorization_header = {"Authorization": "Bearer 123456"}
-
-            original_request = Request(
-                "https://example.com",
-                headers={**safe_headers, **cookie_header, **authorization_header},
-            )
-
-            # Redirects to the same origin (same scheme, same domain, same port)
-            # keep all headers.
-            internal_response = self.get_response(
-                original_request, "https://example.com/a"
-            )
-            internal_redirect_request = self.mw.process_response(
-                original_request, internal_response
-            )
-            assert isinstance(internal_redirect_request, Request)
-            assert original_request.headers == internal_redirect_request.headers
-
-            # Redirects to the same origin (same scheme, same domain, same port)
-            # keep all headers also when the scheme is http.
-            http_request = Request(
-                "http://example.com",
-                headers={**safe_headers, **cookie_header, **authorization_header},
-            )
-            http_response = self.get_response(http_request, "http://example.com/a")
-            http_redirect_request = self.mw.process_response(
-                http_request, http_response
-            )
-            assert isinstance(http_redirect_request, Request)
-            assert http_request.headers == http_redirect_request.headers
-
-            # For default ports, whether the port is explicit or implicit does not
-            # affect the outcome, it is still the same origin.
-            to_explicit_port_response = self.get_response(
-                original_request, "https://example.com:443/a"
-            )
-            to_explicit_port_redirect_request = self.mw.process_response(
-                original_request, to_explicit_port_response
-            )
-            assert isinstance(to_explicit_port_redirect_request, Request)
-            assert original_request.headers == to_explicit_port_redirect_request.headers
-
-            # For default ports, whether the port is explicit or implicit does not
-            # affect the outcome, it is still the same origin.
-            to_implicit_port_response = self.get_response(
-                original_request, "https://example.com/a"
-            )
-            to_implicit_port_redirect_request = self.mw.process_response(
-                original_request, to_implicit_port_response
-            )
-            assert isinstance(to_implicit_port_redirect_request, Request)
-            assert original_request.headers == to_implicit_port_redirect_request.headers
-
-            # A port change drops the Authorization header because the origin
-            # changes, but keeps the Cookie header because the domain remains the
-            # same.
-            different_port_response = self.get_response(
-                original_request, "https://example.com:8080/a"
-            )
-            different_port_redirect_request = self.mw.process_response(
-                original_request, different_port_response
-            )
-            assert isinstance(different_port_redirect_request, Request)
-            assert {
-                **safe_headers,
-                **cookie_header,
-            } == different_port_redirect_request.headers.to_unicode_dict()
-
-            # A domain change drops both the Authorization and the Cookie header.
-            external_response = self.get_response(
-                original_request, "https://example.org/a"
-            )
-            external_redirect_request = self.mw.process_response(
-                original_request, external_response
-            )
-            assert isinstance(external_redirect_request, Request)
-            assert safe_headers == external_redirect_request.headers.to_unicode_dict()
-
-            # A scheme upgrade (http → https) drops the Authorization header
-            # because the origin changes, but keeps the Cookie header because the
-            # domain remains the same.
-            upgrade_response = self.get_response(http_request, "https://example.com/a")
-            upgrade_redirect_request = self.mw.process_response(
-                http_request, upgrade_response
-            )
-            assert isinstance(upgrade_redirect_request, Request)
-            assert {
-                **safe_headers,
-                **cookie_header,
-            } == upgrade_redirect_request.headers.to_unicode_dict()
-
-            # A scheme downgrade (https → http) drops the Authorization header
-            # because the origin changes, and the Cookie header because its value
-            # cannot indicate whether the cookies were secure (HTTPS-only) or not.
-            #
-            # Note: If the Cookie header is set by the cookie management
-            # middleware, as recommended in the docs, the dropping of Cookie on
-            # scheme downgrade is not an issue, because the cookie management
-            # middleware will add again the Cookie header to the new request if
-            # appropriate.
-            downgrade_response = self.get_response(
-                original_request, "http://example.com/a"
-            )
-            downgrade_redirect_request = self.mw.process_response(
-                original_request, downgrade_response
-            )
-            assert isinstance(downgrade_redirect_request, Request)
-            assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
-
-        def test_meta_proxy_http_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_http_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_http_to_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("http://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_meta_proxy_https_to_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            meta = {"proxy": "https://a:@a.example"}
-            request1 = Request("https://example.com", meta=meta)
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_http_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_http_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_https_absolute(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_https_relative(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "/a")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "/a")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_proxied_http_to_proxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request2.meta["_auth_proxy"] == "https://b.example"
-            assert request2.meta["proxy"] == "https://b.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_proxied_http_to_unproxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request1.meta["_auth_proxy"] == "https://a.example"
-            assert request1.meta["proxy"] == "https://a.example"
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request3.meta["_auth_proxy"] == "https://a.example"
-            assert request3.meta["proxy"] == "https://a.example"
-
-        def test_system_proxy_unproxied_http_to_proxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request2.meta["_auth_proxy"] == "https://b.example"
-            assert request2.meta["proxy"] == "https://b.example"
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_unproxied_http_to_unproxied_https(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("http://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "https://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "http://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_proxied_https_to_proxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request1.meta["_auth_proxy"] == "https://b.example"
-            assert request1.meta["proxy"] == "https://b.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request3.meta["_auth_proxy"] == "https://b.example"
-            assert request3.meta["proxy"] == "https://b.example"
-
-        def test_system_proxy_proxied_https_to_unproxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "https_proxy": "https://b:@b.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request1.meta["_auth_proxy"] == "https://b.example"
-            assert request1.meta["proxy"] == "https://b.example"
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
-            assert request3.meta["_auth_proxy"] == "https://b.example"
-            assert request3.meta["proxy"] == "https://b.example"
-
-        def test_system_proxy_unproxied_https_to_proxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            env = {
-                "http_proxy": "https://a:@a.example",
-            }
-            with set_environ(**env):
-                proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
-            assert request2.meta["_auth_proxy"] == "https://a.example"
-            assert request2.meta["proxy"] == "https://a.example"
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-        def test_system_proxy_unproxied_https_to_unproxied_http(self):
-            crawler = get_crawler()
-            redirect_mw = self.mwcls.from_crawler(crawler)
-            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
-
-            request1 = Request("https://example.com")
-            proxy_mw.process_request(request1)
-
-            assert "Proxy-Authorization" not in request1.headers
-            assert "_auth_proxy" not in request1.meta
-            assert "proxy" not in request1.meta
-
-            response1 = self.get_response(request1, "http://example.com")
-            request2 = redirect_mw.process_response(request1, response1)
-
-            assert isinstance(request2, Request)
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            proxy_mw.process_request(request2)
-
-            assert "Proxy-Authorization" not in request2.headers
-            assert "_auth_proxy" not in request2.meta
-            assert "proxy" not in request2.meta
-
-            response2 = self.get_response(request2, "https://example.com")
-            request3 = redirect_mw.process_response(request2, response2)
-
-            assert isinstance(request3, Request)
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
-
-            proxy_mw.process_request(request3)
-
-            assert "Proxy-Authorization" not in request3.headers
-            assert "_auth_proxy" not in request3.meta
-            assert "proxy" not in request3.meta
diff --git a/tests/test_downloadermiddleware_redirect_metarefresh.py b/tests/test_downloadermiddleware_redirect_metarefresh.py
index b5d39080c..aeae759a0 100644
--- a/tests/test_downloadermiddleware_redirect_metarefresh.py
+++ b/tests/test_downloadermiddleware_redirect_metarefresh.py
@@ -12,7 +12,7 @@ from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
-from tests.test_downloadermiddleware_redirect_base import Base
+from tests.utils.bases.redirect import TestRedirectBase
 from tests.utils.redirect import (
     HTTP_SCHEMES,
     NON_HTTP_SCHEMES,
@@ -26,7 +26,7 @@ def meta_refresh_body(url, interval=5):
     return html.encode("utf-8")
 
 
-class TestMetaRefreshMiddleware(Base.Test):
+class TestMetaRefreshMiddleware(TestRedirectBase):
     mwcls = MetaRefreshMiddleware
     reason = "meta refresh"
 
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 3ab8f6d1b..87f0265a0 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -1,115 +1,45 @@
 from __future__ import annotations
 
 import asyncio
-import re
 import subprocess
 import sys
-from collections import defaultdict
-from dataclasses import dataclass
 from typing import TYPE_CHECKING, Any
 from unittest.mock import Mock
-from urllib.parse import urlparse
 
-import attr
 import pytest
-from itemadapter import ItemAdapter
-from pydispatch import dispatcher
 from testfixtures import LogCapture
-from twisted.internet import defer
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine, _Slot
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.exceptions import CloseSpider, IgnoreRequest
-from scrapy.http import Headers, Request, Response
-from scrapy.item import Field, Item
-from scrapy.linkextractors import LinkExtractor
+from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.utils.defer import (
-    _schedule_coro,
-    deferred_from_coro,
-    maybe_deferred_to_future,
-)
-from scrapy.utils.signal import disconnect_all
+from scrapy.utils.defer import _schedule_coro, deferred_from_coro
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests import get_testdata
+from tests.utils.bases.engine import TestEngineBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.engine import (
+    AttrsItemsSpider,
+    CrawlerRun,
+    DataClassItemsSpider,
+    DictItemsSpider,
+    MySpider,
+)
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
 
-    from twisted.python.failure import Failure
-
     from tests.mockserver.http import MockServer
 
 
-class MyItem(Item):
-    name = Field()
-    url = Field()
-    price = Field()
-
-
-@attr.s
-class AttrsItem:
-    name = attr.ib(default="")
-    url = attr.ib(default="")
-    price = attr.ib(default=0)
-
-
-@dataclass
-class DataClassItem:
-    name: str = ""
-    url: str = ""
-    price: int = 0
-
-
-class MySpider(Spider):
-    name = "scrapytest.org"
-
-    itemurl_re = re.compile(r"item\d+.html")
-    name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
-    price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
-
-    item_cls: type = MyItem
-
-    def parse(self, response):
-        xlink = LinkExtractor()
-        itemre = re.compile(self.itemurl_re)
-        for link in xlink.extract_links(response):
-            if itemre.search(link.url):
-                yield Request(url=link.url, callback=self.parse_item)
-
-    def parse_item(self, response):
-        adapter = ItemAdapter(self.item_cls())
-        m = self.name_re.search(response.text)
-        if m:
-            adapter["name"] = m.group(1)
-        adapter["url"] = response.url
-        m = self.price_re.search(response.text)
-        if m:
-            adapter["price"] = m.group(1)
-        return adapter.item
-
-
 class DupeFilterSpider(MySpider):
     async def start(self):
         for url in self.start_urls:
             yield Request(url)  # no dont_filter=True
 
 
-class DictItemsSpider(MySpider):
-    item_cls = dict
-
-
-class AttrsItemsSpider(MySpider):
-    item_cls = AttrsItem
-
-
-class DataClassItemsSpider(MySpider):
-    item_cls = DataClassItem
-
-
 class ItemZeroDivisionErrorSpider(MySpider):
     custom_settings = {
         "ITEM_PIPELINES": {
@@ -130,253 +60,6 @@ class ChangeCloseReasonSpider(MySpider):
         raise CloseSpider(reason="custom_reason")
 
 
-class CrawlerRun:
-    """A class to run the crawler and keep track of events occurred"""
-
-    def __init__(self, spider_class: type[Spider]):
-        self.respplug: list[tuple[Response, Spider]] = []
-        self.reqplug: list[tuple[Request, Spider]] = []
-        self.reqdropped: list[tuple[Request, Spider]] = []
-        self.reqreached: list[tuple[Request, Spider]] = []
-        self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
-        self.itemresp: list[tuple[Any, Response]] = []
-        self.headers: dict[Request, Headers] = {}
-        self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
-        self.signals_caught: dict[Any, dict[str, Any]] = {}
-        self.spider_class = spider_class
-
-    async def run(self, mockserver: MockServer) -> None:
-        self.mockserver = mockserver
-
-        start_urls = [
-            self.geturl("/static/"),
-            self.geturl("/redirect"),
-            self.geturl("/redirect"),  # duplicate
-            self.geturl("/numbers"),
-        ]
-
-        for name, signal in vars(signals).items():
-            if not name.startswith("_"):
-                dispatcher.connect(self.record_signal, signal)
-
-        self.crawler = get_crawler(self.spider_class)
-        self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
-        self.crawler.signals.connect(self.item_error, signals.item_error)
-        self.crawler.signals.connect(self.headers_received, signals.headers_received)
-        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
-        self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
-        self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
-        self.crawler.signals.connect(
-            self.request_reached, signals.request_reached_downloader
-        )
-        self.crawler.signals.connect(
-            self.response_downloaded, signals.response_downloaded
-        )
-        self.crawler.crawl(start_urls=start_urls)
-
-        self.deferred: defer.Deferred[None] = defer.Deferred()
-        dispatcher.connect(self.stop, signals.engine_stopped)
-        await maybe_deferred_to_future(self.deferred)
-
-    async def stop(self):
-        for name, signal in vars(signals).items():
-            if not name.startswith("_"):
-                disconnect_all(signal)
-        self.deferred.callback(None)
-        await self.crawler.stop_async()
-
-    def geturl(self, path: str) -> str:
-        return self.mockserver.url(path)
-
-    def getpath(self, url: str) -> str:
-        u = urlparse(url)
-        return u.path
-
-    def item_error(
-        self, item: Any, response: Response, spider: Spider, failure: Failure
-    ) -> None:
-        self.itemerror.append((item, response, spider, failure))
-
-    def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
-        self.itemresp.append((item, response))
-
-    def headers_received(
-        self, headers: Headers, body_length: int, request: Request, spider: Spider
-    ) -> None:
-        self.headers[request] = headers
-
-    def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
-        self.bytes[request].append(data)
-
-    def request_scheduled(self, request: Request, spider: Spider) -> None:
-        self.reqplug.append((request, spider))
-
-    def request_reached(self, request: Request, spider: Spider) -> None:
-        self.reqreached.append((request, spider))
-
-    def request_dropped(self, request: Request, spider: Spider) -> None:
-        self.reqdropped.append((request, spider))
-
-    def response_downloaded(self, response: Response, spider: Spider) -> None:
-        self.respplug.append((response, spider))
-
-    def record_signal(self, *args: Any, **kwargs: Any) -> None:
-        """Record a signal and its parameters"""
-        signalargs = kwargs.copy()
-        sig = signalargs.pop("signal")
-        signalargs.pop("sender", None)
-        self.signals_caught[sig] = signalargs
-
-
-class TestEngineBase:
-    @staticmethod
-    def _assert_visited_urls(run: CrawlerRun) -> None:
-        must_be_visited = [
-            "/static/",
-            "/redirect",
-            "/redirected",
-            "/static/item1.html",
-            "/static/item2.html",
-            "/static/item999.html",
-        ]
-        urls_visited = {rp[0].url for rp in run.respplug}
-        urls_expected = {run.geturl(p) for p in must_be_visited}
-        assert urls_expected <= urls_visited, (
-            f"URLs not visited: {list(urls_expected - urls_visited)}"
-        )
-
-    @staticmethod
-    def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
-        assert len(run.reqplug) == count
-
-        paths_expected = [
-            "/static/item999.html",
-            "/static/item2.html",
-            "/static/item1.html",
-        ]
-
-        urls_requested = {rq[0].url for rq in run.reqplug}
-        urls_expected = {run.geturl(p) for p in paths_expected}
-        assert urls_expected <= urls_requested
-        scheduled_requests_count = len(run.reqplug)
-        dropped_requests_count = len(run.reqdropped)
-        responses_count = len(run.respplug)
-        assert scheduled_requests_count == dropped_requests_count + responses_count
-        assert len(run.reqreached) == responses_count
-
-    @staticmethod
-    def _assert_dropped_requests(run: CrawlerRun) -> None:
-        assert len(run.reqdropped) == 1
-
-    @staticmethod
-    def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
-        # response tests
-        assert len(run.respplug) == count
-        assert len(run.reqreached) == count
-
-        for response, _ in run.respplug:
-            if run.getpath(response.url) == "/static/item999.html":
-                assert response.status == 404
-            if run.getpath(response.url) == "/redirect":
-                assert response.status == 302
-
-    @staticmethod
-    def _assert_items_error(run: CrawlerRun) -> None:
-        assert len(run.itemerror) == 2
-        for item, response, spider, failure in run.itemerror:
-            assert failure.value.__class__ is ZeroDivisionError
-            assert spider == run.crawler.spider
-
-            assert item["url"] == response.url
-            if "item1.html" in item["url"]:
-                assert item["name"] == "Item 1 name"
-                assert item["price"] == "100"
-            if "item2.html" in item["url"]:
-                assert item["name"] == "Item 2 name"
-                assert item["price"] == "200"
-
-    @staticmethod
-    def _assert_scraped_items(run: CrawlerRun) -> None:
-        assert len(run.itemresp) == 2
-        for item_, response in run.itemresp:
-            item = ItemAdapter(item_)
-            assert item["url"] == response.url
-            if "item1.html" in item["url"]:
-                assert item["name"] == "Item 1 name"
-                assert item["price"] == "100"
-            if "item2.html" in item["url"]:
-                assert item["name"] == "Item 2 name"
-                assert item["price"] == "200"
-
-    @staticmethod
-    def _assert_headers_received(run: CrawlerRun) -> None:
-        for headers in run.headers.values():
-            assert b"Server" in headers
-            assert headers[b"Server"]
-            assert b"TwistedWeb" in headers[b"Server"]
-            assert b"Date" in headers
-            assert b"Content-Type" in headers
-
-    @staticmethod
-    def _assert_bytes_received(run: CrawlerRun) -> None:
-        assert len(run.bytes) == 9
-        for request, data in run.bytes.items():
-            joined_data = b"".join(data)
-            if run.getpath(request.url) == "/static/":
-                assert joined_data == get_testdata("test_site", "index.html")
-            elif run.getpath(request.url) == "/static/item1.html":
-                assert joined_data == get_testdata("test_site", "item1.html")
-            elif run.getpath(request.url) == "/static/item2.html":
-                assert joined_data == get_testdata("test_site", "item2.html")
-            elif run.getpath(request.url) == "/redirected":
-                assert joined_data == b"Redirected here"
-            elif run.getpath(request.url) == "/redirect":
-                assert (
-                    joined_data == b"\n<html>\n"
-                    b"    <head>\n"
-                    b'        <meta http-equiv="refresh" content="0;URL=/redirected">\n'
-                    b"    </head>\n"
-                    b'    <body bgcolor="#FFFFFF" text="#000000">\n'
-                    b'    <a href="/redirected">click here</a>\n'
-                    b"    </body>\n"
-                    b"</html>\n"
-                )
-            elif run.getpath(request.url) == "/static/item999.html":
-                assert (
-                    joined_data == b"\n<html>\n"
-                    b"  <head><title>404 - No Such Resource</title></head>\n"
-                    b"  <body>\n"
-                    b"    <h1>No Such Resource</h1>\n"
-                    b"    <p>File not found.</p>\n"
-                    b"  </body>\n"
-                    b"</html>\n"
-                )
-            elif run.getpath(request.url) == "/numbers":
-                # signal was fired multiple times
-                assert len(data) > 1
-                # bytes were received in order
-                numbers = [str(x).encode("utf8") for x in range(2**18)]
-                assert joined_data == b"".join(numbers)
-
-    @staticmethod
-    def _assert_signals_caught(run: CrawlerRun) -> None:
-        assert signals.engine_started in run.signals_caught
-        assert signals.engine_stopped in run.signals_caught
-        assert signals.spider_opened in run.signals_caught
-        assert signals.spider_idle in run.signals_caught
-        assert signals.spider_closed in run.signals_caught
-        assert signals.headers_received in run.signals_caught
-
-        assert {"spider": run.crawler.spider} == run.signals_caught[
-            signals.spider_opened
-        ]
-        assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
-        assert {
-            "spider": run.crawler.spider,
-            "reason": "finished",
-        } == run.signals_caught[signals.spider_closed]
-
-
 class TestEngine(TestEngineBase):
     @coroutine_test
     async def test_crawler(self, mockserver: MockServer) -> None:
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 091485781..27f282af4 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -3,15 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from tests.test_engine import (
+from tests.utils.bases.engine import TestEngineBase
+from tests.utils.decorators import coroutine_test
+from tests.utils.engine import (
     AttrsItemsSpider,
     CrawlerRun,
     DataClassItemsSpider,
     DictItemsSpider,
     MySpider,
-    TestEngineBase,
 )
-from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     import pytest
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index 8ebc948e2..09fa14326 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -3,15 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from tests.test_engine import (
+from tests.utils.bases.engine import TestEngineBase
+from tests.utils.decorators import coroutine_test
+from tests.utils.engine import (
     AttrsItemsSpider,
     CrawlerRun,
     DataClassItemsSpider,
     DictItemsSpider,
     MySpider,
-    TestEngineBase,
 )
-from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     import pytest
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 3f659ee0b..f1c416fbd 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -4,13 +4,9 @@ import csv
 import json
 import marshal
 import pickle
-import random
-import shutil
 import tempfile
-from abc import ABC, abstractmethod
 from logging import getLogger
 from pathlib import Path
-from string import ascii_letters, digits
 from typing import IO, TYPE_CHECKING, Any
 from unittest import mock
 
@@ -32,8 +28,8 @@ from scrapy.extensions.feedexport import (
 )
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 from tests.utils.feedexport import MyItem, MyItem2, path_to_url, printf_escape
 
@@ -99,141 +95,6 @@ class LogOnStoreFileStorage:
         file.close()
 
 
-class TestFeedExportBase(ABC):
-    mockserver: MockServer
-
-    def _random_temp_filename(self, inter_dir="") -> Path:
-        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
-        filename = "".join(chars)
-        return Path(self.temp_dir, inter_dir, filename)
-
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    def setup_method(self):
-        self.temp_dir = tempfile.mkdtemp()
-
-    def teardown_method(self):
-        shutil.rmtree(self.temp_dir, ignore_errors=True)
-
-    async def exported_data(
-        self, items: Iterable[Any], settings: dict[str, Any]
-    ) -> dict[str, Any]:
-        """
-        Return exported data which a spider yielding ``items`` would return.
-        """
-
-        class TestSpider(scrapy.Spider):
-            name = "testspider"
-
-            def parse(self, response):
-                yield from items
-
-        return await self.run_and_export(TestSpider, settings)
-
-    async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
-        """
-        Return exported data which a spider yielding no ``items`` would return.
-        """
-
-        class TestSpider(scrapy.Spider):
-            name = "testspider"
-
-            def parse(self, response):
-                pass
-
-        return await self.run_and_export(TestSpider, settings)
-
-    async def assertExported(
-        self,
-        items: Iterable[Any],
-        header: Iterable[str],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        await self.assertExportedCsv(items, header, rows, settings)
-        await self.assertExportedJsonLines(items, rows, settings)
-        await self.assertExportedXml(items, rows, settings)
-        await self.assertExportedPickle(items, rows, settings)
-        await self.assertExportedMarshal(items, rows, settings)
-        await self.assertExportedMultiple(items, rows, settings)
-
-    async def assertExportedCsv(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        header: Iterable[str],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedJsonLines(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedXml(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedMultiple(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedPickle(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    async def assertExportedMarshal(  # noqa: B027
-        self,
-        items: Iterable[Any],
-        rows: Iterable[dict[str, Any]],
-        settings: dict[str, Any] | None = None,
-    ) -> None:
-        pass
-
-    @abstractmethod
-    async def run_and_export(
-        self, spider_cls: type[Spider], settings: dict[str, Any]
-    ) -> dict[str, Any]:
-        pass
-
-    def _load_until_eof(
-        self, data: bytes, load_func: Callable[[IO[bytes]], Any]
-    ) -> list[Any]:
-        result: list[Any] = []
-        with tempfile.TemporaryFile() as temp:
-            temp.write(data)
-            temp.seek(0)
-            while True:
-                try:
-                    result.append(load_func(temp))
-                except EOFError:
-                    break
-        return result
-
-
 class InstrumentedFeedSlot(FeedSlot):
     """Instrumented FeedSlot subclass for keeping track of calls to
     start_exporting and finish_exporting."""
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 3b50cd492..80ff6229b 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -21,7 +21,7 @@ from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
 from tests.spiders import ItemSpider
-from tests.test_feedexport import TestFeedExportBase
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 from tests.utils.feedexport import MyItem
 
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index 83d976e94..f120ce36f 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
 import pytest
 
 from scrapy.utils.test import get_crawler
-from tests.test_feedexport import TestFeedExportBase
+from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test
 from tests.utils.feedexport import path_to_url, printf_escape
 
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 1941b826f..e58ae8f39 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -1,493 +1,18 @@
-import warnings
 import xmlrpc.client
-from typing import Any
 
 import pytest
 
-from scrapy.http import Headers, Request, XmlRpcRequest
-from scrapy.http.request import NO_CALLBACK
+from scrapy import Request
+from scrapy.http import XmlRpcRequest
 from scrapy.utils.python import to_bytes
+from tests.utils.bases.http_request import TestRequestBase
 
 
-class TestRequest:
+class TestRequest(TestRequestBase):
     request_class = Request
-    default_method = "GET"
-    default_headers: dict[bytes, list[bytes]] = {}
-    default_meta: dict[str, Any] = {}
 
-    def test_init(self):
-        # Request requires url in the __init__ method
-        with pytest.raises(TypeError):
-            self.request_class()
 
-        # url argument must be basestring
-        with pytest.raises(TypeError):
-            self.request_class(123)
-
-        # priority argument must be an integer
-        with pytest.raises(TypeError, match="Request priority not an integer"):
-            self.request_class("http://www.example.com", priority="1")
-
-        r = self.request_class("http://www.example.com")
-        assert isinstance(r.url, str)
-        assert r.url == "http://www.example.com"
-        assert r.method == self.default_method
-
-        assert isinstance(r.headers, Headers)
-        assert r.headers == self.default_headers
-        assert r.meta == self.default_meta
-
-        meta = {"lala": "lolo"}
-        headers = {b"caca": b"coco"}
-        r = self.request_class(
-            "http://www.example.com", meta=meta, headers=headers, body="a body"
-        )
-
-        assert r.meta is not meta
-        assert r.meta == meta
-        assert r.headers is not headers
-        assert r.headers[b"caca"] == b"coco"
-
-    def test_url_scheme(self):
-        # This test passes by not raising any (ValueError) exception
-        self.request_class("http://example.org")
-        self.request_class("https://example.org")
-        self.request_class("s3://example.org")
-        self.request_class("ftp://example.org")
-        self.request_class("about:config")
-        self.request_class("data:,Hello%2C%20World!")
-
-    def test_url_no_scheme(self):
-        msg = "Missing scheme in request url:"
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("foo")
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("/foo/")
-        with pytest.raises(ValueError, match=msg):
-            self.request_class("/foo:bar")
-
-    def test_headers(self):
-        # Different ways of setting headers attribute
-        url = "http://www.scrapy.org"
-        headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
-        r = self.request_class(url=url, headers=headers)
-        p = self.request_class(url=url, headers=r.headers)
-
-        assert r.headers == p.headers
-        assert r.headers is not headers
-        assert p.headers is not r.headers
-
-        # headers must not be unicode
-        h = Headers({"key1": "val1", "key2": "val2"})
-        h["newkey"] = "newval"
-        for k, v in h.items():
-            assert isinstance(k, bytes)
-            for s in v:
-                assert isinstance(s, bytes)
-
-    def test_eq(self):
-        url = "http://www.scrapy.org"
-        r1 = self.request_class(url=url)
-        r2 = self.request_class(url=url)
-        assert r1 != r2
-
-        set_ = set()
-        set_.add(r1)
-        set_.add(r2)
-        assert len(set_) == 2
-
-    def test_url(self):
-        r = self.request_class(url="http://www.scrapy.org/path")
-        assert r.url == "http://www.scrapy.org/path"
-
-    def test_url_quoting(self):
-        r = self.request_class(url="http://www.scrapy.org/blank%20space")
-        assert r.url == "http://www.scrapy.org/blank%20space"
-        r = self.request_class(url="http://www.scrapy.org/blank space")
-        assert r.url == "http://www.scrapy.org/blank%20space"
-
-    def test_url_encoding(self):
-        r = self.request_class(url="http://www.scrapy.org/price/£")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-    def test_url_encoding_other(self):
-        # encoding affects only query part of URI, not path
-        # path part should always be UTF-8 encoded before percent-escaping
-        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
-        assert r.url == "http://www.scrapy.org/price/%C2%A3"
-
-    def test_url_encoding_query(self):
-        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
-        assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
-
-        # should be same as above
-        r2 = self.request_class(
-            url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
-        )
-        assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
-
-    def test_url_encoding_query_latin1(self):
-        # encoding is used for encoding query-string before percent-escaping;
-        # path is still UTF-8 encoded before percent-escaping
-        r3 = self.request_class(
-            url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
-        )
-        assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
-
-    def test_url_encoding_nonutf8_untouched(self):
-        # percent-escaping sequences that do not match valid UTF-8 sequences
-        # should be kept untouched (just upper-cased perhaps)
-        #
-        # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
-        #
-        # "Conversions from URIs to IRIs MUST NOT use any character encoding
-        # other than UTF-8 in steps 3 and 4, even if it might be possible to
-        # guess from the context that another character encoding than UTF-8 was
-        # used in the URI.  For example, the URI
-        # "http://www.example.org/r%E9sum%E9.html" might with some guessing be
-        # interpreted to contain two e-acute characters encoded as iso-8859-1.
-        # It must not be converted to an IRI containing these e-acute
-        # characters.  Otherwise, in the future the IRI will be mapped to
-        # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
-        # URI from "http://www.example.org/r%E9sum%E9.html".
-        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
-        assert r1.url == "http://www.scrapy.org/price/%a3"
-
-        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
-        assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
-
-        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
-        assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
-
-        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
-        assert r4.url == "http://www.example.org/r%E9sum%E9.html"
-
-    def test_url_verbatim(self):
-        r = self.request_class(
-            url="http://www.scrapy.org/price/£",
-            meta={"verbatim_url": True},
-        )
-        assert r.url == "http://www.scrapy.org/price/£"
-
-        r = self.request_class(
-            url="http://www.scrapy.org/blank space",
-            meta={"verbatim_url": True},
-        )
-        assert r.url == "http://www.scrapy.org/blank space"
-
-    def test_body(self):
-        r1 = self.request_class(url="http://www.example.com/")
-        assert r1.body == b""
-
-        r2 = self.request_class(url="http://www.example.com/", body=b"")
-        assert isinstance(r2.body, bytes)
-        assert r2.encoding == "utf-8"  # default encoding
-
-        r3 = self.request_class(
-            url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
-        )
-        assert isinstance(r3.body, bytes)
-        assert r3.body == b"Price: \xc2\xa3100"
-
-        r4 = self.request_class(
-            url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
-        )
-        assert isinstance(r4.body, bytes)
-        assert r4.body == b"Price: \xa3100"
-
-    def test_copy(self):
-        """Test Request copy"""
-
-        def somecallback():
-            pass
-
-        r1 = self.request_class(
-            "http://www.example.com",
-            flags=["f1", "f2"],
-            callback=somecallback,
-            errback=somecallback,
-        )
-        r1.meta["foo"] = "bar"
-        r1.cb_kwargs["key"] = "value"
-        r2 = r1.copy()
-
-        # make sure callbaclks are copied
-        assert r1.callback is somecallback
-        assert r1.errback is somecallback
-        assert r2.callback is r1.callback
-        assert r2.errback is r1.errback
-
-        # make sure flags list is shallow copied
-        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
-        assert r1.flags == r2.flags
-
-        # make sure cb_kwargs dict is shallow copied
-        assert r1.cb_kwargs is not r2.cb_kwargs, (
-            "cb_kwargs must be a shallow copy, not identical"
-        )
-        assert r1.cb_kwargs == r2.cb_kwargs
-
-        # make sure meta dict is shallow copied
-        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
-        assert r1.meta == r2.meta
-
-        # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, (
-            "headers must be a shallow copy, not identical"
-        )
-        assert r1.headers == r2.headers
-        assert r1.encoding == r2.encoding
-        assert r1.dont_filter == r2.dont_filter
-
-        # Request.body can be identical since it's an immutable object (str)
-
-    def test_copy_inherited_classes(self):
-        """Test Request children copies preserve their class"""
-
-        class CustomRequest(self.request_class):
-            pass
-
-        r1 = CustomRequest("http://www.example.com")
-        r2 = r1.copy()
-
-        assert isinstance(r2, CustomRequest)
-
-    def test_replace(self):
-        """Test Request.replace() method"""
-        r1 = self.request_class("http://www.example.com", method="GET")
-        hdrs = Headers(r1.headers)
-        hdrs[b"key"] = b"value"
-        r2 = r1.replace(method="POST", body="New body", headers=hdrs)
-        assert r1.url == r2.url
-        assert (r1.method, r2.method) == ("GET", "POST")
-        assert (r1.body, r2.body) == (b"", b"New body")
-        assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
-
-        # Empty attributes (which may fail if not compared properly)
-        r3 = self.request_class(
-            "http://www.example.com", meta={"a": 1}, dont_filter=True
-        )
-        r4 = r3.replace(
-            url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
-        )
-        assert r4.url == "http://www.example.com/2"
-        assert r4.body == b""
-        assert r4.meta == {}
-        assert r4.dont_filter is False
-
-        # the cls argument allows changing the resulting class
-        custom_request_cls = type("CustomRequest", (self.request_class,), {})
-        r5 = r1.replace(cls=custom_request_cls)
-        assert isinstance(r5, custom_request_cls)
-        assert r5.url == r1.url
-
-    def test_method_always_str(self):
-        r = self.request_class("http://www.example.com", method="POST")
-        assert isinstance(r.method, str)
-
-    def test_immutable_attributes(self):
-        r = self.request_class("http://example.com")
-        with pytest.raises(AttributeError):
-            r.url = "http://example2.com"
-        with pytest.raises(AttributeError):
-            r.body = "xxx"
-
-    def test_callback_and_errback(self):
-        def a_function():
-            pass
-
-        r1 = self.request_class("http://example.com")
-        assert r1.callback is None
-        assert r1.errback is None
-
-        r2 = self.request_class("http://example.com", callback=a_function)
-        assert r2.callback is a_function
-        assert r2.errback is None
-
-        r3 = self.request_class("http://example.com", errback=a_function)
-        assert r3.callback is None
-        assert r3.errback is a_function
-
-        r4 = self.request_class(
-            url="http://example.com",
-            callback=a_function,
-            errback=a_function,
-        )
-        assert r4.callback is a_function
-        assert r4.errback is a_function
-
-        r5 = self.request_class(
-            url="http://example.com",
-            callback=NO_CALLBACK,
-            errback=NO_CALLBACK,
-        )
-        assert r5.callback is NO_CALLBACK
-        assert r5.errback is NO_CALLBACK
-
-    def test_callback_and_errback_type(self):
-        with pytest.raises(TypeError):
-            self.request_class("http://example.com", callback="a_function")
-        with pytest.raises(TypeError):
-            self.request_class("http://example.com", errback="a_function")
-        with pytest.raises(TypeError):
-            self.request_class(
-                url="http://example.com",
-                callback="a_function",
-                errback="a_function",
-            )
-
-    def test_setters(self):
-        request = self.request_class("http://example.com")
-
-        request.flags = ["f1"]
-        assert request.flags == ["f1"]
-
-        request.cookies = {"sid": "1"}
-        assert request.cookies == {"sid": "1"}
-
-        headers = Headers({b"X-Test": b"1"})
-        request.headers = headers
-        assert request._headers is headers
-        request.headers = {b"A": b"b"}
-        assert isinstance(request.headers, Headers)
-        assert request._headers[b"A"] == b"b"
-
-    def test_setter_mutable_lazy_loading(self):
-        """Mutable attributes are set internally to None only until they are
-        read, then they always return the same falsy instance of the
-        corresponding mutable structure.
-
-        Setting them to None causes the next read to return a different object.
-        """
-
-        request = self.request_class("http://example.com")
-
-        assert request._flags is None
-        assert request.flags == []
-        assert request.flags is request.flags
-        assert request._flags == []
-        original_flags = request.flags
-        request.flags = None
-        assert request._flags is None
-        assert request.flags == []
-        assert request.flags is not original_flags
-
-        assert request._cookies is None
-        assert request.cookies == {}
-        assert request.cookies is request.cookies
-        assert request._cookies == {}
-        original_cookies = request.cookies
-        request.cookies = None
-        assert request._cookies is None
-        assert request.cookies == {}
-        assert request.cookies is not original_cookies
-
-        if self.default_headers:
-            assert request._headers == self.default_headers
-            assert request._headers is not self.default_headers
-            assert request.headers == self.default_headers
-        else:
-            assert request._headers is None
-            assert request.headers == {}
-        assert request.headers is request.headers
-        assert isinstance(request.headers, Headers)
-        assert isinstance(request._headers, Headers)
-        original_headers = request.headers
-        request.headers = None
-        assert request._headers is None
-        assert request.headers == {}
-        assert request._headers == {}
-        assert request.headers is not original_headers
-
-    def test_no_callback(self):
-        with pytest.raises(RuntimeError):
-            NO_CALLBACK()
-
-    def test_from_curl(self):
-        # Note: more curated tests regarding curl conversion are in
-        # `test_utils_curl.py`
-        curl_command = (
-            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
-            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
-            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
-            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
-            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
-            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
-            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
-            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
-            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
-            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
-            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
-            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
-            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
-            "2%3A15&comments=' --compressed"
-        )
-        r = self.request_class.from_curl(curl_command)
-        assert r.method == "POST"
-        assert r.url == "http://httpbin.org/post"
-        assert (
-            r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
-            b"example.org&size=small&topping=cheese&topping=onion"
-            b"&delivery=12%3A15&comments="
-        )
-        assert r.cookies == {
-            "_gauges_unique_year": "1",
-            "_gauges_unique": "1",
-            "_gauges_unique_month": "1",
-            "_gauges_unique_hour": "1",
-            "_gauges_unique_day": "1",
-        }
-        assert r.headers == {
-            b"Origin": [b"http://httpbin.org"],
-            b"Accept-Encoding": [b"gzip, deflate"],
-            b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
-            b"Upgrade-Insecure-Requests": [b"1"],
-            b"User-Agent": [
-                b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
-                b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
-                b".75 Chrome/62.0.3202.75 Safari/537.36"
-            ],
-            b"Content-Type": [b"application /x-www-form-urlencoded"],
-            b"Accept": [
-                b"text/html,application/xhtml+xml,application/xml;q=0."
-                b"9,image/webp,image/apng,*/*;q=0.8"
-            ],
-            b"Cache-Control": [b"max-age=0"],
-            b"Referer": [b"http://httpbin.org/forms/post"],
-            b"Connection": [b"keep-alive"],
-        }
-
-    def test_from_curl_with_kwargs(self):
-        r = self.request_class.from_curl(
-            'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
-        )
-        assert r.method == "POST"
-        assert r.meta == {"key": "value"}
-
-    def test_from_curl_ignore_unknown_options(self):
-        # By default: it works and ignores the unknown options: --foo and -z
-        with warnings.catch_warnings():  # avoid warning when executing tests
-            warnings.filterwarnings(
-                "ignore", category=UserWarning, message="Unrecognized options:"
-            )
-            r = self.request_class.from_curl(
-                'curl -X DELETE "http://example.org" --foo -z',
-            )
-        assert r.method == "DELETE"
-
-        # If `ignore_unknown_options` is set to `False` it raises an error with
-        # the unknown options: --foo and -z
-        with pytest.raises(ValueError, match="Unrecognized options:"):
-            self.request_class.from_curl(
-                'curl -X PATCH "http://example.org" --foo -z',
-                ignore_unknown_options=False,
-            )
-
-
-class TestXmlRpcRequest(TestRequest):
+class TestXmlRpcRequest(TestRequestBase):
     request_class = XmlRpcRequest
     default_method = "POST"
     default_headers = {b"Content-Type": [b"text/xml"]}
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index af86b35c0..5e965e8dc 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -10,7 +10,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import FormRequest, HtmlResponse
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode
-from tests.test_http_request import TestRequest
+from tests.utils.bases.http_request import TestRequestBase
 
 
 def _buildresponse(body, **kwargs):
@@ -31,7 +31,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
 # FormRequest.from_response() is deprecated in favor of form2request, so the
 # many tests below that exercise it ignore the resulting deprecation warning.
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestFormRequest(TestRequest):
+class TestFormRequest(TestRequestBase):
     request_class = FormRequest
 
     def assertQueryEqual(self, first, second, msg=None):
diff --git a/tests/test_http_request_json.py b/tests/test_http_request_json.py
index 65022afe1..1bfbbbdf5 100644
--- a/tests/test_http_request_json.py
+++ b/tests/test_http_request_json.py
@@ -8,10 +8,10 @@ import pytest
 
 from scrapy.http import JsonRequest
 from scrapy.utils.python import to_bytes
-from tests.test_http_request import TestRequest
+from tests.utils.bases.http_request import TestRequestBase
 
 
-class TestJsonRequest(TestRequest):
+class TestJsonRequest(TestRequestBase):
     request_class = JsonRequest
     default_method = "GET"
     default_headers = {
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index a8ea4920e..369c2495f 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,413 +1,8 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
-import pytest
-from w3lib.encoding import resolve_encoding
-
-from scrapy.exceptions import NotSupported
-from scrapy.http import Headers, Request, Response
-from scrapy.link import Link
-from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
-from tests import get_testdata
-
-if TYPE_CHECKING:
-    from collections.abc import Iterable
+from scrapy.http import Response
+from tests.utils.bases.http_response import TestResponseBase
 
 
-class TestResponse:
+class TestResponse(TestResponseBase):
     response_class = Response
-
-    def test_init(self):
-        # Response requires url in the constructor
-        with pytest.raises(TypeError):
-            self.response_class()
-        assert isinstance(
-            self.response_class("http://example.com/"), self.response_class
-        )
-        with pytest.raises(TypeError):
-            self.response_class(b"http://example.com")
-        with pytest.raises(TypeError):
-            self.response_class(url="http://example.com", body={})
-        # body can be str or None
-        assert isinstance(
-            self.response_class("http://example.com/", body=b""),
-            self.response_class,
-        )
-        assert isinstance(
-            self.response_class("http://example.com/", body=b"body"),
-            self.response_class,
-        )
-        # test presence of all optional parameters
-        assert isinstance(
-            self.response_class(
-                "http://example.com/", body=b"", headers={}, status=200
-            ),
-            self.response_class,
-        )
-
-        r = self.response_class("http://www.example.com")
-        assert isinstance(r.url, str)
-        assert r.url == "http://www.example.com"
-        assert r.status == 200
-
-        assert isinstance(r.headers, Headers)
-        assert not r.headers
-
-        headers = {"foo": "bar"}
-        body = b"a body"
-        r = self.response_class("http://www.example.com", headers=headers, body=body)
-
-        assert r.headers is not headers
-        assert r.headers[b"foo"] == b"bar"
-
-        r = self.response_class("http://www.example.com", status=301)
-        assert r.status == 301
-        r = self.response_class("http://www.example.com", status="301")
-        assert r.status == 301
-        with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
-            self.response_class("http://example.com", status="lala200")
-
-    def test_copy(self):
-        """Test Response copy"""
-
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        r1.flags.append("cached")
-        r2 = r1.copy()
-
-        assert r1.status == r2.status
-        assert r1.body == r2.body
-
-        # make sure flags list is shallow copied
-        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
-        assert r1.flags == r2.flags
-
-        # make sure headers attribute is shallow copied
-        assert r1.headers is not r2.headers, (
-            "headers must be a shallow copy, not identical"
-        )
-        assert r1.headers == r2.headers
-
-    def test_copy_meta(self):
-        req = Request("http://www.example.com")
-        req.meta["foo"] = "bar"
-        r1 = self.response_class(
-            "http://www.example.com", body=b"Some body", request=req
-        )
-        assert r1.meta is req.meta
-
-    def test_copy_cb_kwargs(self):
-        req = Request("http://www.example.com")
-        req.cb_kwargs["foo"] = "bar"
-        r1 = self.response_class(
-            "http://www.example.com", body=b"Some body", request=req
-        )
-        assert r1.cb_kwargs is req.cb_kwargs
-
-    def test_unavailable_meta(self):
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with pytest.raises(AttributeError, match=r"Response\.meta not available"):
-            r1.meta
-
-    def test_unavailable_cb_kwargs(self):
-        r1 = self.response_class("http://www.example.com", body=b"Some body")
-        with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
-            r1.cb_kwargs
-
-    def test_copy_inherited_classes(self):
-        """Test Response children copies preserve their class"""
-
-        class CustomResponse(self.response_class):
-            pass
-
-        r1 = CustomResponse("http://www.example.com")
-        r2 = r1.copy()
-
-        assert isinstance(r2, CustomResponse)
-
-    def test_replace(self):
-        """Test Response.replace() method"""
-        hdrs = Headers({"key": "value"})
-        r1 = self.response_class("http://www.example.com")
-        r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
-        assert r1.body == b""
-        assert r1.url == r2.url
-        assert (r1.status, r2.status) == (200, 301)
-        assert (r1.body, r2.body) == (b"", b"New body")
-        assert (r1.headers, r2.headers) == ({}, hdrs)
-
-        # Empty attributes (which may fail if not compared properly)
-        r3 = self.response_class("http://www.example.com", flags=["cached"])
-        r4 = r3.replace(body=b"", flags=[])
-        assert r4.body == b""
-        assert not r4.flags
-
-    def _assert_response_values(self, response, encoding, body):
-        if isinstance(body, str):
-            body_unicode = body
-            body_bytes = body.encode(encoding)
-        else:
-            body_unicode = body.decode(encoding)
-            body_bytes = body
-
-        assert isinstance(response.body, bytes)
-        assert isinstance(response.text, str)
-        self._assert_response_encoding(response, encoding)
-        assert response.body == body_bytes
-        assert response.text == body_unicode
-
-    def _assert_response_encoding(self, response, encoding):
-        assert response.encoding == resolve_encoding(encoding)
-
-    def test_immutable_attributes(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(AttributeError):
-            r.url = "http://example2.com"
-        with pytest.raises(AttributeError):
-            r.body = "xxx"
-
-    def test_setter_mutable_lazy_loading(self):
-        """Mutable attributes are set internally to None only until they are
-        read, then they always return the same falsy instance of the
-        corresponding mutable structure.
-
-        Setting them to None causes the next read to return a different object.
-        """
-
-        response = self.response_class("http://example.com")
-
-        response.request = Request("http://example.com")
-
-        assert response._flags is None
-        assert response.flags == []
-        assert response.flags is response.flags
-        assert response._flags == []
-        original_flags = response.flags
-        response.flags = None
-        assert response._flags is None
-        assert response.flags == []
-        assert response.flags is not original_flags
-
-        assert response._headers is None
-        assert response.headers == {}
-        assert response.headers is response.headers
-        assert isinstance(response.headers, Headers)
-        assert isinstance(response._headers, Headers)
-        original_headers = response.headers
-        response.headers = None
-        assert response._headers is None
-        assert response.headers == {}
-        assert response._headers == {}
-        assert response.headers is not original_headers
-
-    def test_setters(self):
-        response = self.response_class("http://example.com")
-
-        response.flags = ["f1"]
-        assert response.flags == ["f1"]
-
-        headers = Headers({b"X-Test": b"1"})
-        response.headers = headers
-        assert response._headers is headers
-        response.headers = {b"A": b"b"}
-        assert isinstance(response.headers, Headers)
-        assert response._headers[b"A"] == b"b"
-
-    def test_urljoin(self):
-        """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
-        joined = self.response_class("http://www.example.com").urljoin("/test")
-        absolute = "http://www.example.com/test"
-        assert joined == absolute
-
-    def test_shortcut_attributes(self):
-        r = self.response_class("http://example.com", body=b"hello")
-        if self.response_class == Response:
-            msg = "Response content isn't text"
-            with pytest.raises(AttributeError, match=msg):
-                r.text
-            with pytest.raises(NotSupported, match=msg):
-                r.css("body")
-            with pytest.raises(NotSupported, match=msg):
-                r.xpath("//body")
-            with pytest.raises(NotSupported, match=msg):
-                r.jmespath("body")
-        else:
-            r.text
-            r.css("body")
-            r.xpath("//body")
-
-    # Response.follow
-
-    def test_follow_url_absolute(self):
-        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
-
-    def test_follow_url_relative(self):
-        self._assert_followed_url("foo", "http://example.com/foo")
-
-    def test_follow_link(self):
-        self._assert_followed_url(
-            Link("http://example.com/foo"), "http://example.com/foo"
-        )
-
-    def test_follow_None_url(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(ValueError, match="url can't be None"):
-            r.follow(None)
-
-    def test_follow_None_encoding(self):
-        r = self.response_class("http://example.com")
-        with pytest.raises(ValueError, match="encoding can't be None"):
-            r.follow("foo", encoding=None)
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_whitespace_url(self):
-        self._assert_followed_url("foo ", "http://example.com/foo")
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_whitespace_link(self):
-        self._assert_followed_url(
-            Link("http://example.com/foo "), "http://example.com/foo"
-        )
-
-    def test_follow_flags(self):
-        res = self.response_class("http://example.com/")
-        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
-        assert fol.flags == ["cached", "allowed"]
-
-    # Response.follow_all
-
-    def test_follow_all_absolute(self):
-        url_list = [
-            "http://example.org",
-            "http://www.example.org",
-            "http://example.com",
-            "http://www.example.com",
-        ]
-        self._assert_followed_all_urls(url_list, url_list)
-
-    def test_follow_all_relative(self):
-        relative = ["foo", "bar", "foo/bar", "bar/foo"]
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        self._assert_followed_all_urls(relative, absolute)
-
-    def test_follow_all_links(self):
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        links = map(Link, absolute)
-        self._assert_followed_all_urls(links, absolute)
-
-    def test_follow_all_empty(self):
-        r = self.response_class("http://example.com")
-        assert not list(r.follow_all([]))
-
-    def test_follow_all_invalid(self):
-        r = self.response_class("http://example.com")
-        if self.response_class == Response:
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=None))
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=12345))
-            with pytest.raises(ValueError, match="url can't be None"):
-                list(r.follow_all(urls=[None]))
-        else:
-            with pytest.raises(
-                ValueError, match="Please supply exactly one of the following arguments"
-            ):
-                list(r.follow_all(urls=None))
-            with pytest.raises(TypeError):
-                list(r.follow_all(urls=12345))
-            with pytest.raises(ValueError, match="url can't be None"):
-                list(r.follow_all(urls=[None]))
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_all_whitespace(self):
-        relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
-        absolute = [
-            "http://example.com/foo",
-            "http://example.com/bar",
-            "http://example.com/foo/bar",
-            "http://example.com/bar/foo",
-        ]
-        self._assert_followed_all_urls(relative, absolute)
-
-    @pytest.mark.xfail(
-        not W3LIB_STRIPS_URLS,
-        reason="https://github.com/scrapy/w3lib/pull/207",
-        strict=True,
-    )
-    def test_follow_all_whitespace_links(self):
-        absolute = [
-            "http://example.com/foo ",
-            "http://example.com/bar ",
-            "http://example.com/foo/bar ",
-            "http://example.com/bar/foo ",
-        ]
-        links = [Link(u) for u in absolute]
-        expected = [u.strip() for u in absolute]
-        self._assert_followed_all_urls(links, expected)
-
-    def test_follow_all_flags(self):
-        re = self.response_class("http://www.example.com/")
-        urls = [
-            "http://www.example.com/",
-            "http://www.example.com/2",
-            "http://www.example.com/foo",
-        ]
-        fol = re.follow_all(urls, flags=["cached", "allowed"])
-        for req in fol:
-            assert req.flags == ["cached", "allowed"]
-
-    def _assert_followed_url(
-        self,
-        follow_obj: str | Link,
-        target_url: str,
-        response: Response | None = None,
-        encoding: str | None = None,
-    ) -> None:
-        if response is None:
-            response = self._links_response()
-        req = response.follow(follow_obj)
-        assert req.url == target_url
-        if encoding is not None:
-            assert req.encoding == encoding
-
-    def _assert_followed_all_urls(
-        self,
-        follow_obj: Iterable[str | Link],
-        target_urls: Iterable[str],
-        response: Response | None = None,
-    ) -> None:
-        if response is None:
-            response = self._links_response()
-        followed = response.follow_all(follow_obj)
-        for req, target in zip(followed, target_urls, strict=True):
-            assert req.url == target
-
-    def _links_response(self) -> Response:
-        body = get_testdata("link_extractor", "linkextractor.html")
-        return self.response_class("http://example.com/index", body=body)
-
-    def _links_response_no_href(self) -> Response:
-        body = get_testdata("link_extractor", "linkextractor_no_href.html")
-        return self.response_class("http://example.com/index", body=body)
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index daf6e66e5..04315ad89 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -8,10 +8,10 @@ import pytest
 from scrapy.http import HtmlResponse, TextResponse, XmlResponse
 from scrapy.selector import Selector
 from scrapy.utils.python import to_unicode
-from tests.test_http_response import TestResponse
+from tests.utils.bases.http_response import TestResponseBase
 
 
-class TestTextResponse(TestResponse):
+class TestTextResponse(TestResponseBase):
     response_class = TextResponse
 
     def test_follow_None_encoding(self):
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0d9aac830..a0ae3635b 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -39,8 +39,8 @@ from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
-from .test_pipeline_media import _mocked_download_func
 from .utils.cloud import mock_google_cloud_storage
+from .utils.media_pipelines import mocked_download_func
 
 # required by persist_file() and stat_file(), but as some stores don't use the argument
 # we can pass this singleton to keep type hints correct
@@ -94,7 +94,7 @@ class TestFilesPipeline:
         settings_dict = {"FILES_STORE": self.tempdir}
         crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
         crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=_mocked_download_func)
+        crawler.engine = MagicMock(download_async=mocked_download_func)
         self.pipeline = FilesPipeline.from_crawler(crawler)
         self.pipeline.open_spider()
 
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 6862c34eb..ee7a576db 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -9,7 +9,6 @@ from twisted.python.failure import Failure
 from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
-from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.files import FileException
 from scrapy.pipelines.media import MediaPipeline
 from scrapy.utils.defer import _defer_sleep_async
@@ -18,16 +17,7 @@ from scrapy.utils.signal import disconnect_all
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
-
-
-async def _mocked_download_func(request):
-    assert request.callback is NO_CALLBACK
-    response = request.meta.get("response")
-    if callable(response):
-        response = await response()
-    if isinstance(response, Exception):
-        raise response
-    return response
+from tests.utils.media_pipelines import mocked_download_func
 
 
 class UserDefinedPipeline(MediaPipeline):
@@ -54,7 +44,7 @@ class TestBaseMediaPipeline:
     def setup_method(self):
         crawler = get_crawler(DefaultSpider, self.settings)
         crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=_mocked_download_func)
+        crawler.engine = MagicMock(download_async=mocked_download_func)
         self.pipe = self.pipeline_class.from_crawler(crawler)
         self.pipe.open_spider()
         self.info = self.pipe.spiderinfo
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index 6c6a6584a..85fefd172 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -11,7 +11,7 @@ from scrapy.spiders import Spider
 from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.test import get_crawler
-from tests.test_scheduler import MockDownloader
+from tests.utils.downloader import MockDownloader
 
 
 class TestPriorityQueue:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 4fb594444..d7c21bdbd 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import warnings
 from abc import ABC, abstractmethod
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from typing import TYPE_CHECKING, Any, NamedTuple, cast
+from typing import TYPE_CHECKING
 from unittest.mock import Mock
 
 import pytest
@@ -15,43 +15,17 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.spiders import Spider
 from scrapy.utils.defer import ensure_awaitable
-from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.downloader import MockDownloader
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
     from pathlib import Path
 
 
-class MockSlot(NamedTuple):
-    active: list[Any]
-
-
-class MockDownloader:
-    def __init__(self) -> None:
-        self.slots: dict[str, MockSlot] = {}
-
-    def get_slot_key(self, request: Request) -> str:
-        if Downloader.DOWNLOAD_SLOT in request.meta:
-            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
-
-        return urlparse_cached(request).hostname or ""
-
-    def increment(self, slot_key: str) -> None:
-        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
-        slot.active.append(1)
-
-    def decrement(self, slot_key: str) -> None:
-        slot = self.slots[slot_key]
-        slot.active.pop()
-
-    def close(self) -> None:
-        pass
-
-
 class MockCrawler(Crawler):
     def __init__(self, priority_queue_cls: str, jobdir: Path | None):
         settings = {
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 7b2727370..03d17199f 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,133 +1,18 @@
 from __future__ import annotations
 
-from typing import Any
-from unittest import mock
-
 import pytest
-from testfixtures import LogCapture
 
-from scrapy import signals
-from scrapy.crawler import Crawler
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, XmlResponse
-from scrapy.settings import Settings
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
-from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import get_testdata
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.bases.spider import TestSpiderBase
 
 
-class TestSpider:
+class TestSpider(TestSpiderBase):
     spider_class = Spider
 
-    def test_base_spider(self):
-        spider = self.spider_class("example.com")
-        assert spider.name == "example.com"
-        assert spider.start_urls == []
 
-    def test_spider_args(self):
-        """``__init__`` method arguments are assigned to spider attributes"""
-        spider = self.spider_class("example.com", foo="bar")
-        assert spider.foo == "bar"
-
-    def test_spider_without_name(self):
-        """``__init__`` raises when the name is not provided."""
-        msg = "must have a name"
-        with pytest.raises(ValueError, match=msg):
-            self.spider_class()
-        with pytest.raises(ValueError, match=msg):
-            self.spider_class(somearg="foo")
-
-    def test_from_crawler_crawler_and_settings_population(self):
-        crawler = get_crawler()
-        spider = self.spider_class.from_crawler(crawler, "example.com")
-        assert hasattr(spider, "crawler")
-        assert spider.crawler is crawler
-        assert hasattr(spider, "settings")
-        assert spider.settings is crawler.settings
-
-    def test_from_crawler_init_call(self):
-        with mock.patch.object(
-            self.spider_class, "__init__", return_value=None
-        ) as mock_init:
-            self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
-            mock_init.assert_called_once_with("example.com", foo="bar")
-
-    def test_closed_signal_call(self):
-        class TestSpider(self.spider_class):
-            closed_called = False
-
-            def closed(self, reason):
-                self.closed_called = True
-
-        crawler = get_crawler()
-        spider = TestSpider.from_crawler(crawler, "example.com")
-        crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
-        crawler.signals.send_catch_log(
-            signal=signals.spider_closed, spider=spider, reason=None
-        )
-        assert spider.closed_called
-
-    def test_update_settings(self):
-        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {"TEST1": "project", "TEST3": "project"}
-        self.spider_class.custom_settings = spider_settings
-        settings = Settings(project_settings, priority="project")
-
-        self.spider_class.update_settings(settings)
-        assert settings.get("TEST1") == "spider"
-        assert settings.get("TEST2") == "spider"
-        assert settings.get("TEST3") == "project"
-
-    @inline_callbacks_test
-    def test_settings_in_from_crawler(self):
-        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
-        project_settings = {
-            "TEST1": "project",
-            "TEST3": "project",
-            **get_reactor_settings(),
-        }
-
-        class TestSpider(self.spider_class):
-            name = "test"
-            custom_settings = spider_settings
-
-            @classmethod
-            def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
-                spider = super().from_crawler(crawler, *args, **kwargs)
-                spider.settings.set("TEST1", "spider_instance", priority="spider")
-                return spider
-
-        crawler = Crawler(TestSpider, project_settings)
-        assert crawler.settings.get("TEST1") == "spider"
-        assert crawler.settings.get("TEST2") == "spider"
-        assert crawler.settings.get("TEST3") == "project"
-        yield crawler.crawl()
-        assert crawler.settings.get("TEST1") == "spider_instance"
-
-    def test_logger(self):
-        spider = self.spider_class("example.com")
-        with LogCapture() as lc:
-            spider.logger.info("test log msg")
-        lc.check(("example.com", "INFO", "test log msg"))
-
-        record = lc.records[0]
-        assert "spider" in record.__dict__
-        assert record.spider is spider
-
-    def test_log(self):
-        spider = self.spider_class("example.com")
-        with (
-            mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
-            ),
-        ):
-            spider.log("test log msg", "INFO")
-        mock_logger.log.assert_called_once_with("INFO", "test log msg")
-
-
-class TestXMLFeedSpider(TestSpider):
+class TestXMLFeedSpider(TestSpiderBase):
     spider_class = XMLFeedSpider
 
     def test_register_namespace(self):
@@ -176,7 +61,7 @@ class TestXMLFeedSpider(TestSpider):
             ], iterator
 
 
-class TestCSVFeedSpider(TestSpider):
+class TestCSVFeedSpider(TestSpiderBase):
     spider_class = CSVFeedSpider
 
     def test_parse_rows(self):
diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py
index 63010e195..f34f9add9 100644
--- a/tests/test_spider_crawl.py
+++ b/tests/test_spider_crawl.py
@@ -11,10 +11,10 @@ from scrapy.http import HtmlResponse, Request, TextResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
 from scrapy.utils.test import get_crawler
-from tests.test_spider import TestSpider
+from tests.utils.bases.spider import TestSpiderBase
 
 
-class TestCrawlSpider(TestSpider):
+class TestCrawlSpider(TestSpiderBase):
     test_body = b"""<html><head><title>Page title</title></head>
     <body>
     <p><a href="item/12.html">Item 12</a></p>
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 0af99ab6d..c8cfd364e 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -14,11 +14,11 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon
 from scrapy.spiders import SitemapSpider
 from scrapy.utils.test import get_crawler
 from tests import tests_datadir
-from tests.test_spider import TestSpider
+from tests.utils.bases.spider import TestSpiderBase
 from tests.utils.decorators import coroutine_test
 
 
-class TestSitemapSpider(TestSpider):
+class TestSitemapSpider(TestSpiderBase):
     spider_class = SitemapSpider
 
     BODY = b"SITEMAP"
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index 67ce3a920..321139da3 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -5,7 +5,6 @@ import pytest
 from scrapy import Spider, signals
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
-from tests.test_spider_start import SLEEP_SECONDS
 
 from .utils import twisted_sleep
 from .utils.decorators import coroutine_test
@@ -14,6 +13,8 @@ ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}
 ITEM_C = {"id": "c"}
 
+SLEEP_SECONDS = 0.1
+
 
 class AsyncioSleepSpiderMiddleware:
     async def process_start(self, start):
diff --git a/tests/utils/bases/__init__.py b/tests/utils/bases/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/utils/base_commands.py b/tests/utils/bases/commands.py
similarity index 100%
rename from tests/utils/base_commands.py
rename to tests/utils/bases/commands.py
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/utils/bases/download_handlers_http.py
similarity index 99%
rename from tests/test_downloader_handlers_http_base.py
rename to tests/utils/bases/download_handlers_http.py
index 29e1d9757..e44f9bcb8 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -1,5 +1,3 @@
-"""Base classes for HTTP download handler tests."""
-
 from __future__ import annotations
 
 import gzip
diff --git a/tests/utils/bases/engine.py b/tests/utils/bases/engine.py
new file mode 100644
index 000000000..1cabbb6ef
--- /dev/null
+++ b/tests/utils/bases/engine.py
@@ -0,0 +1,160 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from itemadapter import ItemAdapter
+
+from scrapy import signals
+from tests import get_testdata
+
+if TYPE_CHECKING:
+    from tests.utils.engine import CrawlerRun
+
+
+class TestEngineBase:
+    @staticmethod
+    def _assert_visited_urls(run: CrawlerRun) -> None:
+        must_be_visited = [
+            "/static/",
+            "/redirect",
+            "/redirected",
+            "/static/item1.html",
+            "/static/item2.html",
+            "/static/item999.html",
+        ]
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
+        assert urls_expected <= urls_visited, (
+            f"URLs not visited: {list(urls_expected - urls_visited)}"
+        )
+
+    @staticmethod
+    def _assert_scheduled_requests(run: CrawlerRun, count: int) -> None:
+        assert len(run.reqplug) == count
+
+        paths_expected = [
+            "/static/item999.html",
+            "/static/item2.html",
+            "/static/item1.html",
+        ]
+
+        urls_requested = {rq[0].url for rq in run.reqplug}
+        urls_expected = {run.geturl(p) for p in paths_expected}
+        assert urls_expected <= urls_requested
+        scheduled_requests_count = len(run.reqplug)
+        dropped_requests_count = len(run.reqdropped)
+        responses_count = len(run.respplug)
+        assert scheduled_requests_count == dropped_requests_count + responses_count
+        assert len(run.reqreached) == responses_count
+
+    @staticmethod
+    def _assert_dropped_requests(run: CrawlerRun) -> None:
+        assert len(run.reqdropped) == 1
+
+    @staticmethod
+    def _assert_downloaded_responses(run: CrawlerRun, count: int) -> None:
+        # response tests
+        assert len(run.respplug) == count
+        assert len(run.reqreached) == count
+
+        for response, _ in run.respplug:
+            if run.getpath(response.url) == "/static/item999.html":
+                assert response.status == 404
+            if run.getpath(response.url) == "/redirect":
+                assert response.status == 302
+
+    @staticmethod
+    def _assert_items_error(run: CrawlerRun) -> None:
+        assert len(run.itemerror) == 2
+        for item, response, spider, failure in run.itemerror:
+            assert failure.value.__class__ is ZeroDivisionError
+            assert spider == run.crawler.spider
+
+            assert item["url"] == response.url
+            if "item1.html" in item["url"]:
+                assert item["name"] == "Item 1 name"
+                assert item["price"] == "100"
+            if "item2.html" in item["url"]:
+                assert item["name"] == "Item 2 name"
+                assert item["price"] == "200"
+
+    @staticmethod
+    def _assert_scraped_items(run: CrawlerRun) -> None:
+        assert len(run.itemresp) == 2
+        for item_, response in run.itemresp:
+            item = ItemAdapter(item_)
+            assert item["url"] == response.url
+            if "item1.html" in item["url"]:
+                assert item["name"] == "Item 1 name"
+                assert item["price"] == "100"
+            if "item2.html" in item["url"]:
+                assert item["name"] == "Item 2 name"
+                assert item["price"] == "200"
+
+    @staticmethod
+    def _assert_headers_received(run: CrawlerRun) -> None:
+        for headers in run.headers.values():
+            assert b"Server" in headers
+            assert headers[b"Server"]
+            assert b"TwistedWeb" in headers[b"Server"]
+            assert b"Date" in headers
+            assert b"Content-Type" in headers
+
+    @staticmethod
+    def _assert_bytes_received(run: CrawlerRun) -> None:
+        assert len(run.bytes) == 9
+        for request, data in run.bytes.items():
+            joined_data = b"".join(data)
+            if run.getpath(request.url) == "/static/":
+                assert joined_data == get_testdata("test_site", "index.html")
+            elif run.getpath(request.url) == "/static/item1.html":
+                assert joined_data == get_testdata("test_site", "item1.html")
+            elif run.getpath(request.url) == "/static/item2.html":
+                assert joined_data == get_testdata("test_site", "item2.html")
+            elif run.getpath(request.url) == "/redirected":
+                assert joined_data == b"Redirected here"
+            elif run.getpath(request.url) == "/redirect":
+                assert (
+                    joined_data == b"\n<html>\n"
+                    b"    <head>\n"
+                    b'        <meta http-equiv="refresh" content="0;URL=/redirected">\n'
+                    b"    </head>\n"
+                    b'    <body bgcolor="#FFFFFF" text="#000000">\n'
+                    b'    <a href="/redirected">click here</a>\n'
+                    b"    </body>\n"
+                    b"</html>\n"
+                )
+            elif run.getpath(request.url) == "/static/item999.html":
+                assert (
+                    joined_data == b"\n<html>\n"
+                    b"  <head><title>404 - No Such Resource</title></head>\n"
+                    b"  <body>\n"
+                    b"    <h1>No Such Resource</h1>\n"
+                    b"    <p>File not found.</p>\n"
+                    b"  </body>\n"
+                    b"</html>\n"
+                )
+            elif run.getpath(request.url) == "/numbers":
+                # signal was fired multiple times
+                assert len(data) > 1
+                # bytes were received in order
+                numbers = [str(x).encode("utf8") for x in range(2**18)]
+                assert joined_data == b"".join(numbers)
+
+    @staticmethod
+    def _assert_signals_caught(run: CrawlerRun) -> None:
+        assert signals.engine_started in run.signals_caught
+        assert signals.engine_stopped in run.signals_caught
+        assert signals.spider_opened in run.signals_caught
+        assert signals.spider_idle in run.signals_caught
+        assert signals.spider_closed in run.signals_caught
+        assert signals.headers_received in run.signals_caught
+
+        assert {"spider": run.crawler.spider} == run.signals_caught[
+            signals.spider_opened
+        ]
+        assert {"spider": run.crawler.spider} == run.signals_caught[signals.spider_idle]
+        assert {
+            "spider": run.crawler.spider,
+            "reason": "finished",
+        } == run.signals_caught[signals.spider_closed]
diff --git a/tests/utils/bases/feedexport.py b/tests/utils/bases/feedexport.py
new file mode 100644
index 000000000..f03ac1fd4
--- /dev/null
+++ b/tests/utils/bases/feedexport.py
@@ -0,0 +1,151 @@
+from __future__ import annotations
+
+import random
+import shutil
+import tempfile
+from abc import ABC, abstractmethod
+from pathlib import Path
+from string import ascii_letters, digits
+from typing import IO, TYPE_CHECKING, Any
+
+import scrapy
+from scrapy import Spider
+from tests.mockserver.http import MockServer
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable
+
+
+class TestFeedExportBase(ABC):
+    mockserver: MockServer
+
+    def _random_temp_filename(self, inter_dir="") -> Path:
+        chars = [random.choice(ascii_letters + digits) for _ in range(15)]
+        filename = "".join(chars)
+        return Path(self.temp_dir, inter_dir, filename)
+
+    @classmethod
+    def setup_class(cls):
+        cls.mockserver = MockServer()
+        cls.mockserver.__enter__()  # pylint: disable=unnecessary-dunder-call
+
+    @classmethod
+    def teardown_class(cls):
+        cls.mockserver.__exit__(None, None, None)
+
+    def setup_method(self):
+        self.temp_dir = tempfile.mkdtemp()
+
+    def teardown_method(self):
+        shutil.rmtree(self.temp_dir, ignore_errors=True)
+
+    async def exported_data(
+        self, items: Iterable[Any], settings: dict[str, Any]
+    ) -> dict[str, Any]:
+        """
+        Return exported data which a spider yielding ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                yield from items
+
+        return await self.run_and_export(TestSpider, settings)
+
+    async def exported_no_data(self, settings: dict[str, Any]) -> dict[str, Any]:
+        """
+        Return exported data which a spider yielding no ``items`` would return.
+        """
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+
+            def parse(self, response):
+                pass
+
+        return await self.run_and_export(TestSpider, settings)
+
+    async def assertExported(
+        self,
+        items: Iterable[Any],
+        header: Iterable[str],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        await self.assertExportedCsv(items, header, rows, settings)
+        await self.assertExportedJsonLines(items, rows, settings)
+        await self.assertExportedXml(items, rows, settings)
+        await self.assertExportedPickle(items, rows, settings)
+        await self.assertExportedMarshal(items, rows, settings)
+        await self.assertExportedMultiple(items, rows, settings)
+
+    async def assertExportedCsv(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        header: Iterable[str],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedJsonLines(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedXml(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedMultiple(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedPickle(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    async def assertExportedMarshal(  # noqa: B027
+        self,
+        items: Iterable[Any],
+        rows: Iterable[dict[str, Any]],
+        settings: dict[str, Any] | None = None,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    async def run_and_export(
+        self, spider_cls: type[Spider], settings: dict[str, Any]
+    ) -> dict[str, Any]:
+        pass
+
+    def _load_until_eof(
+        self, data: bytes, load_func: Callable[[IO[bytes]], Any]
+    ) -> list[Any]:
+        result: list[Any] = []
+        with tempfile.TemporaryFile() as temp:
+            temp.write(data)
+            temp.seek(0)
+            while True:
+                try:
+                    result.append(load_func(temp))
+                except EOFError:
+                    break
+        return result
diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py
new file mode 100644
index 000000000..c255b5e4c
--- /dev/null
+++ b/tests/utils/bases/http_request.py
@@ -0,0 +1,490 @@
+import warnings
+from abc import ABC, abstractmethod
+from typing import Any
+
+import pytest
+
+from scrapy.http import Headers, Request
+from scrapy.http.request import NO_CALLBACK
+
+
+class TestRequestBase(ABC):
+    default_method = "GET"
+    default_headers: dict[bytes, list[bytes]] = {}
+    default_meta: dict[str, Any] = {}
+
+    @property
+    @abstractmethod
+    def request_class(self) -> type[Request]:
+        raise NotImplementedError
+
+    def test_init(self):
+        # Request requires url in the __init__ method
+        with pytest.raises(TypeError):
+            self.request_class()
+
+        # url argument must be basestring
+        with pytest.raises(TypeError):
+            self.request_class(123)
+
+        # priority argument must be an integer
+        with pytest.raises(TypeError, match="Request priority not an integer"):
+            self.request_class("http://www.example.com", priority="1")
+
+        r = self.request_class("http://www.example.com")
+        assert isinstance(r.url, str)
+        assert r.url == "http://www.example.com"
+        assert r.method == self.default_method
+
+        assert isinstance(r.headers, Headers)
+        assert r.headers == self.default_headers
+        assert r.meta == self.default_meta
+
+        meta = {"lala": "lolo"}
+        headers = {b"caca": b"coco"}
+        r = self.request_class(
+            "http://www.example.com", meta=meta, headers=headers, body="a body"
+        )
+
+        assert r.meta is not meta
+        assert r.meta == meta
+        assert r.headers is not headers
+        assert r.headers[b"caca"] == b"coco"
+
+    def test_url_scheme(self):
+        # This test passes by not raising any (ValueError) exception
+        self.request_class("http://example.org")
+        self.request_class("https://example.org")
+        self.request_class("s3://example.org")
+        self.request_class("ftp://example.org")
+        self.request_class("about:config")
+        self.request_class("data:,Hello%2C%20World!")
+
+    def test_url_no_scheme(self):
+        msg = "Missing scheme in request url:"
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("foo")
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("/foo/")
+        with pytest.raises(ValueError, match=msg):
+            self.request_class("/foo:bar")
+
+    def test_headers(self):
+        # Different ways of setting headers attribute
+        url = "http://www.scrapy.org"
+        headers = {b"Accept": "gzip", b"Custom-Header": "nothing to tell you"}
+        r = self.request_class(url=url, headers=headers)
+        p = self.request_class(url=url, headers=r.headers)
+
+        assert r.headers == p.headers
+        assert r.headers is not headers
+        assert p.headers is not r.headers
+
+        # headers must not be unicode
+        h = Headers({"key1": "val1", "key2": "val2"})
+        h["newkey"] = "newval"
+        for k, v in h.items():
+            assert isinstance(k, bytes)
+            for s in v:
+                assert isinstance(s, bytes)
+
+    def test_eq(self):
+        url = "http://www.scrapy.org"
+        r1 = self.request_class(url=url)
+        r2 = self.request_class(url=url)
+        assert r1 != r2
+
+        set_ = set()
+        set_.add(r1)
+        set_.add(r2)
+        assert len(set_) == 2
+
+    def test_url(self):
+        r = self.request_class(url="http://www.scrapy.org/path")
+        assert r.url == "http://www.scrapy.org/path"
+
+    def test_url_quoting(self):
+        r = self.request_class(url="http://www.scrapy.org/blank%20space")
+        assert r.url == "http://www.scrapy.org/blank%20space"
+        r = self.request_class(url="http://www.scrapy.org/blank space")
+        assert r.url == "http://www.scrapy.org/blank%20space"
+
+    def test_url_encoding(self):
+        r = self.request_class(url="http://www.scrapy.org/price/£")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+    def test_url_encoding_other(self):
+        # encoding affects only query part of URI, not path
+        # path part should always be UTF-8 encoded before percent-escaping
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="utf-8")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+        r = self.request_class(url="http://www.scrapy.org/price/£", encoding="latin1")
+        assert r.url == "http://www.scrapy.org/price/%C2%A3"
+
+    def test_url_encoding_query(self):
+        r1 = self.request_class(url="http://www.scrapy.org/price/£?unit=µ")
+        assert r1.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
+
+        # should be same as above
+        r2 = self.request_class(
+            url="http://www.scrapy.org/price/£?unit=µ", encoding="utf-8"
+        )
+        assert r2.url == "http://www.scrapy.org/price/%C2%A3?unit=%C2%B5"
+
+    def test_url_encoding_query_latin1(self):
+        # encoding is used for encoding query-string before percent-escaping;
+        # path is still UTF-8 encoded before percent-escaping
+        r3 = self.request_class(
+            url="http://www.scrapy.org/price/µ?currency=£", encoding="latin1"
+        )
+        assert r3.url == "http://www.scrapy.org/price/%C2%B5?currency=%A3"
+
+    def test_url_encoding_nonutf8_untouched(self):
+        # percent-escaping sequences that do not match valid UTF-8 sequences
+        # should be kept untouched (just upper-cased perhaps)
+        #
+        # See https://datatracker.ietf.org/doc/html/rfc3987#section-3.2
+        #
+        # "Conversions from URIs to IRIs MUST NOT use any character encoding
+        # other than UTF-8 in steps 3 and 4, even if it might be possible to
+        # guess from the context that another character encoding than UTF-8 was
+        # used in the URI.  For example, the URI
+        # "http://www.example.org/r%E9sum%E9.html" might with some guessing be
+        # interpreted to contain two e-acute characters encoded as iso-8859-1.
+        # It must not be converted to an IRI containing these e-acute
+        # characters.  Otherwise, in the future the IRI will be mapped to
+        # "http://www.example.org/r%C3%A9sum%C3%A9.html", which is a different
+        # URI from "http://www.example.org/r%E9sum%E9.html".
+        r1 = self.request_class(url="http://www.scrapy.org/price/%a3")
+        assert r1.url == "http://www.scrapy.org/price/%a3"
+
+        r2 = self.request_class(url="http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3")
+        assert r2.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
+
+        r3 = self.request_class(url="http://www.scrapy.org/résumé/%a3")
+        assert r3.url == "http://www.scrapy.org/r%C3%A9sum%C3%A9/%a3"
+
+        r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
+        assert r4.url == "http://www.example.org/r%E9sum%E9.html"
+
+    def test_url_verbatim(self):
+        r = self.request_class(
+            url="http://www.scrapy.org/price/£",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/price/£"
+
+        r = self.request_class(
+            url="http://www.scrapy.org/blank space",
+            meta={"verbatim_url": True},
+        )
+        assert r.url == "http://www.scrapy.org/blank space"
+
+    def test_body(self):
+        r1 = self.request_class(url="http://www.example.com/")
+        assert r1.body == b""
+
+        r2 = self.request_class(url="http://www.example.com/", body=b"")
+        assert isinstance(r2.body, bytes)
+        assert r2.encoding == "utf-8"  # default encoding
+
+        r3 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="utf-8"
+        )
+        assert isinstance(r3.body, bytes)
+        assert r3.body == b"Price: \xc2\xa3100"
+
+        r4 = self.request_class(
+            url="http://www.example.com/", body="Price: \xa3100", encoding="latin1"
+        )
+        assert isinstance(r4.body, bytes)
+        assert r4.body == b"Price: \xa3100"
+
+    def test_copy(self):
+        """Test Request copy"""
+
+        def somecallback():
+            pass
+
+        r1 = self.request_class(
+            "http://www.example.com",
+            flags=["f1", "f2"],
+            callback=somecallback,
+            errback=somecallback,
+        )
+        r1.meta["foo"] = "bar"
+        r1.cb_kwargs["key"] = "value"
+        r2 = r1.copy()
+
+        # make sure callbaclks are copied
+        assert r1.callback is somecallback
+        assert r1.errback is somecallback
+        assert r2.callback is r1.callback
+        assert r2.errback is r1.errback
+
+        # make sure flags list is shallow copied
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        assert r1.flags == r2.flags
+
+        # make sure cb_kwargs dict is shallow copied
+        assert r1.cb_kwargs is not r2.cb_kwargs, (
+            "cb_kwargs must be a shallow copy, not identical"
+        )
+        assert r1.cb_kwargs == r2.cb_kwargs
+
+        # make sure meta dict is shallow copied
+        assert r1.meta is not r2.meta, "meta must be a shallow copy, not identical"
+        assert r1.meta == r2.meta
+
+        # make sure headers attribute is shallow copied
+        assert r1.headers is not r2.headers, (
+            "headers must be a shallow copy, not identical"
+        )
+        assert r1.headers == r2.headers
+        assert r1.encoding == r2.encoding
+        assert r1.dont_filter == r2.dont_filter
+
+        # Request.body can be identical since it's an immutable object (str)
+
+    def test_copy_inherited_classes(self):
+        """Test Request children copies preserve their class"""
+
+        class CustomRequest(self.request_class):
+            pass
+
+        r1 = CustomRequest("http://www.example.com")
+        r2 = r1.copy()
+
+        assert isinstance(r2, CustomRequest)
+
+    def test_replace(self):
+        """Test Request.replace() method"""
+        r1 = self.request_class("http://www.example.com", method="GET")
+        hdrs = Headers(r1.headers)
+        hdrs[b"key"] = b"value"
+        r2 = r1.replace(method="POST", body="New body", headers=hdrs)
+        assert r1.url == r2.url
+        assert (r1.method, r2.method) == ("GET", "POST")
+        assert (r1.body, r2.body) == (b"", b"New body")
+        assert (r1.headers, r2.headers) == (self.default_headers, hdrs)
+
+        # Empty attributes (which may fail if not compared properly)
+        r3 = self.request_class(
+            "http://www.example.com", meta={"a": 1}, dont_filter=True
+        )
+        r4 = r3.replace(
+            url="http://www.example.com/2", body=b"", meta={}, dont_filter=False
+        )
+        assert r4.url == "http://www.example.com/2"
+        assert r4.body == b""
+        assert r4.meta == {}
+        assert r4.dont_filter is False
+
+        # the cls argument allows changing the resulting class
+        custom_request_cls = type("CustomRequest", (self.request_class,), {})
+        r5 = r1.replace(cls=custom_request_cls)
+        assert isinstance(r5, custom_request_cls)
+        assert r5.url == r1.url
+
+    def test_method_always_str(self):
+        r = self.request_class("http://www.example.com", method="POST")
+        assert isinstance(r.method, str)
+
+    def test_immutable_attributes(self):
+        r = self.request_class("http://example.com")
+        with pytest.raises(AttributeError):
+            r.url = "http://example2.com"
+        with pytest.raises(AttributeError):
+            r.body = "xxx"
+
+    def test_callback_and_errback(self):
+        def a_function():
+            pass
+
+        r1 = self.request_class("http://example.com")
+        assert r1.callback is None
+        assert r1.errback is None
+
+        r2 = self.request_class("http://example.com", callback=a_function)
+        assert r2.callback is a_function
+        assert r2.errback is None
+
+        r3 = self.request_class("http://example.com", errback=a_function)
+        assert r3.callback is None
+        assert r3.errback is a_function
+
+        r4 = self.request_class(
+            url="http://example.com",
+            callback=a_function,
+            errback=a_function,
+        )
+        assert r4.callback is a_function
+        assert r4.errback is a_function
+
+        r5 = self.request_class(
+            url="http://example.com",
+            callback=NO_CALLBACK,
+            errback=NO_CALLBACK,
+        )
+        assert r5.callback is NO_CALLBACK
+        assert r5.errback is NO_CALLBACK
+
+    def test_callback_and_errback_type(self):
+        with pytest.raises(TypeError):
+            self.request_class("http://example.com", callback="a_function")
+        with pytest.raises(TypeError):
+            self.request_class("http://example.com", errback="a_function")
+        with pytest.raises(TypeError):
+            self.request_class(
+                url="http://example.com",
+                callback="a_function",
+                errback="a_function",
+            )
+
+    def test_setters(self):
+        request = self.request_class("http://example.com")
+
+        request.flags = ["f1"]
+        assert request.flags == ["f1"]
+
+        request.cookies = {"sid": "1"}
+        assert request.cookies == {"sid": "1"}
+
+        headers = Headers({b"X-Test": b"1"})
+        request.headers = headers
+        assert request._headers is headers
+        request.headers = {b"A": b"b"}
+        assert isinstance(request.headers, Headers)
+        assert request._headers[b"A"] == b"b"
+
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        request = self.request_class("http://example.com")
+
+        assert request._flags is None
+        assert request.flags == []
+        assert request.flags is request.flags
+        assert request._flags == []
+        original_flags = request.flags
+        request.flags = None
+        assert request._flags is None
+        assert request.flags == []
+        assert request.flags is not original_flags
+
+        assert request._cookies is None
+        assert request.cookies == {}
+        assert request.cookies is request.cookies
+        assert request._cookies == {}
+        original_cookies = request.cookies
+        request.cookies = None
+        assert request._cookies is None
+        assert request.cookies == {}
+        assert request.cookies is not original_cookies
+
+        if self.default_headers:
+            assert request._headers == self.default_headers
+            assert request._headers is not self.default_headers
+            assert request.headers == self.default_headers
+        else:
+            assert request._headers is None
+            assert request.headers == {}
+        assert request.headers is request.headers
+        assert isinstance(request.headers, Headers)
+        assert isinstance(request._headers, Headers)
+        original_headers = request.headers
+        request.headers = None
+        assert request._headers is None
+        assert request.headers == {}
+        assert request._headers == {}
+        assert request.headers is not original_headers
+
+    def test_no_callback(self):
+        with pytest.raises(RuntimeError):
+            NO_CALLBACK()
+
+    def test_from_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_curl.py`
+        curl_command = (
+            "curl 'http://httpbin.org/post' -X POST -H 'Cookie: _gauges_unique"
+            "_year=1; _gauges_unique=1; _gauges_unique_month=1; _gauges_unique"
+            "_hour=1; _gauges_unique_day=1' -H 'Origin: http://httpbin.org' -H"
+            " 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: en-US,en;q"
+            "=0.9,ru;q=0.8,es;q=0.7' -H 'Upgrade-Insecure-Requests: 1' -H 'Use"
+            "r-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTM"
+            "L, like Gecko) Ubuntu Chromium/62.0.3202.75 Chrome/62.0.3202.75 S"
+            "afari/537.36' -H 'Content-Type: application /x-www-form-urlencode"
+            "d' -H 'Accept: text/html,application/xhtml+xml,application/xml;q="
+            "0.9,image/webp,image/apng,*/*;q=0.8' -H 'Cache-Control: max-age=0"
+            "' -H 'Referer: http://httpbin.org/forms/post' -H 'Connection: kee"
+            "p-alive' --data 'custname=John+Smith&custtel=500&custemail=jsmith"
+            "%40example.org&size=small&topping=cheese&topping=onion&delivery=1"
+            "2%3A15&comments=' --compressed"
+        )
+        r = self.request_class.from_curl(curl_command)
+        assert r.method == "POST"
+        assert r.url == "http://httpbin.org/post"
+        assert (
+            r.body == b"custname=John+Smith&custtel=500&custemail=jsmith%40"
+            b"example.org&size=small&topping=cheese&topping=onion"
+            b"&delivery=12%3A15&comments="
+        )
+        assert r.cookies == {
+            "_gauges_unique_year": "1",
+            "_gauges_unique": "1",
+            "_gauges_unique_month": "1",
+            "_gauges_unique_hour": "1",
+            "_gauges_unique_day": "1",
+        }
+        assert r.headers == {
+            b"Origin": [b"http://httpbin.org"],
+            b"Accept-Encoding": [b"gzip, deflate"],
+            b"Accept-Language": [b"en-US,en;q=0.9,ru;q=0.8,es;q=0.7"],
+            b"Upgrade-Insecure-Requests": [b"1"],
+            b"User-Agent": [
+                b"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537."
+                b"36 (KHTML, like Gecko) Ubuntu Chromium/62.0.3202"
+                b".75 Chrome/62.0.3202.75 Safari/537.36"
+            ],
+            b"Content-Type": [b"application /x-www-form-urlencoded"],
+            b"Accept": [
+                b"text/html,application/xhtml+xml,application/xml;q=0."
+                b"9,image/webp,image/apng,*/*;q=0.8"
+            ],
+            b"Cache-Control": [b"max-age=0"],
+            b"Referer": [b"http://httpbin.org/forms/post"],
+            b"Connection": [b"keep-alive"],
+        }
+
+    def test_from_curl_with_kwargs(self):
+        r = self.request_class.from_curl(
+            'curl -X PATCH "http://example.org"', method="POST", meta={"key": "value"}
+        )
+        assert r.method == "POST"
+        assert r.meta == {"key": "value"}
+
+    def test_from_curl_ignore_unknown_options(self):
+        # By default: it works and ignores the unknown options: --foo and -z
+        with warnings.catch_warnings():  # avoid warning when executing tests
+            warnings.filterwarnings(
+                "ignore", category=UserWarning, message="Unrecognized options:"
+            )
+            r = self.request_class.from_curl(
+                'curl -X DELETE "http://example.org" --foo -z',
+            )
+        assert r.method == "DELETE"
+
+        # If `ignore_unknown_options` is set to `False` it raises an error with
+        # the unknown options: --foo and -z
+        with pytest.raises(ValueError, match="Unrecognized options:"):
+            self.request_class.from_curl(
+                'curl -X PATCH "http://example.org" --foo -z',
+                ignore_unknown_options=False,
+            )
diff --git a/tests/utils/bases/http_response.py b/tests/utils/bases/http_response.py
new file mode 100644
index 000000000..2fbf6527a
--- /dev/null
+++ b/tests/utils/bases/http_response.py
@@ -0,0 +1,417 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+import pytest
+from w3lib.encoding import resolve_encoding
+
+from scrapy.exceptions import NotSupported
+from scrapy.http import Headers, Request, Response
+from scrapy.link import Link
+from scrapy.utils._deps_compat import W3LIB_STRIPS_URLS
+from tests import get_testdata
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+
+class TestResponseBase(ABC):
+    @property
+    @abstractmethod
+    def response_class(self) -> type[Response]:
+        raise NotImplementedError
+
+    def test_init(self):
+        # Response requires url in the constructor
+        with pytest.raises(TypeError):
+            self.response_class()
+        assert isinstance(
+            self.response_class("http://example.com/"), self.response_class
+        )
+        with pytest.raises(TypeError):
+            self.response_class(b"http://example.com")
+        with pytest.raises(TypeError):
+            self.response_class(url="http://example.com", body={})
+        # body can be str or None
+        assert isinstance(
+            self.response_class("http://example.com/", body=b""),
+            self.response_class,
+        )
+        assert isinstance(
+            self.response_class("http://example.com/", body=b"body"),
+            self.response_class,
+        )
+        # test presence of all optional parameters
+        assert isinstance(
+            self.response_class(
+                "http://example.com/", body=b"", headers={}, status=200
+            ),
+            self.response_class,
+        )
+
+        r = self.response_class("http://www.example.com")
+        assert isinstance(r.url, str)
+        assert r.url == "http://www.example.com"
+        assert r.status == 200
+
+        assert isinstance(r.headers, Headers)
+        assert not r.headers
+
+        headers = {"foo": "bar"}
+        body = b"a body"
+        r = self.response_class("http://www.example.com", headers=headers, body=body)
+
+        assert r.headers is not headers
+        assert r.headers[b"foo"] == b"bar"
+
+        r = self.response_class("http://www.example.com", status=301)
+        assert r.status == 301
+        r = self.response_class("http://www.example.com", status="301")
+        assert r.status == 301
+        with pytest.raises(ValueError, match=r"invalid literal for int\(\)"):
+            self.response_class("http://example.com", status="lala200")
+
+    def test_copy(self):
+        """Test Response copy"""
+
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        r1.flags.append("cached")
+        r2 = r1.copy()
+
+        assert r1.status == r2.status
+        assert r1.body == r2.body
+
+        # make sure flags list is shallow copied
+        assert r1.flags is not r2.flags, "flags must be a shallow copy, not identical"
+        assert r1.flags == r2.flags
+
+        # make sure headers attribute is shallow copied
+        assert r1.headers is not r2.headers, (
+            "headers must be a shallow copy, not identical"
+        )
+        assert r1.headers == r2.headers
+
+    def test_copy_meta(self):
+        req = Request("http://www.example.com")
+        req.meta["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
+        assert r1.meta is req.meta
+
+    def test_copy_cb_kwargs(self):
+        req = Request("http://www.example.com")
+        req.cb_kwargs["foo"] = "bar"
+        r1 = self.response_class(
+            "http://www.example.com", body=b"Some body", request=req
+        )
+        assert r1.cb_kwargs is req.cb_kwargs
+
+    def test_unavailable_meta(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with pytest.raises(AttributeError, match=r"Response\.meta not available"):
+            r1.meta  # pylint: disable=pointless-statement
+
+    def test_unavailable_cb_kwargs(self):
+        r1 = self.response_class("http://www.example.com", body=b"Some body")
+        with pytest.raises(AttributeError, match=r"Response\.cb_kwargs not available"):
+            r1.cb_kwargs  # pylint: disable=pointless-statement
+
+    def test_copy_inherited_classes(self):
+        """Test Response children copies preserve their class"""
+
+        class CustomResponse(self.response_class):
+            pass
+
+        r1 = CustomResponse("http://www.example.com")
+        r2 = r1.copy()
+
+        assert isinstance(r2, CustomResponse)
+
+    def test_replace(self):
+        """Test Response.replace() method"""
+        hdrs = Headers({"key": "value"})
+        r1 = self.response_class("http://www.example.com")
+        r2 = r1.replace(status=301, body=b"New body", headers=hdrs)
+        assert r1.body == b""
+        assert r1.url == r2.url
+        assert (r1.status, r2.status) == (200, 301)
+        assert (r1.body, r2.body) == (b"", b"New body")
+        assert (r1.headers, r2.headers) == ({}, hdrs)
+
+        # Empty attributes (which may fail if not compared properly)
+        r3 = self.response_class("http://www.example.com", flags=["cached"])
+        r4 = r3.replace(body=b"", flags=[])
+        assert r4.body == b""
+        assert not r4.flags
+
+    def _assert_response_values(self, response, encoding, body):
+        if isinstance(body, str):
+            body_unicode = body
+            body_bytes = body.encode(encoding)
+        else:
+            body_unicode = body.decode(encoding)
+            body_bytes = body
+
+        assert isinstance(response.body, bytes)
+        assert isinstance(response.text, str)
+        self._assert_response_encoding(response, encoding)
+        assert response.body == body_bytes
+        assert response.text == body_unicode
+
+    def _assert_response_encoding(self, response, encoding):
+        assert response.encoding == resolve_encoding(encoding)
+
+    def test_immutable_attributes(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(AttributeError):
+            r.url = "http://example2.com"
+        with pytest.raises(AttributeError):
+            r.body = "xxx"
+
+    def test_setter_mutable_lazy_loading(self):
+        """Mutable attributes are set internally to None only until they are
+        read, then they always return the same falsy instance of the
+        corresponding mutable structure.
+
+        Setting them to None causes the next read to return a different object.
+        """
+
+        response = self.response_class("http://example.com")
+
+        response.request = Request("http://example.com")
+
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is response.flags
+        assert response._flags == []
+        original_flags = response.flags
+        response.flags = None
+        assert response._flags is None
+        assert response.flags == []
+        assert response.flags is not original_flags
+
+        assert response._headers is None
+        assert response.headers == {}
+        assert response.headers is response.headers
+        assert isinstance(response.headers, Headers)
+        assert isinstance(response._headers, Headers)
+        original_headers = response.headers
+        response.headers = None
+        assert response._headers is None
+        assert response.headers == {}
+        assert response._headers == {}
+        assert response.headers is not original_headers
+
+    def test_setters(self):
+        response = self.response_class("http://example.com")
+
+        response.flags = ["f1"]
+        assert response.flags == ["f1"]
+
+        headers = Headers({b"X-Test": b"1"})
+        response.headers = headers
+        assert response._headers is headers
+        response.headers = {b"A": b"b"}
+        assert isinstance(response.headers, Headers)
+        assert response._headers[b"A"] == b"b"
+
+    def test_urljoin(self):
+        """Test urljoin shortcut (only for existence, since behavior equals urljoin)"""
+        joined = self.response_class("http://www.example.com").urljoin("/test")
+        absolute = "http://www.example.com/test"
+        assert joined == absolute
+
+    def test_shortcut_attributes(self):
+        r = self.response_class("http://example.com", body=b"hello")
+        if self.response_class == Response:
+            msg = "Response content isn't text"
+            with pytest.raises(AttributeError, match=msg):
+                r.text  # pylint: disable=pointless-statement
+            with pytest.raises(NotSupported, match=msg):
+                r.css("body")
+            with pytest.raises(NotSupported, match=msg):
+                r.xpath("//body")
+            with pytest.raises(NotSupported, match=msg):
+                r.jmespath("body")
+        else:
+            r.text  # pylint: disable=pointless-statement
+            r.css("body")
+            r.xpath("//body")
+
+    # Response.follow
+
+    def test_follow_url_absolute(self):
+        self._assert_followed_url("http://foo.example.com", "http://foo.example.com")
+
+    def test_follow_url_relative(self):
+        self._assert_followed_url("foo", "http://example.com/foo")
+
+    def test_follow_link(self):
+        self._assert_followed_url(
+            Link("http://example.com/foo"), "http://example.com/foo"
+        )
+
+    def test_follow_None_url(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(ValueError, match="url can't be None"):
+            r.follow(None)
+
+    def test_follow_None_encoding(self):
+        r = self.response_class("http://example.com")
+        with pytest.raises(ValueError, match="encoding can't be None"):
+            r.follow("foo", encoding=None)
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_whitespace_url(self):
+        self._assert_followed_url("foo ", "http://example.com/foo")
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_whitespace_link(self):
+        self._assert_followed_url(
+            Link("http://example.com/foo "), "http://example.com/foo"
+        )
+
+    def test_follow_flags(self):
+        res = self.response_class("http://example.com/")
+        fol = res.follow("http://example.com/", flags=["cached", "allowed"])
+        assert fol.flags == ["cached", "allowed"]
+
+    # Response.follow_all
+
+    def test_follow_all_absolute(self):
+        url_list = [
+            "http://example.org",
+            "http://www.example.org",
+            "http://example.com",
+            "http://www.example.com",
+        ]
+        self._assert_followed_all_urls(url_list, url_list)
+
+    def test_follow_all_relative(self):
+        relative = ["foo", "bar", "foo/bar", "bar/foo"]
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    def test_follow_all_links(self):
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        links = map(Link, absolute)
+        self._assert_followed_all_urls(links, absolute)
+
+    def test_follow_all_empty(self):
+        r = self.response_class("http://example.com")
+        assert not list(r.follow_all([]))
+
+    def test_follow_all_invalid(self):
+        r = self.response_class("http://example.com")
+        if self.response_class == Response:
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=None))
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=12345))
+            with pytest.raises(ValueError, match="url can't be None"):
+                list(r.follow_all(urls=[None]))
+        else:
+            with pytest.raises(
+                ValueError, match="Please supply exactly one of the following arguments"
+            ):
+                list(r.follow_all(urls=None))
+            with pytest.raises(TypeError):
+                list(r.follow_all(urls=12345))
+            with pytest.raises(ValueError, match="url can't be None"):
+                list(r.follow_all(urls=[None]))
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_all_whitespace(self):
+        relative = ["foo ", "bar ", "foo/bar ", "bar/foo "]
+        absolute = [
+            "http://example.com/foo",
+            "http://example.com/bar",
+            "http://example.com/foo/bar",
+            "http://example.com/bar/foo",
+        ]
+        self._assert_followed_all_urls(relative, absolute)
+
+    @pytest.mark.xfail(
+        not W3LIB_STRIPS_URLS,
+        reason="https://github.com/scrapy/w3lib/pull/207",
+        strict=True,
+    )
+    def test_follow_all_whitespace_links(self):
+        absolute = [
+            "http://example.com/foo ",
+            "http://example.com/bar ",
+            "http://example.com/foo/bar ",
+            "http://example.com/bar/foo ",
+        ]
+        links = [Link(u) for u in absolute]
+        expected = [u.strip() for u in absolute]
+        self._assert_followed_all_urls(links, expected)
+
+    def test_follow_all_flags(self):
+        re = self.response_class("http://www.example.com/")
+        urls = [
+            "http://www.example.com/",
+            "http://www.example.com/2",
+            "http://www.example.com/foo",
+        ]
+        fol = re.follow_all(urls, flags=["cached", "allowed"])
+        for req in fol:
+            assert req.flags == ["cached", "allowed"]
+
+    def _assert_followed_url(
+        self,
+        follow_obj: str | Link,
+        target_url: str,
+        response: Response | None = None,
+        encoding: str | None = None,
+    ) -> None:
+        if response is None:
+            response = self._links_response()
+        req = response.follow(follow_obj)
+        assert req.url == target_url
+        if encoding is not None:
+            assert req.encoding == encoding
+
+    def _assert_followed_all_urls(
+        self,
+        follow_obj: Iterable[str | Link],
+        target_urls: Iterable[str],
+        response: Response | None = None,
+    ) -> None:
+        if response is None:
+            response = self._links_response()
+        followed = response.follow_all(follow_obj)
+        for req, target in zip(followed, target_urls, strict=True):
+            assert req.url == target
+
+    def _links_response(self) -> Response:
+        body = get_testdata("link_extractor", "linkextractor.html")
+        return self.response_class("http://example.com/index", body=body)
+
+    def _links_response_no_href(self) -> Response:
+        body = get_testdata("link_extractor", "linkextractor_no_href.html")
+        return self.response_class("http://example.com/index", body=body)
diff --git a/tests/utils/bases/redirect.py b/tests/utils/bases/redirect.py
new file mode 100644
index 000000000..e77a30533
--- /dev/null
+++ b/tests/utils/bases/redirect.py
@@ -0,0 +1,988 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+import pytest
+
+from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
+from scrapy.exceptions import IgnoreRequest
+from scrapy.http import Request, Response
+from scrapy.utils.misc import set_environ
+from scrapy.utils.test import get_crawler
+
+
+class TestRedirectBase(ABC):
+    mwcls: type[Any]
+    mw: Any
+    reason: int | str
+
+    @abstractmethod
+    def get_response(
+        self, request: Request, location: str, status: int = 302
+    ) -> Response:
+        raise NotImplementedError
+
+    def test_priority_adjust(self):
+        req = Request("http://a.example")
+        rsp = self.get_response(req, "http://a.example/redirected")
+        req2 = self.mw.process_response(req, rsp)
+        assert req2.priority > req.priority
+
+    def test_dont_redirect(self):
+        url = "http://www.example.com/301"
+        url2 = "http://www.example.com/redirected"
+        req = Request(url, meta={"dont_redirect": True})
+        rsp = self.get_response(req, url2)
+
+        r = self.mw.process_response(req, rsp)
+        assert isinstance(r, Response)
+        assert r is rsp
+
+        # Test that it redirects when dont_redirect is False
+        req = Request(url, meta={"dont_redirect": False})
+        rsp = self.get_response(req, url2)
+
+        r = self.mw.process_response(req, rsp)
+        assert isinstance(r, Request)
+
+    def test_post(self):
+        url = "http://www.example.com/302"
+        url2 = "http://www.example.com/redirected2"
+        req = Request(
+            url,
+            method="POST",
+            body="test",
+            headers={"Content-Type": "text/plain", "Content-length": "4"},
+        )
+        rsp = self.get_response(req, url2)
+
+        req2 = self.mw.process_response(req, rsp)
+        assert isinstance(req2, Request)
+        assert req2.url == url2
+        assert req2.method == "GET"
+        assert "Content-Type" not in req2.headers, (
+            "Content-Type header must not be present in redirected request"
+        )
+        assert "Content-Length" not in req2.headers, (
+            "Content-Length header must not be present in redirected request"
+        )
+        assert not req2.body, f"Redirected body must be empty, not '{req2.body!r}'"
+
+    def test_max_redirect_times(self):
+        self.mw.max_redirect_times = 1
+        req = Request("http://a.example/302")
+        rsp = self.get_response(req, "/redirected")
+
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        assert "redirect_times" in req.meta
+        assert req.meta["redirect_times"] == 1
+        with pytest.raises(IgnoreRequest):
+            self.mw.process_response(req, rsp)
+
+    def test_ttl(self):
+        self.mw.max_redirect_times = 100
+        req = Request("http://a.example/302", meta={"redirect_ttl": 1})
+        rsp = self.get_response(req, "/a")
+
+        req = self.mw.process_response(req, rsp)
+        assert isinstance(req, Request)
+        with pytest.raises(IgnoreRequest):
+            self.mw.process_response(req, rsp)
+
+    def test_redirect_urls(self):
+        req1 = Request("http://a.example/first")
+        rsp1 = self.get_response(req1, "/redirected")
+        req2 = self.mw.process_response(req1, rsp1)
+        rsp2 = self.get_response(req2, "/redirected2")
+        req3 = self.mw.process_response(req2, rsp2)
+
+        assert req2.url == "http://a.example/redirected"
+        assert req2.meta["redirect_urls"] == ["http://a.example/first"]
+        assert req3.url == "http://a.example/redirected2"
+        assert req3.meta["redirect_urls"] == [
+            "http://a.example/first",
+            "http://a.example/redirected",
+        ]
+
+    def test_redirect_reasons(self):
+        req1 = Request("http://a.example/first")
+        rsp1 = self.get_response(req1, "/redirected1")
+        req2 = self.mw.process_response(req1, rsp1)
+        rsp2 = self.get_response(req2, "/redirected2")
+        req3 = self.mw.process_response(req2, rsp2)
+        assert req2.meta["redirect_reasons"] == [self.reason]
+        assert req3.meta["redirect_reasons"] == [self.reason, self.reason]
+
+    def test_cross_origin_header_dropping(self):
+        safe_headers = {"A": "B"}
+        cookie_header = {"Cookie": "a=b"}
+        authorization_header = {"Authorization": "Bearer 123456"}
+
+        original_request = Request(
+            "https://example.com",
+            headers={**safe_headers, **cookie_header, **authorization_header},
+        )
+
+        # Redirects to the same origin (same scheme, same domain, same port)
+        # keep all headers.
+        internal_response = self.get_response(original_request, "https://example.com/a")
+        internal_redirect_request = self.mw.process_response(
+            original_request, internal_response
+        )
+        assert isinstance(internal_redirect_request, Request)
+        assert original_request.headers == internal_redirect_request.headers
+
+        # Redirects to the same origin (same scheme, same domain, same port)
+        # keep all headers also when the scheme is http.
+        http_request = Request(
+            "http://example.com",
+            headers={**safe_headers, **cookie_header, **authorization_header},
+        )
+        http_response = self.get_response(http_request, "http://example.com/a")
+        http_redirect_request = self.mw.process_response(http_request, http_response)
+        assert isinstance(http_redirect_request, Request)
+        assert http_request.headers == http_redirect_request.headers
+
+        # For default ports, whether the port is explicit or implicit does not
+        # affect the outcome, it is still the same origin.
+        to_explicit_port_response = self.get_response(
+            original_request, "https://example.com:443/a"
+        )
+        to_explicit_port_redirect_request = self.mw.process_response(
+            original_request, to_explicit_port_response
+        )
+        assert isinstance(to_explicit_port_redirect_request, Request)
+        assert original_request.headers == to_explicit_port_redirect_request.headers
+
+        # For default ports, whether the port is explicit or implicit does not
+        # affect the outcome, it is still the same origin.
+        to_implicit_port_response = self.get_response(
+            original_request, "https://example.com/a"
+        )
+        to_implicit_port_redirect_request = self.mw.process_response(
+            original_request, to_implicit_port_response
+        )
+        assert isinstance(to_implicit_port_redirect_request, Request)
+        assert original_request.headers == to_implicit_port_redirect_request.headers
+
+        # A port change drops the Authorization header because the origin
+        # changes, but keeps the Cookie header because the domain remains the
+        # same.
+        different_port_response = self.get_response(
+            original_request, "https://example.com:8080/a"
+        )
+        different_port_redirect_request = self.mw.process_response(
+            original_request, different_port_response
+        )
+        assert isinstance(different_port_redirect_request, Request)
+        assert {
+            **safe_headers,
+            **cookie_header,
+        } == different_port_redirect_request.headers.to_unicode_dict()
+
+        # A domain change drops both the Authorization and the Cookie header.
+        external_response = self.get_response(original_request, "https://example.org/a")
+        external_redirect_request = self.mw.process_response(
+            original_request, external_response
+        )
+        assert isinstance(external_redirect_request, Request)
+        assert safe_headers == external_redirect_request.headers.to_unicode_dict()
+
+        # A scheme upgrade (http → https) drops the Authorization header
+        # because the origin changes, but keeps the Cookie header because the
+        # domain remains the same.
+        upgrade_response = self.get_response(http_request, "https://example.com/a")
+        upgrade_redirect_request = self.mw.process_response(
+            http_request, upgrade_response
+        )
+        assert isinstance(upgrade_redirect_request, Request)
+        assert {
+            **safe_headers,
+            **cookie_header,
+        } == upgrade_redirect_request.headers.to_unicode_dict()
+
+        # A scheme downgrade (https → http) drops the Authorization header
+        # because the origin changes, and the Cookie header because its value
+        # cannot indicate whether the cookies were secure (HTTPS-only) or not.
+        #
+        # Note: If the Cookie header is set by the cookie management
+        # middleware, as recommended in the docs, the dropping of Cookie on
+        # scheme downgrade is not an issue, because the cookie management
+        # middleware will add again the Cookie header to the new request if
+        # appropriate.
+        downgrade_response = self.get_response(original_request, "http://example.com/a")
+        downgrade_redirect_request = self.mw.process_response(
+            original_request, downgrade_response
+        )
+        assert isinstance(downgrade_redirect_request, Request)
+        assert safe_headers == downgrade_redirect_request.headers.to_unicode_dict()
+
+    def test_meta_proxy_http_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_http_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_http_to_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("http://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_meta_proxy_https_to_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        meta = {"proxy": "https://a:@a.example"}
+        request1 = Request("https://example.com", meta=meta)
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_http_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_http_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_https_absolute(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_https_relative(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "/a")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "/a")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_proxied_http_to_proxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request2.meta["_auth_proxy"] == "https://b.example"
+        assert request2.meta["proxy"] == "https://b.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_proxied_http_to_unproxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request1.meta["_auth_proxy"] == "https://a.example"
+        assert request1.meta["proxy"] == "https://a.example"
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request3.meta["_auth_proxy"] == "https://a.example"
+        assert request3.meta["proxy"] == "https://a.example"
+
+    def test_system_proxy_unproxied_http_to_proxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request2.meta["_auth_proxy"] == "https://b.example"
+        assert request2.meta["proxy"] == "https://b.example"
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_unproxied_http_to_unproxied_https(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("http://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "https://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "http://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_proxied_https_to_proxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request1.meta["_auth_proxy"] == "https://b.example"
+        assert request1.meta["proxy"] == "https://b.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request3.meta["_auth_proxy"] == "https://b.example"
+        assert request3.meta["proxy"] == "https://b.example"
+
+    def test_system_proxy_proxied_https_to_unproxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "https_proxy": "https://b:@b.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert request1.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request1.meta["_auth_proxy"] == "https://b.example"
+        assert request1.meta["proxy"] == "https://b.example"
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert request3.headers["Proxy-Authorization"] == b"Basic Yjo="
+        assert request3.meta["_auth_proxy"] == "https://b.example"
+        assert request3.meta["proxy"] == "https://b.example"
+
+    def test_system_proxy_unproxied_https_to_proxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        env = {
+            "http_proxy": "https://a:@a.example",
+        }
+        with set_environ(**env):
+            proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert request2.headers["Proxy-Authorization"] == b"Basic YTo="
+        assert request2.meta["_auth_proxy"] == "https://a.example"
+        assert request2.meta["proxy"] == "https://a.example"
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+    def test_system_proxy_unproxied_https_to_unproxied_http(self):
+        crawler = get_crawler()
+        redirect_mw = self.mwcls.from_crawler(crawler)
+        proxy_mw = HttpProxyMiddleware.from_crawler(crawler)
+
+        request1 = Request("https://example.com")
+        proxy_mw.process_request(request1)
+
+        assert "Proxy-Authorization" not in request1.headers
+        assert "_auth_proxy" not in request1.meta
+        assert "proxy" not in request1.meta
+
+        response1 = self.get_response(request1, "http://example.com")
+        request2 = redirect_mw.process_response(request1, response1)
+
+        assert isinstance(request2, Request)
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        proxy_mw.process_request(request2)
+
+        assert "Proxy-Authorization" not in request2.headers
+        assert "_auth_proxy" not in request2.meta
+        assert "proxy" not in request2.meta
+
+        response2 = self.get_response(request2, "https://example.com")
+        request3 = redirect_mw.process_response(request2, response2)
+
+        assert isinstance(request3, Request)
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
+
+        proxy_mw.process_request(request3)
+
+        assert "Proxy-Authorization" not in request3.headers
+        assert "_auth_proxy" not in request3.meta
+        assert "proxy" not in request3.meta
diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py
new file mode 100644
index 000000000..f774fde07
--- /dev/null
+++ b/tests/utils/bases/spider.py
@@ -0,0 +1,131 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Any
+from unittest import mock
+
+import pytest
+from testfixtures import LogCapture
+
+from scrapy import signals
+from scrapy.crawler import Crawler
+from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.utils.decorators import inline_callbacks_test
+
+if TYPE_CHECKING:
+    from scrapy.spiders import Spider
+
+
+class TestSpiderBase(ABC):
+    @property
+    @abstractmethod
+    def spider_class(self) -> type[Spider]:
+        raise NotImplementedError
+
+    def test_base_spider(self):
+        spider = self.spider_class("example.com")
+        assert spider.name == "example.com"
+        assert spider.start_urls == []
+
+    def test_spider_args(self):
+        """``__init__`` method arguments are assigned to spider attributes"""
+        spider = self.spider_class("example.com", foo="bar")
+        assert spider.foo == "bar"
+
+    def test_spider_without_name(self):
+        """``__init__`` raises when the name is not provided."""
+        msg = "must have a name"
+        with pytest.raises(ValueError, match=msg):
+            self.spider_class()
+        with pytest.raises(ValueError, match=msg):
+            self.spider_class(somearg="foo")
+
+    def test_from_crawler_crawler_and_settings_population(self):
+        crawler = get_crawler()
+        spider = self.spider_class.from_crawler(crawler, "example.com")
+        assert hasattr(spider, "crawler")
+        assert spider.crawler is crawler
+        assert hasattr(spider, "settings")
+        assert spider.settings is crawler.settings
+
+    def test_from_crawler_init_call(self):
+        with mock.patch.object(
+            self.spider_class, "__init__", return_value=None
+        ) as mock_init:
+            self.spider_class.from_crawler(get_crawler(), "example.com", foo="bar")
+            mock_init.assert_called_once_with("example.com", foo="bar")
+
+    def test_closed_signal_call(self):
+        class TestSpider(self.spider_class):
+            closed_called = False
+
+            def closed(self, reason):
+                self.closed_called = True
+
+        crawler = get_crawler()
+        spider = TestSpider.from_crawler(crawler, "example.com")
+        crawler.signals.send_catch_log(signal=signals.spider_opened, spider=spider)
+        crawler.signals.send_catch_log(
+            signal=signals.spider_closed, spider=spider, reason=None
+        )
+        assert spider.closed_called
+
+    def test_update_settings(self):
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {"TEST1": "project", "TEST3": "project"}
+        self.spider_class.custom_settings = spider_settings
+        settings = Settings(project_settings, priority="project")
+
+        self.spider_class.update_settings(settings)
+        assert settings.get("TEST1") == "spider"
+        assert settings.get("TEST2") == "spider"
+        assert settings.get("TEST3") == "project"
+
+    @inline_callbacks_test
+    def test_settings_in_from_crawler(self):
+        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+        project_settings = {
+            "TEST1": "project",
+            "TEST3": "project",
+            **get_reactor_settings(),
+        }
+
+        class TestSpider(self.spider_class):
+            name = "test"
+            custom_settings = spider_settings
+
+            @classmethod
+            def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any):
+                spider = super().from_crawler(crawler, *args, **kwargs)
+                spider.settings.set("TEST1", "spider_instance", priority="spider")
+                return spider
+
+        crawler = Crawler(TestSpider, project_settings)
+        assert crawler.settings.get("TEST1") == "spider"
+        assert crawler.settings.get("TEST2") == "spider"
+        assert crawler.settings.get("TEST3") == "project"
+        yield crawler.crawl()
+        assert crawler.settings.get("TEST1") == "spider_instance"
+
+    def test_logger(self):
+        spider = self.spider_class("example.com")
+        with LogCapture() as lc:
+            spider.logger.info("test log msg")
+        lc.check(("example.com", "INFO", "test log msg"))
+
+        record = lc.records[0]
+        assert "spider" in record.__dict__
+        assert record.spider is spider
+
+    def test_log(self):
+        spider = self.spider_class("example.com")
+        with (
+            mock.patch("scrapy.spiders.Spider.logger") as mock_logger,
+            pytest.warns(
+                ScrapyDeprecationWarning, match=r"Spider.log\(\) is deprecated"
+            ),
+        ):
+            spider.log("test log msg", "INFO")
+        mock_logger.log.assert_called_once_with("INFO", "test log msg")
diff --git a/tests/utils/downloader.py b/tests/utils/downloader.py
new file mode 100644
index 000000000..f64c02332
--- /dev/null
+++ b/tests/utils/downloader.py
@@ -0,0 +1,35 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, NamedTuple, cast
+
+from scrapy.core.downloader import Downloader
+from scrapy.utils.httpobj import urlparse_cached
+
+if TYPE_CHECKING:
+    from scrapy.http import Request
+
+
+class MockSlot(NamedTuple):
+    active: list[Any]
+
+
+class MockDownloader:
+    def __init__(self) -> None:
+        self.slots: dict[str, MockSlot] = {}
+
+    def get_slot_key(self, request: Request) -> str:
+        if Downloader.DOWNLOAD_SLOT in request.meta:
+            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
+
+        return urlparse_cached(request).hostname or ""
+
+    def increment(self, slot_key: str) -> None:
+        slot = self.slots.setdefault(slot_key, MockSlot(active=[]))
+        slot.active.append(1)
+
+    def decrement(self, slot_key: str) -> None:
+        slot = self.slots[slot_key]
+        slot.active.pop()
+
+    def close(self) -> None:
+        pass
diff --git a/tests/utils/engine.py b/tests/utils/engine.py
new file mode 100644
index 000000000..c193d39a9
--- /dev/null
+++ b/tests/utils/engine.py
@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlparse
+
+import attr
+from itemadapter import ItemAdapter
+from pydispatch import dispatcher
+from twisted.internet import defer
+
+from scrapy import signals
+from scrapy.http import Headers, Request, Response
+from scrapy.item import Field, Item
+from scrapy.linkextractors import LinkExtractor
+from scrapy.spiders import Spider
+from scrapy.utils.defer import maybe_deferred_to_future
+from scrapy.utils.signal import disconnect_all
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+    from twisted.python.failure import Failure
+
+    from tests.mockserver.http import MockServer
+
+
+class MyItem(Item):
+    name = Field()
+    url = Field()
+    price = Field()
+
+
+@attr.s
+class AttrsItem:
+    name = attr.ib(default="")
+    url = attr.ib(default="")
+    price = attr.ib(default=0)
+
+
+@dataclass
+class DataClassItem:
+    name: str = ""
+    url: str = ""
+    price: int = 0
+
+
+class MySpider(Spider):
+    name = "scrapytest.org"
+
+    itemurl_re = re.compile(r"item\d+.html")
+    name_re = re.compile(r"<h1>(.*?)</h1>", re.MULTILINE)
+    price_re = re.compile(r">Price: \$(.*?)<", re.MULTILINE)
+
+    item_cls: type = MyItem
+
+    def parse(self, response):
+        xlink = LinkExtractor()
+        itemre = re.compile(self.itemurl_re)
+        for link in xlink.extract_links(response):
+            if itemre.search(link.url):
+                yield Request(url=link.url, callback=self.parse_item)
+
+    def parse_item(self, response):
+        adapter = ItemAdapter(self.item_cls())
+        m = self.name_re.search(response.text)
+        if m:
+            adapter["name"] = m.group(1)
+        adapter["url"] = response.url
+        m = self.price_re.search(response.text)
+        if m:
+            adapter["price"] = m.group(1)
+        return adapter.item
+
+
+class DictItemsSpider(MySpider):
+    item_cls = dict
+
+
+class AttrsItemsSpider(MySpider):
+    item_cls = AttrsItem
+
+
+class DataClassItemsSpider(MySpider):
+    item_cls = DataClassItem
+
+
+class CrawlerRun:
+    """A class to run the crawler and keep track of events occurred"""
+
+    def __init__(self, spider_class: type[Spider]):
+        self.respplug: list[tuple[Response, Spider]] = []
+        self.reqplug: list[tuple[Request, Spider]] = []
+        self.reqdropped: list[tuple[Request, Spider]] = []
+        self.reqreached: list[tuple[Request, Spider]] = []
+        self.itemerror: list[tuple[Any, Response, Spider, Failure]] = []
+        self.itemresp: list[tuple[Any, Response]] = []
+        self.headers: dict[Request, Headers] = {}
+        self.bytes: defaultdict[Request, list[bytes]] = defaultdict(list)
+        self.signals_caught: dict[Any, dict[str, Any]] = {}
+        self.spider_class = spider_class
+
+    async def run(self, mockserver: MockServer) -> None:
+        self.mockserver = mockserver
+
+        start_urls = [
+            self.geturl("/static/"),
+            self.geturl("/redirect"),
+            self.geturl("/redirect"),  # duplicate
+            self.geturl("/numbers"),
+        ]
+
+        for name, signal in vars(signals).items():
+            if not name.startswith("_"):
+                dispatcher.connect(self.record_signal, signal)
+
+        self.crawler = get_crawler(self.spider_class)
+        self.crawler.signals.connect(self.item_scraped, signals.item_scraped)
+        self.crawler.signals.connect(self.item_error, signals.item_error)
+        self.crawler.signals.connect(self.headers_received, signals.headers_received)
+        self.crawler.signals.connect(self.bytes_received, signals.bytes_received)
+        self.crawler.signals.connect(self.request_scheduled, signals.request_scheduled)
+        self.crawler.signals.connect(self.request_dropped, signals.request_dropped)
+        self.crawler.signals.connect(
+            self.request_reached, signals.request_reached_downloader
+        )
+        self.crawler.signals.connect(
+            self.response_downloaded, signals.response_downloaded
+        )
+        self.crawler.crawl(start_urls=start_urls)
+
+        self.deferred: defer.Deferred[None] = defer.Deferred()
+        dispatcher.connect(self.stop, signals.engine_stopped)
+        await maybe_deferred_to_future(self.deferred)
+
+    async def stop(self):
+        for name, signal in vars(signals).items():
+            if not name.startswith("_"):
+                disconnect_all(signal)
+        self.deferred.callback(None)
+        await self.crawler.stop_async()
+
+    def geturl(self, path: str) -> str:
+        return self.mockserver.url(path)
+
+    def getpath(self, url: str) -> str:
+        u = urlparse(url)
+        return u.path
+
+    def item_error(
+        self, item: Any, response: Response, spider: Spider, failure: Failure
+    ) -> None:
+        self.itemerror.append((item, response, spider, failure))
+
+    def item_scraped(self, item: Any, spider: Spider, response: Response) -> None:
+        self.itemresp.append((item, response))
+
+    def headers_received(
+        self, headers: Headers, body_length: int, request: Request, spider: Spider
+    ) -> None:
+        self.headers[request] = headers
+
+    def bytes_received(self, data: bytes, request: Request, spider: Spider) -> None:
+        self.bytes[request].append(data)
+
+    def request_scheduled(self, request: Request, spider: Spider) -> None:
+        self.reqplug.append((request, spider))
+
+    def request_reached(self, request: Request, spider: Spider) -> None:
+        self.reqreached.append((request, spider))
+
+    def request_dropped(self, request: Request, spider: Spider) -> None:
+        self.reqdropped.append((request, spider))
+
+    def response_downloaded(self, response: Response, spider: Spider) -> None:
+        self.respplug.append((response, spider))
+
+    def record_signal(self, *args: Any, **kwargs: Any) -> None:
+        """Record a signal and its parameters"""
+        signalargs = kwargs.copy()
+        sig = signalargs.pop("signal")
+        signalargs.pop("sender", None)
+        self.signals_caught[sig] = signalargs
diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py
new file mode 100644
index 000000000..283c95940
--- /dev/null
+++ b/tests/utils/media_pipelines.py
@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from typing import Any
+
+from scrapy.http.request import NO_CALLBACK, Request
+
+
+async def mocked_download_func(request: Request) -> Any:
+    assert request.callback is NO_CALLBACK
+    response = request.meta.get("response")
+    if callable(response):
+        response = await response()
+    if isinstance(response, Exception):
+        raise response
+    return response

From 25e6884e2f24d554e3527cbe20ff001913cc5863 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Mon, 27 Jul 2026 17:34:10 +0200
Subject: [PATCH 44/62] Address recursion and media ignore-request reporting
 issues (#7673)

* Address recursion and media ignore-request reporting issues

* Improve test coverage

* Address feedback
---
 scrapy/core/engine.py                      | 21 ++++---
 scrapy/downloadermiddlewares/offsite.py    |  2 +-
 scrapy/pipelines/files.py                  | 36 ++++++-----
 scrapy/pipelines/media.py                  | 24 +++++++-
 tests/test_downloadermiddleware_offsite.py | 14 +++++
 tests/test_engine_download.py              | 20 ++++++
 tests/test_pipeline_files.py               | 72 +++++++++++++++++++++-
 tests/test_pipeline_media.py               | 18 +++++-
 8 files changed, 176 insertions(+), 31 deletions(-)

diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 1033e874f..efac5f71c 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -470,16 +470,17 @@ class ExecutionEngine:
         """
         if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
-        try:
-            response_or_request = await maybe_deferred_to_future(
-                self._download(request)
-            )
-        finally:
-            assert self._slot is not None
-            self._slot.remove_request(request)
-        if isinstance(response_or_request, Request):
-            return await self.download_async(response_or_request)
-        return response_or_request
+        while True:
+            try:
+                response_or_request = await maybe_deferred_to_future(
+                    self._download(request)
+                )
+            finally:
+                assert self._slot is not None
+                self._slot.remove_request(request)
+            if not isinstance(response_or_request, Request):
+                return response_or_request
+            request = response_or_request
 
     @inlineCallbacks
     def _download(
diff --git a/scrapy/downloadermiddlewares/offsite.py b/scrapy/downloadermiddlewares/offsite.py
index e03dc040e..db85b62a1 100644
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@@ -61,7 +61,7 @@ class OffsiteMiddleware:
             )
             self.stats.inc_value("offsite/domains")
         self.stats.inc_value("offsite/filtered")
-        raise IgnoreRequest
+        raise IgnoreRequest(f"Filtered offsite request to {domain!r}")
 
     def should_follow(self, request: Request, spider: Spider) -> bool:
         regex = self.host_regex
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 70dc36e35..8e8082332 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -27,7 +27,15 @@ from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
-from scrapy.pipelines.media import FileInfo, FileInfoOrError, MediaPipeline
+from scrapy.pipelines.media import (
+    FileException as FileException,  # noqa: PLC0414  # re-exported for backward compatibility
+)
+from scrapy.pipelines.media import (
+    FileInfo,
+    FileInfoOrError,
+    MediaPipeline,
+    _MediaRequestFiltered,
+)
 from scrapy.utils.asyncio import run_in_thread
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
@@ -75,10 +83,6 @@ def _md5sum(file: IO[bytes]) -> str:
     return m.hexdigest()
 
 
-class FileException(Exception):
-    """General media error exception"""
-
-
 class StatInfo(TypedDict, total=False):
     checksum: str
     last_modified: float
@@ -597,20 +601,20 @@ class FilesPipeline(MediaPipeline):
     def media_failed(
         self, failure: Failure, request: Request, info: MediaPipeline.SpiderInfo
     ) -> NoReturn:
-        if not isinstance(failure.value, IgnoreRequest):
-            referer = referer_str(request)
-            logger.warning(
-                "File (unknown-error): Error downloading %(medianame)s from "
-                "%(request)s referred in <%(referer)s>: %(exception)s",
-                {
-                    "medianame": self.MEDIA_NAME,
-                    "request": request,
-                    "referer": referer,
-                    "exception": failure.value,
-                },
+        referer = referer_str(request)
+        if isinstance(failure.value, IgnoreRequest):
+            logger.debug(
+                f"File (filtered): Not downloading {self.MEDIA_NAME} from "
+                f"{request} referred in <{referer}>: {failure.value}",
                 extra={"spider": info.spider},
             )
+            raise _MediaRequestFiltered(str(failure.value)) from failure.value
 
+        logger.warning(
+            f"File (unknown-error): Error downloading {self.MEDIA_NAME} from "
+            f"{request} referred in <{referer}>: {failure.value}",
+            extra={"spider": info.spider},
+        )
         raise FileException
 
     async def media_downloaded(
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 5b5d2dcb2..764f82a78 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -55,6 +55,25 @@ FileInfoOrError: TypeAlias = (
 logger = logging.getLogger(__name__)
 
 
+class FileException(Exception):
+    """General media error exception"""
+
+
+class _MediaRequestFiltered(FileException):
+    """Raised internally by media pipelines when a media request is filtered
+    out (e.g. as an offsite request) instead of being downloaded.
+
+    It is a subclass of :exc:`FileException` for backward compatibility, but
+    unlike an actual download error it is logged at the ``DEBUG`` level and
+    without a traceback, since filtering a request is expected behavior rather
+    than an error.
+    """
+
+
+def _media_request_filtered(failure: Failure) -> bool:
+    return isinstance(failure.value, _MediaRequestFiltered)
+
+
 class MediaPipeline(ABC):
     LOG_FAILED_RESULTS: bool = True
 
@@ -193,7 +212,8 @@ class MediaPipeline(ABC):
                 result = await self._check_media_to_download(request, info, item=item)
         except Exception:
             result = Failure()
-            logger.exception(result)
+            if not _media_request_filtered(result):
+                logger.exception(result)
         self._cache_result_and_execute_waiters(result, fp, info)
         return await maybe_deferred_to_future(wad)  # it must return wad at last
 
@@ -304,6 +324,8 @@ class MediaPipeline(ABC):
             for ok, value in results:
                 if not ok:
                     assert isinstance(value, Failure)
+                    if _media_request_filtered(value):
+                        continue
                     logger.error(
                         "%(class)s found errors processing %(item)s",
                         {"class": self.__class__.__name__, "item": item},
diff --git a/tests/test_downloadermiddleware_offsite.py b/tests/test_downloadermiddleware_offsite.py
index c0b8dc4dd..78efb0191 100644
--- a/tests/test_downloadermiddleware_offsite.py
+++ b/tests/test_downloadermiddleware_offsite.py
@@ -1,3 +1,5 @@
+import re
+
 import pytest
 
 from scrapy import Request, Spider
@@ -231,3 +233,15 @@ def test_repeated_offsite_domain():
         mw.process_request(req2)
     assert crawler.stats.get_value("offsite/domains") == 1  # not incremented again
     assert crawler.stats.get_value("offsite/filtered") == 2
+
+
+def test_ignore_request_reason():
+    crawler = get_crawler(Spider)
+    crawler.spider = crawler._create_spider(name="a", allowed_domains=["example.com"])
+    mw = OffsiteMiddleware.from_crawler(crawler)
+    mw.spider_opened(crawler.spider)
+    request = Request("http://other.org/1")
+    with pytest.raises(
+        IgnoreRequest, match=re.escape("Filtered offsite request to 'other.org'")
+    ):
+        mw.process_request(request)
diff --git a/tests/test_engine_download.py b/tests/test_engine_download.py
index f15bfd5e2..962808d96 100644
--- a/tests/test_engine_download.py
+++ b/tests/test_engine_download.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import sys
 from unittest.mock import Mock, call
 
 import pytest
@@ -73,6 +74,25 @@ class TestEngineDownloadAsync:
             [call(original_request), call(redirect_request)]
         )
 
+    @coroutine_test
+    async def test_download_async_many_redirects(self, engine):
+        """A long chain of requests being replaced by new ones is handled
+        iteratively, without hitting the recursion limit."""
+        count = sys.getrecursionlimit() * 2
+        requests = [Request(f"http://example.com/{i}") for i in range(count)]
+        final_response = Response("http://example.com/final", body=b"done")
+        engine.downloader.fetch.side_effect = [
+            *(defer.succeed(request) for request in requests[1:]),
+            defer.succeed(final_response),
+        ]
+        engine.spider = Mock()
+        engine._slot.add_request = Mock()
+        engine._slot.remove_request = Mock()
+
+        result = await self._download(engine, requests[0])
+        assert result == final_response
+        assert engine.downloader.fetch.call_count == count
+
     @coroutine_test
     async def test_download_async_no_spider(self, engine):
         """Test async download attempt when no spider is available."""
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index a0ae3635b..4f6fa21a0 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -2,6 +2,7 @@ import base64
 import dataclasses
 import logging
 import random
+import re
 import time
 from abc import ABC, abstractmethod
 from datetime import datetime
@@ -19,18 +20,20 @@ import attr
 import pytest
 from itemadapter import ItemAdapter
 from twisted.internet.defer import Deferred
+from twisted.python.failure import Failure
 
-from scrapy.exceptions import NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
 from scrapy.pipelines.files import (
+    FileException,
     FilesPipeline,
     FSFilesStore,
     FTPFilesStore,
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -290,6 +293,71 @@ class TestFilesPipeline:
         request = Request("http://example.com")
         assert file_path(request, item=item) == "full/path-to-store-file"
 
+    def test_media_failed_filtered_request(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A filtered media request (IgnoreRequest) is reported as a
+        _MediaRequestFiltered exception and logged at the DEBUG level, instead
+        of as a download error with a traceback."""
+        request = Request("http://example.com/file.pdf")
+        reason = "Filtered offsite request to 'example.com'"
+        failure = Failure(IgnoreRequest(reason))
+
+        with (
+            caplog.at_level(logging.DEBUG),
+            pytest.raises(_MediaRequestFiltered, match=re.escape(reason)),
+        ):
+            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)
+
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
+        assert record.levelname == "DEBUG"
+        assert record.exc_info is None
+        assert reason in record.getMessage()
+
+    def test_media_failed_download_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A genuine download error is reported as a FileException and logged as
+        a warning."""
+        request = Request("http://example.com/file.pdf")
+        failure = Failure(Exception("boom"))
+
+        with caplog.at_level(logging.WARNING), pytest.raises(FileException):
+            self.pipeline.media_failed(failure, request, self.pipeline.spiderinfo)
+
+        assert len(caplog.records) == 1
+        assert caplog.records[0].levelname == "WARNING"
+
+    @coroutine_test
+    async def test_process_item_filtered_request(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A filtered (e.g. offsite) media request is processed as a failed
+        result without being logged as an error with a traceback."""
+        item_url = "http://example.com/file.pdf"
+        item = _create_item_with_files(item_url)
+        request = Request(
+            item_url,
+            meta={
+                "response": IgnoreRequest("Filtered offsite request to 'example.com'")
+            },
+        )
+        with (
+            caplog.at_level(logging.DEBUG),
+            mock.patch.object(
+                FilesPipeline, "get_media_requests", return_value=[request]
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+
+        assert result["files"] == []
+        assert not any(r.levelname in ("WARNING", "ERROR") for r in caplog.records)
+        assert any(
+            "Filtered offsite request to 'example.com'" in r.getMessage()
+            for r in caplog.records
+        )
+
     @pytest.mark.parametrize(
         "bad_type",
         [
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index ee7a576db..be471e5fe 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from unittest.mock import MagicMock
 
 import pytest
@@ -10,7 +11,7 @@ from scrapy import signals
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.pipelines.files import FileException
-from scrapy.pipelines.media import MediaPipeline
+from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
 from scrapy.utils.defer import _defer_sleep_async
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
@@ -152,6 +153,21 @@ class TestBaseMediaPipeline:
         assert new_item is item
         assert len(log.records) == 0
 
+    def test_item_completed_filtered_request_not_logged(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Filtered media requests (e.g. offsite ones) are not logged as errors
+        by item_completed(), as they are not download errors."""
+        item = {"name": "name"}
+        fail = Failure(_MediaRequestFiltered("Filtered offsite request"))
+        results = [(True, 1), (False, fail)]
+
+        with caplog.at_level(logging.DEBUG):
+            new_item = self.pipe.item_completed(results, item, self.info)
+
+        assert new_item is item
+        assert len(caplog.records) == 0
+
     @coroutine_test
     async def test_default_process_item(self):
         item = {"name": "name"}

From f3693aa8ba7be410c5d892eaf5d11580d6a969b2 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Mon, 27 Jul 2026 18:23:56 +0200
Subject: [PATCH 45/62] Fix port issue with cached DNS (#7772)

* Fix port issue with cached DNS

* Keep ports off the cache

* Complete coverage
---
 scrapy/resolver.py     | 17 +++++++++--
 tests/test_resolver.py | 69 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 82 insertions(+), 4 deletions(-)

diff --git a/scrapy/resolver.py b/scrapy/resolver.py
index 270a7fbf5..0e9775fd1 100644
--- a/scrapy/resolver.py
+++ b/scrapy/resolver.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+import attr
 from twisted.internet import defer
 from twisted.internet.base import ReactorBase, ThreadedResolver
 from twisted.internet.interfaces import (
@@ -70,6 +71,12 @@ class CachingThreadedResolver(ThreadedResolver):
         return result
 
 
+def _address_with_port(address: IAddress, port: int) -> IAddress:
+    if getattr(address, "port", port) == port:
+        return address
+    return attr.evolve(address, port=port)
+
+
 @implementer(IHostResolution)
 class HostResolution:
     def __init__(self, name: str):
@@ -97,7 +104,11 @@ class _CachingResolutionReceiver:
     def resolutionComplete(self) -> None:
         self.resolutionReceiver.resolutionComplete()
         if self.addresses:
-            dnscache[self.hostName] = self.addresses
+            # Name resolution does not depend on the port, so cache entries are
+            # kept port-agnostic and the requested port is set on cache hits.
+            dnscache[self.hostName] = [
+                _address_with_port(address, 0) for address in self.addresses
+            ]
 
 
 @implementer(IHostnameResolver)
@@ -142,7 +153,7 @@ class CachingHostnameResolver:
                 transportSemantics,
             )
         resolutionReceiver.resolutionBegan(HostResolution(hostName))
-        for addr in addresses:
-            resolutionReceiver.addressResolved(addr)
+        for address in addresses:
+            resolutionReceiver.addressResolved(_address_with_port(address, portNumber))
         resolutionReceiver.resolutionComplete()
         return resolutionReceiver
diff --git a/tests/test_resolver.py b/tests/test_resolver.py
index 7cca45ed1..8e69f62a0 100644
--- a/tests/test_resolver.py
+++ b/tests/test_resolver.py
@@ -3,6 +3,7 @@ from __future__ import annotations
 from unittest.mock import Mock
 
 import pytest
+from twisted.internet.address import IPv4Address, IPv6Address
 
 from scrapy.resolver import CachingHostnameResolver, CachingThreadedResolver, dnscache
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -55,11 +56,77 @@ def test_caching_hostname_resolver_no_addresses_not_cached():
     assert "example.com" not in dnscache
 
 
+def test_caching_hostname_resolver_cached_addresses_have_no_port():
+    def fake_resolve(receiver, *_):
+        receiver.resolutionBegan(Mock())
+        receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
+        receiver.addressResolved(IPv6Address("TCP", "::1", 80))
+        receiver.resolutionComplete()
+        return receiver
+
+    reactor = Mock()
+    reactor.nameResolver.resolveHostName.side_effect = fake_resolve
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(reactor, cache_size=10)
+    resolver.resolveHostName(receiver, "example.com", portNumber=80)
+
+    # The port requested on a cache miss is passed through unchanged, but it is
+    # not part of what gets cached.
+    resolved_ports = [
+        call.args[0].port for call in receiver.addressResolved.call_args_list
+    ]
+    assert resolved_ports == [80, 80]
+    assert [address.port for address in dnscache["example.com"]] == [0, 0]
+
+
+def test_caching_hostname_resolver_cache_hit_without_port():
+    cached_addresses = [
+        IPv4Address("TCP", "1.2.3.4", 0),
+        IPv6Address("TCP", "::1", 0),
+    ]
+    dnscache["example.com"] = cached_addresses
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(Mock(), cache_size=10)
+    resolver.resolveHostName(receiver, "example.com")
+
+    # Cached addresses already use the requested port, so they are reused as is.
+    resolved_addresses = [
+        call.args[0] for call in receiver.addressResolved.call_args_list
+    ]
+    assert all(
+        resolved is cached
+        for resolved, cached in zip(resolved_addresses, cached_addresses, strict=True)
+    )
+
+
+def test_caching_hostname_resolver_cache_hit_uses_requested_port():
+    dnscache["example.com"] = [
+        IPv4Address("TCP", "1.2.3.4", 0),
+        IPv6Address("TCP", "::1", 0),
+    ]
+
+    receiver = Mock()
+    resolver = CachingHostnameResolver(Mock(), cache_size=10)
+    resolver.resolveHostName(receiver, "example.com", portNumber=443)
+
+    resolved_addresses = [
+        call.args[0] for call in receiver.addressResolved.call_args_list
+    ]
+    assert resolved_addresses == [
+        IPv4Address("TCP", "1.2.3.4", 443),
+        IPv6Address("TCP", "::1", 443),
+    ]
+    # The cached addresses must not be mutated in place.
+    assert [address.port for address in dnscache["example.com"]] == [0, 0]
+
+
 def test_caching_hostname_resolver_dnscache_disabled_rejects_storage():
 
     def fake_resolve(receiver, *_):
         receiver.resolutionBegan(Mock())
-        receiver.addressResolved(Mock())
+        receiver.addressResolved(IPv4Address("TCP", "1.2.3.4", 80))
         receiver.resolutionComplete()
         return receiver
 

From a5bc43e34c301232571708615f8b8d78cb56a36d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 28 Jul 2026 12:51:52 +0500
Subject: [PATCH 46/62] Don't generate test keys concurrently with xdist.
 (#7792)

---
 conftest.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/conftest.py b/conftest.py
index b8a9dc19e..27c398792 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import os
 from importlib.util import find_spec
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -135,5 +136,6 @@ def pytest_runtest_setup(item):
         pytest.skip("mitmdump is not available")
 
 
-# Generate localhost certificate files, needed by some tests
-generate_keys()
+# Generate localhost certificate files, needed by some tests (but only once if xdist is used)
+if "PYTEST_XDIST_WORKER" not in os.environ:
+    generate_keys()

From 7e8b58a2b2c0c2cf92a81dd7079580b8c603751c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 28 Jul 2026 12:52:52 +0500
Subject: [PATCH 47/62] Migrate away from testfixtures. (#7793)

---
 pyproject.toml                                |   1 -
 .../test_downloader_handler_twisted_http2.py  |  24 +-
 tests/test_downloadermiddleware_cookies.py    | 126 ++++---
 ...st_downloadermiddleware_httpcompression.py | 188 ++++++----
 tests/test_downloadermiddleware_retry.py      | 350 +++++++++---------
 tests/test_dupefilters.py                     | 156 ++++----
 tests/test_engine.py                          |  16 +-
 tests/test_feedexport.py                      |  26 +-
 tests/test_feedexport_storages.py             |  59 ++-
 tests/test_logformatter.py                    |  51 ++-
 tests/test_pipeline_crawl.py                  |  74 ++--
 tests/test_pipeline_media.py                  |  18 +-
 tests/test_request_attribute_binding.py       |  60 +--
 tests/test_request_cb_kwargs.py               |  52 +--
 tests/test_scheduler_base.py                  |  34 +-
 tests/test_spider_sitemap.py                  |  57 +--
 tests/test_spidermiddleware_output_chain.py   | 137 ++++---
 tests/test_utils_log.py                       |   9 +-
 tests/test_utils_signal.py                    |  44 +--
 tests/utils/bases/spider.py                   |  14 +-
 tox.ini                                       |   1 -
 21 files changed, 775 insertions(+), 722 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index ee7f4ecdf..9b1e64121 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -225,7 +225,6 @@ module = [
     "pyftpdlib.*",
     "pytest_twisted",
     "robotexclusionrulesparser",
-    "testfixtures",
     "zope.interface.*",
 ]
 ignore_missing_imports = true
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index daa89df58..449f2d635 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -2,11 +2,11 @@
 
 from __future__ import annotations
 
+import logging
 import sys
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from twisted.web.http import H2_ENABLED
 
 from scrapy import Spider
@@ -130,24 +130,24 @@ class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
         assert response.text == custom_content_length
 
     @coroutine_test
-    async def test_custom_content_length_bad(self, mockserver: MockServer) -> None:
+    async def test_custom_content_length_bad(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         request = Request(mockserver.url("/contentlength", is_secure=self.is_secure))
         actual_content_length = str(len(request.body))
         bad_content_length = str(len(request.body) + 1)
         request.headers["Content-Length"] = bad_content_length
         async with self.get_dh() as download_handler:
-            with LogCapture() as log:
+            with caplog.at_level(logging.DEBUG):
                 response = await download_handler.download_request(request)
         assert response.text == actual_content_length
-        log.check_present(
-            (
-                "scrapy.core.http2.stream",
-                "WARNING",
-                f"Ignoring bad Content-Length header "
-                f"{bad_content_length!r} of request {request}, sending "
-                f"{actual_content_length!r} instead",
-            )
-        )
+        assert (
+            "scrapy.core.http2.stream",
+            logging.WARNING,
+            f"Ignoring bad Content-Length header "
+            f"{bad_content_length!r} of request {request}, sending "
+            f"{actual_content_length!r} instead",
+        ) in caplog.record_tuples
 
     @coroutine_test
     async def test_data_loss_handling(self, mockserver: MockServer) -> None:
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index b4419dc67..7ad103f27 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -2,7 +2,6 @@ import logging
 from collections.abc import Iterable
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.downloadermiddlewares.cookies import CookiesMiddleware
 from scrapy.downloadermiddlewares.defaultheaders import DefaultHeadersMiddleware
@@ -111,14 +110,15 @@ class TestCookiesMiddleware:
             CookiesMiddleware,
         )
 
-    def test_setting_enabled_cookies_debug(self):
+    def test_setting_enabled_cookies_debug(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response(
                 "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
@@ -127,43 +127,44 @@ class TestCookiesMiddleware:
             req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2)
 
-            log.check(
-                (
-                    "scrapy.downloadermiddlewares.cookies",
-                    "DEBUG",
-                    "Received cookies from: <200 http://scrapytest.org/>\n"
-                    "Set-Cookie: C1=value1; path=/\n",
-                ),
-                (
-                    "scrapy.downloadermiddlewares.cookies",
-                    "DEBUG",
-                    "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
-                    "Cookie: C1=value1\n",
-                ),
-            )
+        assert caplog.record_tuples == [
+            (
+                "scrapy.downloadermiddlewares.cookies",
+                logging.DEBUG,
+                "Received cookies from: <200 http://scrapytest.org/>\n"
+                "Set-Cookie: C1=value1; path=/\n",
+            ),
+            (
+                "scrapy.downloadermiddlewares.cookies",
+                logging.DEBUG,
+                "Sending cookies to: <GET http://scrapytest.org/sub1/>\n"
+                "Cookie: C1=value1\n",
+            ),
+        ]
 
-    def test_debug_no_cookies(self):
+    def test_debug_no_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": True})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response("http://scrapytest.org/")  # no Set-Cookie header
             mw.process_response(req, res)
             mw.process_request(req)  # no cookies to send either
-            log.check()  # no log output since cl is empty in both cases
+        # no log output since cl is empty in both cases
+        assert caplog.record_tuples == []
 
-    def test_setting_disabled_cookies_debug(self):
+    def test_setting_disabled_cookies_debug(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler(settings_dict={"COOKIES_DEBUG": False})
         mw = CookiesMiddleware.from_crawler(crawler)
-        with LogCapture(
-            "scrapy.downloadermiddlewares.cookies",
-            propagate=False,
-            level=logging.DEBUG,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            logging.DEBUG, logger="scrapy.downloadermiddlewares.cookies"
+        ):
             req = Request("http://scrapytest.org/")
             res = Response(
                 "http://scrapytest.org/", headers={"Set-Cookie": "C1=value1; path=/"}
@@ -172,7 +173,7 @@ class TestCookiesMiddleware:
             req2 = Request("http://scrapytest.org/sub1/")
             mw.process_request(req2)
 
-            log.check()
+        assert caplog.record_tuples == []
 
     def test_do_not_break_on_non_utf8_header(self):
         req = Request("http://scrapytest.org/")
@@ -420,44 +421,41 @@ class TestCookiesMiddleware:
         assert self.mw.process_request(req3) is None
         self.assertCookieValEqual(req3.headers["Cookie"], b"a=\xc3\xa1")
 
-    def test_invalid_cookies(self):
+    def test_invalid_cookies(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         Invalid cookies are logged as warnings and discarded
         """
-        with LogCapture(
-            "scrapy.utils.request",
-            propagate=False,
-            level=logging.INFO,
-        ) as lc:
+        caplog.clear()
+        with caplog.at_level(logging.INFO, logger="scrapy.utils.request"):
             cookies1 = [{"value": "bar"}, {"name": "key", "value": "value1"}]
-            req1 = Request("http://example.org/1", cookies=cookies1)
+            req1 = Request("http://example.org/1", cookies=cookies1)  # type: ignore[arg-type]
             assert self.mw.process_request(req1) is None
             cookies2 = [{"name": "foo"}, {"name": "key", "value": "value2"}]
-            req2 = Request("http://example.org/2", cookies=cookies2)
+            req2 = Request("http://example.org/2", cookies=cookies2)  # type: ignore[arg-type]
             assert self.mw.process_request(req2) is None
             cookies3 = [{"name": "foo", "value": None}, {"name": "key", "value": ""}]
-            req3 = Request("http://example.org/3", cookies=cookies3)
+            req3 = Request("http://example.org/3", cookies=cookies3)  # type: ignore[arg-type]
             assert self.mw.process_request(req3) is None
-            lc.check(
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/1>:"
-                    " {'value': 'bar', 'secure': False} ('name' is missing)",
-                ),
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/2>:"
-                    " {'name': 'foo', 'secure': False} ('value' is missing)",
-                ),
-                (
-                    "scrapy.utils.request",
-                    "WARNING",
-                    "Invalid cookie found in request <GET http://example.org/3>:"
-                    " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",
-                ),
-            )
+        assert caplog.record_tuples == [
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/1>:"
+                " {'value': 'bar', 'secure': False} ('name' is missing)",
+            ),
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/2>:"
+                " {'name': 'foo', 'secure': False} ('value' is missing)",
+            ),
+            (
+                "scrapy.utils.request",
+                logging.WARNING,
+                "Invalid cookie found in request <GET http://example.org/3>:"
+                " {'name': 'foo', 'value': None, 'secure': False} ('value' is missing)",
+            ),
+        ]
         self.assertCookieValEqual(req1.headers["Cookie"], "key=value1")
         self.assertCookieValEqual(req2.headers["Cookie"], "key=value2")
         self.assertCookieValEqual(req3.headers["Cookie"], "key=")
diff --git a/tests/test_downloadermiddleware_httpcompression.py b/tests/test_downloadermiddleware_httpcompression.py
index ca18c8038..55f06b396 100644
--- a/tests/test_downloadermiddleware_httpcompression.py
+++ b/tests/test_downloadermiddleware_httpcompression.py
@@ -5,7 +5,6 @@ from logging import WARNING
 from pathlib import Path
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.encoding import resolve_encoding
 
 from scrapy.downloadermiddlewares.httpcompression import (
@@ -75,7 +74,7 @@ class TestHttpCompression:
         self.mw = HttpCompressionMiddleware.from_crawler(self.crawler)
         self.crawler.stats.open_spider()
 
-    def _getresponse(self, coding):
+    def _getresponse(self, coding: str) -> Response:
         if coding not in FORMAT:
             raise ValueError
 
@@ -169,29 +168,28 @@ class TestHttpCompression:
         self.assertStatsEqual("httpcompression/response_count", 1)
         self.assertStatsEqual("httpcompression/response_bytes", 74837)
 
-    def test_process_response_br_unsupported(self):
+    def test_process_response_br_unsupported(self, caplog: pytest.LogCaptureFixture):
         if find_spec("brotli") is not None or find_spec("brotlicffi") is not None:
             pytest.skip("Requires not having brotli support")
         response = self._getresponse("br")
         request = response.request
         assert response.headers["Content-Encoding"] == b"br"
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for "
                     "http://scrapytest.org/ from unsupported encoding(s) 'br'. "
                     "You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"br"]
 
@@ -214,29 +212,28 @@ class TestHttpCompression:
             assert newresponse.body.startswith(b"<!DOCTYPE")
             assert "Content-Encoding" not in newresponse.headers
 
-    def test_process_response_zstd_unsupported(self):
+    def test_process_response_zstd_unsupported(self, caplog: pytest.LogCaptureFixture):
         if find_spec("zstandard") is not None:
             pytest.skip("Requires not having zstandard support")
         response = self._getresponse("zstd-static-content-size")
         request = response.request
         assert response.headers["Content-Encoding"] == b"zstd"
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for"
                     " http://scrapytest.org/ from unsupported encoding(s) 'zstd'."
                     " You need to install zstandard to decode 'zstd'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"zstd"]
 
@@ -291,26 +288,27 @@ class TestHttpCompression:
         assert "Content-Encoding" not in newresponse.headers
         assert newresponse.body.startswith(b"<!DOCTYPE")
 
-    def test_multi_compression_single_header_invalid_compression(self):
+    def test_multi_compression_single_header_invalid_compression(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         response = self._getresponse("gzip-deflate")
         response.headers["Content-Encoding"] = [b"gzip, foo, deflate"]
         request = response.request
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             newresponse = self.mw.process_response(request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "HttpCompressionMiddleware cannot decode the response for"
                     " http://scrapytest.org/ from unsupported encoding(s) 'gzip,foo'."
                 ),
             ),
-        )
+        ]
         assert newresponse is not response
         assert newresponse.headers.getlist("Content-Encoding") == [b"gzip", b"foo"]
 
@@ -629,7 +627,9 @@ class TestHttpCompression:
 
         self._test_compression_bomb_request_meta("zstd")
 
-    def _test_download_warnsize_setting(self, compression_id):
+    def _test_download_warnsize_setting(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
         crawler = get_crawler(Spider, settings_dict=settings)
         spider = crawler._create_spider("scrapytest.org")
@@ -637,41 +637,51 @@ class TestHttpCompression:
         mw.open_spider(spider)
         response = self._getresponse(f"bomb-{compression_id}")
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_setting_br(self):
+    def test_download_warnsize_setting_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_setting("br")
+        self._test_download_warnsize_setting(caplog, "br")
 
-    def test_download_warnsize_setting_deflate(self):
-        self._test_download_warnsize_setting("deflate")
+    def test_download_warnsize_setting_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_setting(caplog, "deflate")
 
-    def test_download_warnsize_setting_gzip(self):
-        self._test_download_warnsize_setting("gzip")
+    def test_download_warnsize_setting_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_setting(caplog, "gzip")
 
-    def test_download_warnsize_setting_zstd(self):
+    def test_download_warnsize_setting_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_setting("zstd")
+        self._test_download_warnsize_setting(caplog, "zstd")
 
-    def _test_download_warnsize_spider_attr(self, compression_id):
+    def _test_download_warnsize_spider_attr(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         class DownloadWarnSizeSpider(Spider):
             download_warnsize = 10_000_000
 
@@ -681,45 +691,55 @@ class TestHttpCompression:
         mw.open_spider(spider)
         response = self._getresponse(f"bomb-{compression_id}")
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_br(self):
+    def test_download_warnsize_spider_attr_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_spider_attr("br")
+        self._test_download_warnsize_spider_attr(caplog, "br")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_deflate(self):
-        self._test_download_warnsize_spider_attr("deflate")
+    def test_download_warnsize_spider_attr_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_spider_attr(caplog, "deflate")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_gzip(self):
-        self._test_download_warnsize_spider_attr("gzip")
+    def test_download_warnsize_spider_attr_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_spider_attr(caplog, "gzip")
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr_zstd(self):
+    def test_download_warnsize_spider_attr_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_spider_attr("zstd")
+        self._test_download_warnsize_spider_attr(caplog, "zstd")
 
-    def _test_download_warnsize_request_meta(self, compression_id):
+    def _test_download_warnsize_request_meta(
+        self, caplog: pytest.LogCaptureFixture, compression_id: str
+    ) -> None:
         crawler = get_crawler(Spider)
         spider = crawler._create_spider("scrapytest.org")
         mw = HttpCompressionMiddleware.from_crawler(crawler)
@@ -727,39 +747,47 @@ class TestHttpCompression:
         response = self._getresponse(f"bomb-{compression_id}")
         response.meta["download_warnsize"] = 10_000_000
 
-        with LogCapture(
-            "scrapy.downloadermiddlewares.httpcompression",
-            propagate=False,
-            level=WARNING,
-        ) as log:
+        assert response.request
+        caplog.clear()
+        with caplog.at_level(
+            WARNING, logger="scrapy.downloadermiddlewares.httpcompression"
+        ):
             mw.process_response(response.request, response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.downloadermiddlewares.httpcompression",
-                "WARNING",
+                WARNING,
                 (
                     "<200 http://scrapytest.org/> body size after "
                     "decompression (11511612 B) is larger than the download "
                     "warning size (10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_request_meta_br(self):
+    def test_download_warnsize_request_meta_br(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_br()
 
-        self._test_download_warnsize_request_meta("br")
+        self._test_download_warnsize_request_meta(caplog, "br")
 
-    def test_download_warnsize_request_meta_deflate(self):
-        self._test_download_warnsize_request_meta("deflate")
+    def test_download_warnsize_request_meta_deflate(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_request_meta(caplog, "deflate")
 
-    def test_download_warnsize_request_meta_gzip(self):
-        self._test_download_warnsize_request_meta("gzip")
+    def test_download_warnsize_request_meta_gzip(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        self._test_download_warnsize_request_meta(caplog, "gzip")
 
-    def test_download_warnsize_request_meta_zstd(self):
+    def test_download_warnsize_request_meta_zstd(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         _skip_if_no_zstd()
 
-        self._test_download_warnsize_request_meta("zstd")
+        self._test_download_warnsize_request_meta(caplog, "zstd")
 
     def _get_truncated_response(self, compression_id):
         crawler = get_crawler(Spider)
diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.py
index 825efce2f..410427b84 100644
--- a/tests/test_downloadermiddleware_retry.py
+++ b/tests/test_downloadermiddleware_retry.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import logging
+from typing import Any, cast
 
 import pytest
-from testfixtures import LogCapture
 from twisted.internet.error import ConnectError, ConnectionDone, ConnectionLost
 
 from scrapy.downloadermiddlewares.retry import RetryMiddleware, get_retry_request
@@ -85,7 +87,7 @@ class TestRetry:
         )
         assert self.crawler.stats.get_value("retry/count") == 2
 
-    def test_give_up_log_level_setting(self):
+    def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(
             DefaultSpider, settings_dict={"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"}
         )
@@ -94,29 +96,25 @@ class TestRetry:
         mw.max_retry_times = 0
         req = Request("http://example.com/503")
         rsp = Response("http://example.com/503", body=b"", status=503)
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             assert mw.process_response(req, rsp) is rsp
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_meta(self):
+    def test_give_up_log_level_meta(self, caplog: pytest.LogCaptureFixture) -> None:
         self.mw.max_retry_times = 0
         req = Request("http://example.com/503", meta={"give_up_log_level": "WARNING"})
         rsp = Response("http://example.com/503", body=b"", status=503)
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             assert self.mw.process_response(req, rsp) is rsp
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {req} (failed 1 times): 503 Service Unavailable",
+        ) in caplog.record_tuples
 
     def test_twistederrors(self):
         exceptions = [
@@ -294,14 +292,15 @@ class TestMaxRetryTimes:
 
 
 class TestGetRetryRequest:
-    def get_spider(self, settings=None):
+    @staticmethod
+    def get_spider(settings: dict[str, Any] | None = None) -> Spider:
         crawler = get_crawler(Spider, settings or {})
         return crawler._create_spider("foo")
 
-    def test_basic_usage(self):
+    def test_basic_usage(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
@@ -313,44 +312,42 @@ class TestGetRetryRequest:
         assert new_request.meta["retry_times"] == expected_retry_times
         assert new_request.priority == -1
         expected_reason = "unspecified"
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_max_retries_reached(self):
+    def test_max_retries_reached(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         max_retry_times = 0
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=max_retry_times,
             )
         assert new_request is None
+        assert spider.crawler.stats
         assert spider.crawler.stats.get_value("retry/max_reached") == 1
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed {failure_count} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed {failure_count} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_one_retry(self):
+    def test_one_retry(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             new_request = get_retry_request(
                 request,
                 spider=spider,
@@ -363,28 +360,31 @@ class TestGetRetryRequest:
         assert new_request.meta["retry_times"] == expected_retry_times
         assert new_request.priority == -1
         expected_reason = "unspecified"
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_two_retries(self):
+    def test_two_retries(self, caplog: pytest.LogCaptureFixture) -> None:
         spider = self.get_spider()
         request = Request("https://example.com")
         new_request = request
         max_retry_times = 2
         for index in range(max_retry_times):
-            with LogCapture() as log:
-                new_request = get_retry_request(
-                    new_request,
-                    spider=spider,
-                    max_retry_times=max_retry_times,
+            caplog.clear()
+            with caplog.at_level(logging.DEBUG):
+                new_request = cast(
+                    "Request",
+                    get_retry_request(
+                        new_request,
+                        spider=spider,
+                        max_retry_times=max_retry_times,
+                    ),
                 )
             assert isinstance(new_request, Request)
             assert new_request != request
@@ -393,36 +393,37 @@ class TestGetRetryRequest:
             assert new_request.meta["retry_times"] == expected_retry_times
             assert new_request.priority == -expected_retry_times
             expected_reason = "unspecified"
+            assert spider.crawler.stats
             for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
                 value = spider.crawler.stats.get_value(stat)
                 assert value == expected_retry_times
-            log.check_present(
-                (
-                    "scrapy.downloadermiddlewares.retry",
-                    "DEBUG",
-                    f"Retrying {request} (failed {expected_retry_times} times): "
-                    f"{expected_reason}",
-                )
-            )
+            assert (
+                "scrapy.downloadermiddlewares.retry",
+                logging.DEBUG,
+                f"Retrying {request} (failed {expected_retry_times} times): "
+                f"{expected_reason}",
+            ) in caplog.record_tuples
 
-        with LogCapture() as log:
-            new_request = get_retry_request(
-                new_request,
-                spider=spider,
-                max_retry_times=max_retry_times,
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG):
+            new_request = cast(
+                "Request",
+                get_retry_request(
+                    new_request,
+                    spider=spider,
+                    max_retry_times=max_retry_times,
+                ),
             )
         assert new_request is None
         assert spider.crawler.stats.get_value("retry/max_reached") == 1
         failure_count = max_retry_times + 1
         expected_reason = "unspecified"
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed {failure_count} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed {failure_count} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
     def test_no_spider(self):
         request = Request("https://example.com")
@@ -483,238 +484,231 @@ class TestGetRetryRequest:
         )
         assert new_request.priority == priority_adjust
 
-    def test_log_extra_retry_success(self):
+    def test_log_extra_retry_success(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=("spider",)) as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
             )
-        log.check_present(spider)
+        assert any(getattr(r, "spider", None) is spider for r in caplog.records)
 
-    def test_log_extra_retries_exceeded(self):
+    def test_log_extra_retries_exceeded(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture(attributes=("spider",)) as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(spider)
+        assert any(getattr(r, "spider", None) is spider for r in caplog.records)
 
-    def test_reason_string(self):
+    def test_reason_string(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = "because"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         for stat in ("retry/count", f"retry/reason_count/{expected_reason}"):
             assert spider.crawler.stats.get_value(stat) == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_builtin_exception(self):
+    def test_reason_builtin_exception(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError()
         expected_reason_string = "builtins.NotImplementedError"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_builtin_exception_class(self):
+    def test_reason_builtin_exception_class(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = NotImplementedError
         expected_reason_string = "builtins.NotImplementedError"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_custom_exception(self):
+    def test_reason_custom_exception(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest()
         expected_reason_string = "scrapy.exceptions.IgnoreRequest"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_reason_custom_exception_class(self):
+    def test_reason_custom_exception_class(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = IgnoreRequest
         expected_reason_string = "scrapy.exceptions.IgnoreRequest"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
             )
         expected_retry_times = 1
+        assert spider.crawler.stats
         stat = spider.crawler.stats.get_value(
             f"retry/reason_count/{expected_reason_string}"
         )
         assert stat == 1
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "DEBUG",
-                f"Retrying {request} (failed {expected_retry_times} times): "
-                f"{expected_reason}",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.DEBUG,
+            f"Retrying {request} (failed {expected_retry_times} times): "
+            f"{expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_custom_logger(self):
+    def test_custom_logger(self, caplog: pytest.LogCaptureFixture) -> None:
         logger = logging.getLogger("custom-logger")
         request = Request("https://example.com")
         spider = self.get_spider()
         expected_reason = "because"
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             get_retry_request(
                 request,
                 spider=spider,
                 reason=expected_reason,
                 logger=logger,
             )
-        log.check_present(
-            (
-                "custom-logger",
-                "DEBUG",
-                f"Retrying {request} (failed 1 times): {expected_reason}",
-            )
-        )
+        assert (
+            "custom-logger",
+            logging.DEBUG,
+            f"Retrying {request} (failed 1 times): {expected_reason}",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_default(self):
+    def test_give_up_log_level_default(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.ERROR):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "ERROR",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.ERROR,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_argument_name(self):
+    def test_give_up_log_level_argument_name(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
                 give_up_log_level="WARNING",
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_argument_number(self):
+    def test_give_up_log_level_argument_number(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         request = Request("https://example.com")
         spider = self.get_spider()
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
                 give_up_log_level=logging.WARNING,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
-    def test_give_up_log_level_setting(self):
+    def test_give_up_log_level_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         request = Request("https://example.com")
         spider = self.get_spider({"RETRY_GIVE_UP_LOG_LEVEL": "WARNING"})
-        with LogCapture() as log:
+        with caplog.at_level(logging.WARNING):
             get_retry_request(
                 request,
                 spider=spider,
                 max_retry_times=0,
             )
-        log.check_present(
-            (
-                "scrapy.downloadermiddlewares.retry",
-                "WARNING",
-                f"Gave up retrying {request} (failed 1 times): unspecified",
-            )
-        )
+        assert (
+            "scrapy.downloadermiddlewares.retry",
+            logging.WARNING,
+            f"Gave up retrying {request} (failed 1 times): unspecified",
+        ) in caplog.record_tuples
 
     def test_give_up_log_level_invalid(self):
         request = Request("https://example.com")
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 5d79691b2..479332eae 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -1,11 +1,14 @@
+from __future__ import annotations
+
 import hashlib
+import logging
 import shutil
 import sys
 import tempfile
 from pathlib import Path
+from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.core.scheduler import Scheduler
 from scrapy.dupefilters import BaseDupeFilter, RFPDupeFilter
@@ -15,8 +18,16 @@ from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
 
-def _get_dupefilter(*, crawler=None, settings=None, open_=True):
+
+def _get_dupefilter(
+    *,
+    crawler: Crawler | None = None,
+    settings: dict[str, Any] | None = None,
+    open_: bool = True,
+) -> BaseDupeFilter:
     if crawler is None:
         crawler = get_crawler(settings_dict=settings)
     scheduler = Scheduler.from_crawler(crawler)
@@ -151,108 +162,71 @@ class TestRFPDupeFilter:
         finally:
             shutil.rmtree(path)
 
-    def test_log(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": False,
-                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
+    def test_log(self, caplog: pytest.LogCaptureFixture) -> None:
+        settings = {
+            "DUPEFILTER_DEBUG": False,
+            "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
+        }
+        crawler = get_crawler(SimpleSpider, settings_dict=settings)
+        spider = SimpleSpider.from_crawler(crawler)
+        dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request("http://scrapytest.org/index.html")
+        r1 = Request("http://scrapytest.org/index.html")
+        r2 = Request("http://scrapytest.org/index.html")
 
+        with caplog.at_level(logging.DEBUG):
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
-                    " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
-                )
-            )
+        assert crawler.stats
+        assert crawler.stats.get_value("dupefilter/filtered") == 2
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html> - no more"
+            " duplicates will be shown (see DUPEFILTER_DEBUG to show all duplicates)",
+        ) in caplog.record_tuples
 
-            dupefilter.close("finished")
+        dupefilter.close("finished")
 
-    def test_log_debug(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": True,
-                "DUPEFILTER_CLASS": FromCrawlerRFPDupeFilter,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
+    @pytest.mark.parametrize("df", [None, FromCrawlerRFPDupeFilter])
+    def test_log_debug(
+        self, caplog: pytest.LogCaptureFixture, df: type[BaseDupeFilter] | None
+    ) -> None:
+        settings: dict[str, Any] = {
+            "DUPEFILTER_DEBUG": True,
+        }
+        if df:
+            settings["DUPEFILTER_CLASS"] = df
+        crawler = get_crawler(SimpleSpider, settings_dict=settings)
+        spider = SimpleSpider.from_crawler(crawler)
+        dupefilter = _get_dupefilter(crawler=crawler)
 
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request(
-                "http://scrapytest.org/index.html",
-                headers={"Referer": "http://scrapytest.org/INDEX.html"},
-            )
+        r1 = Request("http://scrapytest.org/index.html")
+        r2 = Request(
+            "http://scrapytest.org/index.html",
+            headers={"Referer": "http://scrapytest.org/INDEX.html"},
+        )
 
+        with caplog.at_level(logging.DEBUG):
             dupefilter.log(r1, spider)
             dupefilter.log(r2, spider)
 
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
-                )
-            )
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
-                    " (referer: http://scrapytest.org/INDEX.html)",
-                )
-            )
+        assert crawler.stats
+        assert crawler.stats.get_value("dupefilter/filtered") == 2
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
+        ) in caplog.record_tuples
+        assert (
+            "scrapy.dupefilters",
+            logging.DEBUG,
+            "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
+            " (referer: http://scrapytest.org/INDEX.html)",
+        ) in caplog.record_tuples
 
-            dupefilter.close("finished")
-
-    def test_log_debug_default_dupefilter(self):
-        with LogCapture() as log:
-            settings = {
-                "DUPEFILTER_DEBUG": True,
-            }
-            crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            spider = SimpleSpider.from_crawler(crawler)
-            dupefilter = _get_dupefilter(crawler=crawler)
-
-            r1 = Request("http://scrapytest.org/index.html")
-            r2 = Request(
-                "http://scrapytest.org/index.html",
-                headers={"Referer": "http://scrapytest.org/INDEX.html"},
-            )
-
-            dupefilter.log(r1, spider)
-            dupefilter.log(r2, spider)
-
-            assert crawler.stats.get_value("dupefilter/filtered") == 2
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)",
-                )
-            )
-            log.check_present(
-                (
-                    "scrapy.dupefilters",
-                    "DEBUG",
-                    "Filtered duplicate request: <GET http://scrapytest.org/index.html>"
-                    " (referer: http://scrapytest.org/INDEX.html)",
-                )
-            )
-
-            dupefilter.close("finished")
+        dupefilter.close("finished")
 
 
 class TestBaseDupeFilter:
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 87f0265a0..8a3bceccb 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import subprocess
 import sys
 from typing import TYPE_CHECKING, Any
 from unittest.mock import Mock
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine, _Slot
@@ -145,8 +145,10 @@ class TestEngine(TestEngineBase):
             await asyncio.gather(e.start_async(), e.start_async())
         await e.stop_async()
 
-    @inline_callbacks_test
-    def test_start_request_processing_exception(self):
+    @coroutine_test
+    async def test_start_request_processing_exception(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         class BadRequestFingerprinter:
             def fingerprint(self, request):
                 raise ValueError  # to make Scheduler.enqueue_request() fail
@@ -160,10 +162,10 @@ class TestEngine(TestEngineBase):
         crawler = get_crawler(
             SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter}
         )
-        with LogCapture() as log:
-            yield crawler.crawl()
-        assert "Error while processing requests from start()" in str(log)
-        assert "Spider closed (shutdown)" in str(log)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async()
+        assert "Error while processing requests from start()" in caplog.text
+        assert "Spider closed (shutdown)" in caplog.text
 
     def test_short_timeout(self):
         args = (
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f1c416fbd..92c414bef 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import csv
 import json
+import logging
 import marshal
 import pickle
 import tempfile
@@ -12,7 +13,6 @@ from unittest import mock
 
 import lxml.etree
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import file_uri_to_path
 
 import scrapy
@@ -563,7 +563,9 @@ class TestFeedExport(TestFeedExportBase):
             assert expctd == data[fmt]
 
     @coroutine_test
-    async def test_export_no_items_multiple_feeds(self):
+    async def test_export_no_items_multiple_feeds(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """Make sure that `storage.store` is not called."""
         settings = {
             "FEEDS": {
@@ -575,10 +577,10 @@ class TestFeedExport(TestFeedExportBase):
             "FEED_STORE_EMPTY": False,
         }
 
-        with LogCapture() as log:
+        with caplog.at_level(logging.INFO):
             await self.exported_no_data(settings)
 
-        assert str(log).count("Storage.store is called") == 0
+        assert caplog.text.count("Storage.store is called") == 0
 
     @coroutine_test
     async def test_export_multiple_item_classes(self):
@@ -1080,7 +1082,9 @@ class TestFeedExport(TestFeedExportBase):
         assert data["csv"] == b""
 
     @coroutine_test
-    async def test_multiple_feeds_success_logs_blocking_feed_storage(self):
+    async def test_multiple_feeds_success_logs_blocking_feed_storage(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},
@@ -1093,14 +1097,16 @@ class TestFeedExport(TestFeedExportBase):
             {"foo": "bar1", "baz": ""},
             {"foo": "bar2", "baz": "quux"},
         ]
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             await self.exported_data(items, settings)
 
         for fmt in ["json", "xml", "csv"]:
-            assert f"Stored {fmt} feed (2 items)" in str(log)
+            assert f"Stored {fmt} feed (2 items)" in caplog.text
 
     @coroutine_test
-    async def test_multiple_feeds_failing_logs_blocking_feed_storage(self):
+    async def test_multiple_feeds_failing_logs_blocking_feed_storage(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         settings = {
             "FEEDS": {
                 self._random_temp_filename(): {"format": "json"},
@@ -1113,11 +1119,11 @@ class TestFeedExport(TestFeedExportBase):
             {"foo": "bar1", "baz": ""},
             {"foo": "bar2", "baz": "quux"},
         ]
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             await self.exported_data(items, settings)
 
         for fmt in ["json", "xml", "csv"]:
-            assert f"Error storing {fmt} feed (2 items)" in str(log)
+            assert f"Error storing {fmt} feed (2 items)" in caplog.text
 
     @coroutine_test
     async def test_extend_kwargs(self):
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index bdd0d7614..66488540f 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 import os
 import string
 import tempfile
@@ -10,7 +11,6 @@ from unittest import mock
 from urllib.parse import quote
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import path_to_file_uri
 
 import scrapy
@@ -415,23 +415,21 @@ class TestS3FeedStorage:
         acl = storage.s3_client.upload_fileobj.call_args[1]["ExtraArgs"]["ACL"]
         assert acl == "custom-acl"
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
-            S3FeedStorage(
-                "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
-            )
-        assert "S3 does not support appending to files" not in str(log)
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture) -> None:
+        S3FeedStorage(
+            "s3://mybucket/export.csv", "access_key", "secret_key", "custom-acl"
+        )
+        assert "S3 does not support appending to files" not in caplog.text
 
-    def test_overwrite_false(self):
-        with LogCapture() as log:
-            S3FeedStorage(
-                "s3://mybucket/export.csv",
-                "access_key",
-                "secret_key",
-                "custom-acl",
-                feed_options={"overwrite": False},
-            )
-        assert "S3 does not support appending to files" in str(log)
+    def test_overwrite_false(self, caplog: pytest.LogCaptureFixture) -> None:
+        S3FeedStorage(
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
+            "custom-acl",
+            feed_options={"overwrite": False},
+        )
+        assert "S3 does not support appending to files" in caplog.text
 
 
 class TestGCSFeedStorage:
@@ -505,20 +503,20 @@ class TestGCSFeedStorage:
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
             f.close.assert_called_once_with()
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             GCSFeedStorage("gs://mybucket/export.csv", "myproject-123", "custom-acl")
-        assert "GCS does not support appending to files" not in str(log)
+        assert "GCS does not support appending to files" not in caplog.text
 
-    def test_overwrite_false(self):
-        with LogCapture() as log:
+    def test_overwrite_false(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             GCSFeedStorage(
                 "gs://mybucket/export.csv",
                 "myproject-123",
                 "custom-acl",
                 feed_options={"overwrite": False},
             )
-        assert "GCS does not support appending to files" in str(log)
+        assert "GCS does not support appending to files" in caplog.text
 
 
 class TestStdoutFeedStorage:
@@ -530,17 +528,18 @@ class TestStdoutFeedStorage:
         storage.store(file)
         assert out.getvalue() == b"content"
 
-    def test_overwrite_default(self):
-        with LogCapture() as log:
+    def test_overwrite_default(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             StdoutFeedStorage("stdout:")
         assert (
             "Standard output (stdout) storage does not support overwriting"
-            not in str(log)
+            not in caplog.text
         )
 
-    def test_overwrite_true(self):
-        with LogCapture() as log:
+    def test_overwrite_true(self, caplog: pytest.LogCaptureFixture):
+        with caplog.at_level(logging.DEBUG):
             StdoutFeedStorage("stdout:", feed_options={"overwrite": True})
-        assert "Standard output (stdout) storage does not support overwriting" in str(
-            log
+        assert (
+            "Standard output (stdout) storage does not support overwriting"
+            in caplog.text
         )
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 360aa613e..fc6ccf981 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import logging
+from typing import TYPE_CHECKING
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import DropItem
@@ -10,9 +12,11 @@ from scrapy.item import Field, Item
 from scrapy.logformatter import LogFormatter
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 
 class CustomItem(Item):
@@ -254,15 +258,6 @@ class DropSomeItemsPipeline:
 
 
 class TestShowOrSkipMessages:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
     def setup_method(self):
         self.base_settings = {
             "LOG_LEVEL": "DEBUG",
@@ -271,22 +266,26 @@ class TestShowOrSkipMessages:
             },
         }
 
-    @inline_callbacks_test
-    def test_show_messages(self):
+    @coroutine_test
+    async def test_show_messages(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(ItemSpider, self.base_settings)
-        with LogCapture() as lc:
-            yield crawler.crawl(mockserver=self.mockserver)
-        assert "Scraped from <200 http://127.0.0.1:" in str(lc)
-        assert "Crawled (200) <GET http://127.0.0.1:" in str(lc)
-        assert "Dropped: Ignoring item" in str(lc)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert "Scraped from <200 http://127.0.0.1:" in caplog.text
+        assert "Crawled (200) <GET http://127.0.0.1:" in caplog.text
+        assert "Dropped: Ignoring item" in caplog.text
 
-    @inline_callbacks_test
-    def test_skip_messages(self):
+    @coroutine_test
+    async def test_skip_messages(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         settings = self.base_settings.copy()
         settings["LOG_FORMATTER"] = SkipMessagesLogFormatter
         crawler = get_crawler(ItemSpider, settings)
-        with LogCapture() as lc:
-            yield crawler.crawl(mockserver=self.mockserver)
-        assert "Scraped from <200 http://127.0.0.1:" not in str(lc)
-        assert "Crawled (200) <GET http://127.0.0.1:" not in str(lc)
-        assert "Dropped: Ignoring item" not in str(lc)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert "Scraped from <200 http://127.0.0.1:" not in caplog.text
+        assert "Crawled (200) <GET http://127.0.0.1:" not in caplog.text
+        assert "Dropped: Ignoring item" not in caplog.text
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 0376d27c4..0681371ef 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -1,12 +1,12 @@
 from __future__ import annotations
 
+import logging
 import shutil
 from pathlib import Path
 from tempfile import mkdtemp
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from w3lib.url import add_or_replace_parameter
 
 from scrapy import Spider, signals
@@ -14,7 +14,7 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SimpleSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
@@ -58,6 +58,8 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider):
 
 
 class TestFileDownloadCrawl:
+    mockserver: MockServer
+
     pipeline_class = "scrapy.pipelines.files.FilesPipeline"
     store_setting_key = "FILES_STORE"
     media_key = "files"
@@ -102,7 +104,7 @@ class TestFileDownloadCrawl:
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         return crawler
 
-    def _assert_files_downloaded(self, items, logs):
+    def _assert_files_downloaded(self, items: list[Any], logs: str) -> None:
         assert len(items) == 1
         assert self.media_key in items[0]
 
@@ -125,13 +127,16 @@ class TestFileDownloadCrawl:
             for i in item[self.media_key]:
                 assert (self.tmpmediastore / i["path"]).exists()
 
-    def _assert_files_download_failure(self, crawler, items, code, logs):
+    def _assert_files_download_failure(
+        self, crawler: Crawler, items: list[Any], code: int, logs: str
+    ) -> None:
         # check that the item does NOT have the "images/files" field populated
         assert len(items) == 1
         assert self.media_key in items[0]
         assert not items[0][self.media_key]
 
         # check that there was 1 successful fetch and 3 other responses with non-200 code
+        assert crawler.stats
         assert crawler.stats.get_value("downloader/request_method_count/GET") == 4
         assert crawler.stats.get_value("downloader/response_count") == 4
         assert crawler.stats.get_value("downloader/response_status_count/200") == 1
@@ -144,62 +149,71 @@ class TestFileDownloadCrawl:
         # check that no files were written to the media store
         assert not list(self.tmpmediastore.iterdir())
 
-    @inline_callbacks_test
-    def test_download_media(self):
+    @coroutine_test
+    async def test_download_media(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = self._create_crawler(MediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
             )
-        self._assert_files_downloaded(self.items, str(log))
+        self._assert_files_downloaded(self.items, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_wrong_urls(self):
+    @coroutine_test
+    async def test_download_media_wrong_urls(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
             )
-        self._assert_files_download_failure(crawler, self.items, 404, str(log))
+        self._assert_files_download_failure(crawler, self.items, 404, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_redirected_default_failure(self):
+    @coroutine_test
+    async def test_download_media_redirected_default_failure(
+        self, caplog: pytest.LogCaptureFixture
+    ):
         crawler = self._create_crawler(RedirectedMediaDownloadSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        self._assert_files_download_failure(crawler, self.items, 302, str(log))
+        self._assert_files_download_failure(crawler, self.items, 302, caplog.text)
 
-    @inline_callbacks_test
-    def test_download_media_redirected_allowed(self):
+    @coroutine_test
+    async def test_download_media_redirected_allowed(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         settings = {
             **self.settings,
             "MEDIA_ALLOW_REDIRECTS": True,
         }
         crawler = self._create_crawler(RedirectedMediaDownloadSpider, settings)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        self._assert_files_downloaded(self.items, str(log))
+        self._assert_files_downloaded(self.items, caplog.text)
+        assert crawler.stats
         assert crawler.stats.get_value("downloader/response_status_count/302") == 3
 
-    @inline_callbacks_test
-    def test_download_media_file_path_error(self):
+    @coroutine_test
+    async def test_download_media_file_path_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         cls = load_object(self.pipeline_class)
 
-        class ExceptionRaisingMediaPipeline(cls):
+        class ExceptionRaisingMediaPipeline(cls):  # type: ignore[misc,valid-type]
             def file_path(self, request, response=None, info=None, *, item=None):
                 return 1 / 0
 
@@ -208,14 +222,14 @@ class TestFileDownloadCrawl:
             "ITEM_PIPELINES": {ExceptionRaisingMediaPipeline: 1},
         }
         crawler = self._create_crawler(MediaDownloadSpider, settings)
-        with LogCapture() as log:
-            yield crawler.crawl(
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
                 self.mockserver.url("/static/files/images/"),
                 media_key=self.media_key,
                 media_urls_key=self.media_urls_key,
                 mockserver=self.mockserver,
             )
-        assert "ZeroDivisionError" in str(log)
+        assert "ZeroDivisionError" in caplog.text
 
 
 pillow_available: bool
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index be471e5fe..23c19e4be 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -4,7 +4,6 @@ import logging
 from unittest.mock import MagicMock
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy import signals
@@ -129,7 +128,7 @@ class TestBaseMediaPipeline:
         context = getattr(info.downloaded[fp].value, "__context__", None)
         assert context is None
 
-    def test_default_item_completed(self):
+    def test_default_item_completed(self, caplog: pytest.LogCaptureFixture) -> None:
         item = {"name": "name"}
         assert self.pipe.item_completed([], item, self.info) is item
 
@@ -137,21 +136,20 @@ class TestBaseMediaPipeline:
         fail = Failure(Exception())
         results = [(True, 1), (False, fail)]
 
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
-
+        caplog.clear()
+        new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert record.levelname == "ERROR"
         assert record.exc_info == failure_to_exc_info(fail)
 
         # disable failure logging and check again
+        caplog.clear()
         self.pipe.LOG_FAILED_RESULTS = False
-        with LogCapture() as log:
-            new_item = self.pipe.item_completed(results, item, self.info)
+        new_item = self.pipe.item_completed(results, item, self.info)
         assert new_item is item
-        assert len(log.records) == 0
+        assert len(caplog.records) == 0
 
     def test_item_completed_filtered_request_not_logged(
         self, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 38d56e9bd..a624d2097 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -1,11 +1,17 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy import Request, signals
 from scrapy.http.response import Response
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SingleRequestSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test, inline_callbacks_test
+
+if TYPE_CHECKING:
+    import pytest
 
 OVERRIDDEN_URL = "https://example.org"
 
@@ -63,6 +69,8 @@ class AlternativeCallbacksMiddleware:
 
 
 class TestCrawl:
+    mockserver: MockServer
+
     @classmethod
     def setup_class(cls):
         cls.mockserver = MockServer()
@@ -107,8 +115,10 @@ class TestCrawl:
         assert failure.request.url == url
         assert isinstance(failure.value, ZeroDivisionError)
 
-    @inline_callbacks_test
-    def test_downloader_middleware_override_request_in_process_response(self):
+    @coroutine_test
+    async def test_downloader_middleware_override_request_in_process_response(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Downloader middleware which returns a response with an specific 'request' attribute.
 
@@ -133,22 +143,21 @@ class TestCrawl:
         )
         crawler.signals.connect(signal_handler, signal=signals.response_received)
 
-        with LogCapture() as log:
-            yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(seed=url, mockserver=self.mockserver)
 
+        assert isinstance(crawler.spider, SingleRequestSpider)
         response = crawler.spider.meta["responses"][0]
         assert response.request.url == OVERRIDDEN_URL
 
         assert signal_params["response"].url == url
         assert signal_params["request"].url == OVERRIDDEN_URL
 
-        log.check_present(
-            (
-                "scrapy.core.engine",
-                "DEBUG",
-                f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
-            ),
-        )
+        assert (
+            "scrapy.core.engine",
+            logging.DEBUG,
+            f"Crawled (200) <GET {OVERRIDDEN_URL}> (referer: None)",
+        ) in caplog.record_tuples
 
     @inline_callbacks_test
     def test_downloader_middleware_override_in_process_exception(self):
@@ -196,8 +205,10 @@ class TestCrawl:
         assert response.body == b"Caught ZeroDivisionError"
         assert response.request.url == url
 
-    @inline_callbacks_test
-    def test_downloader_middleware_alternative_callback(self):
+    @coroutine_test
+    async def test_downloader_middleware_alternative_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Downloader middleware which returns a response with a
         specific 'request' attribute, with an alternative callback
@@ -211,14 +222,11 @@ class TestCrawl:
             },
         )
 
-        with LogCapture() as log:
-            url = self.mockserver.url("/status?n=200")
-            yield crawler.crawl(seed=url, mockserver=self.mockserver)
-
-        log.check_present(
-            (
-                "alternative_callbacks_spider",
-                "INFO",
-                "alt_callback was invoked with foo=bar",
-            ),
-        )
+        url = self.mockserver.url("/status?n=200")
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(seed=url, mockserver=self.mockserver)
+        assert (
+            "alternative_callbacks_spider",
+            logging.INFO,
+            "alt_callback was invoked with foo=bar",
+        ) in caplog.record_tuples
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index ee9d2ed51..b88893b2b 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -1,10 +1,17 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import MockServerSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    import pytest
+
+    from tests.mockserver.http import MockServer
 
 
 class InjectArgumentsDownloaderMiddleware:
@@ -147,33 +154,32 @@ class KeywordArgumentsSpider(MockServerSpider):
 
 
 class TestCallbackKeywordArguments:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    @inline_callbacks_test
-    def test_callback_kwargs(self):
+    @coroutine_test
+    async def test_callback_kwargs(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(KeywordArgumentsSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.ERROR):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, KeywordArgumentsSpider)
         assert all(crawler.spider.checks)
+        assert crawler.stats
         assert len(crawler.spider.checks) == crawler.stats.get_value("boolean_checks")
         # check exceptions for argument mismatch
         exceptions = {}
-        for line in log.records:
+        for line in caplog.records:
             for key in ("takes_less", "takes_more"):
                 if key in line.getMessage():
                     exceptions[key] = line
-        assert exceptions["takes_less"].exc_info[0] is TypeError
-        assert str(exceptions["takes_less"].exc_info[1]).endswith(
+        takes_less_exc_info = exceptions["takes_less"].exc_info
+        assert takes_less_exc_info is not None
+        assert takes_less_exc_info[0] is TypeError
+        assert str(takes_less_exc_info[1]).endswith(
             "parse_takes_less() got an unexpected keyword argument 'number'"
-        ), "Exception message: " + str(exceptions["takes_less"].exc_info[1])
-        assert exceptions["takes_more"].exc_info[0] is TypeError
-        assert str(exceptions["takes_more"].exc_info[1]).endswith(
+        )
+        takes_more_exc_info = exceptions["takes_more"].exc_info
+        assert takes_more_exc_info is not None
+        assert takes_more_exc_info[0] is TypeError
+        assert str(takes_more_exc_info[1]).endswith(
             "parse_takes_more() missing 1 required positional argument: 'other'"
-        ), "Exception message: " + str(exceptions["takes_more"].exc_info[1])
+        )
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 08acacae7..2c2d26d53 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -1,9 +1,10 @@
 from __future__ import annotations
 
+import logging
+from typing import TYPE_CHECKING
 from urllib.parse import urljoin
 
 import pytest
-from testfixtures import LogCapture
 from twisted.internet import defer
 
 from scrapy.core.scheduler import BaseScheduler
@@ -12,8 +13,10 @@ from scrapy.spiders import Spider
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.request import fingerprint
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test, inline_callbacks_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 PATHS = ["/a", "/b", "/c"]
 URLS = [urljoin("https://example.org", p) for p in PATHS]
@@ -147,18 +150,19 @@ class TestSimpleScheduler(InterfaceCheckMixin):
 class TestMinimalSchedulerCrawl:
     scheduler_cls = MinimalScheduler
 
-    @inline_callbacks_test
-    def test_crawl(self):
-        with MockServer() as mockserver:
-            settings = {
-                "SCHEDULER": self.scheduler_cls,
-            }
-            with LogCapture() as log:
-                crawler = get_crawler(PathsSpider, settings)
-                yield crawler.crawl(mockserver)
-            for path in PATHS:
-                assert f"{{'path': '{path}'}}" in str(log)
-            assert f"'item_scraped_count': {len(PATHS)}" in str(log)
+    @coroutine_test
+    async def test_crawl(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        settings = {
+            "SCHEDULER": self.scheduler_cls,
+        }
+        with caplog.at_level(logging.DEBUG):
+            crawler = get_crawler(PathsSpider, settings)
+            await crawler.crawl_async(mockserver)
+        for path in PATHS:
+            assert f"{{'path': '{path}'}}" in caplog.text
+        assert f"'item_scraped_count': {len(PATHS)}" in caplog.text
 
 
 class TestSimpleSchedulerCrawl(TestMinimalSchedulerCrawl):
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index c8cfd364e..fd62e0016 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -8,7 +8,6 @@ from logging import WARNING
 from pathlib import Path
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
 from scrapy.spiders import SitemapSpider
@@ -254,22 +253,23 @@ Sitemap: /sitemap-relative-url.xml
         urls = [req.url for req in spider._parse_sitemap(r)]
         assert urls == result
 
-    def test_parse_sitemap_empty_body(self):
+    def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
         r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
         spider = self.spider_class("example.com")
 
-        with LogCapture() as lc:
+        caplog.clear()
+        with caplog.at_level(WARNING):
             results = list(spider._parse_sitemap(r))
 
         assert not results
 
-        lc.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 "Ignoring invalid sitemap: <200 http://www.example.com/sitemap.xml>",
             )
-        )
+        ]
 
     def test_parse_sitemap_not_sitemap(self):
         body = b"""<?xml version="1.0" encoding="UTF-8"?>
@@ -342,7 +342,7 @@ Sitemap: /sitemap-relative-url.xml
         response = Response(url="https://example.com", body=body, request=request)
         assert spider._get_sitemap_body(response) is None
 
-    def test_download_warnsize_setting(self):
+    def test_download_warnsize_setting(self, caplog: pytest.LogCaptureFixture) -> None:
         settings = {"DOWNLOAD_WARNSIZE": 10_000_000}
         crawler = get_crawler(settings_dict=settings)
         spider = self.spider_class.from_crawler(crawler, "example.com")
@@ -350,25 +350,26 @@ Sitemap: /sitemap-relative-url.xml
         body = body_path.read_bytes()
         request = Request(url="https://example.com")
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-    def test_download_warnsize_spider_attr(self):
-        class DownloadWarnSizeSpider(self.spider_class):
+    def test_download_warnsize_spider_attr(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        class DownloadWarnSizeSpider(self.spider_class):  # type: ignore[name-defined,misc]
             download_warnsize = 10_000_000
 
         crawler = get_crawler()
@@ -379,23 +380,24 @@ Sitemap: /sitemap-relative-url.xml
             url="https://example.com", meta={"download_warnsize": 10_000_000}
         )
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
-    def test_download_warnsize_request_meta(self):
+    def test_download_warnsize_request_meta(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         crawler = get_crawler()
         spider = self.spider_class.from_crawler(crawler, "example.com")
         body_path = Path(tests_datadir, "compressed", "bomb-gzip.bin")
@@ -404,21 +406,20 @@ Sitemap: /sitemap-relative-url.xml
             url="https://example.com", meta={"download_warnsize": 10_000_000}
         )
         response = Response(url="https://example.com", body=body, request=request)
-        with LogCapture(
-            "scrapy.spiders.sitemap", propagate=False, level=WARNING
-        ) as log:
+        caplog.clear()
+        with caplog.at_level(WARNING, logger="scrapy.spiders.sitemap"):
             spider._get_sitemap_body(response)
-        log.check(
+        assert caplog.record_tuples == [
             (
                 "scrapy.spiders.sitemap",
-                "WARNING",
+                WARNING,
                 (
                     "<200 https://example.com> body size after decompression "
                     "(11511612 B) is larger than the download warning size "
                     "(10000000 B)."
                 ),
             ),
-        )
+        ]
 
     @coroutine_test
     async def test_sitemap_urls(self):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index afdfaa322..dacc90b27 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -1,10 +1,16 @@
-from testfixtures import LogCapture
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
 
 from scrapy import Request, Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    import pytest
+
 
 class _BaseSpiderMiddleware:
     def __init__(self, crawler):
@@ -250,113 +256,132 @@ class TestSpiderMiddleware:
     def teardown_class(cls):
         cls.mockserver.__exit__(None, None, None)
 
-    async def crawl_log(self, spider: type[Spider]) -> LogCapture:
+    async def crawl_log(
+        self, spider: type[Spider], caplog: pytest.LogCaptureFixture
+    ) -> str:
         crawler = get_crawler(spider)
-        with LogCapture() as log:
+        caplog.clear()
+        with caplog.at_level(logging.DEBUG):
             await crawler.crawl_async(mockserver=self.mockserver)
-        return log
+        return caplog.text
 
     @coroutine_test
-    async def test_recovery(self):
+    async def test_recovery(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         (0) Recover from an exception in a spider callback. The final item count should be 3
         (one yielded from the callback method before the exception is raised, one directly
         from the recovery middleware and one from the spider when processing the request that
         was enqueued from the recovery middleware)
         """
-        log = await self.crawl_log(RecoverySpider)
-        assert "Middleware: TabError exception caught" in str(log)
-        assert str(log).count("Middleware: TabError exception caught") == 1
-        assert "'item_scraped_count': 3" in str(log)
+        log = await self.crawl_log(RecoverySpider, caplog)
+        assert "Middleware: TabError exception caught" in log
+        assert log.count("Middleware: TabError exception caught") == 1
+        assert "'item_scraped_count': 3" in log
 
     @coroutine_test
-    async def test_recovery_asyncgen(self):
+    async def test_recovery_asyncgen(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         Same as test_recovery but with an async callback.
         """
-        log = await self.crawl_log(RecoveryAsyncGenSpider)
-        assert "Middleware: TabError exception caught" in str(log)
-        assert str(log).count("Middleware: TabError exception caught") == 1
-        assert "'item_scraped_count': 3" in str(log)
+        log = await self.crawl_log(RecoveryAsyncGenSpider, caplog)
+        assert "Middleware: TabError exception caught" in log
+        assert log.count("Middleware: TabError exception caught") == 1
+        assert "'item_scraped_count': 3" in log
 
     @coroutine_test
-    async def test_process_spider_input_without_errback(self):
+    async def test_process_spider_input_without_errback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (1.1) An exception from the process_spider_input chain should be caught by the
         process_spider_exception chain from the start if the Request has no errback
         """
-        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback)
-        assert "Middleware: will raise IndexError" in str(log1)
-        assert "Middleware: IndexError exception caught" in str(log1)
+        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithoutErrback, caplog)
+        assert "Middleware: will raise IndexError" in log1
+        assert "Middleware: IndexError exception caught" in log1
 
     @coroutine_test
-    async def test_process_spider_input_with_errback(self):
+    async def test_process_spider_input_with_errback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (1.2) An exception from the process_spider_input chain should not be caught by the
         process_spider_exception chain if the Request has an errback
         """
-        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback)
-        assert "Middleware: IndexError exception caught" not in str(log1)
-        assert "Middleware: will raise IndexError" in str(log1)
-        assert "Got a Failure on the Request errback" in str(log1)
-        assert "{'from': 'errback'}" in str(log1)
-        assert "{'from': 'callback'}" not in str(log1)
-        assert "'item_scraped_count': 1" in str(log1)
+        log1 = await self.crawl_log(ProcessSpiderInputSpiderWithErrback, caplog)
+        assert "Middleware: IndexError exception caught" not in log1
+        assert "Middleware: will raise IndexError" in log1
+        assert "Got a Failure on the Request errback" in log1
+        assert "{'from': 'errback'}" in log1
+        assert "{'from': 'callback'}" not in log1
+        assert "'item_scraped_count': 1" in log1
 
     @coroutine_test
-    async def test_generator_callback(self):
+    async def test_generator_callback(self, caplog: pytest.LogCaptureFixture) -> None:
         """
         (2) An exception from a spider callback (returning a generator) should
         be caught by the process_spider_exception chain. Items yielded before the
         exception is raised should be processed normally.
         """
-        log2 = await self.crawl_log(GeneratorCallbackSpider)
-        assert "Middleware: ImportError exception caught" in str(log2)
-        assert "'item_scraped_count': 2" in str(log2)
+        log2 = await self.crawl_log(GeneratorCallbackSpider, caplog)
+        assert "Middleware: ImportError exception caught" in log2
+        assert "'item_scraped_count': 2" in log2
 
     @coroutine_test
-    async def test_async_generator_callback(self):
+    async def test_async_generator_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         Same as test_generator_callback but with an async callback.
         """
-        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider)
-        assert "Middleware: ImportError exception caught" in str(log2)
-        assert "'item_scraped_count': 2" in str(log2)
+        log2 = await self.crawl_log(AsyncGeneratorCallbackSpider, caplog)
+        assert "Middleware: ImportError exception caught" in log2
+        assert "'item_scraped_count': 2" in log2
 
     @coroutine_test
-    async def test_generator_callback_right_after_callback(self):
+    async def test_generator_callback_right_after_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (2.1) Special case of (2): Exceptions should be caught
         even if the middleware is placed right after the spider
         """
-        log21 = await self.crawl_log(GeneratorCallbackSpiderMiddlewareRightAfterSpider)
-        assert "Middleware: ImportError exception caught" in str(log21)
-        assert "'item_scraped_count': 2" in str(log21)
+        log21 = await self.crawl_log(
+            GeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog
+        )
+        assert "Middleware: ImportError exception caught" in log21
+        assert "'item_scraped_count': 2" in log21
 
     @coroutine_test
-    async def test_not_a_generator_callback(self):
+    async def test_not_a_generator_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (3) An exception from a spider callback (returning a list) should
         be caught by the process_spider_exception chain. No items should be processed.
         """
-        log3 = await self.crawl_log(NotGeneratorCallbackSpider)
-        assert "Middleware: ZeroDivisionError exception caught" in str(log3)
-        assert "item_scraped_count" not in str(log3)
+        log3 = await self.crawl_log(NotGeneratorCallbackSpider, caplog)
+        assert "Middleware: ZeroDivisionError exception caught" in log3
+        assert "item_scraped_count" not in log3
 
     @coroutine_test
-    async def test_not_a_generator_callback_right_after_callback(self):
+    async def test_not_a_generator_callback_right_after_callback(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (3.1) Special case of (3): Exceptions should be caught
         even if the middleware is placed right after the spider
         """
         log31 = await self.crawl_log(
-            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider
+            NotGeneratorCallbackSpiderMiddlewareRightAfterSpider, caplog
         )
-        assert "Middleware: ZeroDivisionError exception caught" in str(log31)
-        assert "item_scraped_count" not in str(log31)
+        assert "Middleware: ZeroDivisionError exception caught" in log31
+        assert "item_scraped_count" not in log31
 
     @coroutine_test
-    async def test_generator_output_chain(self):
+    async def test_generator_output_chain(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         """
         (4) An exception from a middleware's process_spider_output method should be sent
         to the process_spider_exception method from the next middleware in the chain.
@@ -365,23 +390,23 @@ class TestSpiderMiddleware:
         The final item count should be 2 (one from the spider callback and one from the
         process_spider_exception chain)
         """
-        log4 = await self.crawl_log(GeneratorOutputChainSpider)
-        assert "'item_scraped_count': 2" in str(log4)
+        log4 = await self.crawl_log(GeneratorOutputChainSpider, caplog)
+        assert "'item_scraped_count': 2" in log4
         assert (
             "GeneratorRecoverMiddleware.process_spider_exception: LookupError caught"
-            in str(log4)
+            in log4
         )
         assert (
             "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: LookupError caught"
-            in str(log4)
+            in log4
         )
         assert (
             "GeneratorFailMiddleware.process_spider_exception: LookupError caught"
-            not in str(log4)
+            not in log4
         )
         assert (
             "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: LookupError caught"
-            not in str(log4)
+            not in log4
         )
         item_from_callback = {
             "processed": [
@@ -398,6 +423,6 @@ class TestSpiderMiddleware:
                 "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
             ]
         }
-        assert str(item_from_callback) in str(log4)
-        assert str(item_recovered) in str(log4)
-        assert "parse-second-item" not in str(log4)
+        assert str(item_from_callback) in log4
+        assert str(item_recovered) in log4
+        assert "parse-second-item" not in log4
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index fe57492b6..7f5301387 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -8,7 +8,6 @@ from io import StringIO
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from testfixtures import LogCapture
 from twisted.python.failure import Failure
 
 from scrapy.utils.log import (
@@ -107,15 +106,15 @@ class TestLogCounterHandler:
 
 
 class TestStreamLogger:
-    def test_redirect(self):
+    def test_redirect(self, caplog: pytest.LogCaptureFixture) -> None:
         logger = logging.getLogger("test")
         logger.setLevel(logging.WARNING)
         old_stdout = sys.stdout
         sys.stdout = StreamLogger(logger, logging.ERROR)
 
-        with LogCapture() as log:
-            print("test log msg")
-        log.check(("test", "ERROR", "test log msg"))
+        caplog.clear()
+        print("test log msg")
+        assert caplog.record_tuples == [("test", logging.ERROR, "test log msg")]
 
         sys.stdout = old_stdout
 
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index dc615eca6..dbb9caf2a 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,23 +1,25 @@
 from __future__ import annotations
 
 import asyncio
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pytest
 from pydispatch import dispatcher
-from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.python.failure import Failure
 
 from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
 from scrapy.utils.signal import (
     send_catch_log,
     send_catch_log_async,
     send_catch_log_deferred,
 )
 from scrapy.utils.test import get_from_asyncio_queue
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -27,25 +29,22 @@ class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures
     returns_exceptions: bool = False
 
-    @inline_callbacks_test
-    def test_send_catch_log(self):
+    @coroutine_test
+    async def test_send_catch_log(self, caplog: pytest.LogCaptureFixture) -> None:
         test_signal = object()
         handlers_called: set[Callable[..., None]] = set()
 
         dispatcher.connect(self.error_handler, signal=test_signal)
         dispatcher.connect(self.ok_handler, signal=test_signal)
-        with LogCapture() as log:
-            result = yield defer.maybeDeferred(
-                self._get_result,
-                test_signal,
-                arg="test",
-                handlers_called=handlers_called,
-            )
+        caplog.clear()
+        result = await ensure_awaitable(
+            self._get_result(test_signal, arg="test", handlers_called=handlers_called)
+        )
 
         assert self.error_handler in handlers_called
         assert self.ok_handler in handlers_called
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert "error_handler" in record.getMessage()
         assert record.levelname == "ERROR"
         assert result[0][0] == self.error_handler  # pylint: disable=comparison-with-callable
@@ -57,7 +56,7 @@ class TestSendCatchLog:
         dispatcher.disconnect(self.error_handler, signal=test_signal)
         dispatcher.disconnect(self.ok_handler, signal=test_signal)
 
-    def _get_result(self, signal, *a, **kw):
+    def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:
         return send_catch_log(signal, *a, **kw)
 
     def error_handler(self, arg, handlers_called):
@@ -72,7 +71,7 @@ class TestSendCatchLog:
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
 class TestSendCatchLogDeferred(TestSendCatchLog):
-    def _get_result(self, signal, *a, **kw):
+    def _get_result(self, signal: Any, *a: Any, **kw: Any) -> Any:
         return send_catch_log_deferred(signal, *a, **kw)
 
 
@@ -137,14 +136,15 @@ class TestSendCatchLogAsyncAsyncio(TestSendCatchLogAsync):
 
 
 class TestSendCatchLog2:
-    def test_error_logged_if_deferred_not_supported(self):
+    def test_error_logged_if_deferred_not_supported(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
         def test_handler():
             return defer.Deferred()
 
         test_signal = object()
         dispatcher.connect(test_handler, test_signal)
-        with LogCapture() as log:
-            send_catch_log(test_signal)
-        assert len(log.records) == 1
-        assert "Cannot return deferreds from signal handler" in str(log)
+        send_catch_log(test_signal)
+        assert len(caplog.records) == 1
+        assert "Cannot return deferreds from signal handler" in caplog.text
         dispatcher.disconnect(test_handler, test_signal)
diff --git a/tests/utils/bases/spider.py b/tests/utils/bases/spider.py
index f774fde07..799c1820d 100644
--- a/tests/utils/bases/spider.py
+++ b/tests/utils/bases/spider.py
@@ -1,11 +1,11 @@
 from __future__ import annotations
 
+import logging
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any
 from unittest import mock
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy import signals
 from scrapy.crawler import Crawler
@@ -109,15 +109,15 @@ class TestSpiderBase(ABC):
         yield crawler.crawl()
         assert crawler.settings.get("TEST1") == "spider_instance"
 
-    def test_logger(self):
+    def test_logger(self, caplog: pytest.LogCaptureFixture) -> None:
         spider = self.spider_class("example.com")
-        with LogCapture() as lc:
+        caplog.clear()
+        with caplog.at_level(logging.INFO):
             spider.logger.info("test log msg")
-        lc.check(("example.com", "INFO", "test log msg"))
+        assert caplog.record_tuples == [("example.com", logging.INFO, "test log msg")]
 
-        record = lc.records[0]
-        assert "spider" in record.__dict__
-        assert record.spider is spider
+        record = caplog.records[0]
+        assert getattr(record, "spider", None) is spider
 
     def test_log(self):
         spider = self.spider_class("example.com")
diff --git a/tox.ini b/tox.ini
index d35fe91db..e10a7cc0c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,7 +44,6 @@ deps =
     pytest-cov >= 7.0.0
     pytest-xdist
     sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
-    testfixtures
     pytest-twisted >= 1.14.3
 
 [testenv]

From 59ebb26e60ea1e776ed11d00aef1d20bdb535e98 Mon Sep 17 00:00:00 2001
From: Shadow_Lu <xshadowlu13@gmail.com>
Date: Tue, 28 Jul 2026 15:51:55 +0530
Subject: [PATCH 48/62] Fix CaseInsensitiveDict.copy() sharing state with the
 original (#7783)

* Fix CaseInsensitiveDict.copy() sharing state with the original

* Address review: don't re-normalise in __copy__, keep _keys in sync in __ior__

UserDict.__ior__ writes self.data directly, bypassing __setitem__, so
_keys never learned about the new keys.
---
 scrapy/utils/datatypes.py     | 16 ++++++++++++++++
 tests/test_utils_datatypes.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+)

diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index c020ff4b9..e761a2474 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -132,6 +132,22 @@ class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
+    # UserDict.copy() shallow-copies the instance, which would share self._keys
+    # between the copy and the original.
+    def __copy__(self) -> Self:
+        new = self.__class__()
+        new.data = self.data.copy()
+        new._keys = self._keys.copy()
+        return new
+
+    copy = __copy__
+
+    # UserDict.__ior__ updates self.data directly, which would leave self._keys
+    # out of date.
+    def __ior__(self, other: Any) -> Self:  # type: ignore[override,misc]
+        self.update(other)
+        return self
+
     def _normkey(self, key: str | bytes) -> str | bytes:
         return key
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index fe1f60c7d..f43d20e69 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -204,6 +204,15 @@ class TestCaseInsensitiveDictBase(ABC):
         assert h1.get("header1") == h3.get("header1")
         assert h1.get("header1") == h3.get("HEADER1")
 
+    def test_copy_is_independent(self):
+        h1 = self.dict_class({"header1": "value1", "header2": "value2"})
+        for h2 in (copy.copy(h1), h1.copy()):
+            del h2["header1"]
+            h2["header3"] = "value3"
+            assert "header1" in h1
+            assert "header3" not in h1
+            assert dict(h1) == {"header1": "value1", "header2": "value2"}
+
 
 class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
     dict_class = CaseInsensitiveDict  # type: ignore[assignment]
@@ -220,6 +229,28 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
         assert isinstance(iterkeys, Iterator)
         assert list(iterkeys) == ["AsDf", "FoO"]
 
+    def test_copy_keeps_values(self):
+        class MyDict(self.dict_class):
+            def _normvalue(self, value):
+                return value + 1
+
+        d = MyDict({"key": 1})
+        for copied in (copy.copy(d), d.copy()):
+            assert copied["key"] == 2
+
+    def test_ior(self):
+        d = self.dict_class({"header1": "value1"})
+        d |= {"HEADER1": "value2", "header2": "value3"}
+        assert len(d) == 2
+        assert d["HeAdEr1"] == "value2"
+        assert d["HeAdEr2"] == "value3"
+
+    def test_ior_mapping(self):
+        d = self.dict_class({"header1": "value1"})
+        d |= self.dict_class({"HEADER1": "value2"})
+        assert len(d) == 1
+        assert d["HeAdEr1"] == "value2"
+
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
 class TestCaselessDict(TestCaseInsensitiveDictBase):

From 5a65bdcc18e51a85fbc9eca0a25e4be7aa4ce4e3 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Tue, 28 Jul 2026 12:24:04 +0200
Subject: [PATCH 49/62] Improve spiders coverage (#7768)

* Skip the deprecated scrapy.mail in coverage data

* Improve CrawlSpider coverage

* Improve XMLFeedSpider coverage

* Improve SitemapSpider coverage

* Solve mypy issues

* Align new spider tests with the shared test helper structure
---
 scrapy/mail.py               |   2 +
 scrapy/spiders/feed.py       |  11 +--
 tests/spiders.py             |  46 +++++++++++++
 tests/test_crawl.py          |  16 +++++
 tests/test_spider.py         | 130 ++++++++++++++++++++++++++++++++++-
 tests/test_spider_crawl.py   |  43 ++++++++++++
 tests/test_spider_sitemap.py |  62 +++++++++++++++++
 tests/utils/crawl.py         |  27 ++++++++
 8 files changed, 326 insertions(+), 11 deletions(-)
 create mode 100644 tests/utils/crawl.py

diff --git a/scrapy/mail.py b/scrapy/mail.py
index 97123e63c..0691312a3 100644
--- a/scrapy/mail.py
+++ b/scrapy/mail.py
@@ -2,6 +2,8 @@
 Mail sending helpers
 """
 
+# pragma: no file cover
+
 from __future__ import annotations
 
 import logging
diff --git a/scrapy/spiders/feed.py b/scrapy/spiders/feed.py
index 925f31ede..1e7ac9c34 100644
--- a/scrapy/spiders/feed.py
+++ b/scrapy/spiders/feed.py
@@ -9,7 +9,7 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
-from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.exceptions import NotSupported
 from scrapy.http import Response, TextResponse
 from scrapy.selector import Selector
 from scrapy.spiders import Spider
@@ -76,11 +76,6 @@ class XMLFeedSpider(Spider):
             yield from self.process_results(response, ret)
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
-        if not hasattr(self, "parse_node"):
-            raise NotConfigured(
-                "You must define parse_node method in order to scrape this XML feed"
-            )
-
         response = self.adapt_response(response)
         nodes: Iterable[Selector]
         if self.iterator == "iternodes":
@@ -158,9 +153,5 @@ class CSVFeedSpider(Spider):
             yield from self.process_results(response, ret)
 
     def _parse(self, response: Response, **kwargs: Any) -> Any:
-        if not hasattr(self, "parse_row"):
-            raise NotConfigured(
-                "You must define parse_row method in order to scrape this CSV feed"
-            )
         response = self.adapt_response(response)
         return self.parse_rows(response)
diff --git a/tests/spiders.py b/tests/spiders.py
index da14fdbe3..7c7d3007c 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -38,6 +38,35 @@ class MockServerSpider(Spider):
         self.is_secure = is_secure
 
 
+class RawResponseSpider(MockServerSpider):
+    """Base class for spiders that fetch a response built by the test itself.
+
+    Subclasses return the body from :meth:`raw_body` and request
+    :attr:`raw_url`, which the mock server answers with that body verbatim
+    under :attr:`content_type`. This lets tests reach parsing code that only
+    a specific kind of response triggers while still going through a regular
+    crawl, instead of calling internal parsing methods directly.
+    """
+
+    name = "raw_response"
+    content_type = "text/plain"
+
+    def raw_body(self) -> str:
+        raise NotImplementedError
+
+    @property
+    def raw_url(self) -> str:
+        assert self.mockserver
+        raw = (
+            "HTTP/1.1 200 OK\r\n"
+            f"Content-Type: {self.content_type}\r\n"
+            "Connection: close\r\n"
+            "\r\n"
+            f"{self.raw_body()}"
+        )
+        return self.mockserver.url("/raw?" + urlencode({"raw": raw}))
+
+
 class MetaSpider(MockServerSpider):
     name = "meta"
 
@@ -496,6 +525,23 @@ class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
         self.logger.info("[errback] status %i", failure.value.response.status)
 
 
+class CrawlSpiderWithoutErrback(CrawlSpiderWithParseMethod):
+    name = "crawl_spider_without_errback"
+
+    async def start(self):
+        test_body = b"""
+        <html>
+            <head><title>Page title</title></head>
+            <body>
+                <p><a href="/status?n=200">Item 200</a></p>  <!-- callback -->
+                <p><a href="/status?n=404">Item 404</a></p>  <!-- failure, no errback -->
+            </body>
+        </html>
+        """
+        url = self.mockserver.url("/alpayload")
+        yield Request(url, method="POST", body=test_body)
+
+
 class CrawlSpiderWithProcessRequestCallbackKeywordArguments(CrawlSpiderWithParseMethod):
     name = "crawl_spider_with_process_request_cb_kwargs"
     rules = (
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 9c479068a..d284805be 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -41,6 +41,7 @@ from tests.spiders import (
     CrawlSpiderWithAsyncCallback,
     CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
+    CrawlSpiderWithoutErrback,
     CrawlSpiderWithParseMethod,
     CrawlSpiderWithProcessRequestCallbackKeywordArguments,
     DelaySpider,
@@ -505,6 +506,21 @@ class TestCrawlSpider:
         assert "[errback] status 500" in caplog.text
         assert "[errback] status 501" in caplog.text
 
+    @coroutine_test
+    async def test_crawlspider_without_errback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        crawler = get_crawler(CrawlSpiderWithoutErrback)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
+
+        # The failing request (404) is followed by a rule without an errback,
+        # so the failure is dropped silently and the crawl finishes normally.
+        assert "[parse] status 200 (foo: None)" in caplog.text
+        assert "[errback]" not in caplog.text
+        assert crawler.stats
+        assert crawler.stats.get_value("downloader/response_status_count/404") == 1
+
     @coroutine_test
     async def test_crawlspider_process_request_cb_kwargs(
         self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 03d17199f..38cb8da18 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -1,11 +1,26 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 import pytest
 
-from scrapy.http import Response, TextResponse, XmlResponse
+from scrapy.http import Request, Response, TextResponse, XmlResponse
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
 from tests import get_testdata
+from tests.spiders import RawResponseSpider
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.crawl import crawl_items
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
+
+class RawFeedSpider(RawResponseSpider):
+    content_type = "text/xml"
+
+    async def start(self):
+        yield Request(self.raw_url)
 
 
 class TestSpider(TestSpiderBase):
@@ -60,6 +75,89 @@ class TestXMLFeedSpider(TestSpiderBase):
                 },
             ], iterator
 
+    @coroutine_test
+    async def test_parse_node_uses_parse_item(self, mockserver: MockServer):
+        # parse_node falls back to parse_item for backward compatibility.
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            itertag = "item"
+
+            def raw_body(self):
+                return "<items><item><id>1</id></item></items>"
+
+            def parse_item(self, response, selector):
+                return {"id": selector.xpath("id/text()").get()}
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"id": "1"}]
+
+    @coroutine_test
+    async def test_parse_node_not_defined(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            itertag = "item"
+
+            def raw_body(self):
+                return "<items><item><id>1</id></item></items>"
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
+
+    @coroutine_test
+    async def test_html_iterator(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            iterator = "html"
+            itertag = "item"
+            content_type = "text/html"
+
+            def raw_body(self):
+                return (
+                    "<html><body><item><id>1</id></item>"
+                    "<item><id>2</id></item></body></html>"
+                )
+
+            def parse_node(self, response, selector):
+                return {"id": selector.xpath("id/text()").get()}
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"id": "1"}, {"id": "2"}]
+
+    @coroutine_test
+    async def test_unsupported_iterator(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            iterator = "unsupported"
+
+            def raw_body(self):
+                return "<items><item/></items>"
+
+            def parse_node(self, response, selector):
+                return {}
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotSupported") == 1
+
+    @pytest.mark.parametrize("feed_iterator", ["xml", "html"])
+    @coroutine_test
+    async def test_non_text_response(self, feed_iterator: str, mockserver: MockServer):
+        # The xml and html iterators require a text response.
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "application/octet-stream"
+            iterator = feed_iterator
+
+            def raw_body(self):
+                # A binary (non-text) body, so the response is a plain Response.
+                return "\x00\x01\x02\x03"
+
+            def parse_node(self, response, selector):
+                return {}
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/ValueError") == 1
+
 
 class TestCSVFeedSpider(TestSpiderBase):
     spider_class = CSVFeedSpider
@@ -81,6 +179,36 @@ class TestCSVFeedSpider(TestSpiderBase):
         assert rows[0] == {"id": "1", "name": "alpha", "value": "foobar"}
         assert len(rows) == 4
 
+    @coroutine_test
+    async def test_parse(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "text/csv"
+            delimiter = ","
+            quotechar = "'"
+
+            def raw_body(self):
+                return get_testdata("feeds", "feed-sample6.csv").decode()
+
+            def parse_row(self, response, row):
+                return row
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items[0] == {"id": "1", "name": "alpha", "value": "foobar"}
+        assert len(items) == 4
+
+    @coroutine_test
+    async def test_parse_row_not_defined(self, mockserver: MockServer):
+        class _Spider(RawFeedSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            content_type = "text/csv"
+
+            def raw_body(self):
+                return "id\n1\n"
+
+        items, crawler = await crawl_items(_Spider, mockserver)
+        assert items == []
+        assert crawler.stats
+        assert crawler.stats.get_value("spider_exceptions/NotImplementedError") == 1
+
 
 class TestNoParseMethodSpider:
     spider_class = Spider
diff --git a/tests/test_spider_crawl.py b/tests/test_spider_crawl.py
index f34f9add9..9d5f0548b 100644
--- a/tests/test_spider_crawl.py
+++ b/tests/test_spider_crawl.py
@@ -12,6 +12,7 @@ from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule, Spider
 from scrapy.utils.test import get_crawler
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.decorators import coroutine_test
 
 
 class TestCrawlSpider(TestSpiderBase):
@@ -293,6 +294,48 @@ class TestCrawlSpider(TestSpiderBase):
                 TextResponse(spider.start_urls, body=b""), None, None
             )
 
+    @coroutine_test
+    async def test_parse_with_rules_without_callback(self):
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
+
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            allowed_domains = ["example.org"]
+            rules = (Rule(),)
+
+        spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
+        results = [
+            r async for r in spider.parse_with_rules(response, None, {}, follow=True)
+        ]
+        assert [r.url for r in results] == [
+            "http://example.org/somepage/item/12.html",
+            "http://example.org/about.html",
+            "http://example.org/nofollow.html",
+        ]
+
+    @coroutine_test
+    async def test_parse_with_rules_without_following(self):
+        response = HtmlResponse(
+            "http://example.org/somepage/index.html", body=self.test_body
+        )
+        item = {"name": "item"}
+
+        class _CrawlSpider(CrawlSpider):
+            name = "test"
+            allowed_domains = ["example.org"]
+            rules = (Rule(),)
+
+        spider = _CrawlSpider.from_crawler(get_crawler(_CrawlSpider))
+        results = [
+            r
+            async for r in spider.parse_with_rules(
+                response, lambda response: [item], {}, follow=False
+            )
+        ]
+        assert results == [item]
+
 
 class TestDeprecation:
     def test_crawl_spider(self):
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index fd62e0016..2f1ccab81 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -6,6 +6,7 @@ from datetime import datetime
 from io import BytesIO
 from logging import WARNING
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
@@ -13,9 +14,30 @@ from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlRespon
 from scrapy.spiders import SitemapSpider
 from scrapy.utils.test import get_crawler
 from tests import tests_datadir
+from tests.spiders import RawResponseSpider
 from tests.utils.bases.spider import TestSpiderBase
+from tests.utils.crawl import crawl_items
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
+
+class RawSitemapSpider(RawResponseSpider):
+    """Feeds :meth:`raw_body` to :class:`~scrapy.spiders.SitemapSpider` as a
+    sitemap, so that it is fetched and followed through a regular crawl.
+
+    Subclasses build the document in :meth:`raw_body`, typically using
+    :attr:`mockserver` to point ``<loc>`` entries at real endpoints.
+    """
+
+    content_type = "application/xml"
+
+    async def start(self):
+        self.sitemap_urls = [self.raw_url]
+        async for request in super().start():
+            yield request
+
 
 class TestSitemapSpider(TestSpiderBase):
     spider_class = SitemapSpider
@@ -253,6 +275,46 @@ Sitemap: /sitemap-relative-url.xml
         urls = [req.url for req in spider._parse_sitemap(r)]
         assert urls == result
 
+    @coroutine_test
+    async def test_sitemap_rules_with_callable(self, mockserver: MockServer):
+        # A sitemap_rules entry may hold a callable instead of a method name.
+        def parse_item(response):
+            yield {"url": response.url}
+
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            sitemap_rules = [("", parse_item)]
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+                    f"<url><loc>{loc}</loc></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"url": mockserver.url("/text")}]
+
+    @coroutine_test
+    async def test_sitemap_empty_loc(self, mockserver: MockServer):
+        class _Spider(RawSitemapSpider, self.spider_class):  # type: ignore[name-defined,misc]
+            def parse(self, response):
+                yield {"url": response.url}
+
+            def raw_body(self):
+                loc = self.mockserver.url("/text")
+                return (
+                    '<?xml version="1.0" encoding="UTF-8"?>'
+                    '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+                    "<url><loc></loc></url>"
+                    f"<url><loc>{loc}</loc></url>"
+                    "</urlset>"
+                )
+
+        items, _ = await crawl_items(_Spider, mockserver)
+        assert items == [{"url": mockserver.url("/text")}]
+
     def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
         r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
         spider = self.spider_class("example.com")
diff --git a/tests/utils/crawl.py b/tests/utils/crawl.py
new file mode 100644
index 000000000..4631d909d
--- /dev/null
+++ b/tests/utils/crawl.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy import signals
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+    from scrapy.spiders import Spider
+    from tests.mockserver.http import MockServer
+
+
+async def crawl_items(
+    spider_cls: type[Spider], mockserver: MockServer, **kwargs: Any
+) -> tuple[list[Any], Crawler]:
+    """Run *spider_cls* against *mockserver* and return the scraped items along
+    with the crawler, which gives tests access to the resulting stats."""
+    items: list[Any] = []
+
+    def collect(item: Any) -> None:
+        items.append(item)
+
+    crawler = get_crawler(spider_cls)
+    crawler.signals.connect(collect, signals.item_scraped)
+    await crawler.crawl_async(mockserver=mockserver, **kwargs)
+    return items, crawler

From ad816d2b3a00c04c86ac8b3fd85e19a7e1b355f8 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Tue, 28 Jul 2026 14:04:54 +0200
Subject: [PATCH 50/62] Improve test coverage for scrapy.cmdline (#7795)

---
 scrapy/cmdline.py             |   6 +-
 tests/test_commands.py        | 244 ++++++++++++++++++++++++++++++----
 tests/utils/bases/commands.py |   6 +
 3 files changed, 226 insertions(+), 30 deletions(-)

diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 6c306afdb..e6d5ff96a 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -225,13 +225,11 @@ def _run_command(cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace)
 def _run_command_profiled(
     cmd: ScrapyCommand, args: list[str], opts: argparse.Namespace
 ) -> None:
-    if opts.profile:
-        sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
+    sys.stderr.write(f"scrapy: writing cProfile stats to {opts.profile!r}\n")
     loc = locals()
     p = cProfile.Profile()
     p.runctx("cmd.run(args, opts)", globals(), loc)
-    if opts.profile:
-        p.dump_stats(opts.profile)
+    p.dump_stats(opts.profile)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 51f98db1b..3e687e811 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -3,14 +3,12 @@ from __future__ import annotations
 import argparse
 import json
 import sys
-from io import StringIO
 from typing import TYPE_CHECKING
-from unittest import mock
 
 import pytest
 
 import scrapy
-from scrapy.cmdline import _pop_command_name, _print_unknown_command_msg
+from scrapy.cmdline import _pop_command_name, execute
 from scrapy.commands import ScrapyCommand, ScrapyHelpFormatter, view
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
@@ -153,12 +151,6 @@ class MySpider(scrapy.Spider):
 
         self._append_settings(proj_mod_path, "LOG_LEVEL = 'DEBUG'\n")
 
-    @staticmethod
-    def _append_settings(proj_mod_path: Path, text: str) -> None:
-        """Add text to the end of the project settings.py."""
-        with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
-            f.write(text)
-
     @staticmethod
     def _replace_custom_settings(
         proj_mod_path: Path, spider_name: str, text: str
@@ -347,23 +339,223 @@ class TestMiscCommands(TestProjectBase):
         subdir.mkdir(exist_ok=True)
         assert call("list", cwd=subdir) == 0
 
-    def test_command_not_found(self) -> None:
-        na_msg = """
-The list command is not available from this location.
-These commands are only available from within a project: check, crawl, edit, list, parse.
-"""
-        not_found_msg = """
-Unknown command: abc
-"""
-        params = [
-            ("list", False, na_msg),
-            ("abc", False, not_found_msg),
-            ("abc", True, not_found_msg),
-        ]
-        for cmdname, inproject, message in params:
-            with mock.patch("sys.stdout", new=StringIO()) as out:
-                _print_unknown_command_msg(Settings(), cmdname, inproject)
-                assert out.getvalue().strip() == message.strip()
+
+class TestCommandListing(TestProjectBase):
+    """Tests for the command list that ``scrapy`` prints when called without a
+    command name."""
+
+    def test_outside_project(self) -> None:
+        returncode, out, err = proc()
+        assert returncode == 0, err
+        assert f"Scrapy {scrapy.__version__} - no active project" in out
+        assert "Available commands:" in out
+        assert "Create new project" in out
+        assert "More commands available when run from project directory" in out
+        assert 'Use "scrapy <command> -h" to see more info about a command' in out
+
+    def test_inside_project(self, proj_path: Path) -> None:
+        returncode, out, err = proc(cwd=proj_path)
+        assert returncode == 0, err
+        assert (
+            f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+        )
+        assert "List available spiders" in out
+        assert "More commands available when run from project directory" not in out
+
+
+class TestUnknownCommand(TestProjectBase):
+    def test_outside_project(self) -> None:
+        returncode, out, err = proc("abc")
+        assert returncode == 2, err
+        assert f"Scrapy {scrapy.__version__} - no active project" in out
+        assert "Unknown command: abc" in out
+        assert 'Use "scrapy" to see available commands' in out
+
+    def test_inside_project(self, proj_path: Path) -> None:
+        returncode, out, err = proc("abc", cwd=proj_path)
+        assert returncode == 2, err
+        assert (
+            f"Scrapy {scrapy.__version__} - active project: {self.project_name}" in out
+        )
+        assert "Unknown command: abc" in out
+
+    def test_project_only_command_outside_project(self) -> None:
+        returncode, out, err = proc("list")
+        assert returncode == 2, err
+        assert "The list command is not available from this location." in out
+        assert (
+            "These commands are only available from within a project: "
+            "check, crawl, edit, list, parse." in out
+        )
+
+
+class TestCommandsModule(TestProjectBase):
+    """Tests for commands defined in the module of the COMMANDS_MODULE setting."""
+
+    @pytest.fixture
+    def proj_path_with_commands(self, proj_path: Path) -> Path:
+        commands_path = proj_path / self.project_name / "commands"
+        commands_path.mkdir()
+        (commands_path / "__init__.py").touch()
+        (commands_path / "mycmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My custom command"
+
+    def run(self, args, opts):
+        print("My custom command ran")
+""",
+            encoding="utf-8",
+        )
+        (commands_path / "helpcmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My command that asks for its help message"
+
+    def run(self, args, opts):
+        raise UsageError
+""",
+            encoding="utf-8",
+        )
+        (commands_path / "silentcmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My command that fails silently"
+
+    def run(self, args, opts):
+        raise UsageError(print_help=False)
+""",
+            encoding="utf-8",
+        )
+        self._append_settings(
+            proj_path / self.project_name,
+            f'\nCOMMANDS_MODULE = "{self.project_name}.commands"\n',
+        )
+        return proj_path
+
+    def test_listed(self, proj_path_with_commands: Path) -> None:
+        returncode, out, err = proc(cwd=proj_path_with_commands)
+        assert returncode == 0, err
+        assert "My custom command" in out
+
+    def test_run(self, proj_path_with_commands: Path) -> None:
+        returncode, out, err = proc("mycmd", cwd=proj_path_with_commands)
+        assert returncode == 0, err
+        assert "My custom command ran" in out
+
+    def test_usage_error(self, proj_path_with_commands: Path) -> None:
+        """A message-less UsageError makes the help message be printed."""
+        returncode, out, err = proc("helpcmd", cwd=proj_path_with_commands)
+        assert returncode == 2, err
+        assert "scrapy helpcmd" in out
+
+    def test_usage_error_without_help(self, proj_path_with_commands: Path) -> None:
+        """A message-less UsageError with print_help disabled prints nothing."""
+        returncode, out, err = proc("silentcmd", cwd=proj_path_with_commands)
+        assert returncode == 2, err
+        assert not out
+
+
+class TestEntryPointCommands:
+    """Tests for commands defined in the scrapy.commands entry point group."""
+
+    @staticmethod
+    def _write_dist(path: Path, entry_point: str) -> None:
+        """Write into *path* a package with a command and a function, and the
+        metadata of an installed distribution that declares *entry_point* in
+        the scrapy.commands entry point group.
+
+        Since ``python -m scrapy.cmdline`` puts the current working directory
+        in the import path, running it with *path* as the working directory
+        makes Scrapy find that entry point.
+        """
+        package_path = path / "mycmds"
+        package_path.mkdir()
+        (package_path / "__init__.py").touch()
+        (package_path / "mycmd.py").write_text(
+            """
+from scrapy.commands import ScrapyCommand
+
+
+class Command(ScrapyCommand):
+    requires_crawler_process = False
+
+    def short_desc(self):
+        return "My entry point command"
+
+    def run(self, args, opts):
+        print("My entry point command ran")
+
+
+def not_a_command():
+    pass
+""",
+            encoding="utf-8",
+        )
+        dist_info_path = path / "mycmds-1.0.dist-info"
+        dist_info_path.mkdir()
+        (dist_info_path / "METADATA").write_text(
+            "Metadata-Version: 2.1\nName: mycmds\nVersion: 1.0\n", encoding="utf-8"
+        )
+        (dist_info_path / "entry_points.txt").write_text(
+            f"[scrapy.commands]\n{entry_point}\n", encoding="utf-8"
+        )
+
+    def test_listed(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+        returncode, out, err = proc(cwd=tmp_path)
+        assert returncode == 0, err
+        assert "My entry point command" in out
+
+    def test_run(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:Command")
+        returncode, out, err = proc("mycmd", cwd=tmp_path)
+        assert returncode == 0, err
+        assert "My entry point command ran" in out
+
+    def test_not_a_class(self, tmp_path: Path) -> None:
+        self._write_dist(tmp_path, "mycmd = mycmds.mycmd:not_a_command")
+        returncode, _, err = proc("version", cwd=tmp_path)
+        assert returncode == 1
+        assert "ValueError: Invalid entry point mycmd" in err
+
+
+class TestExecute:
+    """Tests for calls to scrapy.cmdline.execute() from Python code, which the
+    command line does not cover."""
+
+    def test_argv(self, capsys: pytest.CaptureFixture[str]) -> None:
+        with pytest.raises(SystemExit) as exc_info:
+            execute(["scrapy", "version"])
+        assert exc_info.value.code == 0
+        assert scrapy.__version__ in capsys.readouterr().out
+
+    def test_settings(self, capsys: pytest.CaptureFixture[str]) -> None:
+        settings = Settings()
+        with pytest.raises(SystemExit) as exc_info:
+            execute(["scrapy", "settings", "--get", "BOT_NAME"], settings=settings)
+        assert exc_info.value.code == 0
+        assert capsys.readouterr().out.strip() == "scrapybot"
 
 
 class TestBenchCommand:
diff --git a/tests/utils/bases/commands.py b/tests/utils/bases/commands.py
index 594544c83..55ef686fa 100644
--- a/tests/utils/bases/commands.py
+++ b/tests/utils/bases/commands.py
@@ -32,3 +32,9 @@ class TestProjectBase:
         proj_path = tmp_path / self.project_name
         copytree(_proj_path_cached, proj_path)
         return proj_path
+
+    @staticmethod
+    def _append_settings(proj_mod_path: Path, text: str) -> None:
+        """Add text to the end of the project settings.py."""
+        with (proj_mod_path / "settings.py").open("a", encoding="utf-8") as f:
+            f.write(text)

From e7d8b34e73ef2598b27ea0bca31a292e008fff76 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 28 Jul 2026 17:24:34 +0500
Subject: [PATCH 51/62] Next refactoring pass of test_utils_*. (#7797)

---
 pyproject.toml                                |   7 -
 scrapy/core/downloader/handlers/ftp.py        |   3 +-
 scrapy/http/headers.py                        |  26 +-
 scrapy/http/request/__init__.py               |  12 +-
 scrapy/http/response/__init__.py              |  23 +-
 scrapy/http/response/text.py                  |  12 +-
 scrapy/utils/datatypes.py                     |  40 +-
 scrapy/utils/decorators.py                    |  17 +-
 tests/test_utils_asyncgen.py                  |  24 +-
 tests/test_utils_asyncio.py                   |   9 +-
 tests/test_utils_curl.py                      |   2 +
 tests/test_utils_datatypes.py                 |  60 +--
 tests/test_utils_decorators.py                |  20 +-
 tests/test_utils_defer.py                     |  24 +-
 tests/test_utils_deprecate.py                 |  46 +-
 tests/test_utils_display.py                   |  16 +-
 tests/test_utils_gz.py                        |   2 +
 tests/test_utils_httpobj.py                   |   2 +
 tests/test_utils_misc/__init__.py             | 261 +++++------
 ...t_return_with_argument_inside_generator.py | 421 +++++++++---------
 tests/test_utils_project.py                   |  10 +-
 tests/test_utils_python.py                    |  21 +-
 tests/test_utils_reactor.py                   |   2 +
 tests/test_utils_request.py                   |  26 +-
 tests/test_utils_response.py                  |   6 +-
 tests/test_utils_serialize.py                 |   2 +
 tests/test_utils_signal.py                    |   3 -
 tests/test_utils_sitemap.py                   |   2 +
 tests/test_utils_template.py                  |   9 +-
 tests/test_utils_trackref.py                  |   6 +-
 tests/test_utils_url.py                       |   4 +-
 tests_typing/test_http_request.mypy-testing   |   2 +-
 tests_typing/test_http_response.mypy-testing  |   2 +-
 33 files changed, 619 insertions(+), 503 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 9b1e64121..576a42e5c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -178,13 +178,6 @@ module = [
     "tests.test_squeues",
     "tests.test_squeues_request",
     "tests.test_stats",
-    "tests.test_utils_datatypes",
-    "tests.test_utils_decorators",
-    "tests.test_utils_defer",
-    "tests.test_utils_deprecate",
-    "tests.test_utils_misc.test_return_with_argument_inside_generator",
-    "tests.test_utils_python",
-    "tests.test_utils_request",
     "tests.utils.bases.http_request",
     "tests.utils.bases.http_response",
     "tests.utils.bases.spider",
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 07ff4a74e..29b3e3c0f 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -126,5 +126,4 @@ class FTPDownloadHandler(BaseDownloadHandler):
         headers = {"local filename": protocol.filename or b"", "size": protocol.size}
         body = protocol.filename or protocol.body.read()
         respcls = responsetypes.from_args(url=request.url, body=body)
-        # hints for Headers-related types may need to be fixed to not use AnyStr
-        return respcls(url=request.url, status=200, body=body, headers=headers)  # type: ignore[arg-type]
+        return respcls(url=request.url, status=200, body=body, headers=headers)
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 34d4ec6f2..b55ef6191 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 
 from w3lib.http import headers_dict_to_raw
 
@@ -25,14 +25,20 @@ class Headers(CaselessDict):
 
     def __init__(
         self,
-        seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         encoding: str = "utf-8",
     ):
         self.encoding: str = encoding
         super().__init__(seq)
 
     def update(  # type: ignore[override]
-        self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]
+        self,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]],
     ) -> None:
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq: dict[bytes, list[bytes]] = {}
@@ -40,7 +46,7 @@ class Headers(CaselessDict):
             iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
         super().update(iseq)
 
-    def normkey(self, key: AnyStr) -> bytes:  # type: ignore[override]
+    def normkey(self, key: str | bytes) -> bytes:
         """Normalize key to bytes"""
         return self._tobytes(key.title())
 
@@ -67,19 +73,19 @@ class Headers(CaselessDict):
             return str(x).encode(self.encoding)
         raise TypeError(f"Unsupported value type: {type(x)}")
 
-    def __getitem__(self, key: AnyStr) -> bytes | None:
+    def __getitem__(self, key: str | bytes) -> bytes | None:
         try:
             return cast("list[bytes]", super().__getitem__(key))[-1]
         except IndexError:
             return None
 
-    def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
+    def get(self, key: str | bytes, def_val: Any = None) -> bytes | None:
         try:
             return cast("list[bytes]", super().get(key, def_val))[-1]
         except IndexError:
             return None
 
-    def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
+    def getlist(self, key: str | bytes, def_val: Any = None) -> list[bytes]:
         try:
             return cast("list[bytes]", super().__getitem__(key))
         except KeyError:
@@ -87,15 +93,15 @@ class Headers(CaselessDict):
                 return self.normvalue(def_val)
             return []
 
-    def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
+    def setlist(self, key: str | bytes, list_: Iterable[_RawValue]) -> None:
         self[key] = list_
 
     def setlistdefault(
-        self, key: AnyStr, default_list: Iterable[_RawValue] = ()
+        self, key: str | bytes, default_list: Iterable[_RawValue] = ()
     ) -> Any:
         return self.setdefault(key, default_list)
 
-    def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
+    def appendlist(self, key: str | bytes, value: Iterable[_RawValue]) -> None:
         lst = self.getlist(key)
         lst.extend(self.normvalue(value))
         self[key] = lst
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 73c2e7dd4..57517a65b 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -11,7 +11,6 @@ import inspect
 from typing import (
     TYPE_CHECKING,
     Any,
-    AnyStr,
     Concatenate,
     NoReturn,
     TypeAlias,
@@ -125,7 +124,10 @@ class Request(object_ref):
         url: str,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -310,7 +312,11 @@ class Request(object_ref):
 
     @headers.setter
     def headers(
-        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+        self,
+        value: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None,
     ) -> None:
         if isinstance(value, Headers):
             self._headers = value
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index 09b1c8b32..f1db11488 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -7,7 +7,7 @@ See documentation in docs/topics/request-response.rst
 
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, overload
+from typing import TYPE_CHECKING, Any, TypeVar, overload
 from urllib.parse import urljoin
 
 from scrapy.exceptions import NotSupported
@@ -72,7 +72,10 @@ class Response(object_ref):
         self,
         url: str,
         status: int = 200,
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes = b"",
         flags: list[str] | None = None,
         request: Request | None = None,
@@ -145,7 +148,11 @@ class Response(object_ref):
 
     @headers.setter
     def headers(
-        self, value: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None
+        self,
+        value: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None,
     ) -> None:
         if isinstance(value, Headers):
             self._headers = value
@@ -222,7 +229,10 @@ class Response(object_ref):
         url: str | Link,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -272,7 +282,10 @@ class Response(object_ref):
         urls: Iterable[str | Link],
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 6876e35e8..d01e23e47 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -9,7 +9,7 @@ from __future__ import annotations
 
 import json
 from contextlib import suppress
-from typing import TYPE_CHECKING, Any, AnyStr, cast
+from typing import TYPE_CHECKING, Any, cast
 from urllib.parse import urljoin
 
 import parsel
@@ -170,7 +170,10 @@ class TextResponse(Response):
         url: str | Link | parsel.Selector,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
@@ -223,7 +226,10 @@ class TextResponse(Response):
         urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
         callback: CallbackT | None = None,
         method: str = "GET",
-        headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        headers: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
         body: bytes | str | None = None,
         cookies: CookiesT | None = None,
         meta: dict[str, Any] | None = None,
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index e761a2474..9a945c61c 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -11,12 +11,12 @@ import warnings
 import weakref
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeVar, cast
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Sequence
+    from collections.abc import Container, Iterable
 
     # typing.Self requires Python 3.11
     from typing_extensions import Self
@@ -44,22 +44,25 @@ class CaselessDict(dict):  # type: ignore[type-arg]
 
     def __init__(
         self,
-        seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]]
+        | None = None,
     ):
         super().__init__()
         if seq:
             self.update(seq)
 
-    def __getitem__(self, key: AnyStr) -> Any:
+    def __getitem__(self, key: str | bytes) -> Any:
         return dict.__getitem__(self, self.normkey(key))
 
-    def __setitem__(self, key: AnyStr, value: Any) -> None:
+    def __setitem__(self, key: str | bytes, value: Any) -> None:
         dict.__setitem__(self, self.normkey(key), self.normvalue(value))
 
-    def __delitem__(self, key: AnyStr) -> None:
+    def __delitem__(self, key: str | bytes) -> None:
         dict.__delitem__(self, self.normkey(key))
 
-    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+    def __contains__(self, key: str | bytes) -> bool:  # type: ignore[override]
         return dict.__contains__(self, self.normkey(key))
 
     has_key = __contains__
@@ -69,7 +72,7 @@ class CaselessDict(dict):  # type: ignore[type-arg]
 
     copy = __copy__
 
-    def normkey(self, key: AnyStr) -> AnyStr:
+    def normkey(self, key: str | bytes) -> str | bytes:
         """Method to normalize dictionary key access"""
         return key.lower()
 
@@ -77,23 +80,28 @@ class CaselessDict(dict):  # type: ignore[type-arg]
         """Method to normalize values prior to be set"""
         return value
 
-    def get(self, key: AnyStr, def_val: Any = None) -> Any:
+    def get(self, key: str | bytes, def_val: Any = None) -> Any:
         return dict.get(self, self.normkey(key), self.normvalue(def_val))
 
-    def setdefault(self, key: AnyStr, def_val: Any = None) -> Any:
+    def setdefault(self, key: str | bytes, def_val: Any = None) -> Any:
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     # doesn't fully implement MutableMapping.update()
-    def update(self, seq: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]]) -> None:  # type: ignore[override]
+    def update(  # type: ignore[override]
+        self,
+        seq: Mapping[str, Any]
+        | Mapping[bytes, Any]
+        | Iterable[tuple[str | bytes, Any]],
+    ) -> None:
         seq = seq.items() if isinstance(seq, Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super().update(iseq)
 
     @classmethod
-    def fromkeys(cls, keys: Iterable[AnyStr], value: Any = None) -> Self:  # type: ignore[override]
-        return cls((k, value) for k in keys)  # type: ignore[misc]
+    def fromkeys(cls, keys: Iterable[str | bytes], value: Any = None) -> Self:  # type: ignore[override]
+        return cls((k, value) for k in keys)
 
-    def pop(self, key: AnyStr, *args: Any) -> Any:
+    def pop(self, key: str | bytes, *args: Any) -> Any:
         return dict.pop(self, self.normkey(key), *args)
 
 
@@ -205,8 +213,8 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
 class SequenceExclude:
     """Object to test if an item is NOT within some sequence."""
 
-    def __init__(self, seq: Sequence[Any]):
-        self.seq: Sequence[Any] = seq
+    def __init__(self, seq: Container[Any]):
+        self.seq: Container[Any] = seq
 
     def __contains__(self, item: Any) -> bool:
         return item not in self.seq
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index a5bb6fa24..4960dc27a 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -19,9 +19,19 @@ _T = TypeVar("_T")
 _P = ParamSpec("_P")
 
 
+@overload
+def deprecated(use_instead: Callable[_P, _T]) -> Callable[_P, _T]: ...
+
+
+@overload
 def deprecated(
-    use_instead: Any = None,
-) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]:
+    use_instead: str | None = None,
+) -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...
+
+
+def deprecated(
+    use_instead: Callable[_P, _T] | str | None = None,
+) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:
     """This is a decorator which can be used to mark functions
     as deprecated. It will result in a warning being emitted
     when the function is used."""
@@ -38,8 +48,9 @@ def deprecated(
         return wrapped
 
     if callable(use_instead):
-        deco = deco(use_instead)
+        func = use_instead
         use_instead = None
+        return deco(func)
     return deco
 
 
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index fc4e1c487..1d36a66fc 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,16 +1,18 @@
+from __future__ import annotations
+
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from tests.utils.decorators import coroutine_test
 
 
-class TestAsyncgenUtils:
-    @coroutine_test
-    async def test_as_async_generator(self):
-        ag = as_async_generator(range(42))
-        results = [i async for i in ag]
-        assert results == list(range(42))
+@coroutine_test
+async def test_as_async_generator():
+    ag = as_async_generator(range(42))
+    results = [i async for i in ag]
+    assert results == list(range(42))
 
-    @coroutine_test
-    async def test_collect_asyncgen(self):
-        ag = as_async_generator(range(42))
-        results = await collect_asyncgen(ag)
-        assert results == list(range(42))
+
+@coroutine_test
+async def test_collect_asyncgen():
+    ag = as_async_generator(range(42))
+    results = await collect_asyncgen(ag)
+    assert results == list(range(42))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
index 7528dc51a..9b7eb22fa 100644
--- a/tests/test_utils_asyncio.py
+++ b/tests/test_utils_asyncio.py
@@ -20,11 +20,10 @@ if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
 
 
-class TestAsyncio:
-    @coroutine_test
-    async def test_is_asyncio_available(self, reactor_pytest: str) -> None:
-        # the result should depend only on the pytest --reactor argument
-        assert is_asyncio_available() == (reactor_pytest != "default")
+@coroutine_test
+async def test_is_asyncio_available(reactor_pytest: str) -> None:
+    # the result should depend only on the pytest --reactor argument
+    assert is_asyncio_available() == (reactor_pytest != "default")
 
 
 @pytest.mark.only_asyncio
diff --git a/tests/test_utils_curl.py b/tests/test_utils_curl.py
index fce9fc984..6b30744bb 100644
--- a/tests/test_utils_curl.py
+++ b/tests/test_utils_curl.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import warnings
 from typing import Any
 
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index f43d20e69..af203ca61 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -1,7 +1,9 @@
+from __future__ import annotations
+
 import copy
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, MutableMapping
-from typing import Any
+from typing import Any, Generic, TypeVar
 
 import pytest
 
@@ -16,11 +18,13 @@ from scrapy.utils.datatypes import (
 )
 from scrapy.utils.python import garbage_collect
 
+_DictT = TypeVar("_DictT", bound="CaselessDict | CaseInsensitiveDict")
 
-class TestCaseInsensitiveDictBase(ABC):
+
+class TestCaseInsensitiveDictBase(ABC, Generic[_DictT]):
     @property
     @abstractmethod
-    def dict_class(self) -> type[MutableMapping[str, Any]]:
+    def dict_class(self) -> type[_DictT]:
         raise NotImplementedError
 
     def test_init_dict(self):
@@ -36,17 +40,17 @@ class TestCaseInsensitiveDictBase(ABC):
         assert d["black"] == 3
 
     def test_init_mapping(self):
-        class MyMapping(Mapping):
-            def __init__(self, **kwargs):
+        class MyMapping(Mapping[str, int]):
+            def __init__(self, **kwargs: int) -> None:
                 self._d = kwargs
 
-            def __getitem__(self, key):
+            def __getitem__(self, key: str) -> int:
                 return self._d[key]
 
-            def __iter__(self):
+            def __iter__(self) -> Iterator[str]:
                 return iter(self._d)
 
-            def __len__(self):
+            def __len__(self) -> int:
                 return len(self._d)
 
         seq = MyMapping(red=1, black=3)
@@ -55,23 +59,23 @@ class TestCaseInsensitiveDictBase(ABC):
         assert d["black"] == 3
 
     def test_init_mutable_mapping(self):
-        class MyMutableMapping(MutableMapping):
-            def __init__(self, **kwargs):
+        class MyMutableMapping(MutableMapping[str, int]):
+            def __init__(self, **kwargs: int) -> None:
                 self._d = kwargs
 
-            def __getitem__(self, key):
+            def __getitem__(self, key: str) -> int:
                 return self._d[key]
 
-            def __setitem__(self, key, value):
+            def __setitem__(self, key: str, value: int) -> None:
                 self._d[key] = value
 
-            def __delitem__(self, key):
+            def __delitem__(self, key: str) -> None:
                 del self._d[key]
 
-            def __iter__(self):
+            def __iter__(self) -> Iterator[str]:
                 return iter(self._d)
 
-            def __len__(self):
+            def __len__(self) -> int:
                 return len(self._d)
 
         seq = MyMutableMapping(red=1, black=3)
@@ -149,7 +153,7 @@ class TestCaseInsensitiveDictBase(ABC):
             d.pop("A")
 
     def test_normkey(self):
-        class MyDict(self.dict_class):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
             def _normkey(self, key):
                 return key.title()
 
@@ -160,7 +164,7 @@ class TestCaseInsensitiveDictBase(ABC):
         assert list(d.keys()) == ["Key-One"]
 
     def test_normvalue(self):
-        class MyDict(self.dict_class):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
             def _normvalue(self, value):
                 if value is not None:
                     return value + 1
@@ -214,8 +218,8 @@ class TestCaseInsensitiveDictBase(ABC):
             assert dict(h1) == {"header1": "value1", "header2": "value2"}
 
 
-class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
-    dict_class = CaseInsensitiveDict  # type: ignore[assignment]
+class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase[CaseInsensitiveDict]):
+    dict_class = CaseInsensitiveDict
 
     def test_repr(self):
         d1 = self.dict_class({"foo": "bar"})
@@ -230,7 +234,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
         assert list(iterkeys) == ["AsDf", "FoO"]
 
     def test_copy_keeps_values(self):
-        class MyDict(self.dict_class):
+        class MyDict(self.dict_class):  # type: ignore[misc,name-defined]
             def _normvalue(self, value):
                 return value + 1
 
@@ -253,7 +257,7 @@ class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
 
 
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestCaselessDict(TestCaseInsensitiveDictBase):
+class TestCaselessDict(TestCaseInsensitiveDictBase[CaselessDict]):
     dict_class = CaselessDict
 
     def test_deprecation_message(self):
@@ -319,7 +323,7 @@ class TestSequenceExclude:
 
 class TestLocalCache:
     def test_cache_with_limit(self):
-        cache = LocalCache(limit=2)
+        cache: LocalCache[str, int] = LocalCache(limit=2)
         cache["a"] = 1
         cache["b"] = 2
         cache["c"] = 3
@@ -332,7 +336,7 @@ class TestLocalCache:
 
     def test_cache_without_limit(self):
         maximum = 10**4
-        cache = LocalCache()
+        cache: LocalCache[str, int] = LocalCache()
         for x in range(maximum):
             cache[str(x)] = x
         assert len(cache) == maximum
@@ -341,7 +345,7 @@ class TestLocalCache:
             assert cache[str(x)] == x
 
     def test_cache_with_zero_limit(self):
-        cache = LocalCache(limit=0)
+        cache: LocalCache[str, int] = LocalCache(limit=0)
         cache["a"] = 1
         cache["b"] = 2
         cache["c"] = 3
@@ -353,7 +357,9 @@ class TestLocalCache:
 
 class TestLocalWeakReferencedCache:
     def test_cache_with_limit(self):
-        cache = LocalWeakReferencedCache(limit=2)
+        cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache(
+            limit=2
+        )
         r1 = Request("https://example.org")
         r2 = Request("https://example.com")
         r3 = Request("https://example.net")
@@ -375,7 +381,7 @@ class TestLocalWeakReferencedCache:
         assert len(cache) == 1
 
     def test_cache_non_weak_referenceable_objects(self):
-        cache = LocalWeakReferencedCache()
+        cache: LocalWeakReferencedCache[Any, int] = LocalWeakReferencedCache()
         k1 = None
         k2 = 1
         k3 = [1, 2, 3]
@@ -389,7 +395,7 @@ class TestLocalWeakReferencedCache:
 
     def test_cache_without_limit(self):
         maximum = 10**4
-        cache = LocalWeakReferencedCache()
+        cache: LocalWeakReferencedCache[Request, int] = LocalWeakReferencedCache()
         refs = []
         for x in range(maximum):
             refs.append(Request(f"https://example.org/{x}"))
diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py
index 9743e1a50..807294a57 100644
--- a/tests/test_utils_decorators.py
+++ b/tests/test_utils_decorators.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import warnings
+from typing import TYPE_CHECKING
 
 import pytest
 from twisted.internet.defer import Deferred
@@ -10,11 +11,14 @@ from scrapy.utils.decorators import _warn_spider_arg, deprecated, inthread
 from scrapy.utils.defer import maybe_deferred_to_future
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
 
 class TestDeprecated:
     def test_warns_and_still_calls(self):
         @deprecated()
-        def add(a, b):
+        def add(a: int, b: int) -> int:
             return a + b
 
         with pytest.warns(
@@ -26,7 +30,7 @@ class TestDeprecated:
 
     def test_use_instead_in_message(self):
         @deprecated(use_instead="other_function")
-        def old():
+        def old() -> None:
             return None
 
         with pytest.warns(
@@ -37,7 +41,7 @@ class TestDeprecated:
 
     def test_applied_without_parentheses(self):
         @deprecated
-        def square(x):
+        def square(x: int) -> int:
             return x * x
 
         with pytest.warns(
@@ -65,7 +69,7 @@ class TestInthread:
 class TestWarnSpiderArg:
     def test_sync_warns_with_spider_arg(self):
         @_warn_spider_arg
-        def parse(response, spider=None):
+        def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with pytest.warns(
@@ -75,7 +79,7 @@ class TestWarnSpiderArg:
 
     def test_sync_no_warning_without_spider_arg(self):
         @_warn_spider_arg
-        def parse(response, spider=None):
+        def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with warnings.catch_warnings():
@@ -85,7 +89,7 @@ class TestWarnSpiderArg:
     @coroutine_test
     async def test_async_warns_with_spider_arg(self):
         @_warn_spider_arg
-        async def parse(response, spider=None):
+        async def parse(response: str, spider: str | None = None) -> str:
             return response
 
         with pytest.warns(
@@ -96,7 +100,9 @@ class TestWarnSpiderArg:
     @coroutine_test
     async def test_asyncgen_warns_with_spider_arg(self):
         @_warn_spider_arg
-        async def parse(response, spider=None):
+        async def parse(
+            response: str, spider: str | None = None
+        ) -> AsyncGenerator[str]:
             yield response
 
         with pytest.warns(
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 978c24f5a..175a4fe03 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -24,6 +24,8 @@ from tests.utils.decorators import coroutine_test, inline_callbacks_test
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
 
+    from twisted.python.failure import Failure
+
 
 @pytest.mark.requires_reactor  # mustbe_deferred() requires a reactor
 @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
@@ -70,7 +72,7 @@ class TestIterErrback:
         def itergood() -> Generator[int, None, None]:
             yield from range(10)
 
-        errors = []
+        errors: list[Failure] = []
         out = list(iter_errback(itergood(), errors.append))
         assert out == list(range(10))
         assert not errors
@@ -82,7 +84,7 @@ class TestIterErrback:
                     1 / 0
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = list(iter_errback(iterbad(), errors.append))
         assert out == [0, 1, 2, 3, 4]
         assert len(errors) == 1
@@ -96,7 +98,7 @@ class TestAiterErrback:
             for x in range(10):
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
         assert out == list(range(10))
         assert not errors
@@ -109,7 +111,7 @@ class TestAiterErrback:
                     1 / 0
                 yield x
 
-        errors = []
+        errors: list[Failure] = []
         out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
         assert out == [0, 1, 2, 3, 4]
         assert len(errors) == 1
@@ -202,7 +204,7 @@ class TestParallelAsync:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable(length)
             dl = parallel_async(
                 ait,
@@ -222,7 +224,7 @@ class TestParallelAsync:
         for length in [20, 50, 100]:
             parallel_count = [0]
             max_parallel_count = [0]
-            results = []
+            results: list[int] = []
             ait = self.get_async_iterable_with_delays(length)
             dl = parallel_async(
                 ait,
@@ -240,7 +242,7 @@ class TestParallelAsync:
 
 class TestDeferredFromCoro:
     def test_deferred(self):
-        d = Deferred()
+        d: Deferred[None] = Deferred()
         result = deferred_from_coro(d)
         assert isinstance(result, Deferred)
         assert result is d
@@ -274,7 +276,7 @@ class TestDeferredFromCoro:
     @pytest.mark.only_asyncio
     @inline_callbacks_test
     def test_future(self):
-        future = Future()
+        future: Future[int] = Future()
         result = deferred_from_coro(future)
         assert isinstance(result, Deferred)
         future.set_result(42)
@@ -324,7 +326,7 @@ class TestDeferredFFromCoroF:
 class TestDeferredToFuture:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = deferred_to_future(d)
         assert isinstance(result, Future)
         d.callback(42)
@@ -359,7 +361,7 @@ class TestDeferredToFuture:
 class TestMaybeDeferredToFutureAsyncio:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = maybe_deferred_to_future(d)
         assert isinstance(result, Future)
         d.callback(42)
@@ -394,7 +396,7 @@ class TestMaybeDeferredToFutureAsyncio:
 class TestMaybeDeferredToFutureNotAsyncio:
     @coroutine_test
     async def test_deferred(self):
-        d = Deferred()
+        d: Deferred[int] = Deferred()
         result = maybe_deferred_to_future(d)
         assert isinstance(result, Deferred)
         assert result is d
diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py
index 0706fec99..4c8585916 100644
--- a/tests/test_utils_deprecate.py
+++ b/tests/test_utils_deprecate.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import inspect
 import warnings
 from unittest import mock
@@ -38,7 +40,7 @@ class TestWarnWhenSubclassed:
         )
         with pytest.warns(MyWarning, match=msg) as w:
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         assert w[0].lineno == inspect.getsourcelines(UserClass)[1]
@@ -57,7 +59,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with pytest.warns(
@@ -76,7 +78,7 @@ class TestWarnWhenSubclassed:
             match="UserClass inherits from deprecated class",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
@@ -95,16 +97,16 @@ class TestWarnWhenSubclassed:
             match="UserClass inherits from deprecated class",
         ):
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
             warnings.simplefilter("error", MyWarning)
 
-            class FooClass(Deprecated):
+            class FooClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
-            class BarClass(Deprecated):
+            class BarClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_warning_on_instance(self):
@@ -112,22 +114,20 @@ class TestWarnWhenSubclassed:
             "Deprecated", NewName, warn_category=MyWarning
         )
 
-        with pytest.warns(MyWarning) as w:
-            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())
-
-        w = [x for x in w if x.category is MyWarning]
+        with pytest.warns(
+            MyWarning,
+            match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "
+            r"instantiate tests\.test_utils_deprecate\.NewName instead\.",
+        ) as w:
+            _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe())  # type: ignore[arg-type]
         assert len(w) == 1
-        assert (
-            str(w[0].message) == "tests.test_utils_deprecate.Deprecated is deprecated, "
-            "instantiate tests.test_utils_deprecate.NewName instead."
-        )
         assert w[0].lineno == lineno
 
         # ignore subclassing warnings
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", MyWarning)
 
-            class UserClass(Deprecated):
+            class UserClass(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
         with warnings.catch_warnings():
@@ -141,7 +141,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",
         ):
 
-            class UserClass2(Deprecated):
+            class UserClass2(Deprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_issubclass(self):
@@ -155,10 +155,10 @@ class TestWarnWhenSubclassed:
             class UpdatedUserClass1a(NewName):
                 pass
 
-            class OutdatedUserClass1(DeprecatedName):
+            class OutdatedUserClass1(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
-            class OutdatedUserClass1a(DeprecatedName):
+            class OutdatedUserClass1a(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
             class UnrelatedClass:
@@ -174,7 +174,7 @@ class TestWarnWhenSubclassed:
         assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)
 
         with pytest.raises(TypeError):
-            issubclass(object(), DeprecatedName)
+            issubclass(object(), DeprecatedName)  # type: ignore[arg-type]
 
     def test_isinstance(self):
         with warnings.catch_warnings():
@@ -187,10 +187,10 @@ class TestWarnWhenSubclassed:
             class UpdatedUserClass2a(NewName):
                 pass
 
-            class OutdatedUserClass2(DeprecatedName):
+            class OutdatedUserClass2(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
-            class OutdatedUserClass2a(DeprecatedName):
+            class OutdatedUserClass2a(DeprecatedName):  # type: ignore[misc, valid-type]
                 pass
 
             class UnrelatedClass:
@@ -211,7 +211,7 @@ class TestWarnWhenSubclassed:
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
             Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})
 
-        assert Deprecated.foo == "bar"
+        assert Deprecated.foo == "bar"  # type: ignore[attr-defined]
 
     def test_deprecate_a_class_with_custom_metaclass(self):
         Meta1 = type("Meta1", (type,), {})
@@ -242,7 +242,7 @@ class TestWarnWhenSubclassed:
             match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",
         ):
 
-            class UserClass(AlsoDeprecated):
+            class UserClass(AlsoDeprecated):  # type: ignore[misc, valid-type]
                 pass
 
     def test_inspect_stack(self):
diff --git a/tests/test_utils_display.py b/tests/test_utils_display.py
index 9f9e24957..a87816c20 100644
--- a/tests/test_utils_display.py
+++ b/tests/test_utils_display.py
@@ -31,13 +31,13 @@ plain_string = "{'a': 1}"
 
 @mock.patch("sys.platform", "linux")
 @mock.patch("sys.stdout.isatty")
-def test_pformat(isatty):
+def test_pformat(isatty: mock.Mock) -> None:
     isatty.return_value = True
     assert pformat(value) in colorized_strings
 
 
 @mock.patch("sys.stdout.isatty")
-def test_pformat_dont_colorize(isatty):
+def test_pformat_dont_colorize(isatty: mock.Mock) -> None:
     isatty.return_value = True
     assert pformat(value, colorize=False) == plain_string
 
@@ -49,7 +49,7 @@ def test_pformat_not_tty():
 @mock.patch("sys.platform", "win32")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_old_windows(isatty, version):
+def test_pformat_old_windows(isatty: mock.Mock, version: mock.Mock) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14392"
     assert pformat(value) in colorized_strings
@@ -59,7 +59,9 @@ def test_pformat_old_windows(isatty, version):
 @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_windows_no_terminal_processing(isatty, version, terminal_processing):
+def test_pformat_windows_no_terminal_processing(
+    isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14393"
     terminal_processing.return_value = False
@@ -70,7 +72,9 @@ def test_pformat_windows_no_terminal_processing(isatty, version, terminal_proces
 @mock.patch("scrapy.utils.display._enable_windows_terminal_processing")
 @mock.patch("platform.version")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_windows(isatty, version, terminal_processing):
+def test_pformat_windows(
+    isatty: mock.Mock, version: mock.Mock, terminal_processing: mock.Mock
+) -> None:
     isatty.return_value = True
     version.return_value = "10.0.14393"
     terminal_processing.return_value = True
@@ -79,7 +83,7 @@ def test_pformat_windows(isatty, version, terminal_processing):
 
 @mock.patch("sys.platform", "linux")
 @mock.patch("sys.stdout.isatty")
-def test_pformat_no_pygments(isatty):
+def test_pformat_no_pygments(isatty: mock.Mock) -> None:
     isatty.return_value = True
 
     real_import = builtins.__import__
diff --git a/tests/test_utils_gz.py b/tests/test_utils_gz.py
index 06fdf9cba..75f8be6f6 100644
--- a/tests/test_utils_gz.py
+++ b/tests/test_utils_gz.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from gzip import BadGzipFile
 from pathlib import Path
 
diff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.py
index 0eb330461..610c463ec 100644
--- a/tests/test_utils_httpobj.py
+++ b/tests/test_utils_httpobj.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from urllib.parse import urlparse
 
 from scrapy.http import Request
diff --git a/tests/test_utils_misc/__init__.py b/tests/test_utils_misc/__init__.py
index ab6965ed5..0775ec608 100644
--- a/tests/test_utils_misc/__init__.py
+++ b/tests/test_utils_misc/__init__.py
@@ -20,150 +20,157 @@ from scrapy.utils.misc import (
 )
 
 
-class TestUtilsMisc:
-    def test_load_object_class(self):
-        obj = load_object(Field)
-        assert obj is Field
-        obj = load_object("scrapy.item.Field")
-        assert obj is Field
+def test_load_object_class() -> None:
+    obj = load_object(Field)
+    assert obj is Field
+    obj = load_object("scrapy.item.Field")
+    assert obj is Field
 
-    def test_load_object_function(self):
-        obj = load_object(load_object)
-        assert obj is load_object
-        obj = load_object("scrapy.utils.misc.load_object")
-        assert obj is load_object
 
-    def test_load_object_exceptions(self):
-        with pytest.raises(ImportError):
-            load_object("nomodule999.mod.function")
-        with pytest.raises(NameError):
-            load_object("scrapy.utils.misc.load_object999")
-        with pytest.raises(TypeError):
-            load_object({})  # type: ignore[arg-type]
+def test_load_object_function() -> None:
+    obj = load_object(load_object)
+    assert obj is load_object
+    obj = load_object("scrapy.utils.misc.load_object")
+    assert obj is load_object
 
-    def test_walk_modules(self):
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+
+def test_load_object_exceptions() -> None:
+    with pytest.raises(ImportError):
+        load_object("nomodule999.mod.function")
+    with pytest.raises(NameError):
+        load_object("scrapy.utils.misc.load_object999")
+    with pytest.raises(TypeError):
+        load_object({})  # type: ignore[arg-type]
+
+
+def test_walk_modules() -> None:
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules",
+        "tests.test_utils_misc.test_walk_modules.mod",
+        "tests.test_utils_misc.test_walk_modules.mod.mod0",
+        "tests.test_utils_misc.test_walk_modules.mod1",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules.mod",
+        "tests.test_utils_misc.test_walk_modules.mod.mod0",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
+    expected = [
+        "tests.test_utils_misc.test_walk_modules.mod1",
+    ]
+    assert {m.__name__ for m in mods} == set(expected)
+
+    with pytest.raises(ImportError):
+        for _ in walk_modules_iter("nomodule999"):
+            pass
+    with (
+        pytest.raises(ImportError),
+        pytest.warns(
+            ScrapyDeprecationWarning,
+            match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
+            "removed in a future version of Scrapy. "
+            "Use scrapy.utils.misc.walk_modules_iter instead.",
+        ),
+    ):
+        walk_modules("nomodule999")
+
+
+def test_walk_modules_egg() -> None:
+    egg = str(Path(__file__).parent / "test.egg")
+    sys.path.append(egg)
+    try:
+        mods = walk_modules_iter("testegg")
         expected = [
-            "tests.test_utils_misc.test_walk_modules",
-            "tests.test_utils_misc.test_walk_modules.mod",
-            "tests.test_utils_misc.test_walk_modules.mod.mod0",
-            "tests.test_utils_misc.test_walk_modules.mod1",
+            "testegg.spiders",
+            "testegg.spiders.a",
+            "testegg.spiders.b",
+            "testegg",
         ]
         assert {m.__name__ for m in mods} == set(expected)
+    finally:
+        sys.path.remove(egg)
 
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod")
-        expected = [
-            "tests.test_utils_misc.test_walk_modules.mod",
-            "tests.test_utils_misc.test_walk_modules.mod.mod0",
-        ]
-        assert {m.__name__ for m in mods} == set(expected)
 
-        mods = walk_modules_iter("tests.test_utils_misc.test_walk_modules.mod1")
-        expected = [
-            "tests.test_utils_misc.test_walk_modules.mod1",
-        ]
-        assert {m.__name__ for m in mods} == set(expected)
+def test_arg_to_iter() -> None:
+    class TestItem(Item):
+        name = Field()
 
-        with pytest.raises(ImportError):
-            for _ in walk_modules_iter("nomodule999"):
-                pass
-        with (
-            pytest.raises(ImportError),
-            pytest.warns(
-                ScrapyDeprecationWarning,
-                match="The scrapy.utils.misc.walk_modules function is deprecated and will be "
-                "removed in a future version of Scrapy. "
-                "Use scrapy.utils.misc.walk_modules_iter instead.",
-            ),
-        ):
-            walk_modules("nomodule999")
+    assert hasattr(arg_to_iter(None), "__iter__")
+    assert hasattr(arg_to_iter(100), "__iter__")
+    assert hasattr(arg_to_iter("lala"), "__iter__")
+    assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
+    assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
 
-    def test_walk_modules_egg(self):
-        egg = str(Path(__file__).parent / "test.egg")
-        sys.path.append(egg)
-        try:
-            mods = walk_modules_iter("testegg")
-            expected = [
-                "testegg.spiders",
-                "testegg.spiders.a",
-                "testegg.spiders.b",
-                "testegg",
-            ]
-            assert {m.__name__ for m in mods} == set(expected)
-        finally:
-            sys.path.remove(egg)
+    assert not list(arg_to_iter(None))
+    assert list(arg_to_iter("lala")) == ["lala"]
+    assert list(arg_to_iter(100)) == [100]
+    assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
+    assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
+    assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
+    assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
 
-    def test_arg_to_iter(self):
-        class TestItem(Item):
-            name = Field()
 
-        assert hasattr(arg_to_iter(None), "__iter__")
-        assert hasattr(arg_to_iter(100), "__iter__")
-        assert hasattr(arg_to_iter("lala"), "__iter__")
-        assert hasattr(arg_to_iter([1, 2, 3]), "__iter__")
-        assert hasattr(arg_to_iter(c for c in "abcd"), "__iter__")
+def test_build_from_crawler() -> None:
+    crawler = mock.MagicMock(spec_set=["settings"])
+    args = (True, 100.0)
+    kwargs = {"key": "val"}
 
-        assert not list(arg_to_iter(None))
-        assert list(arg_to_iter("lala")) == ["lala"]
-        assert list(arg_to_iter(100)) == [100]
-        assert list(arg_to_iter(c for c in "abc")) == ["a", "b", "c"]
-        assert list(arg_to_iter([1, 2, 3])) == [1, 2, 3]
-        assert list(arg_to_iter({"a": 1})) == [{"a": 1}]
-        assert list(arg_to_iter(TestItem(name="john"))) == [TestItem(name="john")]
+    def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
+        build_from_crawler(mock, crawler, *args, **kwargs)
+        if hasattr(mock, "from_crawler"):
+            mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
+            assert mock.call_count == 0
+        else:
+            mock.assert_called_once_with(*args, **kwargs)
 
-    def test_build_from_crawler(self):
-        crawler = mock.MagicMock(spec_set=["settings"])
-        args = (True, 100.0)
-        kwargs = {"key": "val"}
+    # Check usage of correct constructor using 2 mocks:
+    #   1. with no alternative constructors
+    #   2. with from_crawler() constructor
+    spec_sets = (
+        ["__qualname__"],
+        ["__qualname__", "from_crawler"],
+    )
+    for specs in spec_sets:
+        m = mock.MagicMock(spec_set=specs)
+        _test_with_crawler(m, crawler)
+        m.reset_mock()
 
-        def _test_with_crawler(mock: mock.MagicMock, crawler: mock.MagicMock) -> None:
-            build_from_crawler(mock, crawler, *args, **kwargs)
-            if hasattr(mock, "from_crawler"):
-                mock.from_crawler.assert_called_once_with(crawler, *args, **kwargs)
-                assert mock.call_count == 0
-            else:
-                mock.assert_called_once_with(*args, **kwargs)
+    # Check adoption of crawler
+    m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
+    m.from_crawler.return_value = None
+    with pytest.raises(TypeError):
+        build_from_crawler(m, crawler, *args, **kwargs)
 
-        # Check usage of correct constructor using 2 mocks:
-        #   1. with no alternative constructors
-        #   2. with from_crawler() constructor
-        spec_sets = (
-            ["__qualname__"],
-            ["__qualname__", "from_crawler"],
-        )
-        for specs in spec_sets:
-            m = mock.MagicMock(spec_set=specs)
-            _test_with_crawler(m, crawler)
-            m.reset_mock()
 
-        # Check adoption of crawler
-        m = mock.MagicMock(spec_set=["__qualname__", "from_crawler"])
-        m.from_crawler.return_value = None
-        with pytest.raises(TypeError):
-            build_from_crawler(m, crawler, *args, **kwargs)
+def test_set_environ() -> None:
+    assert os.environ.get("some_test_environ") is None
+    with set_environ(some_test_environ="test_value"):
+        assert os.environ.get("some_test_environ") == "test_value"
+    assert os.environ.get("some_test_environ") is None
 
-    def test_set_environ(self):
-        assert os.environ.get("some_test_environ") is None
-        with set_environ(some_test_environ="test_value"):
-            assert os.environ.get("some_test_environ") == "test_value"
-        assert os.environ.get("some_test_environ") is None
+    os.environ["some_test_environ"] = "test"
+    assert os.environ.get("some_test_environ") == "test"
+    with set_environ(some_test_environ="test_value"):
+        assert os.environ.get("some_test_environ") == "test_value"
+    assert os.environ.get("some_test_environ") == "test"
 
-        os.environ["some_test_environ"] = "test"
-        assert os.environ.get("some_test_environ") == "test"
-        with set_environ(some_test_environ="test_value"):
-            assert os.environ.get("some_test_environ") == "test_value"
-        assert os.environ.get("some_test_environ") == "test"
 
-    def test_rel_has_nofollow(self):
-        assert rel_has_nofollow("ugc nofollow") is True
-        assert rel_has_nofollow("ugc,nofollow") is True
-        assert rel_has_nofollow("ugc") is False
-        assert rel_has_nofollow("nofollow") is True
-        assert rel_has_nofollow("nofollowfoo") is False
-        assert rel_has_nofollow("foonofollow") is False
-        assert rel_has_nofollow("ugc,  ,  nofollow") is True
-        # rel attribute values are ASCII case-insensitive per the HTML spec
-        assert rel_has_nofollow("NoFollow") is True
-        assert rel_has_nofollow("NOFOLLOW") is True
-        assert rel_has_nofollow("UGC NoFollow") is True
-        assert rel_has_nofollow("ugc,NoFollow") is True
+def test_rel_has_nofollow() -> None:
+    assert rel_has_nofollow("ugc nofollow") is True
+    assert rel_has_nofollow("ugc,nofollow") is True
+    assert rel_has_nofollow("ugc") is False
+    assert rel_has_nofollow("nofollow") is True
+    assert rel_has_nofollow("nofollowfoo") is False
+    assert rel_has_nofollow("foonofollow") is False
+    assert rel_has_nofollow("ugc,  ,  nofollow") is True
+    # rel attribute values are ASCII case-insensitive per the HTML spec
+    assert rel_has_nofollow("NoFollow") is True
+    assert rel_has_nofollow("NOFOLLOW") is True
+    assert rel_has_nofollow("UGC NoFollow") is True
+    assert rel_has_nofollow("ugc,NoFollow") is True
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 1acc3aac2..7343a135a 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -1,5 +1,8 @@
+from __future__ import annotations
+
 import warnings
 from functools import partial
+from typing import TYPE_CHECKING, Any
 from unittest import mock
 
 import pytest
@@ -9,6 +12,11 @@ from scrapy.utils.misc import (
     warn_on_generator_with_return_value,
 )
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
+    from scrapy import Spider
+
 
 def _indentation_error(*args, **kwargs):
     raise IndentationError
@@ -35,244 +43,239 @@ https://example.org
     yield url
 
 
-def generator_that_returns_stuff():
+def generator_that_returns_stuff() -> Generator[int, None, int]:
     yield 1
     yield 2
     return 3
 
 
-class TestUtilsMisc:
-    @pytest.fixture
-    def mock_spider(self):
-        class MockSettings:
-            def __init__(self, settings_dict=None):
-                self.settings_dict = settings_dict or {
-                    "WARN_ON_GENERATOR_RETURN_VALUE": True
-                }
+@pytest.fixture
+def mock_spider() -> Spider:
+    class MockSettings:
+        def __init__(self, settings_dict: dict[str, Any] | None = None):
+            self.settings_dict = settings_dict or {
+                "WARN_ON_GENERATOR_RETURN_VALUE": True
+            }
 
-            def getbool(self, name, default=False):
-                return self.settings_dict.get(name, default)
+        def getbool(self, name, default=False):
+            return self.settings_dict.get(name, default)
 
-        class MockSpider:
-            def __init__(self):
-                self.settings = MockSettings()
+    class MockSpider:
+        def __init__(self) -> None:
+            self.settings = MockSettings()
 
-        return MockSpider()
+    return MockSpider()  # type: ignore[return-value]
 
-    def test_generators_return_something(self, mock_spider):
-        def f1():
-            yield 1
-            return 2
 
-        def g1():
-            yield 1
-            return "asdf"
+def test_generators_return_something(mock_spider):
+    def f1():
+        yield 1
+        return 2
 
-        def h1():
-            yield 1
+    def g1():
+        yield 1
+        return "asdf"
 
-            def helper():
-                return 0
+    def h1():
+        yield 1
 
-            yield helper()
-            return 2
+        def helper() -> int:
+            return 0
 
-        def i1():
-            """
-            docstring
-            """
-            url = """
-https://example.org
+        yield helper()
+        return 2
+
+    def i1():
         """
-            yield url
-            return 1
-
-        assert is_generator_with_return_value(top_level_return_something)
-        assert is_generator_with_return_value(f1)
-        assert is_generator_with_return_value(g1)
-        assert is_generator_with_return_value(h1)
-        assert is_generator_with_return_value(i1)
-
-        with pytest.warns(
-            UserWarning,
-            match='The "MockSpider.top_level_return_something" method is a generator',
-        ):
-            warn_on_generator_with_return_value(mock_spider, top_level_return_something)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.f1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, f1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.g1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, g1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.h1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, h1)
-        with pytest.warns(
-            UserWarning, match='The "MockSpider.i1" method is a generator'
-        ):
-            warn_on_generator_with_return_value(mock_spider, i1)
-
-    def test_generators_return_none(self, mock_spider):
-        def f2():
-            yield 1
-
-        def g2():
-            yield 1
-
-        def h2():
-            yield 1
-
-        def i2():
-            yield 1
-            yield from generator_that_returns_stuff()
-
-        def j2():
-            yield 1
-
-            def helper():
-                return 0
-
-            yield helper()
-
-        def k2():
-            """
-            docstring
-            """
-            url = """
-https://example.org
+        docstring
         """
-            yield url
-
-        def l2():
-            return
-
-        assert not is_generator_with_return_value(top_level_return_none)
-        assert not is_generator_with_return_value(f2)
-        assert not is_generator_with_return_value(g2)
-        assert not is_generator_with_return_value(h2)
-        assert not is_generator_with_return_value(i2)
-        assert not is_generator_with_return_value(j2)  # not recursive
-        assert not is_generator_with_return_value(k2)  # not recursive
-        assert not is_generator_with_return_value(l2)
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            warn_on_generator_with_return_value(mock_spider, f2)
-            warn_on_generator_with_return_value(mock_spider, g2)
-            warn_on_generator_with_return_value(mock_spider, h2)
-            warn_on_generator_with_return_value(mock_spider, i2)
-            warn_on_generator_with_return_value(mock_spider, j2)
-            warn_on_generator_with_return_value(mock_spider, k2)
-            warn_on_generator_with_return_value(mock_spider, l2)
-
-    def test_generators_return_none_with_decorator(self, mock_spider):
-        def decorator(func):
-            def inner_func():
-                func()
-
-            return inner_func
-
-        @decorator
-        def f3():
-            yield 1
-
-        @decorator
-        def g3():
-            yield 1
-
-        @decorator
-        def h3():
-            yield 1
-
-        @decorator
-        def i3():
-            yield 1
-            yield from generator_that_returns_stuff()
-
-        @decorator
-        def j3():
-            yield 1
-
-            def helper():
-                return 0
-
-            yield helper()
-
-        @decorator
-        def k3():
-            """
-            docstring
-            """
-            url = """
+        url = """
 https://example.org
+    """
+        yield url
+        return 1
+
+    assert is_generator_with_return_value(top_level_return_something)
+    assert is_generator_with_return_value(f1)
+    assert is_generator_with_return_value(g1)
+    assert is_generator_with_return_value(h1)
+    assert is_generator_with_return_value(i1)
+
+    with pytest.warns(
+        UserWarning,
+        match='The "MockSpider.top_level_return_something" method is a generator',
+    ):
+        warn_on_generator_with_return_value(mock_spider, top_level_return_something)
+    with pytest.warns(UserWarning, match='The "MockSpider.f1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, f1)
+    with pytest.warns(UserWarning, match='The "MockSpider.g1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, g1)
+    with pytest.warns(UserWarning, match='The "MockSpider.h1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, h1)
+    with pytest.warns(UserWarning, match='The "MockSpider.i1" method is a generator'):
+        warn_on_generator_with_return_value(mock_spider, i1)
+
+
+def test_generators_return_none(mock_spider):
+    def f2():
+        yield 1
+
+    def g2():
+        yield 1
+
+    def h2():
+        yield 1
+
+    def i2():
+        yield 1
+        yield from generator_that_returns_stuff()
+
+    def j2():
+        yield 1
+
+        def helper() -> int:
+            return 0
+
+        yield helper()
+
+    def k2():
         """
-            yield url
+        docstring
+        """
+        url = """
+https://example.org
+    """
+        yield url
 
-        @decorator
-        def l3():
-            return
+    def l2():
+        return
 
-        assert not is_generator_with_return_value(top_level_return_none)
-        assert not is_generator_with_return_value(f3)
-        assert not is_generator_with_return_value(g3)
-        assert not is_generator_with_return_value(h3)
-        assert not is_generator_with_return_value(i3)
-        assert not is_generator_with_return_value(j3)  # not recursive
-        assert not is_generator_with_return_value(k3)  # not recursive
-        assert not is_generator_with_return_value(l3)
+    assert not is_generator_with_return_value(top_level_return_none)
+    assert not is_generator_with_return_value(f2)
+    assert not is_generator_with_return_value(g2)
+    assert not is_generator_with_return_value(h2)
+    assert not is_generator_with_return_value(i2)
+    assert not is_generator_with_return_value(j2)  # not recursive
+    assert not is_generator_with_return_value(k2)  # not recursive
+    assert not is_generator_with_return_value(l2)
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
-            warn_on_generator_with_return_value(mock_spider, f3)
-            warn_on_generator_with_return_value(mock_spider, g3)
-            warn_on_generator_with_return_value(mock_spider, h3)
-            warn_on_generator_with_return_value(mock_spider, i3)
-            warn_on_generator_with_return_value(mock_spider, j3)
-            warn_on_generator_with_return_value(mock_spider, k3)
-            warn_on_generator_with_return_value(mock_spider, l3)
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+        warn_on_generator_with_return_value(mock_spider, f2)
+        warn_on_generator_with_return_value(mock_spider, g2)
+        warn_on_generator_with_return_value(mock_spider, h2)
+        warn_on_generator_with_return_value(mock_spider, i2)
+        warn_on_generator_with_return_value(mock_spider, j2)
+        warn_on_generator_with_return_value(mock_spider, k2)
+        warn_on_generator_with_return_value(mock_spider, l2)
 
-    @mock.patch(
-        "scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error
-    )
-    def test_indentation_error(self, mock_spider):
-        with pytest.warns(UserWarning, match="Unable to determine"):
-            warn_on_generator_with_return_value(mock_spider, top_level_return_none)
 
-    def test_partial(self):
-        def cb(arg1, arg2):
-            yield {}
+def test_generators_return_none_with_decorator(mock_spider):
+    def decorator(func):
+        def inner_func():
+            func()
 
-        partial_cb = partial(cb, arg1=42)
-        assert not is_generator_with_return_value(partial_cb)
+        return inner_func
 
-    def test_warn_on_generator_with_return_value_settings_disabled(self):
-        class MockSettings:
-            def __init__(self, settings_dict=None):
-                self.settings_dict = settings_dict or {}
+    @decorator
+    def f3():
+        yield 1
 
-            def getbool(self, name, default=False):
-                return self.settings_dict.get(name, default)
+    @decorator
+    def g3():
+        yield 1
 
-        class MockSpider:
-            def __init__(self):
-                self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
+    @decorator
+    def h3():
+        yield 1
 
-        spider = MockSpider()
+    @decorator
+    def i3():
+        yield 1
+        yield from generator_that_returns_stuff()
 
-        def gen_with_return():
-            yield 1
-            return "value"
+    @decorator
+    def j3():
+        yield 1
 
-        with warnings.catch_warnings():
-            warnings.simplefilter("error", UserWarning)
-            warn_on_generator_with_return_value(spider, gen_with_return)
+        def helper() -> int:
+            return 0
 
-        spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
+        yield helper()
 
-        with pytest.warns(UserWarning, match="is a generator"):
-            warn_on_generator_with_return_value(spider, gen_with_return)
+    @decorator
+    def k3():
+        """
+        docstring
+        """
+        url = """
+https://example.org
+    """
+        yield url
+
+    @decorator
+    def l3():
+        return
+
+    assert not is_generator_with_return_value(top_level_return_none)
+    assert not is_generator_with_return_value(f3)
+    assert not is_generator_with_return_value(g3)
+    assert not is_generator_with_return_value(h3)
+    assert not is_generator_with_return_value(i3)
+    assert not is_generator_with_return_value(j3)  # not recursive
+    assert not is_generator_with_return_value(k3)  # not recursive
+    assert not is_generator_with_return_value(l3)
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+        warn_on_generator_with_return_value(mock_spider, f3)
+        warn_on_generator_with_return_value(mock_spider, g3)
+        warn_on_generator_with_return_value(mock_spider, h3)
+        warn_on_generator_with_return_value(mock_spider, i3)
+        warn_on_generator_with_return_value(mock_spider, j3)
+        warn_on_generator_with_return_value(mock_spider, k3)
+        warn_on_generator_with_return_value(mock_spider, l3)
+
+
+@mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
+def test_indentation_error(mock_spider):
+    with pytest.warns(UserWarning, match="Unable to determine"):
+        warn_on_generator_with_return_value(mock_spider, top_level_return_none)
+
+
+def test_partial() -> None:
+    def cb(arg1, arg2):
+        yield {}
+
+    partial_cb = partial(cb, arg1=42)
+    assert not is_generator_with_return_value(partial_cb)
+
+
+def test_warn_on_generator_with_return_value_settings_disabled() -> None:
+    class MockSettings:
+        def __init__(self, settings_dict: dict[str, Any] | None = None):
+            self.settings_dict = settings_dict or {}
+
+        def getbool(self, name, default=False):
+            return self.settings_dict.get(name, default)
+
+    class MockSpider:
+        def __init__(self) -> None:
+            self.settings = MockSettings({"WARN_ON_GENERATOR_RETURN_VALUE": False})
+
+    spider = MockSpider()
+
+    def gen_with_return():
+        yield 1
+        return "value"
+
+    with warnings.catch_warnings():
+        warnings.simplefilter("error", UserWarning)
+        warn_on_generator_with_return_value(spider, gen_with_return)  # type: ignore[arg-type]
+
+    spider.settings.settings_dict["WARN_ON_GENERATOR_RETURN_VALUE"] = True
+
+    with pytest.warns(UserWarning, match="is a generator"):
+        warn_on_generator_with_return_value(spider, gen_with_return)  # type: ignore[arg-type]
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 5333a55cb..7eade6463 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -1,14 +1,20 @@
+from __future__ import annotations
+
 import os
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import pytest
 
 from scrapy.utils.misc import set_environ
 from scrapy.utils.project import data_path, get_project_settings
 
+if TYPE_CHECKING:
+    from collections.abc import Generator
+
 
 @pytest.fixture
-def proj_path(tmp_path):
+def proj_path(tmp_path: Path) -> Generator[Path]:
     prev_dir = Path.cwd()
     project_dir = tmp_path
 
@@ -21,7 +27,7 @@ def proj_path(tmp_path):
         os.chdir(prev_dir)
 
 
-def test_data_path_outside_project():
+def test_data_path_outside_project() -> None:
     assert str(Path(".scrapy", "somepath")) == data_path("somepath")
     abspath = str(Path(os.path.sep, "absolute", "path"))
     assert abspath == data_path(abspath)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 2e8047e2d..c3b5dfc99 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -22,8 +22,7 @@ from scrapy.utils.python import (
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
-    from collections.abc import Iterable, Mapping
-
+    from collections.abc import AsyncIterator, Iterable, Mapping
 
 _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
@@ -31,22 +30,22 @@ _VT = TypeVar("_VT")
 
 class TestMutableAsyncChain:
     @staticmethod
-    async def g1():
+    async def g1() -> AsyncIterator[int]:
         for i in range(3):
             yield i
 
     @staticmethod
-    async def g2():
+    async def g2() -> AsyncIterator[int]:
         return
         yield
 
     @staticmethod
-    async def g3():
+    async def g3() -> AsyncIterator[int]:
         for i in range(7, 10):
             yield i
 
     @staticmethod
-    async def g4():
+    async def g4() -> AsyncIterator[int]:
         for i in range(3, 5):
             yield i
         1 / 0
@@ -85,7 +84,7 @@ class TestToUnicode:
 
     def test_converting_a_strange_object_should_raise_type_error(self):
         with pytest.raises(TypeError):
-            to_unicode(423)
+            to_unicode(423)  # type: ignore[arg-type]
 
     def test_errors_argument(self):
         assert to_unicode(b"a\xedb", "utf-8", errors="replace") == "a\ufffdb"
@@ -103,7 +102,7 @@ class TestToBytes:
 
     def test_converting_a_strange_object_should_raise_type_error(self):
         with pytest.raises(TypeError):
-            to_bytes(pytest)
+            to_bytes(pytest)  # type: ignore[arg-type]
 
     def test_errors_argument(self):
         assert to_bytes("a\ufffdb", "latin-1", errors="replace") == b"a?b"
@@ -112,10 +111,10 @@ class TestToBytes:
 def test_memoizemethod_noargs():
     class A:
         @memoizemethod_noargs
-        def cached(self):
+        def cached(self) -> object:
             return object()
 
-        def noncached(self):
+        def noncached(self) -> object:
             return object()
 
     a = A()
@@ -150,7 +149,7 @@ def test_get_func_args():
         pass
 
     class A:
-        def __init__(self, a, b, c):
+        def __init__(self, a: int, b: int, c: int):
             pass
 
         def method(self, a, b, c):
diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py
index 7d39a478e..44cb5c306 100644
--- a/tests/test_utils_reactor.py
+++ b/tests/test_utils_reactor.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 
 import pytest
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 1642a932b..eb95be06c 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import json
 from hashlib import sha1
-from typing import Any
+from typing import TYPE_CHECKING, Any, Protocol
 from weakref import WeakKeyDictionary
 
 import pytest
@@ -17,6 +17,9 @@ from scrapy.utils.request import (
 )
 from scrapy.utils.test import get_crawler
 
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
 
 @pytest.mark.parametrize(
     ("r", "expected"),
@@ -56,8 +59,18 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
     request_httprepr(r)
 
 
+class _FingerprintFunction(Protocol):
+    def __call__(
+        self,
+        request: Request,
+        *,
+        include_headers: Iterable[bytes | str] | None = None,
+        keep_fragments: bool = False,
+    ) -> bytes: ...
+
+
 class TestFingerprint:
-    function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
+    function: _FingerprintFunction = staticmethod(fingerprint)
     cache: (
         WeakKeyDictionary[
             Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
@@ -261,6 +274,7 @@ class TestRequestFingerprinter:
     def test_fingerprint(self):
         crawler = get_crawler()
         request = Request("https://example.com")
+        assert crawler.request_fingerprinter
         assert crawler.request_fingerprinter.fingerprint(request) == fingerprint(
             request
         )
@@ -277,6 +291,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com", headers={"X-ID": "1"})
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com", headers={"X-ID": "2"})
@@ -285,9 +300,9 @@ class TestCustomRequestFingerprinter:
 
     def test_dont_canonicalize(self):
         class RequestFingerprinter:
-            cache = WeakKeyDictionary()
+            cache: WeakKeyDictionary[Request, bytes] = WeakKeyDictionary()
 
-            def fingerprint(self, request):
+            def fingerprint(self, request: Request) -> bytes:
                 if request not in self.cache:
                     fp = sha1()
                     fp.update(to_bytes(request.url))
@@ -299,6 +314,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com?a=1&a=2")
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com?a=2&a=1")
@@ -317,6 +333,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         r1 = Request("http://www.example.com")
         fp1 = crawler.request_fingerprinter.fingerprint(r1)
         r2 = Request("http://www.example.com", meta={"fingerprint": "a"})
@@ -348,6 +365,7 @@ class TestCustomRequestFingerprinter:
         }
         crawler = get_crawler(settings_dict=settings)
 
+        assert crawler.request_fingerprinter
         request = Request("http://www.example.com")
         fingerprint = crawler.request_fingerprinter.fingerprint(request)
         assert fingerprint == settings["FINGERPRINT"]
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 146cdb802..608b2bbd9 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from pathlib import Path
 from time import process_time
 from urllib.parse import urlparse
@@ -15,7 +17,7 @@ from scrapy.utils.response import (
 )
 
 
-def _read_browser_output(burl: str):
+def _read_browser_output(burl: str) -> bytes:
     path = urlparse(burl).path
     if not path or not Path(path).exists():
         path = burl.replace("file://", "")
@@ -224,7 +226,7 @@ def test_open_in_browser_redos_head():
         (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
     ],
 )
-def test_remove_html_comments(input_body, output_body):
+def test_remove_html_comments(input_body: bytes, output_body: bytes) -> None:
     assert _remove_html_comments(input_body) == output_body
 
 
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index 2e6a790f8..2702c2cce 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import dataclasses
 import datetime
 import json
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index dbb9caf2a..3109bc149 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -21,9 +21,6 @@ from tests.utils.decorators import coroutine_test
 if TYPE_CHECKING:
     from collections.abc import Callable
 
-if TYPE_CHECKING:
-    from collections.abc import Callable
-
 
 class TestSendCatchLog:
     # whether the function being tested returns exceptions or failures
diff --git a/tests/test_utils_sitemap.py b/tests/test_utils_sitemap.py
index ac57e1739..9f2dce6d6 100644
--- a/tests/test_utils_sitemap.py
+++ b/tests/test_utils_sitemap.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.exceptions import ScrapyDeprecationWarning
diff --git a/tests/test_utils_template.py b/tests/test_utils_template.py
index 4515ce36e..076edf3db 100644
--- a/tests/test_utils_template.py
+++ b/tests/test_utils_template.py
@@ -1,7 +1,14 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
 from scrapy.utils.template import render_templatefile
 
+if TYPE_CHECKING:
+    from pathlib import Path
 
-def test_simple_render(tmp_path):
+
+def test_simple_render(tmp_path: Path) -> None:
     context = {"project_name": "proj", "name": "spi", "classname": "TheSpider"}
     template = "from ${project_name}.spiders.${name} import ${classname}"
     rendered = "from proj.spiders.spi import TheSpider"
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index 2334c76e9..5458aa603 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import sys
 from io import StringIO
 from unittest import mock
@@ -46,13 +48,13 @@ Bar                                 1   oldest: 0s ago
 
 
 @mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_empty(stdout):
+def test_print_live_refs_empty(stdout: StringIO) -> None:
     trackref.print_live_refs()
     assert stdout.getvalue() == "Live References\n\n\n"
 
 
 @mock.patch("sys.stdout", new_callable=StringIO)
-def test_print_live_refs_with_objects(stdout):
+def test_print_live_refs_with_objects(stdout: StringIO) -> None:
     o1 = Foo()  # noqa: F841
     trackref.print_live_refs()
     assert (
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 5b98131a1..d9d162c23 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from scrapy.linkextractors import IGNORED_EXTENSIONS
@@ -191,7 +193,7 @@ def test_guess_scheme(url: str, expected: str):
         ),
     ],
 )
-def test_guess_scheme_skipped(url: str, expected: str, reason: str):
+def test_guess_scheme_skipped(url: str, expected: str, reason: str) -> None:
     pytest.skip(reason)
 
 
diff --git a/tests_typing/test_http_request.mypy-testing b/tests_typing/test_http_request.mypy-testing
index a431091d5..4ff562dff 100644
--- a/tests_typing/test_http_request.mypy-testing
+++ b/tests_typing/test_http_request.mypy-testing
@@ -16,7 +16,7 @@ class MyRequest2(Request):
 
 @pytest.mark.mypy_testing
 def mypy_test_headers() -> None:
-    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Request("data:,", headers=1)  # E: Argument "headers" to "Request" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
     Request("data:,", headers=None)
     Request("data:,", headers={})
     Request("data:,", headers=[])
diff --git a/tests_typing/test_http_response.mypy-testing b/tests_typing/test_http_response.mypy-testing
index d497c2470..1c157328c 100644
--- a/tests_typing/test_http_response.mypy-testing
+++ b/tests_typing/test_http_response.mypy-testing
@@ -7,7 +7,7 @@ from scrapy.http import HtmlResponse, Response, TextResponse
 
 @pytest.mark.mypy_testing
 def mypy_test_headers() -> None:
-    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Iterable[tuple[str, Any]] | None"
+    Response("data:,", headers=1)  # E: Argument "headers" to "Response" has incompatible type "int"; expected "Mapping[str, Any] | Mapping[bytes, Any] | Iterable[tuple[str | bytes, Any]] | None"
     Response("data:,", headers=None)
     Response("data:,", headers={})
     Response("data:,", headers=[])

From bc5b5fb1f6d8fef2f57ef74a728dd609b5c00cf4 Mon Sep 17 00:00:00 2001
From: Youssef Mohamed <114195599+MegumiinUwU@users.noreply.github.com>
Date: Wed, 29 Jul 2026 11:47:15 +0300
Subject: [PATCH 52/62] Add Request.to_curl() (#7743) (#7802)

---
 docs/topics/request-response.rst  |  2 ++
 scrapy/http/request/__init__.py   | 14 ++++++++++++++
 tests/test_utils_request.py       | 11 +++++++++++
 tests/utils/bases/http_request.py |  9 +++++++++
 4 files changed, 36 insertions(+)

diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b83a04032..8e565907f 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -260,6 +260,8 @@ Request objects
 
     .. automethod:: from_curl
 
+    .. automethod:: to_curl
+
     .. automethod:: to_dict
 
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 57517a65b..68847283a 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -387,6 +387,20 @@ class Request(object_ref):
         request_kwargs.update(kwargs)
         return cls(**request_kwargs)
 
+    def to_curl(self) -> str:
+        """Return a string with a `cURL <https://curl.se/>`_ command equivalent
+        to this request.
+
+        Inverse of :meth:`from_curl`. See also
+        :func:`scrapy.utils.request.request_to_curl`.
+
+        .. versionadded:: VERSION
+        """
+        # Imported here to avoid a circular import.
+        from scrapy.utils.request import request_to_curl  # noqa: PLC0415
+
+        return request_to_curl(self)
+
     def to_dict(self, *, spider: scrapy.Spider | None = None) -> dict[str, Any]:
         """Return a dictionary containing the Request's data.
 
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index eb95be06c..935447bc4 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -475,3 +475,14 @@ class TestRequestToCurl:
             " --data-raw '{\"foo\": \"bar\"}' --cookie 'foo=1'"
         )
         self._test_request(request_object, expected_curl_command)
+
+    def test_request_to_curl_method(self) -> None:
+        request_object = Request(
+            "https://www.httpbin.org/post",
+            method="POST",
+            body=json.dumps({"foo": "bar"}),
+        )
+        expected_curl_command = (
+            'curl -X POST https://www.httpbin.org/post --data-raw \'{"foo": "bar"}\''
+        )
+        assert request_object.to_curl() == expected_curl_command
diff --git a/tests/utils/bases/http_request.py b/tests/utils/bases/http_request.py
index c255b5e4c..3a1e588ef 100644
--- a/tests/utils/bases/http_request.py
+++ b/tests/utils/bases/http_request.py
@@ -6,6 +6,7 @@ import pytest
 
 from scrapy.http import Headers, Request
 from scrapy.http.request import NO_CALLBACK
+from scrapy.utils.request import request_to_curl
 
 
 class TestRequestBase(ABC):
@@ -488,3 +489,11 @@ class TestRequestBase(ABC):
                 'curl -X PATCH "http://example.org" --foo -z',
                 ignore_unknown_options=False,
             )
+
+    def test_to_curl(self):
+        # Note: more curated tests regarding curl conversion are in
+        # `test_utils_request.py`
+        r = self.request_class(
+            "http://www.example.com/", method="POST", body=b"foo=bar"
+        )
+        assert r.to_curl() == request_to_curl(r)

From 8b5147ae2ec62bca66e3b957a8d0f1c3654ecf2e Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Wed, 29 Jul 2026 11:32:18 +0200
Subject: [PATCH 53/62] Improve test coverage for scrapy.pipelines (#7798)

* Improve test coverage for scrapy.pipelines

* Restore old Pillow support
---
 tests/test_pipeline_files.py   | 220 ++++++++++++++++++++++++++++++---
 tests/test_pipeline_images.py  |  52 +++++++-
 tests/test_pipeline_media.py   |  60 +++++++++
 tests/utils/media_pipelines.py |   6 +
 4 files changed, 321 insertions(+), 17 deletions(-)

diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4f6fa21a0..97daa514e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -33,7 +33,7 @@ from scrapy.pipelines.files import (
     GCSFilesStore,
     S3FilesStore,
 )
-from scrapy.pipelines.media import MediaPipeline, _MediaRequestFiltered
+from scrapy.pipelines.media import _MediaRequestFiltered
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.defer import maybe_deferred_to_future
@@ -43,11 +43,7 @@ from tests.mockserver.ftp import MockFTPServer
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 from .utils.cloud import mock_google_cloud_storage
-from .utils.media_pipelines import mocked_download_func
-
-# required by persist_file() and stat_file(), but as some stores don't use the argument
-# we can pass this singleton to keep type hints correct
-DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
+from .utils.media_pipelines import DUMMY_SPIDER_INFO, mocked_download_func
 
 
 def get_ftp_content_and_delete(
@@ -94,16 +90,19 @@ class DeferredFSFilesStore(FSFilesStore):
 class TestFilesPipeline:
     def setup_method(self):
         self.tempdir = mkdtemp()
-        settings_dict = {"FILES_STORE": self.tempdir}
-        crawler = get_crawler(DefaultSpider, settings_dict=settings_dict)
-        crawler.spider = crawler._create_spider()
-        crawler.engine = MagicMock(download_async=mocked_download_func)
-        self.pipeline = FilesPipeline.from_crawler(crawler)
-        self.pipeline.open_spider()
+        self.pipeline = self._create_pipeline(FilesPipeline)
 
     def teardown_method(self):
         rmtree(self.tempdir)
 
+    def _create_pipeline(self, pipeline_cls: type[FilesPipeline]) -> FilesPipeline:
+        crawler = get_crawler(DefaultSpider, {"FILES_STORE": self.tempdir})
+        crawler.spider = crawler._create_spider()
+        crawler.engine = MagicMock(download_async=mocked_download_func)
+        pipeline = pipeline_cls.from_crawler(crawler)
+        pipeline.open_spider()
+        return pipeline
+
     def test_file_path_query_parameters(self):
         file_path = self.pipeline.file_path
 
@@ -254,6 +253,107 @@ class TestFilesPipeline:
         assert result["files"][0]["checksum"] != "abc"
         assert result["files"][0]["status"] == "cached"
 
+    @coroutine_test
+    async def test_file_stat_without_last_modified(self) -> None:
+        """A stat result without a last modification time forces a download."""
+        item_url = "http://example.com/file4.pdf"
+        item = _create_item_with_files(item_url)
+        with (
+            mock.patch.object(FilesPipeline, "inc_stats", return_value=True),
+            mock.patch.object(
+                FSFilesStore, "stat_file", return_value={"checksum": "abc"}
+            ),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+        assert result["files"][0]["checksum"] != "abc"
+        assert result["files"][0]["status"] == "downloaded"
+
+    @coroutine_test
+    async def test_file_empty_content(self, caplog: pytest.LogCaptureFixture) -> None:
+        item_url = "http://example.com/empty.pdf"
+        item = _create_item_with_files(item_url)
+        request = Request(
+            item_url, meta={"response": Response(item_url, status=200, body=b"")}
+        )
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline, "get_media_requests", return_value=[request]
+            ),
+        ):
+            result = await self.pipeline.process_item(item)
+        assert result["files"] == []
+        assert "File (empty-content): Empty file from" in caplog.text
+
+    @coroutine_test
+    async def test_file_downloaded_file_exception(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """A FileException from file_downloaded() is logged as a warning and
+        kept as is."""
+
+        class FailingFilesPipeline(FilesPipeline):
+            def file_downloaded(self, response, request, info, *, item=None):
+                raise FileException("boom")
+
+        item_url = "http://example.com/file5.pdf"
+        item = _create_item_with_files(item_url)
+        pipeline = self._create_pipeline(FailingFilesPipeline)
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await pipeline.process_item(item)
+        assert result["files"] == []
+        records = [
+            r for r in caplog.records if "Error processing file" in r.getMessage()
+        ]
+        assert len(records) == 1
+        assert records[0].levelname == "WARNING"
+        assert "boom" in records[0].getMessage()
+
+    @coroutine_test
+    async def test_file_downloaded_unknown_error(
+        self, caplog: pytest.LogCaptureFixture
+    ) -> None:
+        """Any other exception from file_downloaded() is logged as an error and
+        reported as a FileException."""
+
+        class FailingFilesPipeline(FilesPipeline):
+            def file_downloaded(self, response, request, info, *, item=None):
+                raise RuntimeError("boom")
+
+        item_url = "http://example.com/file6.pdf"
+        item = _create_item_with_files(item_url)
+        pipeline = self._create_pipeline(FailingFilesPipeline)
+        with (
+            caplog.at_level(logging.WARNING),
+            mock.patch.object(
+                FilesPipeline,
+                "get_media_requests",
+                return_value=[_prepare_request_object(item_url)],
+            ),
+        ):
+            result = await pipeline.process_item(item)
+        assert result["files"] == []
+        records = [
+            r for r in caplog.records if "Error processing file" in r.getMessage()
+        ]
+        assert len(records) == 1
+        assert records[0].levelname == "ERROR"
+        exc_info = records[0].exc_info
+        assert exc_info is not None
+        assert exc_info[0] is RuntimeError
+
     @coroutine_test
     async def test_async_store(self) -> None:
         """Test that async persist_file() works and is awaited."""
@@ -648,9 +748,24 @@ class TestFilesPipelineCustomSettings:
         request = Request("http://example.com/image01.jpg")
         assert pipeline.file_path(request) == Path("subdir/image01.jpg")
 
-    def test_files_store_constructor_with_pathlike_object(self, tmp_path):
-        fs_store = FSFilesStore(tmp_path)
-        assert fs_store.basedir == str(tmp_path)
+
+class TestFSFilesStore:
+    def test_constructor_with_pathlike_object(self, tmp_path: Path) -> None:
+        assert FSFilesStore(tmp_path).basedir == str(tmp_path)
+
+    def test_constructor_with_uri(self, tmp_path: Path) -> None:
+        assert FSFilesStore(f"file://{tmp_path}").basedir == str(tmp_path)
+
+    def test_stat_file(self, tmp_path: Path) -> None:
+        store = FSFilesStore(tmp_path)
+        store.persist_file("full/filename", BytesIO(b"data"), DUMMY_SPIDER_INFO)
+        stat = store.stat_file("full/filename", DUMMY_SPIDER_INFO)
+        assert stat["checksum"] == "8d777f385d3dfec8815d20f7496026dc"
+        assert stat["last_modified"] == pytest.approx(time.time(), abs=60)
+
+    def test_stat_missing_file(self, tmp_path: Path) -> None:
+        store = FSFilesStore(tmp_path)
+        assert store.stat_file("full/filename", DUMMY_SPIDER_INFO) == {}
 
 
 @pytest.mark.requires_botocore
@@ -695,6 +810,59 @@ class TestS3FilesStore:
             # The call to read does not happen with Stubber
             assert buffer.method_calls == [mock.call.seek(0)]
 
+    @inline_callbacks_test
+    def test_persist_without_headers(self):
+        """Without custom headers only the default ones are sent."""
+        bucket = "mybucket"
+        key = "export.csv"
+        buffer = mock.MagicMock()
+
+        store = S3FilesStore(f"s3://{bucket}/{key}")
+        from botocore.stub import Stubber  # noqa: PLC0415
+
+        with Stubber(store.s3_client) as stub:
+            stub.add_response(
+                "put_object",
+                expected_params={
+                    "ACL": S3FilesStore.POLICY,
+                    "Body": buffer,
+                    "Bucket": bucket,
+                    "CacheControl": S3FilesStore.HEADERS["Cache-Control"],
+                    "Key": key,
+                    "Metadata": {},
+                },
+                service_response={},
+            )
+
+            yield store.persist_file("", buffer, info=DUMMY_SPIDER_INFO)
+
+            stub.assert_no_pending_responses()
+
+    def test_missing_botocore(self):
+        with (
+            mock.patch(
+                "scrapy.pipelines.files.is_botocore_available", return_value=False
+            ),
+            pytest.raises(NotConfigured, match="missing botocore library"),
+        ):
+            S3FilesStore("s3://mybucket/key")
+
+    def test_wrong_uri_scheme(self):
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "Incorrect URI scheme in ftp://mybucket/key, expected 's3'"
+            ),
+        ):
+            S3FilesStore("ftp://mybucket/key")
+
+    def test_unsupported_header(self):
+        store = S3FilesStore("s3://mybucket/key")
+        with pytest.raises(
+            TypeError, match='Header "X-Custom" is not supported by botocore'
+        ):
+            store._headers_to_botocore_kwargs({"X-Custom": "value"})
+
     @inline_callbacks_test
     def test_stat(self):
         bucket = "mybucket"
@@ -901,6 +1069,28 @@ class TestFTPFileStore:
             )
         assert data == content
 
+    @inline_callbacks_test
+    def test_persist_active_mode(self, monkeypatch: pytest.MonkeyPatch):
+        data = b"active mode"
+        path = "full/filename"
+        monkeypatch.setattr(FTPFilesStore, "FTP_USERNAME", "anonymous")
+        monkeypatch.setattr(FTPFilesStore, "FTP_PASSWORD", "guest")
+        monkeypatch.setattr(FTPFilesStore, "USE_ACTIVE_MODE", True)
+        with MockFTPServer() as ftp_server:
+            store = FTPFilesStore(ftp_server.url("/"))
+            yield store.persist_file(path, BytesIO(data), info=DUMMY_SPIDER_INFO)
+            stat = yield store.stat_file(path, info=DUMMY_SPIDER_INFO)
+        assert stat["checksum"] == "ff1575649a39a27c13faa0d37c84bab3"
+
+    def test_wrong_uri_scheme(self):
+        with pytest.raises(
+            ValueError,
+            match=re.escape(
+                "Incorrect URI scheme in http://example.com/, expected 'ftp'"
+            ),
+        ):
+            FTPFilesStore("http://example.com/")
+
 
 class ItemWithFiles(Item):
     file_urls = Field()
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 1b73dd157..19e61579f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -3,20 +3,26 @@ from __future__ import annotations
 import dataclasses
 import io
 import random
+import sys
 from abc import ABC, abstractmethod
+from pathlib import Path
 from shutil import rmtree
 from tempfile import mkdtemp
+from types import SimpleNamespace
 from typing import Any
 
 import attr
 import pytest
 from itemadapter import ItemAdapter
 
+from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
 from scrapy.item import Field, Item
-from scrapy.pipelines.files import GCSFilesStore, S3FilesStore
+from scrapy.pipelines.files import GCSFilesStore, S3FilesStore, _md5sum
 from scrapy.pipelines.images import ImageException, ImagesPipeline
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
+from tests.utils.media_pipelines import DUMMY_SPIDER_INFO
 
 try:
     from PIL import Image
@@ -40,6 +46,11 @@ class TestImagesPipeline:
     def teardown_method(self):
         rmtree(self.tempdir)
 
+    def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:
+        monkeypatch.setitem(sys.modules, "PIL", None)
+        with pytest.raises(NotConfigured, match="requires installing Pillow"):
+            ImagesPipeline(self.tempdir, crawler=get_crawler())
+
     def test_file_path(self):
         file_path = self.pipeline.file_path
         assert (
@@ -197,6 +208,25 @@ class TestImagesPipeline:
         assert path == "full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
         assert new_im.getpixel((0, 0)) == (255, 0, 0)
 
+    @coroutine_test
+    async def test_image_downloaded(self) -> None:
+        """The image and its thumbnails are stored, and the checksum of the
+        full-size image is returned."""
+        self.pipeline.thumbs = {"small": (20, 20)}
+        _, buf = _create_image("JPEG", "RGB", (50, 50), (0, 0, 0))
+        url = "https://dev.mydeco.com/mydeco.gif"
+        response = Response(url=url, body=buf.getvalue())
+
+        checksum = await self.pipeline.image_downloaded(
+            response, Request(url=url), DUMMY_SPIDER_INFO
+        )
+
+        buf.seek(0)
+        assert checksum == _md5sum(buf)
+        name = "3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg"
+        assert Path(self.tempdir, "full", name).read_bytes() == buf.getvalue()
+        assert Path(self.tempdir, "thumbs", "small", name).exists()
+
     def test_convert_image(self):
         SIZE = (100, 100)
         # straight forward case: RGB and JPEG
@@ -230,6 +260,24 @@ class TestImagesPipeline:
         assert converted.mode == "RGB"
         assert converted.getcolors() == [(10000, (205, 230, 255))]
 
+    def test_convert_image_legacy_resampling_filter(
+        self, monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        """Pillow older than 9.1.0 has Image.ANTIALIAS instead of
+        Image.Resampling.LANCZOS."""
+        # Image.LANCZOS is the only spelling that exists in every supported
+        # Pillow version, but Pillow defines it dynamically, hence the ignore.
+        monkeypatch.setattr(
+            self.pipeline,
+            "_Image",
+            SimpleNamespace(ANTIALIAS=Image.LANCZOS),  # type: ignore[attr-defined]
+        )
+        im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
+
+        thumbnail, _ = self.pipeline.convert_image(im, size=(10, 25), response_body=buf)
+
+        assert thumbnail.size == (10, 10)
+
     @pytest.mark.parametrize(
         "bad_type",
         [
@@ -581,7 +629,7 @@ class TestImagesPipelineCustomSettings:
             GCSFilesStore.POLICY = old_policy
 
 
-def _create_image(format_, *a, **kw):
+def _create_image(format_: str, *a: Any, **kw: Any) -> tuple[Image.Image, io.BytesIO]:
     buf = io.BytesIO()
     Image.new(*a, **kw).save(buf, format_)
     buf.seek(0)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 23c19e4be..ba1c18006 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -319,6 +319,42 @@ class TestMediaPipeline(TestBaseMediaPipeline):
         assert self.fingerprint(req1) == self.fingerprint(req2)
         assert new_item["results"] == [(True, {})]
 
+    @coroutine_test
+    async def test_failures_are_cached_across_multiple_items(self):
+        self.pipe.LOG_FAILED_RESULTS = False
+        exc = Exception("foo")
+        req1 = Request("http://url1", meta={"response": exc})
+        new_item = await self.pipe.process_item({"requests": req1})
+        assert new_item["results"][0][1].value is exc
+
+        # rsp2 is ignored, the cached failure must be reused because request
+        # fingerprints are the same
+        req2 = Request(
+            req1.url, meta={"response": Response("http://donot.download.me")}
+        )
+        new_item = await self.pipe.process_item({"requests": req2})
+        assert new_item["results"][0][0] is False
+        assert new_item["results"][0][1].value is exc
+        assert self.pipe._mockcalled.count("media_to_download") == 1
+
+    @coroutine_test
+    async def test_cached_failure_calls_errback(self):
+        """The errback of a request is called for a cached failure as well."""
+        self.pipe.LOG_FAILED_RESULTS = False
+        exc = Exception("foo")
+        await self.pipe.process_item(
+            {"requests": Request("http://url1", meta={"response": exc})}
+        )
+
+        def errback(failure):
+            self.pipe._mockcalled.append("request_errback")
+            return {"recovered": failure.value}
+
+        req = Request("http://url1", errback=errback)
+        new_item = await self.pipe.process_item({"requests": req})
+        assert new_item["results"] == [(True, {"recovered": exc})]
+        assert self.pipe._mockcalled.count("request_errback") == 1
+
     @coroutine_test
     async def test_results_are_cached_for_requests_of_single_item(self):
         rsp1 = Response("http://url1")
@@ -472,6 +508,30 @@ class TestBuildFromCrawler:
         assert pipe._from_crawler_called
 
 
+class MediaFailedNonePipeline(MockedMediaPipeline):
+    def media_failed(self, failure, request, info):
+        self._mockcalled.append("media_failed")
+
+
+class TestMediaFailedNone(TestBaseMediaPipeline):
+    """Test what happens when media_failed() neither raises an exception nor
+    returns a failure."""
+
+    pipeline_class = MediaFailedNonePipeline
+
+    @coroutine_test
+    async def test_result_none(self):
+        req = Request("http://url1", meta={"response": Exception("foo")})
+        new_item = await self.pipe.process_item({"requests": req})
+        assert new_item["results"] == [(True, None)]
+        assert self.pipe._mockcalled == [
+            "get_media_requests",
+            "media_to_download",
+            "media_failed",
+            "item_completed",
+        ]
+
+
 class MediaFailedFailurePipeline(MockedMediaPipeline):
     def media_failed(self, failure, request, info):
         self._mockcalled.append("media_failed")
diff --git a/tests/utils/media_pipelines.py b/tests/utils/media_pipelines.py
index 283c95940..7e013b797 100644
--- a/tests/utils/media_pipelines.py
+++ b/tests/utils/media_pipelines.py
@@ -3,6 +3,12 @@ from __future__ import annotations
 from typing import Any
 
 from scrapy.http.request import NO_CALLBACK, Request
+from scrapy.pipelines.media import MediaPipeline
+from scrapy.utils.spider import DefaultSpider
+
+# required by persist_file() and stat_file(), but as some stores don't use the argument
+# we can pass this singleton to keep type hints correct
+DUMMY_SPIDER_INFO = MediaPipeline.SpiderInfo(DefaultSpider())
 
 
 async def mocked_download_func(request: Request) -> Any:

From 0cbb20e8e83996a4d8a3339254c2d7e9b763ce7f Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Wed, 29 Jul 2026 11:39:27 +0200
Subject: [PATCH 54/62] Treat broken cache records as cache misses (#7805)

---
 docs/topics/downloader-middleware.rst        |  4 +
 scrapy/downloadermiddlewares/httpcache.py    | 21 +++++-
 tests/test_downloadermiddleware_httpcache.py | 77 +++++++++++++++++---
 3 files changed, 90 insertions(+), 12 deletions(-)

diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 934eb19ac..fcfe7fd29 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -564,6 +564,10 @@ defines the methods described below.
 
       Return response if present in cache, or ``None`` otherwise.
 
+      If this method raises an exception, e.g. because the cache entry is
+      corrupted, the middleware logs a warning and handles the request as a
+      cache miss.
+
       :param spider: the spider which generated the request
       :type spider: :class:`~scrapy.Spider` object
 
diff --git a/scrapy/downloadermiddlewares/httpcache.py b/scrapy/downloadermiddlewares/httpcache.py
index c6c811809..e7ca0ac0e 100644
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import logging
 from email.utils import formatdate
 from typing import TYPE_CHECKING
 
@@ -28,6 +29,9 @@ if TYPE_CHECKING:
     from scrapy.statscollectors import StatsCollector
 
 
+logger = logging.getLogger(__name__)
+
+
 class HttpCacheMiddleware:
     DOWNLOAD_EXCEPTIONS = (
         ConnectionDone,
@@ -77,9 +81,20 @@ class HttpCacheMiddleware:
             return None
 
         # Look for cached response and check if expired
-        cachedresponse: Response | None = self.storage.retrieve_response(
-            self.crawler.spider, request
-        )
+        cachedresponse: Response | None
+        try:
+            cachedresponse = self.storage.retrieve_response(
+                self.crawler.spider, request
+            )
+        except Exception:
+            self.stats.inc_value("httpcache/retrieve_error")
+            logger.warning(
+                f"Could not read the cache entry for {request}, treating it as a "
+                f"cache miss.",
+                exc_info=True,
+                extra={"spider": self.crawler.spider},
+            )
+            cachedresponse = None
         if cachedresponse is None:
             self.stats.inc_value("httpcache/miss")
             if self.ignore_missing:
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 9d5d6874e..ce56ee11d 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -1,10 +1,12 @@
 from __future__ import annotations
 
 import email.utils
+import logging
 import shutil
 import tempfile
 import time
 from contextlib import contextmanager
+from pathlib import Path
 from typing import TYPE_CHECKING, Any
 from unittest import mock
 
@@ -93,6 +95,12 @@ class TestBase:
 class StorageTestMixin:
     """Mixin containing storage-specific test methods."""
 
+    def _corrupt_cache_entry(
+        self, storage: Any, spider: Spider, request: Request
+    ) -> None:
+        """Make the cache entry of *request* unreadable for *storage*."""
+        raise NotImplementedError
+
     def test_storage(self):
         with self._storage(HTTPCACHE_EXPIRATION_SECS=1) as (storage, crawler):
             request2 = self.request.copy()
@@ -115,6 +123,38 @@ class StorageTestMixin:
             with mock.patch("scrapy.extensions.httpcache.time", return_value=future):
                 assert storage.retrieve_response(crawler.spider, self.request)
 
+    def test_corrupted_cache_entry_is_a_miss(self, caplog):
+        with self._middleware() as mw:
+            spider = mw.crawler.spider
+            mw.storage.store_response(spider, self.request, self.response)
+            self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+            caplog.clear()
+            with caplog.at_level(logging.WARNING):
+                assert mw.process_request(self.request) is None
+
+            assert "treating it as a cache miss" in caplog.text
+            assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+            assert mw.crawler.stats.get_value("httpcache/miss") == 1
+
+            # Storing the response again replaces the corrupted cache entry.
+            mw.storage.store_response(spider, self.request, self.response)
+            self.assertEqualResponse(
+                self.response, mw.storage.retrieve_response(spider, self.request)
+            )
+
+    def test_corrupted_cache_entry_ignore_missing(self):
+        with self._middleware(HTTPCACHE_IGNORE_MISSING=True) as mw:
+            spider = mw.crawler.spider
+            mw.storage.store_response(spider, self.request, self.response)
+            self._corrupt_cache_entry(mw.storage, spider, self.request)
+
+            with pytest.raises(IgnoreRequest):
+                mw.process_request(self.request)
+
+            assert mw.crawler.stats.get_value("httpcache/retrieve_error") == 1
+            assert mw.crawler.stats.get_value("httpcache/ignore") == 1
+
     def test_storage_no_content_type_header(self):
         """Test that the response body is used to get the right response class
         even if there is no Content-Type header"""
@@ -556,29 +596,43 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
 # Concrete test classes that combine storage and policy mixins
 
 
-class TestFilesystemStorageWithDummyPolicy(
-    TestBase, StorageTestMixin, DummyPolicyTestMixin
-):
+class FilesystemStorageTestMixin(StorageTestMixin):
     storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        rpath = Path(storage._get_request_path(spider, request))
+        (rpath / "response_body").unlink()
+
+
+class DbmStorageTestMixin(StorageTestMixin):
+    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        key = storage._fingerprinter.fingerprint(request).hex()
+        storage.db[f"{key}_data"] = b"not a pickle"
+
+
+class TestFilesystemStorageWithDummyPolicy(
+    TestBase, FilesystemStorageTestMixin, DummyPolicyTestMixin
+):
     policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
 
 class TestFilesystemStorageWithRFC2616Policy(
-    TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+    TestBase, FilesystemStorageTestMixin, RFC2616PolicyTestMixin
 ):
-    storage_class = "scrapy.extensions.httpcache.FilesystemCacheStorage"
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
 
-class TestDbmStorageWithDummyPolicy(TestBase, StorageTestMixin, DummyPolicyTestMixin):
-    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
+class TestDbmStorageWithDummyPolicy(
+    TestBase, DbmStorageTestMixin, DummyPolicyTestMixin
+):
     policy_class = "scrapy.extensions.httpcache.DummyPolicy"
 
 
 class TestDbmStorageWithRFC2616Policy(
-    TestBase, StorageTestMixin, RFC2616PolicyTestMixin
+    TestBase, DbmStorageTestMixin, RFC2616PolicyTestMixin
 ):
-    storage_class = "scrapy.extensions.httpcache.DbmCacheStorage"
     policy_class = "scrapy.extensions.httpcache.RFC2616Policy"
 
 
@@ -599,3 +653,8 @@ class TestFilesystemStorageGzipWithDummyPolicy(TestFilesystemStorageWithDummyPol
     def _get_settings(self, **new_settings) -> dict[str, Any]:
         new_settings.setdefault("HTTPCACHE_GZIP", True)
         return super()._get_settings(**new_settings)
+
+    def _corrupt_cache_entry(self, storage, spider, request) -> None:
+        # A spider killed while writing a gzip file leaves it truncated.
+        body_path = Path(storage._get_request_path(spider, request), "response_body")
+        body_path.write_bytes(body_path.read_bytes()[:-5])

From 01447f996500367fb999560d08a40cd2ff68815a Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Wed, 29 Jul 2026 08:00:24 -0300
Subject: [PATCH 55/62] fix(commands/parse): Restore request callback before
 invoking spider (#7803)

* fix(commands/parse): restore request callback before invoking spider

* Add other test
---
 scrapy/commands/parse.py    |  2 ++
 tests/test_command_parse.py | 56 +++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+)

diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 2ac65bf3f..93194ded7 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -346,6 +346,8 @@ class Command(BaseRunSpiderCommand):
                 self.first_response = response
 
             cb = self._get_callback(spider=spider, opts=opts, response=response)
+            assert response.request
+            response.request.callback = cb
 
             # parse items and requests
             depth: int = response.meta["_depth"]
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index 9b7131c7a..772cc82e2 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -126,6 +126,32 @@ class MySpider(scrapy.Spider):
         else:
             self.logger.debug('It Does Not Work :(')
 
+class RetryRequestSpider(BaseSpider):
+    name = 'retry_request'
+
+    def parse(self, response):
+        if response.meta.get('retried'):
+            yield {{'retried': True}}
+            return
+        response.meta['retried'] = True
+        yield response.request.replace(dont_filter=True)
+
+class CustomCallbackRetryRequestSpider(BaseSpider):
+    name = 'retry_request_custom_callback'
+
+    def parse(self, response):
+        yield response.request.replace(
+            callback=self.parse_retry,
+            dont_filter=True,
+        )
+
+    def parse_retry(self, response):
+        if response.meta.get('retried'):
+            yield {{'retried_with_custom_callback': True}}
+            return
+        response.meta['retried'] = True
+        yield response.request.replace(dont_filter=True)
+
 class MyGoodCrawlSpider(CrawlSpider):
     name = 'goodcrawl{self.spider_name}'
 
@@ -381,6 +407,36 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         )
         assert "[{}, {'foo': 'bar'}]" in out
 
+    def test_retry_response_request(
+        self, proj_path: Path, mockserver: MockServer
+    ) -> None:
+        _, out, stderr = proc(
+            "parse",
+            "--spider",
+            "retry_request",
+            "-d",
+            "2",
+            mockserver.url("/html"),
+            cwd=proj_path,
+        )
+        assert "RecursionError" not in stderr
+        assert "{'retried': True}" in out
+
+    def test_retry_response_request_with_custom_callback(
+        self, proj_path: Path, mockserver: MockServer
+    ) -> None:
+        _, out, stderr = proc(
+            "parse",
+            "--spider",
+            "retry_request_custom_callback",
+            "-d",
+            "3",
+            mockserver.url("/html"),
+            cwd=proj_path,
+        )
+        assert "RecursionError" not in stderr
+        assert "{'retried_with_custom_callback': True}" in out
+
     def test_wrong_callback_passed(
         self, proj_path: Path, mockserver: MockServer
     ) -> None:

From b2d4eedea8873700b2597a44dabafe7c9d169275 Mon Sep 17 00:00:00 2001
From: Fandu <113630375+mrfandu1@users.noreply.github.com>
Date: Wed, 29 Jul 2026 20:15:03 +0545
Subject: [PATCH 56/62] Fix immediate delivery of full feed export batches
 (#7730) (#7733)

* Store full feed batches before spider closes (#7730)

Start closing and storing each batch as soon as it reaches the configured item count. Track unfinished close tasks so spider shutdown still waits for all deliveries before emitting the exporter-closed signal.

Add an end-to-end regression test that verifies the first batch is stored while the crawl is still running.

* Remove the issue reference

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 scrapy/extensions/feedexport.py | 39 +++++++++++++++++++++++--------
 tests/test_feedexport_batch.py  | 41 +++++++++++++++++++++++++++++++++
 2 files changed, 71 insertions(+), 9 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 678a29e2e..c2997921d 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -13,7 +13,7 @@ import re
 import sys
 import warnings
 from abc import ABC, abstractmethod
-from collections.abc import Callable, Coroutine
+from collections.abc import Callable
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
@@ -475,7 +475,7 @@ class FeedExporter:
         self.feeds = {}
         self.slots: list[FeedSlot] = []
         self.filters: dict[str, ItemFilter] = {}
-        self._pending_close_coros: list[Coroutine[Any, Any, None]] = []
+        self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []
 
         if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:
             raise NotConfigured
@@ -539,23 +539,44 @@ class FeedExporter:
             )
 
     async def close_spider(self, spider: Spider) -> None:
-        self._pending_close_coros.extend(
-            self._close_slot(slot, spider) for slot in self.slots
-        )
+        for slot in self.slots:
+            self._schedule_slot_close(slot, spider)
 
-        if self._pending_close_coros:
+        if self._pending_close_tasks:
             if is_asyncio_available():
                 await asyncio.wait(
-                    [asyncio.create_task(coro) for coro in self._pending_close_coros]
+                    cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
                 )
             else:
                 await DeferredList(
-                    deferred_from_coro(coro) for coro in self._pending_close_coros
+                    cast("list[Deferred[None]]", list(self._pending_close_tasks))
                 )
 
         # Send FEED_EXPORTER_CLOSED signal
         await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
 
+    def _schedule_slot_close(
+        self, slot: FeedSlot, spider: Spider
+    ) -> asyncio.Task[None] | Deferred[None]:
+        """Start closing the slot without waiting for it to finish, keeping
+        track of the pending work so that it can be awaited in
+        :meth:`close_spider` if it hasn't finished by then."""
+        aw: asyncio.Task[None] | Deferred[None]
+        coro = self._close_slot(slot, spider)
+        if is_asyncio_available():
+            aw = asyncio.create_task(coro)
+            self._pending_close_tasks.append(aw)
+            aw.add_done_callback(self._pending_close_tasks.remove)
+        else:
+            aw = deferred_from_coro(coro)
+            self._pending_close_tasks.append(aw)
+            aw.addBoth(self._untrack_pending_close_task, aw)
+        return aw
+
+    def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:
+        self._pending_close_tasks.remove(aw)
+        return result
+
     @staticmethod
     def _get_file(slot_: FeedSlot) -> IO[bytes]:
         assert slot_.file
@@ -652,7 +673,7 @@ class FeedExporter:
                 uri_params = self._get_uri_params(
                     spider, self.feeds[slot.uri_template]["uri_params"], slot
                 )
-                self._pending_close_coros.append(self._close_slot(slot, spider))
+                self._schedule_slot_close(slot, spider)
                 slots.append(
                     self._start_new_batch(
                         batch_id=slot.batch_id + 1,
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 80ff6229b..4b0962c43 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase):
         header = MyItem.fields.keys()
         await self.assertExported(items, header, rows, settings=settings)
 
+    @coroutine_test
+    async def test_batch_delivered_when_full(self):
+        """Full batches must be finalized and delivered as soon as they are
+        full, instead of when the spider closes."""
+        dir_path = self._random_temp_filename()
+        batch1_path = Path(dir_path, "1.json")
+        mockserver_url = self.mockserver.url("/")
+        batch1_contents: list[bytes | None] = []
+
+        class TestSpider(scrapy.Spider):
+            name = "testspider"
+            start_urls = [mockserver_url]
+
+            def parse(self, response):
+                yield {"foo": "bar1"}
+                yield {"foo": "bar2"}
+                yield scrapy.Request(
+                    mockserver_url, callback=self.parse2, dont_filter=True
+                )
+
+            def parse2(self, response):
+                # the first batch was full after the second item, so it must
+                # have been delivered by now
+                batch1_contents.append(
+                    batch1_path.read_bytes() if batch1_path.exists() else None
+                )
+                yield {"foo": "bar3"}
+
+        settings = {
+            "FEEDS": {
+                build_url(dir_path / "%(batch_id)d.json"): {"format": "json"},
+            },
+            "FEED_EXPORT_BATCH_ITEM_COUNT": 2,
+        }
+        crawler = get_crawler(TestSpider, settings)
+        await crawler.crawl_async()
+
+        assert batch1_contents, "the second request was not processed"
+        assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"
+        assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]
+
     def test_wrong_path(self):
         """If path is without %(batch_time)s and %(batch_id) an exception must be raised"""
         settings = {

From 5b4828a012fcd136a8f46915e19be07a5029e57e Mon Sep 17 00:00:00 2001
From: Laerte Pereira <5853172+Laerte@users.noreply.github.com>
Date: Thu, 30 Jul 2026 03:02:54 -0300
Subject: [PATCH 57/62] docs(practices): Remove scrapoxy mention (#7817)

---
 docs/topics/practices.rst | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 23738c98c..dfa1e21f6 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -533,8 +533,7 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * if possible, use `Common Crawl`_ to fetch pages, instead of hitting the sites
   directly
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
-  services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
-  super proxy that you can attach your own proxies to.
+  services like `ProxyMesh`_.
 * for HTTPS websites, if blocking appears related to TLS behavior, consider
   adjusting the :setting:`DOWNLOAD_TLS_MIN_VERSION` and
   :setting:`DOWNLOAD_TLS_MAX_VERSION` settings, since some websites may respond
@@ -559,5 +558,4 @@ projects that detects common mistakes and anti-patterns.
 .. _ProxyMesh: https://proxymesh.com/
 .. _Common Crawl: https://commoncrawl.org/
 .. _testspiders: https://github.com/scrapinghub/testspiders
-.. _scrapoxy: https://scrapoxy.io/
 .. _Zyte API: https://docs.zyte.com/zyte-api/get-started.html

From 98696efa809ddde93f78cabd555db832f270553f Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 30 Jul 2026 12:46:45 +0200
Subject: [PATCH 58/62] Export item fields in declaration order (#7824)

---
 docs/topics/exporters.rst | 10 ++++++++++
 scrapy/exporters.py       | 17 ++++++++++++++++-
 tests/test_exporters.py   | 12 ++++++++++++
 tests/test_feedexport.py  | 12 ++++++------
 4 files changed, 44 insertions(+), 7 deletions(-)

diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index ecd154122..c43b7e20f 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -211,6 +211,16 @@ BaseItemExporter
 
       -   ``None`` (all fields [2]_, default)
 
+          Fields are exported in declaration order, i.e. the order in which
+          they are defined in the :ref:`item class <item-types>`. For
+          :class:`dict` items, which have no declared fields, the key order of
+          each item is used instead.
+
+          .. versionchanged:: VERSION
+             Fields of non-\ :class:`dict` items used to be exported in the
+             order in which they had been populated, except in
+             :class:`CsvItemExporter`, which has always used declaration order.
+
       -   A list of fields:
 
           .. code-block:: python
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 5a11df833..ea600d1a8 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -74,6 +74,17 @@ class BaseItemExporter(ABC):
     def finish_exporting(self) -> None:  # noqa: B027
         pass
 
+    @staticmethod
+    def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]:
+        """Return the populated field names of *adapter*, in declaration order.
+
+        Populated fields that are not declared, which some item types allow,
+        come last, in item order.
+        """
+        populated = set(adapter.keys())
+        declared = (name for name in adapter.field_names() if name in populated)
+        return dict.fromkeys([*declared, *adapter.keys()])
+
     def _get_serialized_fields(
         self, item: Any, default_value: Any = None, include_empty: bool | None = None
     ) -> Iterable[tuple[str, Any]]:
@@ -86,7 +97,11 @@ class BaseItemExporter(ABC):
             include_empty = self.export_empty_fields
 
         if self.fields_to_export is None:
-            field_iter = item.field_names() if include_empty else item.keys()
+            field_iter = (
+                item.field_names()
+                if include_empty
+                else self._get_populated_field_names(item)
+            )
         elif isinstance(self.fields_to_export, Mapping):
             if include_empty:
                 field_iter = self.fields_to_export.items()
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 3b997767e..b857728ba 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC):
         ie = self._get_exporter(fields_to_export={"name": "名稱"})
         assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
 
+    def test_field_order(self):
+        item = self.item_class(age="22", name="John\xa3")
+        ie = self._get_exporter()
+        assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]
+
+    def test_field_order_dict_item(self):
+        ie = self._get_exporter()
+        assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]
+        assert [
+            name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})
+        ] == ["age", "name"]
+
     def test_field_custom_serializer(self):
         i = self.custom_field_item_class(name="John\xa3", age="22")
         a = ItemAdapter(i)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 92c414bef..1cca287af 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -675,14 +675,14 @@ class TestFeedExport(TestFeedExportBase):
 
         formats = {
             "csv": b"foo,egg,baz\r\nbar1,spam1,\r\n",
-            "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+            "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]',
             "jsonlines": (
-                b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n'
+                b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n'
             ),
             "xml": (
                 b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
-                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
-                b"world2</hello><foo>bar2</foo></item>\n<item><hello>world3"
+                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
+                b"bar2</foo><hello>world2</hello></item>\n<item><hello>world3"
                 b"</hello><egg>spam3</egg></item>\n</items>"
             ),
         }
@@ -740,8 +740,8 @@ class TestFeedExport(TestFeedExportBase):
             "json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
             "xml": (
                 b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
-                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
-                b"world2</hello><foo>bar2</foo></item>\n</items>"
+                b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
+                b"bar2</foo><hello>world2</hello></item>\n</items>"
             ),
             "jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n',
         }

From 433603e6cab4ccea6aee18843deaa03f61e216b8 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 30 Jul 2026 15:51:45 +0200
Subject: [PATCH 59/62] Add AWS_MAX_POOL_CONNECTIONS (#7794)

---
 docs/topics/feed-exports.rst        |  5 +++--
 docs/topics/media-pipeline.rst      |  3 +++
 docs/topics/settings.rst            | 20 +++++++++++++++++
 scrapy/extensions/feedexport.py     | 11 +++++++++
 scrapy/pipelines/files.py           | 13 ++++++++++-
 scrapy/settings/default_settings.py |  2 ++
 scrapy/utils/boto.py                | 15 +++++++++++++
 tests/test_feedexport_storages.py   | 35 +++++++++++++++++++++++++++++
 tests/test_pipeline_files.py        | 27 ++++++++++++++++++++++
 9 files changed, 128 insertions(+), 3 deletions(-)

diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 66768c97b..2f686fd0f 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -218,12 +218,13 @@ passed through the following settings:
 
 .. _temporary security credentials: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html
 
-You can also define a custom ACL, custom endpoint, and region name for exported
-feeds using these settings:
+You can also define a custom ACL, custom endpoint, region name and connection
+pool size for exported feeds using these settings:
 
 -   :setting:`FEED_STORAGE_S3_ACL`
 -   :setting:`AWS_ENDPOINT_URL`
 -   :setting:`AWS_REGION_NAME`
+-   :setting:`AWS_MAX_POOL_CONNECTIONS`
 
 The default value for the ``overwrite`` key in the :setting:`FEEDS` for this
 storage backend is: ``True``.
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 4ceb4732a..b16066d0c 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -268,6 +268,9 @@ For self-hosting you also might feel the need not to use SSL and not to verify S
     AWS_USE_SSL = False  # or True (None by default)
     AWS_VERIFY = False  # or True (None by default)
 
+To reuse connections for as many files as you check or upload in parallel, set
+:setting:`AWS_MAX_POOL_CONNECTIONS` accordingly.
+
 .. _canned ACLs: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
 .. _Minio: https://github.com/minio/minio
 .. _Zenko CloudServer: https://www.zenko.io/cloudserver/
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 8068824e3..1b6851d04 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -458,6 +458,26 @@ Default: ``None``
 
 Endpoint URL used for S3-like storage, for example Minio or s3.scality.
 
+.. setting:: AWS_MAX_POOL_CONNECTIONS
+
+AWS_MAX_POOL_CONNECTIONS
+------------------------
+
+.. versionadded:: VERSION
+
+Default: ``None``
+
+Maximum number of connections that AWS clients, such as those of the
+:ref:`S3 feed storage backend <topics-feed-storage-s3>` and of the
+:ref:`S3 media pipeline storage backend <media-pipelines-s3>`, keep in their
+connection pool.
+
+If ``None``, the value of :setting:`REACTOR_THREADPOOL_MAXSIZE` is used.
+
+Values lower than the number of parallel AWS calls do not limit those calls, but
+their connections are closed instead of reused, which hurts performance, and
+``Connection pool is full, discarding connection`` warnings are logged.
+
 .. setting:: AWS_REGION_NAME
 
 AWS_REGION_NAME
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index c2997921d..448279546 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -28,6 +28,7 @@ from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.asyncio import is_asyncio_available, run_in_thread
+from scrapy.utils.boto import _get_max_pool_connections
 from scrapy.utils.conf import feed_complete_default_values_from_settings
 from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
 from scrapy.utils.ftp import ftp_store_file
@@ -213,11 +214,14 @@ class S3FeedStorage(BlockingFeedStorage):
         feed_options: dict[str, Any] | None = None,
         session_token: str | None = None,
         region_name: str | None = None,
+        max_pool_connections: int | None = None,
     ):
         try:
             import boto3.session  # noqa: PLC0415
         except ImportError:
             raise NotConfigured("missing boto3 library") from None
+        from botocore.config import Config  # noqa: PLC0415
+
         u = urlparse(uri)
         assert u.hostname
         self.bucketname: str = u.hostname
@@ -228,6 +232,7 @@ class S3FeedStorage(BlockingFeedStorage):
         self.acl: str | None = acl
         self.endpoint_url: str | None = endpoint_url
         self.region_name: str | None = region_name
+        self.max_pool_connections: int | None = max_pool_connections
 
         boto3_session = boto3.session.Session()
         self.s3_client = boto3_session.client(
@@ -237,6 +242,11 @@ class S3FeedStorage(BlockingFeedStorage):
             aws_session_token=self.session_token,
             endpoint_url=self.endpoint_url,
             region_name=self.region_name,
+            config=(
+                Config(max_pool_connections=self.max_pool_connections)
+                if self.max_pool_connections is not None
+                else None
+            ),
         )
 
         if feed_options and feed_options.get("overwrite", True) is False:
@@ -262,6 +272,7 @@ class S3FeedStorage(BlockingFeedStorage):
             acl=crawler.settings["FEED_STORAGE_S3_ACL"] or None,
             endpoint_url=crawler.settings["AWS_ENDPOINT_URL"] or None,
             region_name=crawler.settings["AWS_REGION_NAME"] or None,
+            max_pool_connections=_get_max_pool_connections(crawler.settings),
             feed_options=feed_options,
         )
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 8e8082332..55a3676e5 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -37,7 +37,7 @@ from scrapy.pipelines.media import (
     _MediaRequestFiltered,
 )
 from scrapy.utils.asyncio import run_in_thread
-from scrapy.utils.boto import is_botocore_available
+from scrapy.utils.boto import _get_max_pool_connections, is_botocore_available
 from scrapy.utils.datatypes import CaseInsensitiveDict
 from scrapy.utils.defer import deferred_from_coro, ensure_awaitable
 from scrapy.utils.ftp import ftp_store_file
@@ -164,6 +164,9 @@ class S3FilesStore:
     AWS_REGION_NAME = None
     AWS_USE_SSL = None
     AWS_VERIFY = None
+    # Overridden from settings.AWS_MAX_POOL_CONNECTIONS in
+    # FilesPipeline.from_crawler(); None means the botocore default
+    AWS_MAX_POOL_CONNECTIONS: int | None = None
 
     POLICY = "private"  # Overridden from settings.FILES_STORE_S3_ACL in FilesPipeline.from_crawler()
     HEADERS: ClassVar[dict[str, str]] = {
@@ -174,7 +177,13 @@ class S3FilesStore:
         if not is_botocore_available():
             raise NotConfigured("missing botocore library")
         import botocore.session  # noqa: PLC0415
+        from botocore.config import Config  # noqa: PLC0415
 
+        config = (
+            Config(max_pool_connections=self.AWS_MAX_POOL_CONNECTIONS)
+            if self.AWS_MAX_POOL_CONNECTIONS is not None
+            else None
+        )
         session = botocore.session.get_session()
         self.s3_client = session.create_client(
             "s3",
@@ -185,6 +194,7 @@ class S3FilesStore:
             region_name=self.AWS_REGION_NAME,
             use_ssl=self.AWS_USE_SSL,
             verify=self.AWS_VERIFY,
+            config=config,
         )
         if not uri.startswith("s3://"):
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
@@ -522,6 +532,7 @@ class FilesPipeline(MediaPipeline):
         s3store.AWS_REGION_NAME = settings["AWS_REGION_NAME"]
         s3store.AWS_USE_SSL = settings["AWS_USE_SSL"]
         s3store.AWS_VERIFY = settings["AWS_VERIFY"]
+        s3store.AWS_MAX_POOL_CONNECTIONS = _get_max_pool_connections(settings)
         s3store.POLICY = settings["FILES_STORE_S3_ACL"]
 
         gcs_store: type[GCSFilesStore] = cast(
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 993f2436d..a44b36c8a 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -28,6 +28,7 @@ __all__ = [
     "AUTOTHROTTLE_TARGET_CONCURRENCY",
     "AWS_ACCESS_KEY_ID",
     "AWS_ENDPOINT_URL",
+    "AWS_MAX_POOL_CONNECTIONS",
     "AWS_REGION_NAME",
     "AWS_SECRET_ACCESS_KEY",
     "AWS_SESSION_TOKEN",
@@ -229,6 +230,7 @@ AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
 AWS_ACCESS_KEY_ID = None
 AWS_SECRET_ACCESS_KEY = None
 AWS_ENDPOINT_URL = None
+AWS_MAX_POOL_CONNECTIONS = None
 AWS_REGION_NAME = None
 AWS_SESSION_TOKEN = None
 AWS_USE_SSL = None
diff --git a/scrapy/utils/boto.py b/scrapy/utils/boto.py
index 2a77ee2ac..76ee0e7ec 100644
--- a/scrapy/utils/boto.py
+++ b/scrapy/utils/boto.py
@@ -1,7 +1,22 @@
 """Boto/botocore helpers"""
 
+from __future__ import annotations
+
 from importlib.util import find_spec
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from scrapy.settings import BaseSettings
 
 
 def is_botocore_available() -> bool:
     return find_spec("botocore") is not None
+
+
+def _get_max_pool_connections(settings: BaseSettings) -> int:
+    """Return the maximum number of connections that AWS clients may keep in
+    their connection pool.
+    """
+    return settings.getint("AWS_MAX_POOL_CONNECTIONS") or settings.getint(
+        "REACTOR_THREADPOOL_MAXSIZE"
+    )
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 66488540f..b1e3787fd 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -381,6 +381,41 @@ class TestS3FeedStorage:
         assert storage.region_name == region_name
         assert storage.s3_client._client_config.region_name == region_name
 
+    def test_init_without_max_pool_connections(self) -> None:
+        storage = S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
+        assert storage.max_pool_connections is None
+        config: Any = storage.s3_client.meta.config
+        assert config.max_pool_connections == 10
+
+    def test_init_with_max_pool_connections(self) -> None:
+        storage = S3FeedStorage(
+            "s3://mybucket/export.csv",
+            "access_key",
+            "secret_key",
+            max_pool_connections=30,
+        )
+        assert storage.max_pool_connections == 30
+        config: Any = storage.s3_client.meta.config
+        assert config.max_pool_connections == 30
+
+    @pytest.mark.parametrize(
+        ("settings", "expected"),
+        [
+            ({}, 10),
+            ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20),
+            ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30),
+            ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30),
+        ],
+    )
+    def test_from_crawler_max_pool_connections(
+        self, settings: dict[str, Any], expected: int
+    ) -> None:
+        crawler = get_crawler(settings_dict=settings)
+        storage = S3FeedStorage.from_crawler(crawler, "s3://mybucket/export.csv")
+        assert storage.max_pool_connections == expected
+        config: Any = storage.s3_client.meta.config
+        assert config.max_pool_connections == expected
+
     @coroutine_test
     async def test_store_without_acl(self):
         storage = S3FeedStorage(
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 97daa514e..4e7fb118b 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -895,6 +895,33 @@ class TestS3FilesStore:
 
             stub.assert_no_pending_responses()
 
+    def test_default_max_pool_connections(self) -> None:
+        store = S3FilesStore("s3://mybucket/prefix/")
+        config: Any = store.s3_client.meta.config
+        assert config.max_pool_connections == 10
+
+    @pytest.mark.parametrize(
+        ("settings", "expected"),
+        [
+            ({}, 10),
+            ({"REACTOR_THREADPOOL_MAXSIZE": 20}, 20),
+            ({"AWS_MAX_POOL_CONNECTIONS": 30}, 30),
+            ({"AWS_MAX_POOL_CONNECTIONS": 30, "REACTOR_THREADPOOL_MAXSIZE": 20}, 30),
+        ],
+    )
+    def test_max_pool_connections(
+        self, monkeypatch: pytest.MonkeyPatch, settings: dict[str, Any], expected: int
+    ) -> None:
+        # restores the value that FilesPipeline.from_crawler() sets on the class
+        monkeypatch.setattr(S3FilesStore, "AWS_MAX_POOL_CONNECTIONS", None)
+        crawler = get_crawler(
+            settings_dict={"FILES_STORE": "s3://mybucket/prefix/", **settings}
+        )
+        store = FilesPipeline.from_crawler(crawler).store
+        assert isinstance(store, S3FilesStore)
+        config: Any = store.s3_client.meta.config
+        assert config.max_pool_connections == expected
+
 
 class TestGCSFilesStore:
     @staticmethod

From f02a99fe71dd0ff2fde1ef7a533cd61904fa1c9f Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 30 Jul 2026 17:15:15 +0200
Subject: [PATCH 60/62] Add doc sections for callbacks and errbacks (#7821)

---
 docs/faq.rst                     |   2 +-
 docs/intro/tutorial.rst          |   2 +-
 docs/topics/coroutines.rst       |   4 +-
 docs/topics/jobs.rst             |  10 +-
 docs/topics/request-response.rst | 480 ++++++++++++++++++++-----------
 docs/topics/spiders.rst          |  67 ++---
 scrapy/http/request/__init__.py  |   8 +-
 scrapy/spiders/__init__.py       |  16 ++
 8 files changed, 372 insertions(+), 217 deletions(-)

diff --git a/docs/faq.rst b/docs/faq.rst
index 0446a6868..1a574e5da 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -97,7 +97,7 @@ handler documentation.
 How can I scrape an item with attributes in different pages?
 ------------------------------------------------------------
 
-See :ref:`topics-request-response-ref-request-callback-arguments`.
+See :ref:`callback-data`.
 
 How can I simulate a user login in my spider?
 ---------------------------------------------
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index c4e04364b..eaf492c95 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -769,7 +769,7 @@ crawlers on top of it.
 
 Also, a common pattern is to build an item with data from more than one page,
 using a :ref:`trick to pass additional data to the callbacks
-<topics-request-response-ref-request-callback-arguments>`.
+<callback-data>`.
 
 
 Using spider arguments
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 9dcd9d69c..b7ddb0a57 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -21,7 +21,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
     .. versionadded:: 2.13
 
--   :class:`~scrapy.Request` callbacks.
+-   :class:`~scrapy.Request` :ref:`callbacks <callbacks>`, which may
+    also be defined as :term:`asynchronous generators <asynchronous
+    generator>`.
 
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
diff --git a/docs/topics/jobs.rst b/docs/topics/jobs.rst
index c9916110d..dcff10772 100644
--- a/docs/topics/jobs.rst
+++ b/docs/topics/jobs.rst
@@ -96,9 +96,13 @@ Request serialization
 ---------------------
 
 For persistence to work, :class:`~scrapy.Request` objects must be
-serializable with :mod:`pickle`, except for the ``callback`` and ``errback``
-values passed to their ``__init__`` method, which must be methods of the
-running :class:`~scrapy.Spider` class.
+serializable with :mod:`pickle`, except for the :ref:`callback
+<callbacks>` and :ref:`errback
+<errbacks>` values passed to their ``__init__``
+method, which must be methods of the running :class:`~scrapy.Spider` class.
+
+Requests that cannot be serialized are kept in memory only: they are still
+sent, but they are lost when the crawl is paused.
 
 If you wish to log the requests that couldn't be serialized, you can set the
 :setting:`SCHEDULER_DEBUG` setting to ``True`` in the project's settings page.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 8e565907f..75158440b 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -205,10 +205,11 @@ Request objects
         Request metadata can also be accessed through the
         :attr:`~scrapy.http.Response.meta` attribute of a response.
 
-        To pass data from one spider callback to another, consider using
-        :attr:`cb_kwargs` instead. However, request metadata may be the right
-        choice in certain scenarios, such as to maintain some debugging data
-        across all follow-up requests (e.g. the source URL).
+        To pass your own data from one spider callback to another, use
+        :attr:`cb_kwargs` instead, see :ref:`callback-data`. However, request
+        metadata may be the right choice in certain scenarios, such as to
+        maintain some debugging data across all follow-up requests (e.g. the
+        source URL).
 
         A common use of request metadata is to define request-specific
         parameters for Scrapy components (extensions, middlewares, etc.). For
@@ -248,7 +249,7 @@ Request objects
     .. method:: Request.copy()
 
        Return a new Request which is a copy of this Request. See also:
-       :ref:`topics-request-response-ref-request-callback-arguments`.
+       :ref:`callback-data`.
 
     .. method:: Request.replace([url, method, headers, body, cookies, meta, flags, encoding, priority, dont_filter, callback, errback, cb_kwargs, cls])
 
@@ -256,7 +257,7 @@ Request objects
        given new values by whichever keyword arguments are specified. The
        :attr:`~scrapy.Request.cb_kwargs` and :attr:`~scrapy.Request.meta` attributes are shallow
        copied by default (unless new values are given as arguments). See also
-       :ref:`topics-request-response-ref-request-callback-arguments`.
+       :ref:`callback-data`.
 
     .. automethod:: from_curl
 
@@ -347,160 +348,6 @@ Other functions related to requests
 .. autofunction:: scrapy.utils.httpobj.urlparse_cached
 
 
-.. _topics-request-response-ref-request-callback-arguments:
-
-Passing additional data to callback functions
----------------------------------------------
-
-The callback of a request is a function that will be called when the response
-of that request is downloaded. The callback function will be called with the
-downloaded :class:`Response` object as its first argument.
-
-Example:
-
-.. code-block:: python
-
-    def parse_page1(self, response):
-        return scrapy.Request(
-            "http://www.example.com/some_page.html", callback=self.parse_page2
-        )
-
-
-    def parse_page2(self, response):
-        # this would log http://www.example.com/some_page.html
-        self.logger.info("Visited %s", response.url)
-
-In some cases you may be interested in passing arguments to those callback
-functions so you can receive the arguments later, in the second callback.
-The following example shows how to achieve this by using the
-:attr:`.Request.cb_kwargs` attribute:
-
-.. code-block:: python
-
-    def parse(self, response):
-        request = scrapy.Request(
-            "http://www.example.com/index.html",
-            callback=self.parse_page2,
-            cb_kwargs=dict(main_url=response.url),
-        )
-        request.cb_kwargs["foo"] = "bar"  # add more arguments for the callback
-        yield request
-
-
-    def parse_page2(self, response, main_url, foo):
-        yield dict(
-            main_url=main_url,
-            other_url=response.url,
-            foo=foo,
-        )
-
-.. caution:: :attr:`.Request.cb_kwargs` was introduced in version ``1.7``.
-   Prior to that, using :attr:`.Request.meta` was recommended for passing
-   information around callbacks. After ``1.7``, :attr:`.Request.cb_kwargs`
-   became the preferred way for handling user information, leaving :attr:`.Request.meta`
-   for communication with components like middlewares and extensions.
-
-.. _topics-request-response-ref-errbacks:
-
-Using errbacks to catch exceptions in request processing
---------------------------------------------------------
-
-The errback of a request is a function that will be called when an exception
-is raise while processing it.
-
-It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
-be used to track connection establishment timeouts, DNS errors etc.
-
-Here's an example spider logging all errors and catching some specific
-errors if needed:
-
-.. code-block:: python
-
-    import scrapy
-
-    from scrapy.spidermiddlewares.httperror import HttpError
-    from twisted.internet.error import DNSLookupError
-    from twisted.internet.error import TimeoutError, TCPTimedOutError
-
-
-    class ErrbackSpider(scrapy.Spider):
-        name = "errback_example"
-        start_urls = [
-            "http://www.httpbin.org/",  # HTTP 200 expected
-            "http://www.httpbin.org/status/404",  # Not found error
-            "http://www.httpbin.org/status/500",  # server issue
-            "http://www.httpbin.org:12345/",  # non-responding host, timeout expected
-            "https://example.invalid/",  # DNS error expected
-        ]
-
-        async def start(self):
-            for u in self.start_urls:
-                yield scrapy.Request(
-                    u,
-                    callback=self.parse_httpbin,
-                    errback=self.errback_httpbin,
-                    dont_filter=True,
-                )
-
-        def parse_httpbin(self, response):
-            self.logger.info(f"Got successful response from {response.url}")
-            # do something useful here...
-
-        def errback_httpbin(self, failure):
-            # log all failures
-            self.logger.error(repr(failure))
-
-            # in case you want to do something special for some errors,
-            # you may need the failure's type:
-
-            if failure.check(HttpError):
-                # these exceptions come from HttpError spider middleware
-                # you can get the non-200 response
-                response = failure.value.response
-                self.logger.error("HttpError on %s", response.url)
-
-            elif failure.check(DNSLookupError):
-                # this is the original request
-                request = failure.request
-                self.logger.error("DNSLookupError on %s", request.url)
-
-            elif failure.check(TimeoutError, TCPTimedOutError):
-                request = failure.request
-                self.logger.error("TimeoutError on %s", request.url)
-
-
-.. _errback-cb_kwargs:
-
-Accessing additional data in errback functions
-----------------------------------------------
-
-In case of a failure to process the request, you may be interested in
-accessing arguments to the callback functions so you can process further
-based on the arguments in the errback. The following example shows how to
-achieve this by using ``Failure.request.cb_kwargs``:
-
-.. code-block:: python
-
-    def parse(self, response):
-        request = scrapy.Request(
-            "http://www.example.com/index.html",
-            callback=self.parse_page2,
-            errback=self.errback_page2,
-            cb_kwargs=dict(main_url=response.url),
-        )
-        yield request
-
-
-    def parse_page2(self, response, main_url):
-        pass
-
-
-    def errback_page2(self, failure):
-        yield dict(
-            main_url=failure.request.cb_kwargs["main_url"],
-        )
-
-
 .. _request-fingerprints:
 
 Request fingerprints
@@ -702,6 +549,319 @@ The following built-in Scrapy components have such restrictions:
     45-character-long keys must be supported.
 
 
+.. _callbacks:
+
+Callbacks
+=========
+
+A callback is a function that Scrapy calls with the :class:`Response` of a
+:class:`~scrapy.Request` once that request has been downloaded, so that you can
+extract data from that response and generate additional requests to continue
+the crawl:
+
+.. code-block:: python
+
+    from scrapy import Request, Spider
+
+
+    class BookSpider(Spider):
+        name = "books"
+
+        async def start(self):
+            yield Request("https://books.toscrape.com/", callback=self.parse_home)
+
+        def parse_home(self, response):
+            for url in response.css("h3 a::attr(href)").getall():
+                yield Request(response.urljoin(url), callback=self.parse_book)
+
+        def parse_book(self, response):
+            yield {"title": response.css("h1::text").get()}
+
+Requests may also define an :ref:`errback <errbacks>`, which Scrapy calls
+instead of the callback when an exception is raised while processing the
+request or its response, e.g. a connection error or, by default, a non-2xx
+response.
+
+
+.. _callback-assignment:
+
+Assigning a callback to a request
+---------------------------------
+
+To assign a callback to a request, use the ``callback`` parameter of
+:class:`~scrapy.Request`, which sets the :attr:`.Request.callback` attribute:
+
+.. code-block:: python
+
+    from scrapy import Request
+
+
+    def parse_home(response): ...
+
+
+    request = Request("https://books.toscrape.com/", callback=parse_home)
+
+Requests with no callback, i.e. with :attr:`~scrapy.Request.callback` set to
+``None``, are handled by the :meth:`~scrapy.Spider.parse` method of the spider:
+
+.. code-block:: python
+
+    request = Request("https://books.toscrape.com/")  # Handled by parse()
+
+If a request is never meant to reach a spider callback, e.g. because a
+:ref:`component <topics-components>` sends it and handles its response itself,
+assign the special :func:`~scrapy.http.request.NO_CALLBACK` value to it
+instead, so that :ref:`downloader middlewares <topics-downloader-middleware>`
+can tell such requests apart.
+
+While :attr:`~scrapy.Request.callback` only accepts callables, some spider
+classes let you also define a callback by name: both :attr:`CrawlSpider.rules
+<scrapy.spiders.CrawlSpider.rules>` and :attr:`SitemapSpider.sitemap_rules
+<scrapy.spiders.SitemapSpider.sitemap_rules>` accept the name of a spider
+method as a string.
+
+
+.. _writing-callbacks:
+
+Writing a callback
+------------------
+
+Any callable can be a callback, as long as it takes the response as its first
+positional parameter, and any :ref:`additional callback data <callback-data>`
+as keyword parameters. Spider methods are the most common choice, but plain
+functions, lambda expressions and other callable objects work as well.
+
+.. note:: If you enable :ref:`job persistence <topics-jobs>` through the
+    :setting:`JOBDIR` setting, callbacks must be methods of the running spider.
+    Requests with any other callback cannot be serialized, so they are kept in
+    memory only and lost when you pause the crawl. See
+    :ref:`request-serialization`.
+
+A callback can be:
+
+-   A regular function:
+
+    .. code-block:: python
+
+        def parse(self, response):
+            return {"url": response.url}
+
+-   A generator function:
+
+    .. code-block:: python
+
+        def parse(self, response):
+            yield {"url": response.url}
+
+-   A coroutine function, i.e. defined with ``async def``:
+
+    .. code-block:: python
+
+        async def parse(self, response):
+            return {"url": response.url}
+
+-   An asynchronous generator function:
+
+    .. code-block:: python
+
+        async def parse(self, response):
+            yield {"url": response.url}
+
+The last two allow using ``await``, ``async for`` and ``async with`` in your
+callback. See :ref:`topics-coroutines`.
+
+
+.. _callback-output:
+
+Callback output
+---------------
+
+A callback may return or yield any of the following:
+
+-   ``None``, which does nothing.
+
+    Callbacks that produce no output at all, e.g. callbacks that only log
+    information about the response, are perfectly valid. ``None`` values within
+    an iterable of callback output are ignored as well.
+
+-   A :class:`~scrapy.Request` object, which Scrapy schedules, downloads and
+    eventually sends to its own callback.
+
+-   An :ref:`item object <topics-items>`, which Scrapy sends to the
+    :ref:`item pipelines <topics-item-pipeline>`.
+
+    Any object that is neither ``None`` nor a :class:`~scrapy.Request` object
+    is treated as an item.
+
+-   An iterable of any of the values above, e.g. a list or, more commonly, a
+    generator.
+
+    :term:`Asynchronous iterables <asynchronous iterable>`, e.g. an
+    :term:`asynchronous generator`, are also supported.
+
+.. note:: When a callback *returns* an object, Scrapy iterates that object if
+    it supports iteration, except for :class:`dict`, :class:`~scrapy.Item`,
+    :class:`str` and :class:`bytes` objects, which are always handled as single
+    items.
+
+.. note:: In a generator callback, a ``return`` statement with a value does not
+    produce any output, since such a value is not part of what the generator
+    yields. Scrapy logs a warning when it detects such a callback, see
+    :setting:`WARN_ON_GENERATOR_RETURN_VALUE`.
+
+Before Scrapy acts on the output of a callback, that output goes through the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of your :ref:`spider middlewares <topics-spider-middleware>`, which may modify
+it or drop part of it.
+
+If a callback raises an exception, the :attr:`~scrapy.Request.errback` of the
+request is *not* called. The exception goes through the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+method of your spider middlewares instead and, unless one of them handles it,
+Scrapy logs it and sends the :signal:`spider_error` signal.
+
+
+.. _callback-data:
+.. _topics-request-response-ref-request-callback-arguments:
+
+Passing additional data to callback functions
+---------------------------------------------
+
+In some cases you may be interested in passing data to a callback in addition
+to the response, e.g. data extracted from the response that triggered the
+request. The following example shows how to achieve this by using the
+:attr:`.Request.cb_kwargs` attribute:
+
+.. code-block:: python
+
+    from scrapy import Request
+
+
+    def parse(self, response):
+        request = Request(
+            "http://www.example.com/index.html",
+            callback=self.parse_page2,
+            cb_kwargs=dict(main_url=response.url),
+        )
+        request.cb_kwargs["foo"] = "bar"  # add more arguments for the callback
+        yield request
+
+
+    def parse_page2(self, response, main_url, foo):
+        yield dict(
+            main_url=main_url,
+            other_url=response.url,
+            foo=foo,
+        )
+
+:attr:`.Request.cb_kwargs` is the recommended way to pass your own data to a
+callback. Use :attr:`.Request.meta` only for data aimed at :ref:`components
+<topics-components>`, such as middlewares and extensions.
+
+.. _errbacks:
+.. _topics-request-response-ref-errbacks:
+
+Errbacks
+========
+
+The errback of a request is a function that will be called when an exception
+is raise while processing it.
+
+It receives a :exc:`~twisted.python.failure.Failure` as first parameter and can
+be used to track connection establishment timeouts, DNS errors etc.
+
+Here's an example spider logging all errors and catching some specific
+errors if needed:
+
+.. code-block:: python
+
+    from scrapy import Request, Spider
+    from scrapy.spidermiddlewares.httperror import HttpError
+    from twisted.internet.error import DNSLookupError
+    from twisted.internet.error import TimeoutError, TCPTimedOutError
+
+
+    class ErrbackSpider(Spider):
+        name = "errback_example"
+        start_urls = [
+            "http://www.httpbin.org/",  # HTTP 200 expected
+            "http://www.httpbin.org/status/404",  # Not found error
+            "http://www.httpbin.org/status/500",  # server issue
+            "http://www.httpbin.org:12345/",  # non-responding host, timeout expected
+            "https://example.invalid/",  # DNS error expected
+        ]
+
+        async def start(self):
+            for u in self.start_urls:
+                yield Request(
+                    u,
+                    callback=self.parse_httpbin,
+                    errback=self.errback_httpbin,
+                    dont_filter=True,
+                )
+
+        def parse_httpbin(self, response):
+            self.logger.info(f"Got successful response from {response.url}")
+            # do something useful here...
+
+        def errback_httpbin(self, failure):
+            # log all failures
+            self.logger.error(repr(failure))
+
+            # in case you want to do something special for some errors,
+            # you may need the failure's type:
+
+            if failure.check(HttpError):
+                # these exceptions come from HttpError spider middleware
+                # you can get the non-200 response
+                response = failure.value.response
+                self.logger.error("HttpError on %s", response.url)
+
+            elif failure.check(DNSLookupError):
+                # this is the original request
+                request = failure.request
+                self.logger.error("DNSLookupError on %s", request.url)
+
+            elif failure.check(TimeoutError, TCPTimedOutError):
+                request = failure.request
+                self.logger.error("TimeoutError on %s", request.url)
+
+
+.. _errback-cb_kwargs:
+
+Accessing additional data in errback functions
+----------------------------------------------
+
+In case of a failure to process the request, you may be interested in
+accessing arguments to the callback functions so you can process further
+based on the arguments in the errback. The following example shows how to
+achieve this by using ``Failure.request.cb_kwargs``:
+
+.. code-block:: python
+
+    from scrapy import Request
+
+
+    def parse(self, response):
+        request = Request(
+            "http://www.example.com/index.html",
+            callback=self.parse_page2,
+            errback=self.errback_page2,
+            cb_kwargs=dict(main_url=response.url),
+        )
+        yield request
+
+
+    def parse_page2(self, response, main_url):
+        pass
+
+
+    def errback_page2(self, failure):
+        yield dict(
+            main_url=failure.request.cb_kwargs["main_url"],
+        )
+
+
 .. _topics-request-meta:
 
 Request.meta special keys
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index 69ff08fa1..8fbf0c52d 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -4,43 +4,31 @@
 Spiders
 =======
 
-Spiders are classes which define how a certain site (or a group of sites) will be
-scraped, including how to perform the crawl (i.e. follow links) and how to
-extract structured data from their pages (i.e. scraping items). In other words,
-Spiders are the place where you define the custom behaviour for crawling and
-parsing pages for a particular site (or, in some cases, a group of sites).
+Spiders are classes that define how a site, or a group of sites, is scraped:
+which requests to send, and how to parse their responses to extract data and to
+send additional requests.
 
-For spiders, the scraping cycle goes through something like this:
+A crawl goes as follows:
 
-1. You start by generating the initial requests to crawl the first URLs, and
-   specify a callback function to be called with the response downloaded from
-   those requests.
+1.  Scrapy iterates the :meth:`~scrapy.Spider.start` method of the spider to
+    get the initial requests. By default, that method yields a
+    :class:`~scrapy.Request` object for each URL in
+    :attr:`~scrapy.Spider.start_urls`, with :meth:`~scrapy.Spider.parse` as
+    :ref:`callback <callbacks>`.
 
-   The first requests to perform are obtained by iterating the
-   :meth:`~scrapy.Spider.start` method, which by default yields a
-   :class:`~scrapy.Request` object for each URL in the
-   :attr:`~scrapy.Spider.start_urls` spider attribute, with the
-   :attr:`~scrapy.Spider.parse` method set as :attr:`~scrapy.Request.callback`
-   function to handle each :class:`~scrapy.http.Response`.
+2.  Scrapy downloads each request and calls its callback with the resulting
+    :class:`~scrapy.http.Response`.
 
-2. In the callback function, you parse the response (web page) and return
-   :ref:`item objects <topics-items>`,
-   :class:`~scrapy.Request` objects, or an iterable of these objects.
-   Those Requests will also contain a callback (maybe
-   the same) and will then be downloaded by Scrapy and then their
-   response handled by the specified callback.
+3.  Callbacks parse the response, typically using :ref:`topics-selectors`, and
+    return or yield :ref:`item objects <topics-items>` with the extracted data
+    and :class:`~scrapy.Request` objects to continue the crawl, which go back
+    to step 2. See :ref:`callback-output`.
 
-3. In callback functions, you parse the page contents, typically using
-   :ref:`topics-selectors` (but you can also use BeautifulSoup, lxml or whatever
-   mechanism you prefer) and generate items with the parsed data.
+4.  Items go through :ref:`item pipelines <topics-item-pipeline>`, and are
+    usually stored through :ref:`topics-feed-exports`.
 
-4. Finally, the items returned from the spider will be typically persisted to a
-   database (in some :ref:`Item Pipeline <topics-item-pipeline>`) or written to
-   a file using :ref:`topics-feed-exports`.
-
-Even though this cycle applies (more or less) to any kind of spider, there are
-different kinds of default spiders bundled into Scrapy for different purposes.
-We will talk about those types here.
+Scrapy includes different spider classes for different purposes, described
+below.
 
 .. _topics-spiders-ref:
 
@@ -191,22 +179,7 @@ scrapy.Spider
 
    .. automethod:: start
 
-   .. method:: parse(response)
-
-       This is the default callback used by Scrapy to process downloaded
-       responses, when their requests don't specify a callback.
-
-       The ``parse`` method is in charge of processing the response and returning
-       scraped data and/or more URLs to follow. Other Requests callbacks have
-       the same requirements as the :class:`~scrapy.Spider` class.
-
-       This method, as well as any other Request callback, must return a
-       :class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
-       iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
-       <topics-items>`, or ``None``.
-
-       :param response: the response to parse
-       :type response: :class:`~scrapy.http.Response`
+   .. automethod:: parse
 
    .. method:: closed(reason)
 
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 68847283a..7d67bb6d7 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -169,7 +169,8 @@ class Request(object_ref):
         #:
         #: The callable must expect the response as its first parameter, and
         #: support any additional keyword arguments set through
-        #: :attr:`cb_kwargs`.
+        #: :attr:`cb_kwargs`. See :ref:`writing-callbacks` and
+        #: :ref:`callback-output`.
         #:
         #: In addition to an arbitrary callable, the following values are also
         #: supported:
@@ -190,8 +191,7 @@ class Request(object_ref):
         #:     raises exceptions for non-2xx responses by default, sending them
         #:     to the :attr:`errback` instead.
         #:
-        #: .. seealso::
-        #:     :ref:`topics-request-response-ref-request-callback-arguments`
+        #: .. seealso:: :ref:`callbacks`
         self.callback: CallbackT | None = callback
 
         #: :class:`~collections.abc.Callable` to handle exceptions raised
@@ -200,7 +200,7 @@ class Request(object_ref):
         #: The callable must expect a :exc:`~twisted.python.failure.Failure` as
         #: its first parameter.
         #:
-        #: .. seealso:: :ref:`topics-request-response-ref-errbacks`
+        #: .. seealso:: :ref:`errbacks`
         self.errback: Callable[[Failure], Any] | None = errback
 
         self._cookies: CookiesT | None = cookies or None
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 02dfa2ac6..6244e3264 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -143,6 +143,22 @@ class Spider(object_ref):
     else:
 
         def parse(self, response: Response, **kwargs: Any) -> Any:
+            """Process *response*, i.e. extract data from it and generate new
+            requests.
+
+            This is the default :ref:`callback <callbacks>`: Scrapy uses
+            it for the response to any request that does not define a
+            :attr:`~scrapy.Request.callback`, such as the requests that
+            :meth:`start` yields by default.
+
+            Any :attr:`~scrapy.Request.cb_kwargs` of the request are passed as
+            keyword parameters.
+
+            Spiders must define this method, unless every request that they
+            send defines a callback.
+
+            See :ref:`callback-output` about the supported return values.
+            """
             raise NotImplementedError(
                 f"{self.__class__.__name__}.parse callback is not defined"
             )

From 3fc7148c5ec7537c000af27b705e33d797429a3a Mon Sep 17 00:00:00 2001
From: Janit Rajkarnikar <108281535+aniJani@users.noreply.github.com>
Date: Thu, 30 Jul 2026 10:18:45 -0500
Subject: [PATCH 61/62] Don't evaluate annotations when inspecting signatures
 (#7818)

Since Python 3.14 (PEP 649) annotations are evaluated lazily, so
inspect.signature() raises NameError for callables whose annotations
reference names imported only under TYPE_CHECKING. This broke
middleware registration (via argument_is_required()) and custom stats
collectors (via _warn_spider_arg) for user code with such annotations.

Use annotation_format=Format.FORWARDREF on 3.14+: parameter names,
kinds and defaults are unchanged, and unresolvable annotations become
ForwardRef proxies instead of raising.

Resolves #7796.
---
 scrapy/utils/decorators.py     |  3 ++-
 scrapy/utils/python.py         | 21 ++++++++++++++++++++-
 tests/test_utils_decorators.py | 30 ++++++++++++++++++++++++++++--
 tests/test_utils_python.py     | 21 ++++++++++++++++++++-
 4 files changed, 70 insertions(+), 5 deletions(-)

diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index 4960dc27a..2924c81f9 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -10,6 +10,7 @@ from twisted.internet.defer import Deferred, maybeDeferred
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncio import run_in_thread
 from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.python import _signature
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Callable, Coroutine
@@ -109,7 +110,7 @@ def _warn_spider_arg(
 ):
     """Decorator to warn if a ``spider`` argument is passed to a function."""
 
-    sig = inspect.signature(func)
+    sig = _signature(func)
 
     def check_args(*args: _P.args, **kwargs: _P.kwargs) -> None:
         bound = sig.bind(*args, **kwargs)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 8a7517c1d..40fc05257 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -178,11 +178,30 @@ def binary_is_text(data: bytes) -> bool:
     return all(c not in _BINARYCHARS for c in data)
 
 
+# PEP 649 (Python 3.14+) made annotation evaluation lazy, so inspect.signature()
+# can raise NameError for names imported only under TYPE_CHECKING. We only need
+# parameter names, kinds and defaults, so leave such annotations as ForwardRefs.
+if sys.version_info >= (3, 14):
+    from annotationlib import Format
+
+    def _signature(func: Callable[..., Any]) -> inspect.Signature:
+        return inspect.signature(func, annotation_format=Format.FORWARDREF)
+
+else:
+
+    def _signature(func: Callable[..., Any]) -> inspect.Signature:
+        return inspect.signature(func)
+
+
 def get_func_args_dict(
     func: Callable[..., Any], stripself: bool = False
 ) -> Mapping[str, inspect.Parameter]:
     """Return the argument dict of a callable object.
 
+    Annotations are not evaluated, so on Python 3.14 and later the ``annotation``
+    attribute of the returned parameters may be a ``ForwardRef`` instead of the
+    resolved type.
+
     .. versionadded:: 2.14
     """
     if not callable(func):
@@ -190,7 +209,7 @@ def get_func_args_dict(
 
     args: Mapping[str, inspect.Parameter]
     try:
-        sig = inspect.signature(func)
+        sig = _signature(func)
     except ValueError:
         return {}
 
diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.py
index 807294a57..4c29d2917 100644
--- a/tests/test_utils_decorators.py
+++ b/tests/test_utils_decorators.py
@@ -1,7 +1,8 @@
 from __future__ import annotations
 
+import sys
 import warnings
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 import pytest
 from twisted.internet.defer import Deferred
@@ -12,7 +13,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncGenerator
+    from collections.abc import AsyncGenerator, Callable
 
 
 class TestDeprecated:
@@ -77,6 +78,31 @@ class TestWarnSpiderArg:
         ):
             assert parse("response", spider="spider") == "response"
 
+    @pytest.mark.skipif(
+        sys.version_info < (3, 14),
+        reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)",
+    )
+    def test_sync_warns_with_unresolvable_annotations(self):
+        # dont_inherit=True, or the module's future import stringizes the annotations
+        namespace: dict[str, Any] = {}
+        exec(  # pylint: disable=exec-used
+            compile(
+                "def parse(response: OnlyAtTypeCheckingTime,"
+                " spider: OnlyAtTypeCheckingTime | None = None): return response",
+                "<test>",
+                "exec",
+                dont_inherit=True,
+            ),
+            namespace,
+        )
+        parse_func: Callable[..., str] = namespace["parse"]
+        parse = _warn_spider_arg(parse_func)
+
+        with pytest.warns(
+            ScrapyDeprecationWarning, match=r"Passing a 'spider' argument"
+        ):
+            assert parse("response", spider="spider") == "response"
+
     def test_sync_no_warning_without_spider_arg(self):
         @_warn_spider_arg
         def parse(response: str, spider: str | None = None) -> str:
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index c3b5dfc99..099b5ccc2 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -4,7 +4,7 @@ import functools
 import operator
 import platform
 import sys
-from typing import TYPE_CHECKING, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
 
 import pytest
 
@@ -190,6 +190,25 @@ def test_get_func_args():
         ]
 
 
+@pytest.mark.skipif(
+    sys.version_info < (3, 14),
+    reason="annotations are only lazily evaluated since Python 3.14 (PEP 649)",
+)
+def test_get_func_args_unresolvable_annotations():
+    # dont_inherit=True, or the module's future import stringizes the annotations
+    namespace: dict[str, Any] = {}
+    exec(  # pylint: disable=exec-used
+        compile(
+            "def f(a: OnlyAtTypeCheckingTime, b: int = 1) -> OnlyAtTypeCheckingTime: pass",
+            "<test>",
+            "exec",
+            dont_inherit=True,
+        ),
+        namespace,
+    )
+    assert get_func_args(namespace["f"]) == ["a", "b"]
+
+
 @pytest.mark.parametrize(
     ("value", "expected"),
     [

From 7436afc95f521482b1b1473c88f6b5ab1d430bea Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 30 Jul 2026 17:27:27 +0200
Subject: [PATCH 62/62] Improve test coverage for scrapy.extensions (#7809)

---
 scrapy/extensions/closespider.py             |  2 +-
 scrapy/extensions/feedexport.py              |  4 +-
 scrapy/extensions/memusage.py                |  2 +-
 scrapy/extensions/periodic_log.py            |  3 +-
 tests/test_downloadermiddleware_httpcache.py | 71 ++++++++++++++++++++
 tests/test_extension_memusage.py             | 66 +++++++++++++++++-
 tests/test_extension_periodic_log.py         | 36 ++++++++++
 tests/test_extension_telnet.py               | 55 ++++++++++++++-
 tests/test_feedexport.py                     | 26 +++++++
 tests/test_feedexport_postprocess.py         | 10 +++
 tests/test_feedexport_storages.py            | 15 +++++
 11 files changed, 278 insertions(+), 12 deletions(-)

diff --git a/scrapy/extensions/closespider.py b/scrapy/extensions/closespider.py
index a4362b182..9cb792e30 100644
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@@ -119,7 +119,7 @@ class CloseSpider:
             self.task = None
 
         if self.task_no_item:
-            if self.task_no_item.running:
+            if self.task_no_item.running:  # pragma: no branch
                 self.task_no_item.stop()
             self.task_no_item = None
 
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 448279546..118462bc9 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -465,7 +465,7 @@ class FeedSlot:
         )
 
     def finish_exporting(self) -> None:
-        if self._exporting:
+        if self._exporting:  # pragma: no branch
             assert self.exporter
             self.exporter.finish_exporting()
             self._exporting = False
@@ -553,7 +553,7 @@ class FeedExporter:
         for slot in self.slots:
             self._schedule_slot_close(slot, spider)
 
-        if self._pending_close_tasks:
+        if self._pending_close_tasks:  # pragma: no branch
             if is_asyncio_available():
                 await asyncio.wait(
                     cast("list[asyncio.Task[None]]", list(self._pending_close_tasks))
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 1444c8941..e0e289ce8 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -94,7 +94,7 @@ class MemoryUsage:
 
     def engine_stopped(self) -> None:
         for tsk in self.tasks:
-            if tsk.running:
+            if tsk.running:  # pragma: no branch
                 tsk.stop()
 
     def update(self) -> None:
diff --git a/scrapy/extensions/periodic_log.py b/scrapy/extensions/periodic_log.py
index cd35c8165..cbcc8b70e 100644
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@@ -38,7 +38,6 @@ class PeriodicLog:
     ):
         self.stats: StatsCollector = stats
         self.interval: float = interval
-        self.multiplier: float = 60.0 / self.interval
         self.task: AsyncioLoopingCall | LoopingCall | None = None
         self.encoder: JSONEncoder = ScrapyJSONEncoder(sort_keys=True, indent=4)
         self.ext_stats_enabled: bool = bool(ext_stats)
@@ -165,5 +164,5 @@ class PeriodicLog:
 
     def spider_closed(self, spider: Spider, reason: str) -> None:
         self.log()
-        if self.task and self.task.running:
+        if self.task and self.task.running:  # pragma: no branch
             self.task.stop()
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index ce56ee11d..6e8486eb8 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -14,6 +14,7 @@ import pytest
 
 from scrapy.downloadermiddlewares.httpcache import HttpCacheMiddleware
 from scrapy.exceptions import IgnoreRequest
+from scrapy.extensions.httpcache import DummyPolicy
 from scrapy.http import HtmlResponse, Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
@@ -24,6 +25,14 @@ if TYPE_CHECKING:
     from scrapy.crawler import Crawler
 
 
+class AlwaysStalePolicy(DummyPolicy):
+    """:class:`~scrapy.extensions.httpcache.DummyPolicy` that always
+    revalidates cached responses."""
+
+    def is_cached_response_fresh(self, cachedresponse, request):
+        return False
+
+
 class TestBase:
     """Base class with common setup and helper methods."""
 
@@ -282,6 +291,21 @@ class DummyPolicyTestMixin(PolicyTestMixin):
             self.assertEqualResponse(self.response, response)
             assert "cached" in response.flags
 
+    def test_revalidation_keeps_cached_response(self):
+        # The dummy policy considers every cached response valid, so a policy
+        # that subclasses it to force revalidation always gets the cached
+        # response back, whatever the new response is.
+        with self._middleware(HTTPCACHE_POLICY=AlwaysStalePolicy) as mw:
+            assert mw.process_request(self.request) is None
+            mw.process_response(self.request, self.response)
+
+            assert mw.process_request(self.request) is None
+            fresh_response = self.response.replace(body=b"new body")
+            response = mw.process_response(self.request, fresh_response)
+            self.assertEqualResponse(self.response, response)
+            assert "cached" in response.flags
+            assert mw.stats.get_value("httpcache/revalidate") == 1
+
 
 class RFC2616PolicyTestMixin(PolicyTestMixin):
     """Mixin containing RFC2616 policy specific test methods."""
@@ -553,6 +577,53 @@ class RFC2616PolicyTestMixin(PolicyTestMixin):
                 else:
                     assert "cached" in res5.flags
 
+    def test_middleware_ignore_schemes(self):
+        # file responses are not cached by default
+        req = Request("file:///tmp/t.txt")
+        res = Response(req.url, headers={"Expires": self.tomorrow})
+        with self._middleware() as mw:
+            assert mw.process_request(req) is None
+            mw.process_response(req, res)
+
+            assert mw.storage.retrieve_response(mw.crawler.spider, req) is None
+            assert mw.process_request(req) is None
+
+    def test_max_stale_with_value(self):
+        # A response that expired one day ago.
+        headers = {"Date": self.yesterday, "Expires": self.yesterday}
+        with self._middleware() as mw:
+            req0 = Request("http://example.com")
+            res0 = Response(req0.url, headers=headers)
+            self._process_requestresponse(mw, req0, res0)
+
+            # max-stale greater than the staleness of the cached response
+            req1 = req0.replace(headers={"Cache-Control": "max-stale=172800"})
+            res1 = mw.process_request(req1)
+            assert isinstance(res1, Response)
+            assert "cached" in res1.flags
+
+            # max-stale lower than the staleness of the cached response
+            req2 = req0.replace(headers={"Cache-Control": "max-stale=60"})
+            assert mw.process_request(req2) is None
+
+            # a non-integer max-stale value is ignored
+            req3 = req0.replace(headers={"Cache-Control": "max-stale=soon"})
+            assert mw.process_request(req3) is None
+
+    def test_response_dated_in_the_future(self):
+        # A Date header ahead of the local clock must not make the cached
+        # response look aged.
+        headers = {"Date": self.tomorrow, "Cache-Control": "max-age=10"}
+        with self._middleware() as mw:
+            req0 = Request("http://example.com")
+            res0 = Response(req0.url, headers=headers)
+            res1 = self._process_requestresponse(mw, req0, res0)
+            assert "cached" not in res1.flags
+
+            res2 = self._process_requestresponse(mw, req0, None)
+            self.assertEqualResponse(res1, res2)
+            assert "cached" in res2.flags
+
     def test_process_exception(self):
         with self._middleware() as mw:
             res0 = Response(self.request.url, headers={"Expires": self.yesterday})
diff --git a/tests/test_extension_memusage.py b/tests/test_extension_memusage.py
index a474725d8..76e8ca5d6 100644
--- a/tests/test_extension_memusage.py
+++ b/tests/test_extension_memusage.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import logging
 import sys
+from typing import TYPE_CHECKING
 
 import pytest
 
@@ -13,8 +14,12 @@ from scrapy.extensions.memusage import MemoryUsage
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.utils import OneShotLoop
+from tests.utils.cmdline import proc
 from tests.utils.decorators import coroutine_test
 
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
+
 # MemoryUsage relies on the stdlib 'resource' module (not available on Windows)
 pytestmark = pytest.mark.skipif(
     sys.platform.startswith("win"),
@@ -25,6 +30,14 @@ pytestmark = pytest.mark.skipif(
 MB = 1024 * 1024
 
 
+class TwoShotLoop(OneShotLoop):
+    """Like :class:`OneShotLoop`, but runs the check twice."""
+
+    def start(self, interval: float, now: bool = True) -> None:
+        super().start(interval, now=now)
+        self.func()
+
+
 class _LoopSpider(Spider):
     name = "loop-data-spider"
 
@@ -50,6 +63,49 @@ def test_memusage_disabled() -> None:
         MemoryUsage.from_crawler(get_crawler(settings_dict=settings))
 
 
+def test_memusage_limit_stops_crawler_without_spider(mockserver: MockServer) -> None:
+    # The Scrapy shell starts the engine without opening a spider, so the
+    # whole crawler is stopped instead of a spider being closed.
+    _, out, err = proc(
+        "shell",
+        mockserver.url("/text"),
+        "-c",
+        "response.status",
+        "--set",
+        "MEMUSAGE_LIMIT_MB=1",
+    )
+    assert "Memory usage exceeded 1MiB" in err
+    assert "200" in out
+
+
+@coroutine_test
+async def test_memusage_below_thresholds_logs_peak(
+    caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    settings = {
+        "MEMUSAGE_LIMIT_MB": 100,
+        "MEMUSAGE_WARNING_MB": 50,
+        "MEMUSAGE_CHECK_INTERVAL_SECONDS": 0.01,
+        "TELNETCONSOLE_ENABLED": False,
+        "LOG_LEVEL": "INFO",
+    }
+
+    monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
+    monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda _: 25 * MB)
+
+    crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
+
+    with caplog.at_level(logging.INFO, logger="scrapy.extensions.memusage"):
+        await crawler.crawl_async(url="data:,", loops=1)
+
+    assert crawler.stats
+    assert crawler.stats.get_value("memusage/limit_reached") is None
+    assert crawler.stats.get_value("memusage/warning_reached") is None
+    assert crawler.stats.get_value("memusage/max") == 25 * MB
+    assert crawler.stats.get_value("finish_reason") == "finished"
+    assert any("Peak memory usage is 25MiB" in r.getMessage() for r in caplog.records)
+
+
 @coroutine_test
 async def test_memusage_limit_closes_spider_with_reason_and_error_log(
     caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
@@ -92,8 +148,9 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
         "LOG_LEVEL": "INFO",
     }
 
-    # Avoid background LoopingCall that can log after the test finishes.
-    monkeypatch.setattr(memusage_mod, "create_looping_call", OneShotLoop)
+    # Avoid background LoopingCall that can log after the test finishes; check
+    # twice, since the warning is only meant to be reported once.
+    monkeypatch.setattr(memusage_mod, "create_looping_call", TwoShotLoop)
     monkeypatch.setattr(MemoryUsage, "get_virtual_size", lambda self: 75 * MB)
 
     crawler = get_crawler(spidercls=_LoopSpider, settings_dict=settings)
@@ -112,4 +169,7 @@ async def test_memusage_warning_logs_but_allows_normal_finish(
     assert crawler.stats
     assert crawler.stats.get_value("memusage/warning_reached") == 1
     assert crawler.stats.get_value("finish_reason") == "finished"
-    assert any("memory usage reached" in r.getMessage().lower() for r in caplog.records)
+    warnings_logged = [
+        r for r in caplog.records if "memory usage reached" in r.getMessage().lower()
+    ]
+    assert len(warnings_logged) == 1
diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py
index ffe7a0dc7..723fb6e17 100644
--- a/tests/test_extension_periodic_log.py
+++ b/tests/test_extension_periodic_log.py
@@ -1,8 +1,13 @@
 from __future__ import annotations
 
 import datetime
+import json
+import logging
 from typing import TYPE_CHECKING, Any
 
+import pytest
+
+from scrapy.exceptions import NotConfigured
 from scrapy.extensions.periodic_log import PeriodicLog
 from scrapy.utils.test import get_crawler
 
@@ -86,6 +91,14 @@ class TestPeriodicLog:
         assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
         assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
 
+    def test_no_interval(self):
+        with pytest.raises(NotConfigured):
+            extension({"PERIODIC_LOG_STATS": True, "LOGSTATS_INTERVAL": 0})
+
+    def test_nothing_enabled(self):
+        with pytest.raises(NotConfigured):
+            extension({"LOGSTATS_INTERVAL": 60})
+
     @coroutine_test
     async def test_log_delta(self):
         def emulate(
@@ -212,3 +225,26 @@ class TestPeriodicLog:
             {"PERIODIC_LOG_STATS": {"include": ["downloader/"], "exclude": ["bytes"]}},
             lambda k, v: "downloader/" in k and "bytes" not in k,
         )
+
+    @coroutine_test
+    async def test_log_timing(self, caplog: pytest.LogCaptureFixture) -> None:
+        settings = {
+            "EXTENSIONS": {"scrapy.extensions.periodic_log.PeriodicLog": 0},
+            "PERIODIC_LOG_TIMING_ENABLED": True,
+            "LOGSTATS_INTERVAL": 30,
+        }
+        crawler = get_crawler(MetaSpider, settings)
+        with caplog.at_level(logging.INFO, logger="scrapy.extensions.periodic_log"):
+            await crawler.crawl_async()
+
+        records = [
+            r for r in caplog.records if r.name == "scrapy.extensions.periodic_log"
+        ]
+        assert records, "PeriodicLog logged nothing"
+        # Only the timing section is enabled, and it is logged on spider close.
+        data = json.loads(records[-1].getMessage())
+        assert list(data) == ["time"]
+        assert data["time"]["log_interval"] == 30
+        assert data["time"]["log_interval_real"] >= 0
+        assert data["time"]["elapsed"] >= 0
+        assert data["time"]["start_time"] <= data["time"]["utcnow"]
diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py
index 20c801558..fca0e3153 100644
--- a/tests/test_extension_telnet.py
+++ b/tests/test_extension_telnet.py
@@ -7,7 +7,8 @@ import pytest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
 
-from scrapy.extensions.telnet import TelnetConsole
+from scrapy import Spider
+from scrapy.extensions.telnet import TelnetConsole, update_telnet_vars
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.utils.decorators import coroutine_test
@@ -16,16 +17,20 @@ if TYPE_CHECKING:
     from collections.abc import Generator
 
     from scrapy.crawler import Crawler
+    from scrapy.http import Response
 
 pytestmark = pytest.mark.requires_reactor  # TelnetConsole requires a reactor
 
 
-def _get_crawler(settings_dict: dict[str, Any] | None = None) -> Crawler:
+def _get_crawler(
+    spidercls: type[Spider] | None = None,
+    settings_dict: dict[str, Any] | None = None,
+) -> Crawler:
     settings = {
         "TELNETCONSOLE_ENABLED": True,
         **(settings_dict or {}),
     }
-    return get_crawler(settings_dict=settings)
+    return get_crawler(spidercls, settings_dict=settings)
 
 
 @contextmanager
@@ -84,3 +89,47 @@ def test_invalid_reversed_portrange() -> None:
     console = TelnetConsole(_get_crawler(settings_dict=settings))
     with pytest.raises(ValueError, match=r"invalid portrange: \[2, 1\]"):
         console.start_listening()
+
+
+@coroutine_test
+async def test_telnet_vars() -> None:
+    """Log into the console of a running crawl, which is when the telnet
+    variables are built."""
+    received: list[dict[str, Any]] = []
+
+    def on_update_telnet_vars(telnet_vars: dict[str, Any]) -> None:
+        received.append(telnet_vars)
+
+    class TelnetSpider(Spider):
+        name = "telnet"
+        start_urls = ["data:,"]
+
+        async def parse(self, response: Response) -> None:
+            assert self.crawler.extensions
+            console = next(
+                ext
+                for ext in self.crawler.extensions.middlewares
+                if isinstance(ext, TelnetConsole)
+            )
+            creds = credentials.UsernamePassword(
+                console.username.encode("utf8"), console.password.encode("utf8")
+            )
+            portal = console.protocol().protocolArgs[0]
+            await maybe_deferred_to_future(portal.login(creds, None, ITelnetProtocol))
+
+    crawler = _get_crawler(TelnetSpider)
+    crawler.signals.connect(on_update_telnet_vars, signal=update_telnet_vars)
+    await crawler.crawl_async()
+
+    assert len(received) == 1
+    telnet_vars = received[0]
+    assert telnet_vars["crawler"] is crawler
+    assert telnet_vars["engine"] is crawler.engine
+    assert telnet_vars["spider"] is crawler.spider
+    assert telnet_vars["extensions"] is crawler.extensions
+    assert telnet_vars["stats"] is crawler.stats
+    assert telnet_vars["settings"] is crawler.settings
+    assert callable(telnet_vars["est"])
+    assert callable(telnet_vars["p"])
+    assert callable(telnet_vars["prefs"])
+    assert "telnetconsole.html" in telnet_vars["help"]
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 1cca287af..40a763efd 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -24,6 +24,7 @@ from scrapy.extensions.feedexport import (
     FeedExporter,
     FeedSlot,
     FileFeedStorage,
+    ItemFilter,
     apply_uri_params,
 )
 from scrapy.utils.python import to_unicode
@@ -1289,6 +1290,13 @@ class TestFeedExporterSignals:
         assert self.feed_exporter_closed_received
 
 
+class TestItemFilter:
+    def test_no_feed_options(self):
+        item_filter = ItemFilter(None)
+        assert item_filter.item_classes == ()
+        assert item_filter.accepts(MyItem({"foo": "bar"}))
+
+
 class TestFeedExportInit:
     def test_unsupported_storage(self):
         settings = {
@@ -1300,6 +1308,24 @@ class TestFeedExportInit:
         with pytest.raises(NotConfigured):
             FeedExporter.from_crawler(crawler)
 
+    def test_disabled_storage(self, caplog: pytest.LogCaptureFixture):
+        class DisabledFeedStorage:
+            def __init__(self, uri, *, feed_options=None):
+                raise NotConfigured("not today")
+
+        settings = {
+            "FEED_STORAGES": {"disabled": DisabledFeedStorage},
+            "FEEDS": {
+                "disabled://uri": {},
+            },
+        }
+        crawler = get_crawler(settings_dict=settings)
+        with caplog.at_level(logging.ERROR), pytest.raises(NotConfigured):
+            FeedExporter.from_crawler(crawler)
+        assert (
+            "Disabled feed storage scheme: disabled. Reason: not today" in caplog.text
+        )
+
     def test_unsupported_format(self):
         settings = {
             "FEEDS": {
diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.py
index f120ce36f..36d8586ce 100644
--- a/tests/test_feedexport_postprocess.py
+++ b/tests/test_feedexport_postprocess.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
 
 import pytest
 
+from scrapy.extensions.postprocessing import PostProcessingManager
 from scrapy.utils.test import get_crawler
 from tests.utils.bases.feedexport import TestFeedExportBase
 from tests.utils.decorators import coroutine_test
@@ -87,6 +88,15 @@ class TestFeedPostProcessedExports(TestFeedExportBase):
         data_stream.seek(0)
         return data_stream.read()
 
+    def test_tell_reports_target_file_position(self):
+        """Exporters that wrap the file they get, e.g. through
+        :class:`io.TextIOWrapper`, need it to report a position."""
+        file = BytesIO()
+        manager = PostProcessingManager([self.MyPlugin1], file, {})
+        assert manager.tell() == 0
+        manager.write(b"foo")
+        assert manager.tell() == file.tell() == 3
+
     @coroutine_test
     async def test_gzip_plugin(self):
         filename = self._named_tempfile("gzip_file")
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index b1e3787fd..4d28872b7 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -3,6 +3,7 @@ from __future__ import annotations
 import logging
 import os
 import string
+import sys
 import tempfile
 from io import BytesIO
 from pathlib import Path
@@ -14,6 +15,7 @@ import pytest
 from w3lib.url import path_to_file_uri
 
 import scrapy
+from scrapy.exceptions import NotConfigured
 from scrapy.extensions.feedexport import (
     BlockingFeedStorage,
     FileFeedStorage,
@@ -166,6 +168,12 @@ class TestFTPFeedStorage:
         st = FTPFeedStorage(f"ftp://foo:{pw_quoted}@example.com/some_path", {})
         assert st.password == string.punctuation
 
+    def test_uri_without_hostname(self):
+        with pytest.raises(
+            ValueError, match="Got a storage URI without a hostname: ftp:///some_path"
+        ):
+            FTPFeedStorage("ftp:///some_path")
+
 
 class MyBlockingFeedStorage(BlockingFeedStorage):
     def _store_in_thread(self, file: IO[bytes]) -> None:
@@ -205,6 +213,13 @@ class TestBlockingFeedStorage:
             b.open(spider=spider)
 
 
+def test_s3_without_boto3(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setitem(sys.modules, "boto3", None)
+    monkeypatch.setitem(sys.modules, "boto3.session", None)
+    with pytest.raises(NotConfigured, match="missing boto3 library"):
+        S3FeedStorage("s3://mybucket/export.csv", "access_key", "secret_key")
+
+
 @pytest.mark.requires_boto3
 class TestS3FeedStorage:
     def test_parse_credentials(self):