From 14f49ab63c54007a8914706454ba69db4199845e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 3 May 2026 14:11:55 +0500
Subject: [PATCH 01/36] Remove deprecated test utils.

---
 conftest.py              |   3 --
 pyproject.toml           |   1 -
 scrapy/utils/test.py     | 103 ---------------------------------------
 scrapy/utils/testproc.py |  78 -----------------------------
 scrapy/utils/testsite.py |  65 ------------------------
 5 files changed, 250 deletions(-)
 delete mode 100644 scrapy/utils/testproc.py
 delete mode 100644 scrapy/utils/testsite.py

diff --git a/conftest.py b/conftest.py
index d49901a7c..4f8d32e1b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -23,9 +23,6 @@ def _py_files(folder):
 collect_ignore = [
     # may need extra deps
     "docs/_ext",
-    # not a test, but looks like a test
-    "scrapy/utils/testproc.py",
-    "scrapy/utils/testsite.py",
     # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
     *_py_files("tests/AsyncCrawlerProcess"),
     # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
diff --git a/pyproject.toml b/pyproject.toml
index d96790873..d35b333ff 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -127,7 +127,6 @@ ignore_errors = true
 module = [
     "scrapy.core.downloader.webclient",
     "scrapy.spiders.init",
-    "scrapy.utils.testsite",
     "tests.test_webclient",
 ]
 allow_any_generics = true
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 5c69dc76d..b4e20c3c6 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -7,20 +7,14 @@ from __future__ import annotations
 import asyncio
 import os
 import warnings
-from ftplib import FTP
 from importlib import import_module
 from pathlib import Path
-from posixpath import split
 from typing import TYPE_CHECKING, Any, TypeVar, cast
-from unittest import mock
 
-from twisted.trial.unittest import SkipTest
 from twisted.web.client import Agent
 
 from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed
 from scrapy.utils.spider import DefaultSpider
 
@@ -37,80 +31,6 @@ if TYPE_CHECKING:
 _T = TypeVar("_T")
 
 
-def assert_gcs_environ() -> None:  # pragma: no cover
-    warnings.warn(
-        "The assert_gcs_environ() function is deprecated and will be removed in a future version of Scrapy."
-        " Check GCS_PROJECT_ID directly.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    if "GCS_PROJECT_ID" not in os.environ:
-        raise SkipTest("GCS_PROJECT_ID not found")
-
-
-def skip_if_no_boto() -> None:  # pragma: no cover
-    warnings.warn(
-        "The skip_if_no_boto() function is deprecated and will be removed in a future version of Scrapy."
-        " Check scrapy.utils.boto.is_botocore_available() directly.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    if not is_botocore_available():
-        raise SkipTest("missing botocore library")
-
-
-def get_gcs_content_and_delete(
-    bucket: Any, path: str
-) -> tuple[bytes, list[dict[str, str]], Any]:  # pragma: no cover
-    from google.cloud import storage  # noqa: PLC0415
-
-    warnings.warn(
-        "The get_gcs_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    client = storage.Client(project=os.environ.get("GCS_PROJECT_ID"))
-    bucket = client.get_bucket(bucket)
-    blob = bucket.get_blob(path)
-    content = blob.download_as_string()
-    acl = list(blob.acl)  # loads acl before it will be deleted
-    bucket.delete_blob(path)
-    return content, acl, blob
-
-
-def get_ftp_content_and_delete(
-    path: str,
-    host: str,
-    port: int,
-    username: str,
-    password: str,
-    use_active_mode: bool = False,
-) -> bytes:  # pragma: no cover
-    warnings.warn(
-        "The get_ftp_content_and_delete() function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    ftp = FTP()
-    ftp.connect(host, port)
-    ftp.login(username, password)
-    if use_active_mode:
-        ftp.set_pasv(False)
-    ftp_data: list[bytes] = []
-
-    def buffer_data(data: bytes) -> None:
-        ftp_data.append(data)
-
-    ftp.retrbinary(f"RETR {path}", buffer_data)
-    dirname, filename = split(path)
-    ftp.cwd(dirname)
-    ftp.delete(filename)
-    return b"".join(ftp_data)
-
-
-TestSpider = create_deprecated_class("TestSpider", DefaultSpider)
-
-
 def get_reactor_settings() -> dict[str, Any]:
     """Return a settings dict that works with the installed reactor.
 
@@ -185,29 +105,6 @@ def get_from_asyncio_queue(value: _T) -> Awaitable[_T]:
     return getter
 
 
-def mock_google_cloud_storage() -> tuple[Any, Any, Any]:  # pragma: no cover
-    """Creates autospec mocks for google-cloud-storage Client, Bucket and Blob
-    classes and set their proper return values.
-    """
-    from google.cloud.storage import Blob, Bucket, Client  # noqa: PLC0415
-
-    warnings.warn(
-        "The mock_google_cloud_storage() function is deprecated and will be removed in a future version of Scrapy.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-
-    client_mock = mock.create_autospec(Client)
-
-    bucket_mock = mock.create_autospec(Bucket)
-    client_mock.get_bucket.return_value = bucket_mock
-
-    blob_mock = mock.create_autospec(Blob)
-    bucket_mock.blob.return_value = blob_mock
-
-    return (client_mock, bucket_mock, blob_mock)
-
-
 def get_web_client_agent_req(url: str) -> Deferred[TxResponse]:  # pragma: no cover
     warnings.warn(
         "The get_web_client_agent_req() function is deprecated"
diff --git a/scrapy/utils/testproc.py b/scrapy/utils/testproc.py
deleted file mode 100644
index 7d548524a..000000000
--- a/scrapy/utils/testproc.py
+++ /dev/null
@@ -1,78 +0,0 @@
-# pragma: no file cover
-from __future__ import annotations
-
-import os
-import sys
-import warnings
-from typing import TYPE_CHECKING, ClassVar, cast
-
-from twisted.internet.defer import Deferred
-from twisted.internet.protocol import ProcessProtocol
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-if TYPE_CHECKING:
-    from collections.abc import Iterable
-
-    from twisted.internet.error import ProcessTerminated
-    from twisted.python.failure import Failure
-
-
-warnings.warn(
-    "The scrapy.utils.testproc module is deprecated.",
-    ScrapyDeprecationWarning,
-    stacklevel=2,
-)
-
-
-class ProcessTest:
-    command: str | None = None
-    prefix: ClassVar[list[str]] = [sys.executable, "-m", "scrapy.cmdline"]
-    cwd = os.getcwd()  # trial chdirs to temp dir  # noqa: PTH109
-
-    def execute(
-        self,
-        args: Iterable[str],
-        check_code: bool = True,
-        settings: str | None = None,
-    ) -> Deferred[TestProcessProtocol]:
-        from twisted.internet import reactor
-
-        env = os.environ.copy()
-        if settings is not None:
-            env["SCRAPY_SETTINGS_MODULE"] = settings
-        assert self.command
-        cmd = [*self.prefix, self.command, *args]
-        pp = TestProcessProtocol()
-        pp.deferred.addCallback(self._process_finished, cmd, check_code)
-        reactor.spawnProcess(pp, cmd[0], cmd, env=env, path=self.cwd)
-        return pp.deferred
-
-    def _process_finished(
-        self, pp: TestProcessProtocol, cmd: list[str], check_code: bool
-    ) -> tuple[int, bytes, bytes]:
-        if pp.exitcode and check_code:
-            msg = f"process {cmd} exit with code {pp.exitcode}"
-            msg += f"\n>>> stdout <<<\n{pp.out.decode()}"
-            msg += "\n"
-            msg += f"\n>>> stderr <<<\n{pp.err.decode()}"
-            raise RuntimeError(msg)
-        return cast("int", pp.exitcode), pp.out, pp.err
-
-
-class TestProcessProtocol(ProcessProtocol):
-    def __init__(self) -> None:
-        self.deferred: Deferred[TestProcessProtocol] = Deferred()
-        self.out: bytes = b""
-        self.err: bytes = b""
-        self.exitcode: int | None = None
-
-    def outReceived(self, data: bytes) -> None:
-        self.out += data
-
-    def errReceived(self, data: bytes) -> None:
-        self.err += data
-
-    def processEnded(self, status: Failure) -> None:
-        self.exitcode = cast("ProcessTerminated", status.value).exitCode
-        self.deferred.callback(self)
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
deleted file mode 100644
index 4089e86ed..000000000
--- a/scrapy/utils/testsite.py
+++ /dev/null
@@ -1,65 +0,0 @@
-# pragma: no file cover
-import warnings
-from urllib.parse import urljoin
-
-from twisted.web import resource, server, static, util
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-warnings.warn(
-    "The scrapy.utils.testsite module is deprecated.",
-    ScrapyDeprecationWarning,
-    stacklevel=2,
-)
-
-
-class SiteTest:
-    def setUp(self):
-        from twisted.internet import reactor
-
-        super().setUp()
-        self.site = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
-        self.baseurl = f"http://localhost:{self.site.getHost().port}/"
-
-    def tearDown(self):
-        super().tearDown()
-        self.site.stopListening()
-
-    def url(self, path: str) -> str:
-        return urljoin(self.baseurl, path)
-
-
-class NoMetaRefreshRedirect(util.Redirect):
-    def render(self, request: server.Request) -> bytes:
-        content = util.Redirect.render(self, request)
-        return content.replace(
-            b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
-        )
-
-
-def test_site():
-    r = resource.Resource()
-    r.putChild(b"text", static.Data(b"Works", "text/plain"))
-    r.putChild(
-        b"html",
-        static.Data(
-            b"<body><p class='one'>Works</p><p class='two'>World</p></body>",
-            "text/html",
-        ),
-    )
-    r.putChild(
-        b"enc-gb18030",
-        static.Data(b"<p>gb18030 encoding</p>", "text/html; charset=gb18030"),
-    )
-    r.putChild(b"redirect", util.Redirect(b"/redirected"))
-    r.putChild(b"redirect-no-meta-refresh", NoMetaRefreshRedirect(b"/redirected"))
-    r.putChild(b"redirected", static.Data(b"Redirected here", "text/plain"))
-    return server.Site(r)
-
-
-if __name__ == "__main__":
-    from twisted.internet import reactor  # pylint: disable=ungrouped-imports
-
-    port = reactor.listenTCP(0, test_site(), interface="127.0.0.1")
-    print(f"http://localhost:{port.getHost().port}/")
-    reactor.run()

From 8ecfd20fcd6ee5be2ed1761092a581ff5a9a9ed6 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 3 May 2026 14:12:16 +0500
Subject: [PATCH 02/36] Remove InitSpider.

---
 pyproject.toml         |  2 --
 scrapy/spiders/init.py | 64 ------------------------------------------
 tests/test_spider.py   | 24 +---------------
 3 files changed, 1 insertion(+), 89 deletions(-)
 delete mode 100644 scrapy/spiders/init.py

diff --git a/pyproject.toml b/pyproject.toml
index d35b333ff..ea7893e2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -126,7 +126,6 @@ ignore_errors = true
 [[tool.mypy.overrides]]
 module = [
     "scrapy.core.downloader.webclient",
-    "scrapy.spiders.init",
     "tests.test_webclient",
 ]
 allow_any_generics = true
@@ -138,7 +137,6 @@ warn_return_any = false
 # usually no type hints
 [[tool.mypy.overrides]]
 module = [
-#    "IPython.*",
     "bpython",
     "brotli",
     "brotlicffi",
diff --git a/scrapy/spiders/init.py b/scrapy/spiders/init.py
deleted file mode 100644
index 957bfffd3..000000000
--- a/scrapy/spiders/init.py
+++ /dev/null
@@ -1,64 +0,0 @@
-from __future__ import annotations
-
-import warnings
-from typing import TYPE_CHECKING, Any, cast
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.spiders import Spider
-from scrapy.utils.spider import iterate_spider_output
-
-if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Iterable
-
-    from scrapy import Request
-    from scrapy.http import Response
-
-
-class InitSpider(Spider):
-    """Base Spider with initialization facilities
-
-    .. warning:: This class is deprecated. Copy its code into your project if needed.
-    It will be removed in a future Scrapy version.
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        warnings.warn(
-            "InitSpider is deprecated. Copy its code from Scrapy's source if needed. "
-            "Will be removed in a future version.",
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-
-    async def start(self) -> AsyncIterator[Any]:
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
-            )
-            for item_or_request in self.start_requests():
-                yield item_or_request
-
-    def start_requests(self) -> Iterable[Request]:
-        self._postinit_reqs: Iterable[Request] = super().start_requests()
-        return cast("Iterable[Request]", iterate_spider_output(self.init_request()))
-
-    def initialized(self, response: Response | None = None) -> Any:
-        """This method must be set as the callback of your last initialization
-        request. See self.init_request() docstring for more info.
-        """
-        return self.__dict__.pop("_postinit_reqs")
-
-    def init_request(self) -> Any:
-        """This function should return one initialization request, with the
-        self.initialized method as callback. When the self.initialized method
-        is called this spider is considered initialized. If you need to perform
-        several requests for initializing your spider, you can do so by using
-        different callbacks. The only requirement is that the final callback
-        (of the last initialization request) must be self.initialized.
-
-        The default implementation calls self.initialized immediately, and
-        means that no initialization is needed. This method should be
-        overridden only when you need to perform requests to initialize your
-        spider
-        """
-        return self.initialized()
diff --git a/tests/test_spider.py b/tests/test_spider.py
index ff9aed74f..23efed77a 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -11,10 +11,9 @@ from scrapy.crawler import Crawler
 from scrapy.http import Response, TextResponse, XmlResponse
 from scrapy.settings import Settings
 from scrapy.spiders import CSVFeedSpider, Spider, XMLFeedSpider
-from scrapy.spiders.init import InitSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import get_testdata
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import inline_callbacks_test
 
 
 class TestSpider:
@@ -122,27 +121,6 @@ class TestSpider:
         mock_logger.log.assert_called_once_with("INFO", "test log msg")
 
 
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestInitSpider(TestSpider):
-    spider_class = InitSpider
-
-    @coroutine_test
-    async def test_start_urls(self):
-        responses = []
-
-        class TestSpider(self.spider_class):
-            name = "test"
-            start_urls = ["data:,"]
-
-            async def parse(self, response):
-                responses.append(response)
-
-        crawler = get_crawler(TestSpider)
-        await crawler.crawl_async()
-        assert len(responses) == 1
-        assert responses[0].url == "data:,"
-
-
 class TestXMLFeedSpider(TestSpider):
     spider_class = XMLFeedSpider
 

From af7dcabebb72aafbcd04425eaad3b2423406dd5a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 26 Apr 2026 16:02:30 +0500
Subject: [PATCH 03/36] Drop AjaxCrawlMiddleware and escape_ajax().

---
 docs/topics/settings.rst                      |   1 -
 scrapy/downloadermiddlewares/ajaxcrawl.py     | 114 ------------------
 scrapy/settings/default_settings.py           |   1 -
 scrapy/utils/url.py                           |  37 +-----
 ...test_downloadermiddleware_ajaxcrawlable.py |  62 ----------
 5 files changed, 1 insertion(+), 214 deletions(-)
 delete mode 100644 scrapy/downloadermiddlewares/ajaxcrawl.py
 delete mode 100644 tests/test_downloadermiddleware_ajaxcrawlable.py

diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 84219095b..cf2658d60 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -809,7 +809,6 @@ Default:
         "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
         "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
         "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
-        "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
         "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
         "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
         "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
diff --git a/scrapy/downloadermiddlewares/ajaxcrawl.py b/scrapy/downloadermiddlewares/ajaxcrawl.py
deleted file mode 100644
index a23deaa45..000000000
--- a/scrapy/downloadermiddlewares/ajaxcrawl.py
+++ /dev/null
@@ -1,114 +0,0 @@
-from __future__ import annotations
-
-import logging
-import re
-from typing import TYPE_CHECKING
-from warnings import warn
-
-from w3lib import html
-
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import HtmlResponse, Response
-from scrapy.utils.url import escape_ajax
-
-if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
-
-    from scrapy import Request, Spider
-    from scrapy.crawler import Crawler
-    from scrapy.settings import BaseSettings
-
-
-logger = logging.getLogger(__name__)
-
-
-class AjaxCrawlMiddleware:
-    """
-    Handle 'AJAX crawlable' pages marked as crawlable via meta tag.
-    """
-
-    def __init__(self, settings: BaseSettings):
-        if not settings.getbool("AJAXCRAWL_ENABLED"):
-            raise NotConfigured
-
-        warn(
-            "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware is deprecated"
-            " and will be removed in a future Scrapy version.",
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-
-        # XXX: Google parses at least first 100k bytes; scrapy's redirect
-        # middleware parses first 4k. 4k turns out to be insufficient
-        # for this middleware, and parsing 100k could be slow.
-        # We use something in between (32K) by default.
-        self.lookup_bytes: int = settings.getint("AJAXCRAWL_MAXSIZE")
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler.settings)
-
-    def process_response(
-        self, request: Request, response: Response, spider: Spider
-    ) -> Request | Response:
-        if not isinstance(response, HtmlResponse) or response.status != 200:
-            return response
-
-        if request.method != "GET":
-            # other HTTP methods are either not safe or don't have a body
-            return response
-
-        if "ajax_crawlable" in request.meta:  # prevent loops
-            return response
-
-        if not self._has_ajax_crawlable_variant(response):
-            return response
-
-        ajax_crawl_request = request.replace(url=escape_ajax(request.url + "#!"))
-        logger.debug(
-            "Downloading AJAX crawlable %(ajax_crawl_request)s instead of %(request)s",
-            {"ajax_crawl_request": ajax_crawl_request, "request": request},
-            extra={"spider": spider},
-        )
-
-        ajax_crawl_request.meta["ajax_crawlable"] = True
-        return ajax_crawl_request
-
-    def _has_ajax_crawlable_variant(self, response: Response) -> bool:
-        """
-        Return True if a page without hash fragment could be "AJAX crawlable".
-        """
-        body = response.text[: self.lookup_bytes]
-        return _has_ajaxcrawlable_meta(body)
-
-
-_ajax_crawlable_re: re.Pattern[str] = re.compile(
-    r'<meta\s+name=["\']fragment["\']\s+content=["\']!["\']/?>'
-)
-
-
-def _has_ajaxcrawlable_meta(text: str) -> bool:
-    """
-    >>> _has_ajaxcrawlable_meta('<html><head><meta name="fragment"  content="!"/></head><body></body></html>')
-    True
-    >>> _has_ajaxcrawlable_meta("<html><head><meta name='fragment' content='!'></head></html>")
-    True
-    >>> _has_ajaxcrawlable_meta('<html><head><!--<meta name="fragment"  content="!"/>--></head><body></body></html>')
-    False
-    >>> _has_ajaxcrawlable_meta('<html></html>')
-    False
-    """
-
-    # Stripping scripts and comments is slow (about 20x slower than
-    # just checking if a string is in text); this is a quick fail-fast
-    # path that should work for most pages.
-    if "fragment" not in text:
-        return False
-    if "content" not in text:
-        return False
-
-    text = html.remove_tags_with_content(text, ("script", "noscript"))
-    text = html.replace_entities(text)
-    text = html.remove_comments(text)
-    return _ajax_crawlable_re.search(text) is not None
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index b80e48601..7b54a9c14 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -289,7 +289,6 @@ DOWNLOADER_MIDDLEWARES_BASE = {
     "scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware": 400,
     "scrapy.downloadermiddlewares.useragent.UserAgentMiddleware": 500,
     "scrapy.downloadermiddlewares.retry.RetryMiddleware": 550,
-    "scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware": 560,
     "scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware": 580,
     "scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware": 590,
     "scrapy.downloadermiddlewares.redirect.RedirectMiddleware": 600,
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 39b581ff0..1c05ac20e 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -9,11 +9,9 @@ import re
 import warnings
 from importlib import import_module
 from typing import TYPE_CHECKING, Any, TypeAlias
-from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
-from warnings import warn
+from urllib.parse import ParseResult, urlparse, urlunparse
 
 from w3lib.url import __all__ as _public_w3lib_objects
-from w3lib.url import add_or_replace_parameter as _add_or_replace_parameter
 from w3lib.url import any_to_uri as _any_to_uri
 from w3lib.url import parse_url as _parse_url
 
@@ -70,39 +68,6 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
-def escape_ajax(url: str) -> str:
-    """
-    Return the crawlable url
-
-    >>> escape_ajax("www.example.com/ajax.html#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?k1=v1&k2=v2#!key=value")
-    'www.example.com/ajax.html?k1=v1&k2=v2&_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html?#!key=value")
-    'www.example.com/ajax.html?_escaped_fragment_=key%3Dvalue'
-    >>> escape_ajax("www.example.com/ajax.html#!")
-    'www.example.com/ajax.html?_escaped_fragment_='
-
-    URLs that are not "AJAX crawlable" (according to Google) returned as-is:
-
-    >>> escape_ajax("www.example.com/ajax.html#key=value")
-    'www.example.com/ajax.html#key=value'
-    >>> escape_ajax("www.example.com/ajax.html#")
-    'www.example.com/ajax.html#'
-    >>> escape_ajax("www.example.com/ajax.html")
-    'www.example.com/ajax.html'
-    """
-    warn(
-        "escape_ajax() is deprecated and will be removed in a future Scrapy version.",
-        ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    defrag, frag = urldefrag(url)
-    if not frag.startswith("!"):
-        return url
-    return _add_or_replace_parameter(defrag, "_escaped_fragment_", frag[1:])
-
-
 def add_http_if_no_scheme(url: str) -> str:
     """Add http as the default scheme if it is missing from the url."""
     match = re.match(r"^\w+://", url, flags=re.IGNORECASE)
diff --git a/tests/test_downloadermiddleware_ajaxcrawlable.py b/tests/test_downloadermiddleware_ajaxcrawlable.py
deleted file mode 100644
index 44084f1e8..000000000
--- a/tests/test_downloadermiddleware_ajaxcrawlable.py
+++ /dev/null
@@ -1,62 +0,0 @@
-import pytest
-
-from scrapy.downloadermiddlewares.ajaxcrawl import AjaxCrawlMiddleware
-from scrapy.http import HtmlResponse, Request, Response
-from scrapy.spiders import Spider
-from scrapy.utils.test import get_crawler
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestAjaxCrawlMiddleware:
-    def setup_method(self):
-        crawler = get_crawler(Spider, {"AJAXCRAWL_ENABLED": True})
-        self.spider = crawler._create_spider("foo")
-        self.mw = AjaxCrawlMiddleware.from_crawler(crawler)
-
-    def _ajaxcrawlable_body(self):
-        return b'<html><head><meta name="fragment" content="!"/></head><body></body></html>'
-
-    def _req_resp(self, url, req_kwargs=None, resp_kwargs=None):
-        req = Request(url, **(req_kwargs or {}))
-        resp = HtmlResponse(url, request=req, **(resp_kwargs or {}))
-        return req, resp
-
-    def test_non_get(self):
-        req, resp = self._req_resp("http://example.com/", {"method": "HEAD"})
-        resp2 = self.mw.process_response(req, resp, self.spider)
-        assert resp == resp2
-
-    def test_binary_response(self):
-        req = Request("http://example.com/")
-        resp = Response("http://example.com/", body=b"foobar\x00\x01\x02", request=req)
-        resp2 = self.mw.process_response(req, resp, self.spider)
-        assert resp is resp2
-
-    def test_ajaxcrawl(self):
-        req, resp = self._req_resp(
-            "http://example.com/",
-            {"meta": {"foo": "bar"}},
-            {"body": self._ajaxcrawlable_body()},
-        )
-        req2 = self.mw.process_response(req, resp, self.spider)
-        assert req2.url == "http://example.com/?_escaped_fragment_="
-        assert req2.meta["foo"] == "bar"
-
-    def test_ajaxcrawl_loop(self):
-        req, resp = self._req_resp(
-            "http://example.com/", {}, {"body": self._ajaxcrawlable_body()}
-        )
-        req2 = self.mw.process_response(req, resp, self.spider)
-        resp2 = HtmlResponse(req2.url, body=resp.body, request=req2)
-        resp3 = self.mw.process_response(req2, resp2, self.spider)
-
-        assert isinstance(resp3, HtmlResponse), (resp3.__class__, resp3)
-        assert resp3.request.url == "http://example.com/?_escaped_fragment_="
-        assert resp3 is resp2
-
-    def test_noncrawlable_body(self):
-        req, resp = self._req_resp(
-            "http://example.com/", {}, {"body": b"<html></html>"}
-        )
-        resp2 = self.mw.process_response(req, resp, self.spider)
-        assert resp is resp2

From 5fc40f07f3c3a13fd23ef694c15f0d2e772588d5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 26 Apr 2026 16:15:09 +0500
Subject: [PATCH 04/36] Drop w3lib.url reexports.

---
 scrapy/utils/url.py     | 34 +++++-----------------------------
 tests/test_utils_url.py | 26 +-------------------------
 2 files changed, 6 insertions(+), 54 deletions(-)

diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1c05ac20e..8f75e2618 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,34 +6,10 @@ library.
 from __future__ import annotations
 
 import re
-import warnings
-from importlib import import_module
-from typing import TYPE_CHECKING, Any, TypeAlias
+from typing import TYPE_CHECKING, TypeAlias
 from urllib.parse import ParseResult, urlparse, urlunparse
 
-from w3lib.url import __all__ as _public_w3lib_objects
-from w3lib.url import any_to_uri as _any_to_uri
-from w3lib.url import parse_url as _parse_url
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-_DEPRECATED_NAMES: frozenset[str] = frozenset(
-    {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects}
-)
-
-
-def __getattr__(name: str) -> Any:
-    if name in _DEPRECATED_NAMES:
-        obj_type = "attribute" if name == "_safe_chars" else "function"
-        warnings.warn(
-            f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return getattr(import_module("w3lib.url"), name)
-
-    raise AttributeError
-
+from w3lib.url import any_to_uri, parse_url
 
 if TYPE_CHECKING:
     from collections.abc import Iterable
@@ -45,7 +21,7 @@ UrlT: TypeAlias = str | bytes | ParseResult
 
 def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
     """Return True if the url belongs to any of the given domains"""
-    host = _parse_url(url).netloc.lower()
+    host = parse_url(url).netloc.lower()
     if not host:
         return False
     return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains))
@@ -64,7 +40,7 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
 
 def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
     """Return True if the url ends with one of the extensions provided"""
-    lowercase_path = _parse_url(url).path.lower()
+    lowercase_path = parse_url(url).path.lower()
     return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
@@ -125,7 +101,7 @@ def guess_scheme(url: str) -> str:
     """Add an URL scheme if missing: file:// for filepath-like input or
     http:// otherwise."""
     if _is_filesystem_path(url):
-        return _any_to_uri(url)
+        return any_to_uri(url)
     return add_http_if_no_scheme(url)
 
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index bcbbe74a3..19a31c353 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,13 +1,9 @@
-import warnings
-from importlib import import_module
-
 import pytest
 
 from scrapy.linkextractors import IGNORED_EXTENSIONS
 from scrapy.spiders import Spider
-from scrapy.utils.url import (  # type: ignore[attr-defined]
+from scrapy.utils.url import (
     _is_filesystem_path,
-    _public_w3lib_objects,
     add_http_if_no_scheme,
     guess_scheme,
     strip_url,
@@ -446,23 +442,3 @@ class TestStripUrl:
 )
 def test__is_filesystem_path(path: str, expected: bool) -> None:
     assert _is_filesystem_path(path) == expected
-
-
-@pytest.mark.parametrize(
-    "obj_name",
-    [
-        "_unquotepath",
-        "_safe_chars",
-        "parse_url",
-        *_public_w3lib_objects,
-    ],
-)
-def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
-    with warnings.catch_warnings(record=True) as warns:
-        obj_type = "attribute" if obj_name == "_safe_chars" else "function"
-        message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
-
-        getattr(import_module("scrapy.utils.url"), obj_name)
-
-        assert isinstance(warns[0].message, Warning)
-        assert message in warns[0].message.args

From f24bc749ea5b45e3529c2040b55878623ce3d8fc Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 26 Apr 2026 16:20:53 +0500
Subject: [PATCH 05/36] Require start_queue_cls.

---
 scrapy/core/scheduler.py | 64 ++++++++++------------------------------
 1 file changed, 15 insertions(+), 49 deletions(-)

diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 1b7fb652d..78329460e 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -5,16 +5,13 @@ import logging
 from abc import abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
-from warnings import warn
 
 # working around https://github.com/sphinx-doc/sphinx/issues/10400
 from twisted.internet.defer import Deferred  # noqa: TC002
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.spiders import Spider  # noqa: TC001
 from scrapy.utils.job import job_dir
 from scrapy.utils.misc import build_from_crawler, load_object
-from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
     # requires queuelib >= 1.6.2
@@ -450,28 +447,13 @@ class Scheduler(BaseScheduler):
         """Create a new priority queue instance, with in-memory storage"""
         assert self.crawler
         assert self.pqclass
-        try:
-            return build_from_crawler(
-                self.pqclass,
-                self.crawler,
-                downstream_queue_cls=self.mqclass,
-                key="",
-                start_queue_cls=self._smqclass,
-            )
-        except TypeError:  # pragma: no cover
-            warn(
-                f"The __init__ method of {global_object_name(self.pqclass)} "
-                "does not support a `start_queue_cls` keyword-only "
-                "parameter.",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-            return build_from_crawler(
-                self.pqclass,
-                self.crawler,
-                downstream_queue_cls=self.mqclass,
-                key="",
-            )
+        return build_from_crawler(
+            self.pqclass,
+            self.crawler,
+            downstream_queue_cls=self.mqclass,
+            key="",
+            start_queue_cls=self._smqclass,
+        )
 
     def _dq(self) -> ScrapyPriorityQueue:
         """Create a new priority queue instance, with disk storage"""
@@ -479,30 +461,14 @@ class Scheduler(BaseScheduler):
         assert self.dqdir
         assert self.pqclass
         state = self._read_dqs_state(self.dqdir)
-        try:
-            q = build_from_crawler(
-                self.pqclass,
-                self.crawler,
-                downstream_queue_cls=self.dqclass,
-                key=self.dqdir,
-                startprios=state,
-                start_queue_cls=self._sdqclass,
-            )
-        except TypeError:  # pragma: no cover
-            warn(
-                f"The __init__ method of {global_object_name(self.pqclass)} "
-                "does not support a `start_queue_cls` keyword-only "
-                "parameter.",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-            q = build_from_crawler(
-                self.pqclass,
-                self.crawler,
-                downstream_queue_cls=self.dqclass,
-                key=self.dqdir,
-                startprios=state,
-            )
+        q = build_from_crawler(
+            self.pqclass,
+            self.crawler,
+            downstream_queue_cls=self.dqclass,
+            key=self.dqdir,
+            startprios=state,
+            start_queue_cls=self._sdqclass,
+        )
         if q:
             logger.info(
                 "Resuming crawl (%(queuesize)d requests scheduled)",

From 6cb2fe1fc3b79c643487b0f051afdbe9c8cf7026 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 26 Apr 2026 16:22:01 +0500
Subject: [PATCH 06/36] Drop scrapy_components_versions().

---
 scrapy/utils/versions.py | 14 --------------
 1 file changed, 14 deletions(-)

diff --git a/scrapy/utils/versions.py b/scrapy/utils/versions.py
index 6d3572cb1..6bd96a215 100644
--- a/scrapy/utils/versions.py
+++ b/scrapy/utils/versions.py
@@ -3,11 +3,9 @@ from __future__ import annotations
 import platform
 import sys
 from importlib.metadata import version
-from warnings import warn
 
 import lxml.etree
 
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings.default_settings import LOG_VERSIONS
 from scrapy.utils.ssl import get_openssl_version
 
@@ -32,15 +30,3 @@ def get_versions(
 ) -> list[tuple[str, str]]:
     software = software or _DEFAULT_SOFTWARE
     return [(item, _version(item)) for item in software]
-
-
-def scrapy_components_versions() -> list[tuple[str, str]]:  # pragma: no cover
-    warn(
-        (
-            "scrapy.utils.versions.scrapy_components_versions() is deprecated, "
-            "use scrapy.utils.versions.get_versions() instead."
-        ),
-        ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    return get_versions()

From 9f02f6c16acf94bce00a7047dcce2cfa253a5495 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Sun, 26 Apr 2026 16:28:06 +0500
Subject: [PATCH 07/36] Drop spider args of Scraper methods.

---
 scrapy/core/scraper.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 701f7cb46..e756d27eb 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -23,7 +23,6 @@ from scrapy.exceptions import (
 from scrapy.http import Request, Response
 from scrapy.pipelines import ItemPipelineManager
 from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
-from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.defer import (
     _defer_sleep_async,
     _schedule_coro,
@@ -178,9 +177,7 @@ class Scraper:
                 self.itemproc.open_spider(self.crawler.spider)
             )
 
-    def close_spider(
-        self, spider: Spider | None = None
-    ) -> Deferred[None]:  # pragma: no cover
+    def close_spider(self) -> Deferred[None]:  # pragma: no cover
         warnings.warn(
             "Scraper.close_spider() is deprecated, use close_spider_async() instead",
             ScrapyDeprecationWarning,
@@ -217,9 +214,8 @@ class Scraper:
             self.slot.closing.callback(self.crawler.spider)
 
     @inlineCallbacks
-    @_warn_spider_arg
     def enqueue_scrape(
-        self, result: Response | Failure, request: Request, spider: Spider | None = None
+        self, result: Response | Failure, request: Request
     ) -> Generator[Deferred[Any], Any, None]:
         if self.slot is None:
             raise RuntimeError("Scraper slot not assigned")
@@ -349,13 +345,11 @@ class Scraper:
                 )
         return await ensure_awaitable(iterate_spider_output(output))
 
-    @_warn_spider_arg
     def handle_spider_error(
         self,
         _failure: Failure,
         request: Request,
         response: Response | Failure,
-        spider: Spider | None = None,
     ) -> None:
         """Handle an exception raised by a spider callback or errback."""
         assert self.crawler.spider
@@ -391,7 +385,6 @@ class Scraper:
         result: Iterable[_T] | AsyncIterator[_T],
         request: Request,
         response: Response | Failure,
-        spider: Spider | None = None,
     ) -> Deferred[None]:  # pragma: no cover
         """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
         warnings.warn(

From a9324fbf7618d0750678a43106c6d6576db10c1f Mon Sep 17 00:00:00 2001
From: Sanjay M <sanjay.muthuswamy1@gmail.com>
Date: Mon, 4 May 2026 12:13:21 +0530
Subject: [PATCH 08/36] Fix small documentation wording issues (#7480)

---
 .github/pull_request_template.md | 4 ++--
 docs/_ext/scrapyfixautodoc.py    | 2 +-
 docs/topics/telnetconsole.rst    | 6 +++---
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index dd2edebdd..98a74f8ce 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -12,13 +12,13 @@ Key takeaways:
 > Note: What follows is based on
 > https://raw.githubusercontent.com/jackyzha0/quartz/acfaa472253a432d350e9b6904c0cde14f8c487f/.github/pull_request_template.md
 
-We more than welcome contributions, and are OK with the use of LLMs tools. How
+We more than welcome contributions, and are OK with the use of LLM tools. How
 you use those tools depends on whether or not they make you more productive.
 
 But one thing that bugs us a lot are PRs that are made entirely with these
 tools, without any revision or any effort trying to refine their output
 whatsoever. This is just pure laziness, and unacceptable. Doing so will just
-end up wasting everyone time (ours and yours).
+end up wasting everyone's time (ours and yours).
 
 So to be the most productive for all parties, we would encourage any
 contributors to, at the very least, pay attention to what the model is doing,
diff --git a/docs/_ext/scrapyfixautodoc.py b/docs/_ext/scrapyfixautodoc.py
index a8f2c6b68..e342e92cf 100644
--- a/docs/_ext/scrapyfixautodoc.py
+++ b/docs/_ext/scrapyfixautodoc.py
@@ -11,7 +11,7 @@ from sphinx.application import Sphinx
 
 def maybe_skip_member(app: Sphinx, what, name: str, obj, skip: bool, options) -> bool:
     if not skip:
-        # autodocs was generating a text "alias of" for the following members
+        # autodoc was generating the text "alias of" for the following members
         return name in {"default_item_class", "default_selector_class"}
     return skip
 
diff --git a/docs/topics/telnetconsole.rst b/docs/topics/telnetconsole.rst
index e74afe49b..6d99c756e 100644
--- a/docs/topics/telnetconsole.rst
+++ b/docs/topics/telnetconsole.rst
@@ -46,12 +46,12 @@ the console you need to type::
     Password:
     >>>
 
-By default Username is ``scrapy`` and Password is autogenerated. The
-autogenerated Password can be seen on Scrapy logs like the example below::
+By default, the username is ``scrapy`` and the password is autogenerated. The
+autogenerated password can be seen on Scrapy logs like the example below::
 
     2018-10-16 14:35:21 [scrapy.extensions.telnet] INFO: Telnet Password: 16f92501e8a59326
 
-Default Username and Password can be overridden by the settings
+The default username and password can be overridden by the settings
 :setting:`TELNETCONSOLE_USERNAME` and :setting:`TELNETCONSOLE_PASSWORD`.
 
 .. warning::

From c62a81d7dd5f1a7ae3e745b227bc1a6d1ab51e36 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 4 May 2026 11:50:41 +0500
Subject: [PATCH 09/36] Drop HTTP/1.0 code. (#7486)

---
 pyproject.toml                                |   2 -
 scrapy/core/downloader/contextfactory.py      |   6 +-
 scrapy/core/downloader/handlers/http.py       |   2 -
 scrapy/core/downloader/handlers/http10.py     |  79 ----
 scrapy/core/downloader/webclient.py           | 243 -----------
 scrapy/settings/default_settings.py           |   5 -
 tests/test_downloader_handler_httpx.py        |  24 +-
 .../test_downloader_handler_twisted_http10.py |  55 ---
 .../test_downloader_handler_twisted_http11.py |  22 +-
 .../test_downloader_handler_twisted_http2.py  |  16 +-
 tests/test_downloader_handlers_http_base.py   |  25 +-
 tests/test_webclient.py                       | 404 ------------------
 12 files changed, 44 insertions(+), 839 deletions(-)
 delete mode 100644 scrapy/core/downloader/handlers/http10.py
 delete mode 100644 scrapy/core/downloader/webclient.py
 delete mode 100644 tests/test_downloader_handler_twisted_http10.py
 delete mode 100644 tests/test_webclient.py

diff --git a/pyproject.toml b/pyproject.toml
index d96790873..7ce2f2e4c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -125,10 +125,8 @@ ignore_errors = true
 # deprecated modules
 [[tool.mypy.overrides]]
 module = [
-    "scrapy.core.downloader.webclient",
     "scrapy.spiders.init",
     "scrapy.utils.testsite",
-    "tests.test_webclient",
 ]
 allow_any_generics = true
 allow_untyped_calls = true
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 19fc77959..bbda7efcd 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -114,10 +114,10 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
                 acceptableCiphers=self.tls_ciphers,
             )
 
-    # kept for old-style HTTP/1.0 downloader context twisted calls,
-    # e.g. connectSSL()
     # should be removed together with ScrapyClientContextFactory
-    def getContext(self, hostname: Any = None, port: Any = None) -> SSL.Context:
+    def getContext(
+        self, hostname: Any = None, port: Any = None
+    ) -> SSL.Context:  # pragma: no cover
         return self._get_context()
 
     def _get_context(self) -> SSL.Context:
diff --git a/scrapy/core/downloader/handlers/http.py b/scrapy/core/downloader/handlers/http.py
index a7b592d00..dc1b1e375 100644
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@@ -1,7 +1,6 @@
 # pragma: no file cover
 import warnings
 
-from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import (
     HTTP11DownloadHandler as HTTPDownloadHandler,
 )
@@ -16,6 +15,5 @@ warnings.warn(
 )
 
 __all__ = [
-    "HTTP10DownloadHandler",
     "HTTPDownloadHandler",
 ]
diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.py
deleted file mode 100644
index f5d1bbd76..000000000
--- a/scrapy/core/downloader/handlers/http10.py
+++ /dev/null
@@ -1,79 +0,0 @@
-"""Download handlers for http and https schemes"""
-
-from __future__ import annotations
-
-import warnings
-from typing import TYPE_CHECKING
-
-from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory
-from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.utils.defer import maybe_deferred_to_future
-from scrapy.utils.misc import build_from_crawler, load_object
-from scrapy.utils.python import to_unicode
-
-if TYPE_CHECKING:
-    from twisted.internet.interfaces import IConnector
-
-    # typing.Self requires Python 3.11
-    from typing_extensions import Self
-
-    from scrapy import Request
-    from scrapy.core.downloader.webclient import ScrapyHTTPClientFactory
-    from scrapy.crawler import Crawler
-    from scrapy.http import Response
-    from scrapy.settings import BaseSettings
-
-
-class HTTP10DownloadHandler:
-    lazy = False
-
-    def __init__(self, settings: BaseSettings, crawler: Crawler):
-        warnings.warn(
-            "HTTP10DownloadHandler is deprecated and will be removed in a future Scrapy version.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        if not crawler.settings.getbool("TWISTED_REACTOR_ENABLED"):  # pragma: no cover
-            raise NotConfigured(f"{type(self).__name__} requires a Twisted reactor.")
-        self.HTTPClientFactory: type[ScrapyHTTPClientFactory] = load_object(
-            settings["DOWNLOADER_HTTPCLIENTFACTORY"]
-        )
-        if settings["DOWNLOADER_CLIENTCONTEXTFACTORY"] == "SENTINEL":
-            self.ClientContextFactory: type[_ScrapyClientContextFactory] = (
-                _ScrapyClientContextFactory
-            )
-        else:  # pragma: no cover
-            warnings.warn(
-                "The 'DOWNLOADER_CLIENTCONTEXTFACTORY' setting is deprecated.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-            self.ClientContextFactory = load_object(
-                settings["DOWNLOADER_CLIENTCONTEXTFACTORY"]
-            )
-        self._settings: BaseSettings = settings
-        self._crawler: Crawler = crawler
-
-    @classmethod
-    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler.settings, crawler)
-
-    async def download_request(self, request: Request) -> Response:
-        factory = self.HTTPClientFactory(request)
-        self._connect(factory)
-        return await maybe_deferred_to_future(factory.deferred)
-
-    def _connect(self, factory: ScrapyHTTPClientFactory) -> IConnector:
-        from twisted.internet import reactor
-
-        host, port = to_unicode(factory.host), factory.port
-        if factory.scheme == b"https":
-            client_context_factory = build_from_crawler(
-                self.ClientContextFactory,
-                self._crawler,
-            )
-            return reactor.connectSSL(host, port, factory, client_context_factory)
-        return reactor.connectTCP(host, port, factory)
-
-    async def close(self) -> None:
-        pass
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
deleted file mode 100644
index 2a550cf78..000000000
--- a/scrapy/core/downloader/webclient.py
+++ /dev/null
@@ -1,243 +0,0 @@
-"""Deprecated HTTP/1.0 helper classes used by HTTP10DownloadHandler."""
-
-from __future__ import annotations
-
-import warnings
-from time import monotonic, time
-from typing import TYPE_CHECKING
-from urllib.parse import urldefrag, urlparse, urlunparse
-
-from twisted.internet import defer
-from twisted.internet.protocol import ClientFactory
-from twisted.web.http import HTTPClient
-
-from scrapy.exceptions import DownloadTimeoutError, ScrapyDeprecationWarning
-from scrapy.http import Headers, Response
-from scrapy.responsetypes import responsetypes
-from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes, to_unicode
-
-if TYPE_CHECKING:
-    from scrapy import Request
-
-
-class ScrapyHTTPPageGetter(HTTPClient):
-    delimiter = b"\n"
-
-    def __init__(self):
-        warnings.warn(
-            "ScrapyHTTPPageGetter is deprecated and will be removed in a future Scrapy version.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        super().__init__()
-
-    def connectionMade(self):
-        self.headers = Headers()  # bucket for response headers
-
-        # Method command
-        self.sendCommand(self.factory.method, self.factory.path)
-        # Headers
-        for key, values in self.factory.headers.items():
-            for value in values:
-                self.sendHeader(key, value)
-        self.endHeaders()
-        # Body
-        if self.factory.body is not None:
-            self.transport.write(self.factory.body)
-
-    def lineReceived(self, line):
-        return HTTPClient.lineReceived(self, line.rstrip())
-
-    def handleHeader(self, key, value):
-        self.headers.appendlist(key, value)
-
-    def handleStatus(self, version, status, message):
-        self.factory.gotStatus(version, status, message)
-
-    def handleEndHeaders(self):
-        self.factory.gotHeaders(self.headers)
-
-    def connectionLost(self, reason):
-        self._connection_lost_reason = reason
-        HTTPClient.connectionLost(self, reason)
-        self.factory.noPage(reason)
-
-    def handleResponse(self, response):
-        if self.factory.method.upper() == b"HEAD":
-            self.factory.page(b"")
-        elif self.length is not None and self.length > 0:
-            self.factory.noPage(self._connection_lost_reason)
-        else:
-            self.factory.page(response)
-        self.transport.loseConnection()
-
-    def timeout(self):
-        self.transport.loseConnection()
-
-        # transport cleanup needed for HTTPS connections
-        if self.factory.url.startswith(b"https"):
-            self.transport.stopProducing()
-
-        self.factory.noPage(
-            DownloadTimeoutError(
-                f"Getting {self.factory.url} took longer "
-                f"than {self.factory.timeout} seconds."
-            )
-        )
-
-
-# This class used to inherit from Twisted’s
-# twisted.web.client.HTTPClientFactory. When that class was deprecated in
-# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
-# non-overridden code into this class.
-class ScrapyHTTPClientFactory(ClientFactory):
-    protocol = ScrapyHTTPPageGetter
-
-    waiting = 1
-    noisy = False
-    followRedirect = False
-    afterFoundGet = False
-
-    def _build_response(self, body, request):
-        request.meta["download_latency"] = (
-            self._headers_time_mono - self._start_time_mono
-        )
-        status = int(self.status)
-        headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
-        return respcls(
-            url=self._url,
-            status=status,
-            headers=headers,
-            body=body,
-            protocol=to_unicode(self.version),
-        )
-
-    def _set_connection_attributes(self, request):
-        proxy = request.meta.get("proxy")
-        if proxy:
-            proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii"))
-            self.scheme = proxy_parsed.scheme
-            self.host = proxy_parsed.hostname
-            self.port = proxy_parsed.port
-            self.netloc = proxy_parsed.netloc
-            if self.port is None:
-                self.port = 443 if proxy_parsed.scheme == b"https" else 80
-            self.path = self.url
-        else:
-            parsed = urlparse_cached(request)
-            path_str = urlunparse(
-                ("", "", parsed.path or "/", parsed.params, parsed.query, "")
-            )
-            self.path = to_bytes(path_str, encoding="ascii")
-            assert parsed.hostname is not None
-            self.host = to_bytes(parsed.hostname, encoding="ascii")
-            self.port = parsed.port
-            self.scheme = to_bytes(parsed.scheme, encoding="ascii")
-            self.netloc = to_bytes(parsed.netloc, encoding="ascii")
-            if self.port is None:
-                self.port = 443 if self.scheme == b"https" else 80
-
-    def __init__(self, request: Request, timeout: float = 180):
-        warnings.warn(
-            "ScrapyHTTPClientFactory is deprecated and will be removed in a future Scrapy version.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-
-        self._url: str = urldefrag(request.url)[0]
-        # converting to bytes to comply to Twisted interface
-        self.url: bytes = to_bytes(self._url, encoding="ascii")
-        self.method: bytes = to_bytes(request.method, encoding="ascii")
-        self.body: bytes | None = request.body or None
-        self.headers: Headers = Headers(request.headers)
-        self.response_headers: Headers | None = None
-        self.timeout: float = request.meta.get("download_timeout") or timeout
-        self.start_time: float = time()
-        self._start_time_mono: float = monotonic()
-        self.deferred: defer.Deferred[Response] = defer.Deferred().addCallback(
-            self._build_response, request
-        )
-
-        # Fixes Twisted 11.1.0+ support as HTTPClientFactory is expected
-        # to have _disconnectedDeferred. See Twisted r32329.
-        # As Scrapy implements it's own logic to handle redirects is not
-        # needed to add the callback _waitForDisconnect.
-        # Specifically this avoids the AttributeError exception when
-        # clientConnectionFailed method is called.
-        self._disconnectedDeferred: defer.Deferred[None] = defer.Deferred()
-
-        self._set_connection_attributes(request)
-
-        # set Host header based on url
-        self.headers.setdefault("Host", self.netloc)
-
-        # set Content-Length based len of body
-        if self.body is not None:
-            self.headers["Content-Length"] = len(self.body)
-            # just in case a broken http/1.1 decides to keep connection alive
-            self.headers.setdefault("Connection", "close")
-        # Content-Length must be specified in POST method even with no body
-        elif self.method == b"POST":
-            self.headers["Content-Length"] = 0
-
-    def __repr__(self) -> str:
-        return f"<{self.__class__.__name__}: {self._url}>"
-
-    def _cancelTimeout(self, result, timeoutCall):
-        if timeoutCall.active():
-            timeoutCall.cancel()
-        return result
-
-    def buildProtocol(self, addr):
-        p = ClientFactory.buildProtocol(self, addr)
-        p.followRedirect = self.followRedirect
-        p.afterFoundGet = self.afterFoundGet
-        if self.timeout:
-            from twisted.internet import reactor
-
-            timeoutCall = reactor.callLater(self.timeout, p.timeout)
-            self.deferred.addBoth(self._cancelTimeout, timeoutCall)
-        return p
-
-    def gotHeaders(self, headers):
-        self.headers_time = time()
-        self._headers_time_mono = monotonic()
-        self.response_headers = headers
-
-    def gotStatus(self, version, status, message):
-        """
-        Set the status of the request on us.
-        @param version: The HTTP version.
-        @type version: L{bytes}
-        @param status: The HTTP status code, an integer represented as a
-        bytestring.
-        @type status: L{bytes}
-        @param message: The HTTP status message.
-        @type message: L{bytes}
-        """
-        self.version, self.status, self.message = version, status, message
-
-    def page(self, page):
-        if self.waiting:
-            self.waiting = 0
-            self.deferred.callback(page)
-
-    def noPage(self, reason):
-        if self.waiting:
-            self.waiting = 0
-            self.deferred.errback(reason)
-
-    def clientConnectionFailed(self, _, reason):
-        """
-        When a connection attempt fails, the request cannot be issued.  If no
-        result has yet been provided to the result Deferred, provide the
-        connection failure reason as an error result.
-        """
-        if self.waiting:
-            self.waiting = 0
-            # If the connection attempt failed, there is nothing more to
-            # disconnect, so just fire that Deferred now.
-            self._disconnectedDeferred.callback(None)
-            self.deferred.errback(reason)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index b80e48601..f9297d4cf 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -54,7 +54,6 @@ __all__ = [
     "DOWNLOADER_CLIENT_TLS_CIPHERS",
     "DOWNLOADER_CLIENT_TLS_METHOD",
     "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING",
-    "DOWNLOADER_HTTPCLIENTFACTORY",
     "DOWNLOADER_MIDDLEWARES",
     "DOWNLOADER_MIDDLEWARES_BASE",
     "DOWNLOADER_STATS",
@@ -275,10 +274,6 @@ DOWNLOADER_CLIENT_TLS_CIPHERS = "DEFAULT"
 DOWNLOADER_CLIENT_TLS_METHOD = "TLS"
 DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING = False
 
-DOWNLOADER_HTTPCLIENTFACTORY = (
-    "scrapy.core.downloader.webclient.ScrapyHTTPClientFactory"
-)
-
 DOWNLOADER_MIDDLEWARES = {}
 DOWNLOADER_MIDDLEWARES_BASE = {
     # Engine side
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index 3b1796af7..a0e3fccb3 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -9,9 +9,9 @@ import pytest
 
 from scrapy import Request
 from tests.test_downloader_handlers_http_base import (
-    TestHttp11Base,
+    TestHttpBase,
     TestHttpProxyBase,
-    TestHttps11Base,
+    TestHttpsBase,
     TestHttpsCustomCiphersBase,
     TestHttpsInvalidDNSIdBase,
     TestHttpsInvalidDNSPatternBase,
@@ -42,7 +42,7 @@ class HttpxDownloadHandlerMixin:
         return HttpxDownloadHandler
 
 
-class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
+class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
     handler_supports_bindaddress_meta = False
 
     @pytest.mark.skipif(
@@ -77,7 +77,7 @@ class TestHttp11(HttpxDownloadHandlerMixin, TestHttp11Base):
         )
 
 
-class TestHttps11(HttpxDownloadHandlerMixin, TestHttps11Base):
+class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
     handler_supports_bindaddress_meta = False
     tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
 
@@ -90,25 +90,25 @@ class TestSimpleHttps(HttpxDownloadHandlerMixin, TestSimpleHttpsBase):
     pass
 
 
-class TestHttps11WrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
+class TestHttpsWrongHostname(HttpxDownloadHandlerMixin, TestHttpsWrongHostnameBase):
     pass
 
 
-class TestHttps11InvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+class TestHttpsInvalidDNSId(HttpxDownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
     pass
 
 
-class TestHttps11InvalidDNSPattern(
+class TestHttpsInvalidDNSPattern(
     HttpxDownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
 ):
     pass
 
 
-class TestHttps11CustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
+class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBase):
     pass
 
 
-class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
+class TestHttpWithCrawler(TestHttpWithCrawlerBase):
     @property
     def settings_dict(self) -> dict[str, Any] | None:
         return {
@@ -119,7 +119,7 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
         }
 
 
-class TestHttps11WithCrawler(TestHttp11WithCrawler):
+class TestHttpsWithCrawler(TestHttpWithCrawler):
     is_secure = True
 
     @pytest.mark.skip(reason="response.certificate is not implemented")
@@ -129,10 +129,10 @@ class TestHttps11WithCrawler(TestHttp11WithCrawler):
 
 
 @pytest.mark.skip(reason="Proxy support is not implemented yet")
-class TestHttp11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
+class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
     pass
 
 
 @pytest.mark.skip(reason="Proxy support is not implemented yet")
-class TestHttps11Proxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
+class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
     is_secure = True
diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py
deleted file mode 100644
index 8281337b0..000000000
--- a/tests/test_downloader_handler_twisted_http10.py
+++ /dev/null
@@ -1,55 +0,0 @@
-"""Tests for scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler."""
-
-from __future__ import annotations
-
-from typing import TYPE_CHECKING
-
-import pytest
-
-from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
-from scrapy.http import Request
-from scrapy.utils.defer import deferred_f_from_coro_f
-from tests.test_downloader_handlers_http_base import TestHttpBase, TestHttpProxyBase
-
-if TYPE_CHECKING:
-    from scrapy.core.downloader.handlers import DownloadHandlerProtocol
-    from tests.mockserver.http import MockServer
-
-
-pytestmark = pytest.mark.requires_reactor  # HTTP10DownloadHandler requires a reactor
-
-
-class HTTP10DownloadHandlerMixin:
-    @property
-    def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
-        return HTTP10DownloadHandler
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestHttp10(HTTP10DownloadHandlerMixin, TestHttpBase):
-    """HTTP 1.0 test case"""
-
-    def test_unsupported_scheme(self) -> None:  # type: ignore[override]
-        pytest.skip("Check not implemented")
-
-    @deferred_f_from_coro_f
-    async def test_protocol(self, mockserver: MockServer) -> None:
-        request = Request(mockserver.url("/host", is_secure=self.is_secure))
-        async with self.get_dh() as download_handler:
-            response = await download_handler.download_request(request)
-        assert response.protocol == "HTTP/1.0"
-
-
-class TestHttps10(TestHttp10):
-    is_secure = True
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestHttp10Proxy(HTTP10DownloadHandlerMixin, TestHttpProxyBase):
-    @deferred_f_from_coro_f
-    async def test_download_with_proxy_https_timeout(self):
-        pytest.skip("Not implemented")
-
-    @deferred_f_from_coro_f
-    async def test_download_with_proxy_without_http_scheme(self):
-        pytest.skip("Not implemented")
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index f6e86d3dc..01fb26d85 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -11,9 +11,9 @@ from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.crawler import Crawler
 from scrapy.exceptions import NotConfigured
 from tests.test_downloader_handlers_http_base import (
-    TestHttp11Base,
+    TestHttpBase,
     TestHttpProxyBase,
-    TestHttps11Base,
+    TestHttpsBase,
     TestHttpsCustomCiphersBase,
     TestHttpsInvalidDNSIdBase,
     TestHttpsInvalidDNSPatternBase,
@@ -41,11 +41,11 @@ def test_not_configured_without_reactor() -> None:
         HTTP11DownloadHandler.from_crawler(crawler)
 
 
-class TestHttp11(HTTP11DownloadHandlerMixin, TestHttp11Base):
+class TestHttp(HTTP11DownloadHandlerMixin, TestHttpBase):
     pass
 
 
-class TestHttps11(HTTP11DownloadHandlerMixin, TestHttps11Base):
+class TestHttps(HTTP11DownloadHandlerMixin, TestHttpsBase):
     pass
 
 
@@ -53,25 +53,25 @@ class TestSimpleHttps(HTTP11DownloadHandlerMixin, TestSimpleHttpsBase):
     pass
 
 
-class TestHttps11WrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
+class TestHttpsWrongHostname(HTTP11DownloadHandlerMixin, TestHttpsWrongHostnameBase):
     pass
 
 
-class TestHttps11InvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+class TestHttpsInvalidDNSId(HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
     pass
 
 
-class TestHttps11InvalidDNSPattern(
+class TestHttpsInvalidDNSPattern(
     HTTP11DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
 ):
     pass
 
 
-class TestHttps11CustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
+class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersBase):
     pass
 
 
-class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
+class TestHttpWithCrawler(TestHttpWithCrawlerBase):
     @property
     def settings_dict(self) -> dict[str, Any] | None:
         return {
@@ -82,9 +82,9 @@ class TestHttp11WithCrawler(TestHttpWithCrawlerBase):
         }
 
 
-class TestHttps11WithCrawler(TestHttp11WithCrawler):
+class TestHttpsWithCrawler(TestHttpWithCrawler):
     is_secure = True
 
 
-class TestHttp11Proxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
+class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
     pass
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 86fc071f8..3273a264e 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -19,7 +19,7 @@ from scrapy.http import Request
 from scrapy.utils.defer import maybe_deferred_to_future
 from tests.test_downloader_handlers_http_base import (
     TestHttpProxyBase,
-    TestHttps11Base,
+    TestHttpsBase,
     TestHttpsCustomCiphersBase,
     TestHttpsInvalidDNSIdBase,
     TestHttpsInvalidDNSPatternBase,
@@ -61,7 +61,7 @@ def test_not_configured_without_reactor() -> None:
         H2DownloadHandler.from_crawler(crawler)
 
 
-class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
+class TestHttp2(H2DownloadHandlerMixin, TestHttpsBase):
     http2 = True
     handler_supports_http2_dataloss = False
 
@@ -149,27 +149,25 @@ class TestHttps2(H2DownloadHandlerMixin, TestHttps11Base):
                 await download_handler.download_request(request)
 
 
-class TestHttps2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
+class TestHttp2WrongHostname(H2DownloadHandlerMixin, TestHttpsWrongHostnameBase):
     pass
 
 
-class TestHttps2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
+class TestHttp2InvalidDNSId(H2DownloadHandlerMixin, TestHttpsInvalidDNSIdBase):
     pass
 
 
-class TestHttps2InvalidDNSPattern(
+class TestHttp2InvalidDNSPattern(
     H2DownloadHandlerMixin, TestHttpsInvalidDNSPatternBase
 ):
     pass
 
 
-class TestHttps2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
+class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase):
     pass
 
 
 class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
-    """HTTP 2.0 test case with MockServer"""
-
     @property
     def settings_dict(self) -> dict[str, Any] | None:
         return {
@@ -194,7 +192,7 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
         pytest.skip("headers_received support is not implemented")
 
 
-class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
+class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
     is_secure = True
     expected_http_proxy_request_body = b"/"
 
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 66edf9325..2a781dace 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -55,9 +55,17 @@ if TYPE_CHECKING:
 
 
 class TestHttpBase(ABC):
-    is_secure = False
+    is_secure: bool = False
+    http2: bool = False
     # whether the handler supports per-request bindaddress
-    handler_supports_bindaddress_meta = True
+    handler_supports_bindaddress_meta: bool = True
+    # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
+    # stream error (of type PROTOCOL_ERROR) so the client will send
+    # RST_STREAM. Some libraries do only this while e.g. h2 also closes the
+    # connection (see handling of ProtocolError in
+    # h2.connection.H2Connection.receive_data()), thus closing all streams that
+    # were using it, and we handle this as a normal exception.
+    handler_supports_http2_dataloss: bool = True
     # default headers added by the underlying library that cannot be suppressed
     always_present_req_headers: ClassVar[frozenset[str]] = frozenset()
 
@@ -519,17 +527,6 @@ class TestHttpBase(ABC):
         else:
             assert latency > 0
 
-
-class TestHttp11Base(TestHttpBase):
-    http2: bool = False
-    # RFC 9113 §8.1.1 explicitly says that a Content-Length mismatch is a
-    # stream error (of type PROTOCOL_ERROR) so the client will send
-    # RST_STREAM. Some libraries do only this while e.g. h2 also closes the
-    # connection (see handling of ProtocolError in
-    # h2.connection.H2Connection.receive_data()), thus closing all streams that
-    # were using it, and we handle this as a normal exception.
-    handler_supports_http2_dataloss: bool = True
-
     @coroutine_test
     async def test_download_without_maxsize_limit(self, mockserver: MockServer) -> None:
         request = Request(mockserver.url("/text", is_secure=self.is_secure))
@@ -803,7 +800,7 @@ class TestHttp11Base(TestHttpBase):
             )
 
 
-class TestHttps11Base(TestHttp11Base):
+class TestHttpsBase(TestHttpBase):
     is_secure = True
 
     tls_log_message = (
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
deleted file mode 100644
index ac05d457d..000000000
--- a/tests/test_webclient.py
+++ /dev/null
@@ -1,404 +0,0 @@
-"""
-Tests borrowed from the twisted.web.client tests.
-"""
-
-from __future__ import annotations
-
-from urllib.parse import urlparse
-
-import OpenSSL.SSL
-import pytest
-from pytest_twisted import async_yield_fixture
-from twisted.internet.defer import inlineCallbacks
-from twisted.internet.testing import StringTransport
-from twisted.protocols.policies import WrappingFactory
-from twisted.web import resource, server, static, util
-from twisted.web.client import _makeGetterFactory
-
-from scrapy.core.downloader import webclient as client
-from scrapy.core.downloader.contextfactory import _ScrapyClientContextFactory
-from scrapy.exceptions import DownloadTimeoutError
-from scrapy.http import Headers, Request
-from scrapy.utils.misc import build_from_crawler
-from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.test import get_crawler
-from tests.mockserver.http_resources import (
-    ForeverTakingResource,
-    HostHeaderResource,
-    PayloadResource,
-)
-from tests.mockserver.utils import ssl_context_factory
-from tests.test_core_downloader import TestContextFactoryBase
-
-# these tests are related to the Twisted HTTP code
-pytestmark = pytest.mark.requires_reactor
-
-
-def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
-    """Adapted version of twisted.web.client.getPage"""
-
-    def _clientfactory(url, *args, **kwargs):
-        url = to_unicode(url)
-        timeout = kwargs.pop("timeout", 0)
-        f = client.ScrapyHTTPClientFactory(
-            Request(url, *args, **kwargs), timeout=timeout
-        )
-        f.deferred.addCallback(response_transform or (lambda r: r.body))
-        return f
-
-    return _makeGetterFactory(
-        to_bytes(url),
-        _clientfactory,
-        *args,
-        contextFactory=contextFactory,
-        **kwargs,
-    ).deferred
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestScrapyHTTPPageGetter:
-    def test_earlyHeaders(self):
-        # basic test stolen from twisted HTTPageGetter
-        factory = client.ScrapyHTTPClientFactory(
-            Request(
-                url="http://foo/bar",
-                body="some data",
-                headers={
-                    "Host": "example.net",
-                    "User-Agent": "fooble",
-                    "Cookie": "blah blah",
-                    "Content-Length": "12981",
-                    "Useful": "value",
-                },
-            )
-        )
-
-        self._test(
-            factory,
-            b"GET /bar HTTP/1.0\r\n"
-            b"Content-Length: 9\r\n"
-            b"Useful: value\r\n"
-            b"Connection: close\r\n"
-            b"User-Agent: fooble\r\n"
-            b"Host: example.net\r\n"
-            b"Cookie: blah blah\r\n"
-            b"\r\n"
-            b"some data",
-        )
-
-        # test minimal sent headers
-        factory = client.ScrapyHTTPClientFactory(Request("http://foo/bar"))
-        self._test(factory, b"GET /bar HTTP/1.0\r\nHost: foo\r\n\r\n")
-
-        # test a simple POST with body and content-type
-        factory = client.ScrapyHTTPClientFactory(
-            Request(
-                method="POST",
-                url="http://foo/bar",
-                body="name=value",
-                headers={"Content-Type": "application/x-www-form-urlencoded"},
-            )
-        )
-
-        self._test(
-            factory,
-            b"POST /bar HTTP/1.0\r\n"
-            b"Host: foo\r\n"
-            b"Connection: close\r\n"
-            b"Content-Type: application/x-www-form-urlencoded\r\n"
-            b"Content-Length: 10\r\n"
-            b"\r\n"
-            b"name=value",
-        )
-
-        # test a POST method with no body provided
-        factory = client.ScrapyHTTPClientFactory(
-            Request(method="POST", url="http://foo/bar")
-        )
-
-        self._test(
-            factory,
-            b"POST /bar HTTP/1.0\r\nHost: foo\r\nContent-Length: 0\r\n\r\n",
-        )
-
-        # test with single and multivalued headers
-        factory = client.ScrapyHTTPClientFactory(
-            Request(
-                url="http://foo/bar",
-                headers={
-                    "X-Meta-Single": "single",
-                    "X-Meta-Multivalued": ["value1", "value2"],
-                },
-            )
-        )
-
-        self._test(
-            factory,
-            b"GET /bar HTTP/1.0\r\n"
-            b"Host: foo\r\n"
-            b"X-Meta-Multivalued: value1\r\n"
-            b"X-Meta-Multivalued: value2\r\n"
-            b"X-Meta-Single: single\r\n"
-            b"\r\n",
-        )
-
-        # same test with single and multivalued headers but using Headers class
-        factory = client.ScrapyHTTPClientFactory(
-            Request(
-                url="http://foo/bar",
-                headers=Headers(
-                    {
-                        "X-Meta-Single": "single",
-                        "X-Meta-Multivalued": ["value1", "value2"],
-                    }
-                ),
-            )
-        )
-
-        self._test(
-            factory,
-            b"GET /bar HTTP/1.0\r\n"
-            b"Host: foo\r\n"
-            b"X-Meta-Multivalued: value1\r\n"
-            b"X-Meta-Multivalued: value2\r\n"
-            b"X-Meta-Single: single\r\n"
-            b"\r\n",
-        )
-
-    def _test(self, factory, testvalue):
-        transport = StringTransport()
-        protocol = client.ScrapyHTTPPageGetter()
-        protocol.factory = factory
-        protocol.makeConnection(transport)
-        assert set(transport.value().splitlines()) == set(testvalue.splitlines())
-        return testvalue
-
-    def test_non_standard_line_endings(self):
-        # regression test for: http://dev.scrapy.org/ticket/258
-        factory = client.ScrapyHTTPClientFactory(Request(url="http://foo/bar"))
-        protocol = client.ScrapyHTTPPageGetter()
-        protocol.factory = factory
-        protocol.headers = Headers()
-        protocol.dataReceived(b"HTTP/1.0 200 OK\n")
-        protocol.dataReceived(b"Hello: World\n")
-        protocol.dataReceived(b"Foo: Bar\n")
-        protocol.dataReceived(b"\n")
-        assert protocol.headers == Headers({"Hello": ["World"], "Foo": ["Bar"]})
-
-
-class EncodingResource(resource.Resource):
-    out_encoding = "cp1251"
-
-    def render(self, request):
-        body = to_unicode(request.content.read())
-        request.setHeader(b"content-encoding", self.out_encoding)
-        return body.encode(self.out_encoding)
-
-
-class BrokenDownloadResource(resource.Resource):
-    def render(self, request):
-        # only sends 3 bytes even though it claims to send 5
-        request.setHeader(b"content-length", b"5")
-        request.write(b"abc")
-        return b""
-
-
-class ErrorResource(resource.Resource):
-    def render(self, request):
-        request.setResponseCode(401)
-        if request.args.get(b"showlength"):
-            request.setHeader(b"content-length", b"0")
-        return b""
-
-
-class NoLengthResource(resource.Resource):
-    def render(self, request):
-        return b"nolength"
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestWebClient:
-    def _listen(self, site):
-        from twisted.internet import reactor
-
-        return reactor.listenTCP(0, site, interface="127.0.0.1")
-
-    @pytest.fixture
-    def wrapper(self, tmp_path):
-        (tmp_path / "file").write_bytes(b"0123456789")
-        r = static.File(str(tmp_path))
-        r.putChild(b"redirect", util.Redirect(b"/file"))
-        r.putChild(b"wait", ForeverTakingResource())
-        r.putChild(b"error", ErrorResource())
-        r.putChild(b"nolength", NoLengthResource())
-        r.putChild(b"host", HostHeaderResource())
-        r.putChild(b"payload", PayloadResource())
-        r.putChild(b"broken", BrokenDownloadResource())
-        r.putChild(b"encoding", EncodingResource())
-        site = server.Site(r, timeout=None)
-        return WrappingFactory(site)
-
-    @async_yield_fixture
-    async def server_port(self, wrapper):
-        port = self._listen(wrapper)
-
-        yield port.getHost().port
-
-        await port.stopListening()
-
-    @pytest.fixture
-    def server_url(self, server_port):
-        return f"http://127.0.0.1:{server_port}/"
-
-    @inlineCallbacks
-    def testPayload(self, server_url):
-        s = "0123456789" * 10
-        body = yield getPage(server_url + "payload", body=s)
-        assert body == to_bytes(s)
-
-    @inlineCallbacks
-    def testHostHeader(self, server_port, server_url):
-        # if we pass Host header explicitly, it should be used, otherwise
-        # it should extract from url
-        body = yield getPage(server_url + "host")
-        assert body == to_bytes(f"127.0.0.1:{server_port}")
-        body = yield getPage(server_url + "host", headers={"Host": "www.example.com"})
-        assert body == to_bytes("www.example.com")
-
-    @inlineCallbacks
-    def test_getPage(self, server_url):
-        """
-        L{client.getPage} returns a L{Deferred} which is called back with
-        the body of the response if the default method B{GET} is used.
-        """
-        body = yield getPage(server_url + "file")
-        assert body == b"0123456789"
-
-    @inlineCallbacks
-    def test_getPageHead(self, server_url):
-        """
-        L{client.getPage} returns a L{Deferred} which is called back with
-        the empty string if the method is C{HEAD} and there is a successful
-        response code.
-        """
-
-        def _getPage(method):
-            return getPage(server_url + "file", method=method)
-
-        body = yield _getPage("head")
-        assert body == b""
-        body = yield _getPage("HEAD")
-        assert body == b""
-
-    @inlineCallbacks
-    def test_timeoutNotTriggering(self, server_port, server_url):
-        """
-        When a non-zero timeout is passed to L{getPage} and the page is
-        retrieved before the timeout period elapses, the L{Deferred} is
-        called back with the contents of the page.
-        """
-        body = yield getPage(server_url + "host", timeout=100)
-        assert body == to_bytes(f"127.0.0.1:{server_port}")
-
-    @inlineCallbacks
-    def test_timeoutTriggering(self, wrapper, server_url):
-        """
-        When a non-zero timeout is passed to L{getPage} and that many
-        seconds elapse before the server responds to the request. the
-        L{Deferred} is errbacked with a L{DownloadTimeoutError}.
-        """
-        with pytest.raises(DownloadTimeoutError):
-            yield getPage(server_url + "wait", timeout=0.000001)
-        # Clean up the server which is hanging around not doing
-        # anything.
-        connected = list(wrapper.protocols.keys())
-        # There might be nothing here if the server managed to already see
-        # that the connection was lost.
-        if connected:
-            connected[0].transport.loseConnection()
-
-    @inlineCallbacks
-    def testNotFound(self, server_url):
-        body = yield getPage(server_url + "notsuchfile")
-        assert b"404 - No Such Resource" in body
-
-    @inlineCallbacks
-    def testFactoryInfo(self, server_url):
-        from twisted.internet import reactor
-
-        url = server_url + "file"
-        parsed = urlparse(url)
-        factory = client.ScrapyHTTPClientFactory(Request(url))
-        reactor.connectTCP(parsed.hostname, parsed.port, factory)
-        yield factory.deferred
-        assert factory.status == b"200"
-        assert factory.version.startswith(b"HTTP/")
-        assert factory.message == b"OK"
-        assert factory.response_headers[b"content-length"] == b"10"
-
-    @inlineCallbacks
-    def testRedirect(self, server_url):
-        body = yield getPage(server_url + "redirect")
-        assert (
-            body
-            == b'\n<html>\n    <head>\n        <meta http-equiv="refresh" content="0;URL=/file">\n'
-            b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
-            b'<a href="/file">click here</a>\n    </body>\n</html>\n'
-        )
-
-    @inlineCallbacks
-    def test_encoding(self, server_url):
-        """Test that non-standart body encoding matches
-        Content-Encoding header"""
-        original_body = b"\xd0\x81\xd1\x8e\xd0\xaf"
-        response = yield getPage(
-            server_url + "encoding", body=original_body, response_transform=lambda r: r
-        )
-        content_encoding = to_unicode(response.headers[b"Content-Encoding"])
-        assert content_encoding == EncodingResource.out_encoding
-        assert response.body.decode(content_encoding) == to_unicode(original_body)
-
-
-@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
-class TestWebClientSSL(TestContextFactoryBase):
-    @inlineCallbacks
-    def testPayload(self, server_url):
-        s = "0123456789" * 10
-        body = yield getPage(server_url + "payload", body=s)
-        assert body == to_bytes(s)
-
-
-class TestWebClientCustomCiphersSSL(TestWebClientSSL):
-    # we try to use a cipher that is not enabled by default in OpenSSL
-    custom_ciphers = "CAMELLIA256-SHA"
-    context_factory = ssl_context_factory(cipher_string=custom_ciphers)
-
-    @inlineCallbacks
-    def testPayload(self, server_url):
-        s = "0123456789" * 10
-        crawler = get_crawler(
-            settings_dict={"DOWNLOADER_CLIENT_TLS_CIPHERS": self.custom_ciphers}
-        )
-        client_context_factory = build_from_crawler(
-            _ScrapyClientContextFactory, crawler
-        )
-        body = yield getPage(
-            server_url + "payload", body=s, contextFactory=client_context_factory
-        )
-        assert body == to_bytes(s)
-
-    @inlineCallbacks
-    def testPayloadDisabledCipher(self, server_url):
-        s = "0123456789" * 10
-        crawler = get_crawler(
-            settings_dict={
-                "DOWNLOADER_CLIENT_TLS_CIPHERS": "ECDHE-RSA-AES256-GCM-SHA384"
-            }
-        )
-        client_context_factory = build_from_crawler(
-            _ScrapyClientContextFactory, crawler
-        )
-        with pytest.raises(OpenSSL.SSL.Error):
-            yield getPage(
-                server_url + "payload", body=s, contextFactory=client_context_factory
-            )

From d05b241f648f54a7b881a669ad8a60808d614d43 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 4 May 2026 11:59:27 +0500
Subject: [PATCH 10/36] Fix a code block in leaks.rst. (#7489)

---
 docs/topics/leaks.rst | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index e61f33aed..ff05cd284 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -68,19 +68,21 @@ Response, Item, Spider and Selector objects.
 
 You can enter the telnet console and inspect how many objects (of the classes
 mentioned above) are currently alive using the ``prefs()`` function which is an
-alias to the :func:`~scrapy.utils.trackref.print_live_refs` function::
+alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:
+
+.. code-block:: bash
 
     telnet localhost 6023
 
-    .. code-block:: pycon
+.. code-block:: pycon
 
-        >>> prefs()
-        Live References
+    >>> prefs()
+    Live References
 
-        ExampleSpider                       1   oldest: 15s ago
-        HtmlResponse                       10   oldest: 1s ago
-        Selector                            2   oldest: 0s ago
-        FormRequest                       878   oldest: 7s ago
+    ExampleSpider                       1   oldest: 15s ago
+    HtmlResponse                       10   oldest: 1s ago
+    Selector                            2   oldest: 0s ago
+    FormRequest                       878   oldest: 7s ago
 
 As you can see, that report also shows the "age" of the oldest object in each
 class. If you're running multiple spiders per process chances are you can

From f3868e11fb0ef3b0aeae3ebf64a78b1deb733fed Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 4 May 2026 17:34:24 +0500
Subject: [PATCH 11/36] Drop start_requests() support. (#7490)

---
 extras/qpsclient.py                          |   4 -
 scrapy/core/spidermw.py                      | 151 +-----------
 scrapy/spidermiddlewares/base.py             |   7 -
 scrapy/spiders/__init__.py                   |  22 +-
 scrapy/spiders/sitemap.py                    |   4 -
 tests/test_spider_start.py                   |  99 --------
 tests/test_spidermiddleware_base.py          |  22 +-
 tests/test_spidermiddleware_process_start.py | 236 -------------------
 tests/test_spidermiddleware_start.py         |  17 --
 9 files changed, 18 insertions(+), 544 deletions(-)

diff --git a/extras/qpsclient.py b/extras/qpsclient.py
index 269b27336..8e5001c1d 100644
--- a/extras/qpsclient.py
+++ b/extras/qpsclient.py
@@ -35,10 +35,6 @@ class QPSSpider(Spider):
             self.download_delay = float(self.download_delay)
 
     async def start(self):
-        for item_or_request in self.start_requests():
-            yield item_or_request
-
-    def start_requests(self):
         url = self.benchurl
         if self.latency is not None:
             url += f"?latency={self.latency}"
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 67342099d..790317357 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -33,7 +33,6 @@ from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_n
 if TYPE_CHECKING:
     from collections.abc import Generator
 
-    from scrapy.crawler import Crawler
     from scrapy.settings import BaseSettings
 
 
@@ -60,76 +59,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
             settings.get_component_priority_dict_with_base("SPIDER_MIDDLEWARES")
         )
 
-    def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:
-        self._check_deprecated_process_start_requests_use(middlewares)
-        super().__init__(*middlewares, crawler=crawler)
-
-    def _check_deprecated_process_start_requests_use(
-        self, middlewares: tuple[Any, ...]
-    ) -> None:
-        deprecated_middlewares = [
-            middleware
-            for middleware in middlewares
-            if hasattr(middleware, "process_start_requests")
-            and not hasattr(middleware, "process_start")
-        ]
-        modern_middlewares = [
-            middleware
-            for middleware in middlewares
-            if not hasattr(middleware, "process_start_requests")
-            and hasattr(middleware, "process_start")
-        ]
-        if deprecated_middlewares and modern_middlewares:
-            raise ValueError(
-                "You are trying to combine spider middlewares that only "
-                "define the deprecated process_start_requests() method () "
-                "with spider middlewares that only define the "
-                "process_start() method (). This is not possible. You must "
-                "either disable or make universal 1 of those 2 sets of "
-                "spider middlewares. Making a spider middleware universal "
-                "means having it define both methods. See the release notes "
-                "of Scrapy 2.13 for details: "
-                "https://docs.scrapy.org/en/2.13/news.html"
-            )
-
-        self._use_start_requests = bool(deprecated_middlewares)
-        if self._use_start_requests:
-            deprecated_middleware_list = ", ".join(
-                global_object_name(middleware.__class__)
-                for middleware in deprecated_middlewares
-            )
-            warn(
-                f"The following enabled spider middlewares, directly or "
-                f"through their parent classes, define the deprecated "
-                f"process_start_requests() method: "
-                f"{deprecated_middleware_list}. process_start_requests() has "
-                f"been deprecated in favor of a new method, process_start(), "
-                f"to support asynchronous code execution. "
-                f"process_start_requests() will stop being called in a future "
-                f"version of Scrapy. If you use Scrapy 2.13 or higher "
-                f"only, replace process_start_requests() with "
-                f"process_start(); note that process_start() is a coroutine "
-                f"(async def). If you need to maintain compatibility with "
-                f"lower Scrapy versions, when defining "
-                f"process_start_requests() in a spider middleware class, "
-                f"define process_start() as well. See the release notes of "
-                f"Scrapy 2.13 for details: "
-                f"https://docs.scrapy.org/en/2.13/news.html",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
     def _add_middleware(self, mw: Any) -> None:
         if hasattr(mw, "process_spider_input"):
             self.methods["process_spider_input"].append(mw.process_spider_input)
             self._check_mw_method_spider_arg(mw.process_spider_input)
 
-        if self._use_start_requests:
-            if hasattr(mw, "process_start_requests"):
-                self.methods["process_start_requests"].appendleft(
-                    mw.process_start_requests
-                )
-        elif hasattr(mw, "process_start"):
+        if hasattr(mw, "process_start"):
             self.methods["process_start"].appendleft(mw.process_start)
 
         process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
@@ -451,88 +386,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 )
             warn(msg, category=ScrapyDeprecationWarning, stacklevel=2)
             self._set_compat_spider(spider)
-        self._check_deprecated_start_requests_use()
-        if self._use_start_requests:
-            sync_start = iter(self._spider.start_requests())
-            sync_start = await self._process_chain(
-                "process_start_requests", sync_start, always_add_spider=True
-            )
-            start: AsyncIterator[Any] = as_async_generator(sync_start)
-        else:
-            start = self._spider.start()
-            start = await self._process_chain("process_start", start)
-        return start
-
-    def _check_deprecated_start_requests_use(self) -> None:
-        start_requests_cls = None
-        start_cls = None
-        spidercls = self._spider.__class__
-        mro = spidercls.__mro__
-
-        for cls in mro:
-            cls_dict = cls.__dict__
-            if start_requests_cls is None and "start_requests" in cls_dict:
-                start_requests_cls = cls
-            if start_cls is None and "start" in cls_dict:
-                start_cls = cls
-            if start_requests_cls is not None and start_cls is not None:
-                break
-
-        # Spider defines both, start_requests and start.
-        assert start_requests_cls is not None
-        assert start_cls is not None
-
-        if (
-            start_requests_cls is not Spider
-            and start_cls is not start_requests_cls
-            and mro.index(start_requests_cls) < mro.index(start_cls)
-        ):
-            src = global_object_name(start_requests_cls)
-            if start_requests_cls is not spidercls:
-                src += f" (inherited by {global_object_name(spidercls)})"
-            warn(
-                f"{src} defines the deprecated start_requests() method. "
-                f"start_requests() has been deprecated in favor of a new "
-                f"method, start(), to support asynchronous code "
-                f"execution. start_requests() will stop being called in a "
-                f"future version of Scrapy. If you use Scrapy 2.13 or "
-                f"higher only, replace start_requests() with start(); "
-                f"note that start() is a coroutine (async def). If you "
-                f"need to maintain compatibility with lower Scrapy versions, "
-                f"when overriding start_requests() in a spider class, "
-                f"override start() as well; you can use super() to "
-                f"reuse the inherited start() implementation without "
-                f"copy-pasting. See the release notes of Scrapy 2.13 for "
-                f"details: https://docs.scrapy.org/en/2.13/news.html",
-                ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
-        if (
-            self._use_start_requests
-            and start_cls is not Spider
-            and start_requests_cls is not start_cls
-            and mro.index(start_cls) < mro.index(start_requests_cls)
-        ):
-            src = global_object_name(start_cls)
-            if start_cls is not spidercls:
-                src += f" (inherited by {global_object_name(spidercls)})"
-            raise ValueError(
-                f"{src} does not define the deprecated start_requests() "
-                f"method. However, one or more of your enabled spider "
-                f"middlewares (reported in an earlier deprecation warning) "
-                f"define the process_start_requests() method, and not the "
-                f"process_start() method, making them only compatible with "
-                f"(deprecated) spiders that define the start_requests() "
-                f"method. To solve this issue, disable the offending spider "
-                f"middlewares, upgrade them as described in that earlier "
-                f"deprecation warning, or make your spider compatible with "
-                f"deprecated spider middlewares (and earlier Scrapy versions) "
-                f"by defining a sync start_requests() method that works "
-                f"similarly to its existing start() method. See the "
-                f"release notes of Scrapy 2.13 for details: "
-                f"https://docs.scrapy.org/en/2.13/news.html"
-            )
+        start = self._spider.start()
+        return await self._process_chain("process_start", start)
 
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
diff --git a/scrapy/spidermiddlewares/base.py b/scrapy/spidermiddlewares/base.py
index 889fc6df1..e09f2d10e 100644
--- a/scrapy/spidermiddlewares/base.py
+++ b/scrapy/spidermiddlewares/base.py
@@ -41,13 +41,6 @@ class BaseSpiderMiddleware:
     def from_crawler(cls, crawler: Crawler) -> Self:
         return cls(crawler)
 
-    def process_start_requests(
-        self, start: Iterable[Any], spider: Spider
-    ) -> Iterable[Any]:
-        for o in start:
-            if (o := self._get_processed(o, None)) is not None:
-                yield o
-
     async def process_start(self, start: AsyncIterator[Any]) -> AsyncIterator[Any]:
         async for o in start:
             if (o := self._get_processed(o, None)) is not None:
diff --git a/scrapy/spiders/__init__.py b/scrapy/spiders/__init__.py
index 018e510c5..299a5d43f 100644
--- a/scrapy/spiders/__init__.py
+++ b/scrapy/spiders/__init__.py
@@ -7,17 +7,15 @@ See documentation in docs/topics/spiders.rst
 from __future__ import annotations
 
 import logging
-import warnings
 from typing import TYPE_CHECKING, Any, cast
 
 from scrapy import signals
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.utils.trackref import object_ref
 from scrapy.utils.url import url_is_from_spider
 
 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Iterable
+    from collections.abc import AsyncIterator
 
     from twisted.internet.defer import Deferred
 
@@ -127,24 +125,6 @@ class Spider(object_ref):
 
         .. seealso:: :ref:`start-requests`
         """
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore", category=ScrapyDeprecationWarning, module=r"^scrapy\.spiders$"
-            )
-            for item_or_request in self.start_requests():
-                yield item_or_request
-
-    def start_requests(self) -> Iterable[Any]:
-        warnings.warn(
-            (
-                "The Spider.start_requests() method is deprecated, use "
-                "Spider.start() instead. If you are calling "
-                "super().start_requests() from a Spider.start() override, "
-                "iterate super().start() instead."
-            ),
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
         if not self.start_urls and hasattr(self, "start_url"):
             raise AttributeError(
                 "Crawling could not start: 'start_urls' not found "
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index d87ffccaf..2a80b8d24 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -54,10 +54,6 @@ class SitemapSpider(Spider):
         self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
 
     async def start(self) -> AsyncIterator[Any]:
-        for item_or_request in self.start_requests():
-            yield item_or_request
-
-    def start_requests(self) -> Iterable[Request]:
         for url in self.sitemap_urls:
             yield Request(url, self._parse_sitemap)
 
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 4e359fd33..aef2093ac 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -1,15 +1,12 @@
 from __future__ import annotations
 
-import re
 import warnings
 from asyncio import sleep
 from typing import Any
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy import Spider, signals
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 
@@ -77,81 +74,6 @@ class TestMain:
             warnings.simplefilter("error")
             await self._test_spider(TestSpider, [ITEM_A])
 
-    @coroutine_test
-    async def test_deprecated(self):
-        class TestSpider(Spider):
-            name = "test"
-
-            def start_requests(self):
-                yield ITEM_A
-
-        with pytest.warns(ScrapyDeprecationWarning):
-            await self._test_spider(TestSpider, [ITEM_A])
-
-    @coroutine_test
-    async def test_deprecated_subclass(self):
-        class BaseSpider(Spider):
-            def start_requests(self):
-                yield ITEM_A
-
-        class TestSpider(BaseSpider):
-            name = "test"
-
-        # The warning must be about the base class and not the subclass.
-        with pytest.warns(ScrapyDeprecationWarning, match="BaseSpider"):
-            await self._test_spider(TestSpider, [ITEM_A])
-
-    @coroutine_test
-    async def test_universal(self):
-        class TestSpider(Spider):
-            name = "test"
-
-            async def start(self):
-                yield ITEM_A
-
-            def start_requests(self):
-                yield ITEM_B
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_spider(TestSpider, [ITEM_A])
-
-    @coroutine_test
-    async def test_universal_subclass(self):
-        class BaseSpider(Spider):
-            async def start(self):
-                yield ITEM_A
-
-            def start_requests(self):
-                yield ITEM_B
-
-        class TestSpider(BaseSpider):
-            name = "test"
-
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_spider(TestSpider, [ITEM_A])
-
-    @coroutine_test
-    async def test_start_deprecated_super(self):
-        class TestSpider(Spider):
-            name = "test"
-
-            async def start(self):
-                for item_or_request in super().start_requests():
-                    yield item_or_request
-
-        msg = "use Spider.start() instead"
-        with pytest.warns(ScrapyDeprecationWarning, match=re.escape(msg)) as ws:
-            await self._test_spider(TestSpider, [])
-
-        for w in ws:
-            if isinstance(w.message, ScrapyDeprecationWarning) and msg in str(
-                w.message
-            ):
-                assert w.filename.endswith("test_spider_start.py")
-                break
-
     async def _test_start(self, start_, expected_items=None):
         class TestSpider(Spider):
             name = "test"
@@ -176,24 +98,3 @@ class TestMain:
             yield ITEM_A
 
         await self._test_start(start, [ITEM_A])
-
-    # Exceptions
-
-    @coroutine_test
-    async def test_deprecated_non_generator_exception(self):
-        class TestSpider(Spider):
-            name = "test"
-
-            def start_requests(self):
-                raise RuntimeError
-
-        with (
-            LogCapture() as log,
-            pytest.warns(
-                ScrapyDeprecationWarning,
-                match=r"defines the deprecated start_requests\(\) method",
-            ),
-        ):
-            await self._test_spider(TestSpider, [])
-
-        assert "in start_requests\n    raise RuntimeError" in str(log)
diff --git a/tests/test_spidermiddleware_base.py b/tests/test_spidermiddleware_base.py
index da570904d..70326f3f1 100644
--- a/tests/test_spidermiddleware_base.py
+++ b/tests/test_spidermiddleware_base.py
@@ -7,7 +7,9 @@ import pytest
 from scrapy import Request, Spider
 from scrapy.http import Response
 from scrapy.spidermiddlewares.base import BaseSpiderMiddleware
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler
@@ -18,7 +20,8 @@ def crawler() -> Crawler:
     return get_crawler(Spider)
 
 
-def test_trivial(crawler: Crawler) -> None:
+@coroutine_test
+async def test_trivial(crawler: Crawler) -> None:
     class TrivialSpiderMiddleware(BaseSpiderMiddleware):
         pass
 
@@ -29,12 +32,13 @@ def test_trivial(crawler: Crawler) -> None:
     spider_output = [test_req, {"foo": "bar"}]
     for processed in [
         list(mw.process_spider_output(Response("data:,"), spider_output)),
-        list(mw.process_start_requests(spider_output, None)),  # type: ignore[arg-type]
+        await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
     ]:
         assert processed == [test_req, {"foo": "bar"}]
 
 
-def test_processed_request(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_request(crawler: Crawler) -> None:
     class ProcessReqSpiderMiddleware(BaseSpiderMiddleware):
         def get_processed_request(
             self, request: Request, response: Response | None
@@ -52,7 +56,7 @@ def test_processed_request(crawler: Crawler) -> None:
     spider_output = [test_req1, {"foo": "bar"}, test_req2, test_req3]
     for processed in [
         list(mw.process_spider_output(Response("data:,"), spider_output)),
-        list(mw.process_start_requests(spider_output, None)),  # type: ignore[arg-type]
+        await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
     ]:
         assert len(processed) == 3
         assert isinstance(processed[0], Request)
@@ -62,7 +66,8 @@ def test_processed_request(crawler: Crawler) -> None:
         assert processed[2].url == "data:30,"
 
 
-def test_processed_item(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_item(crawler: Crawler) -> None:
     class ProcessItemSpiderMiddleware(BaseSpiderMiddleware):
         def get_processed_item(self, item: Any, response: Response | None) -> Any:
             if item["foo"] == 2:
@@ -76,12 +81,13 @@ def test_processed_item(crawler: Crawler) -> None:
     spider_output = [{"foo": 1}, {"foo": 2}, test_req, {"foo": 3}]
     for processed in [
         list(mw.process_spider_output(Response("data:,"), spider_output)),
-        list(mw.process_start_requests(spider_output, None)),  # type: ignore[arg-type]
+        await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
     ]:
         assert processed == [{"foo": 1}, test_req, {"foo": 30}]
 
 
-def test_processed_both(crawler: Crawler) -> None:
+@coroutine_test
+async def test_processed_both(crawler: Crawler) -> None:
     class ProcessBothSpiderMiddleware(BaseSpiderMiddleware):
         def get_processed_request(
             self, request: Request, response: Response | None
@@ -113,7 +119,7 @@ def test_processed_both(crawler: Crawler) -> None:
     ]
     for processed in [
         list(mw.process_spider_output(Response("data:,"), spider_output)),
-        list(mw.process_start_requests(spider_output, None)),  # type: ignore[arg-type]
+        await collect_asyncgen(mw.process_start(as_async_generator(spider_output))),
     ]:
         assert len(processed) == 4
         assert isinstance(processed[0], Request)
diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py
index ddece52c4..c907c6d73 100644
--- a/tests/test_spidermiddleware_process_start.py
+++ b/tests/test_spidermiddleware_process_start.py
@@ -4,7 +4,6 @@ from asyncio import sleep
 import pytest
 
 from scrapy import Spider, signals
-from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.test_spider_start import SLEEP_SECONDS
@@ -38,15 +37,6 @@ class TwistedSleepSpiderMiddleware:
             yield item_or_request
 
 
-class UniversalSpiderMiddleware:
-    async def process_start(self, start):
-        async for item_or_request in start:
-            yield item_or_request
-
-    def process_start_requests(self, start_requests, spider):
-        raise NotImplementedError
-
-
 # Spiders and spider middlewares for TestMain._test_wrap
 
 
@@ -61,23 +51,6 @@ class ModernWrapSpiderSubclass(ModernWrapSpider):
     name = "test"
 
 
-class UniversalWrapSpider(Spider):
-    name = "test"
-
-    async def start(self):
-        yield ITEM_B
-
-    def start_requests(self):
-        yield ITEM_D
-
-
-class DeprecatedWrapSpider(Spider):
-    name = "test"
-
-    def start_requests(self):
-        yield ITEM_B
-
-
 class ModernWrapSpiderMiddleware:
     async def process_start(self, start):
         yield ITEM_A
@@ -86,26 +59,6 @@ class ModernWrapSpiderMiddleware:
         yield ITEM_C
 
 
-class UniversalWrapSpiderMiddleware:
-    async def process_start(self, start):
-        yield ITEM_A
-        async for item_or_request in start:
-            yield item_or_request
-        yield ITEM_C
-
-    def process_start_requests(self, start, spider):
-        yield ITEM_A
-        yield from start
-        yield ITEM_C
-
-
-class DeprecatedWrapSpiderMiddleware:
-    def process_start_requests(self, start, spider):
-        yield ITEM_A
-        yield from start
-        yield ITEM_C
-
-
 class TestMain:
     async def _test(self, spider_middlewares, spider_cls, expected_items):
         actual_items = []
@@ -136,195 +89,6 @@ class TestMain:
             warnings.simplefilter("error")
             await self._test_wrap(ModernWrapSpiderMiddleware, ModernWrapSpider)
 
-    @coroutine_test
-    async def test_modern_mw_universal_spider(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_wrap(ModernWrapSpiderMiddleware, UniversalWrapSpider)
-
-    @coroutine_test
-    async def test_modern_mw_deprecated_spider(self):
-        with pytest.warns(
-            ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
-        ):
-            await self._test_wrap(ModernWrapSpiderMiddleware, DeprecatedWrapSpider)
-
-    @coroutine_test
-    async def test_universal_mw_modern_spider(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_wrap(UniversalWrapSpiderMiddleware, ModernWrapSpider)
-
-    @coroutine_test
-    async def test_universal_mw_universal_spider(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_wrap(UniversalWrapSpiderMiddleware, UniversalWrapSpider)
-
-    @coroutine_test
-    async def test_universal_mw_deprecated_spider(self):
-        with pytest.warns(
-            ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
-        ):
-            await self._test_wrap(UniversalWrapSpiderMiddleware, DeprecatedWrapSpider)
-
-    @coroutine_test
-    async def test_deprecated_mw_modern_spider(self):
-        with (
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-            ),
-            pytest.raises(
-                ValueError, match=r"only compatible with \(deprecated\) spiders"
-            ),
-        ):
-            await self._test_wrap(DeprecatedWrapSpiderMiddleware, ModernWrapSpider)
-
-    @coroutine_test
-    async def test_deprecated_mw_modern_spider_subclass(self):
-        with (
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-            ),
-            pytest.raises(
-                ValueError,
-                match=r"^\S+?\.ModernWrapSpider \(inherited by \S+?.ModernWrapSpiderSubclass\) .*? only compatible with \(deprecated\) spiders",
-            ),
-        ):
-            await self._test_wrap(
-                DeprecatedWrapSpiderMiddleware, ModernWrapSpiderSubclass
-            )
-
-    @coroutine_test
-    async def test_deprecated_mw_universal_spider(self):
-        with pytest.warns(
-            ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-        ):
-            await self._test_wrap(
-                DeprecatedWrapSpiderMiddleware,
-                UniversalWrapSpider,
-                [ITEM_A, ITEM_D, ITEM_C],
-            )
-
-    @coroutine_test
-    async def test_deprecated_mw_deprecated_spider(self):
-        with (
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-            ),
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
-            ),
-        ):
-            await self._test_wrap(DeprecatedWrapSpiderMiddleware, DeprecatedWrapSpider)
-
-    @coroutine_test
-    async def test_modern_mw_universal_mw_modern_spider(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                UniversalWrapSpiderMiddleware,
-                ModernWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_modern_mw_deprecated_mw_modern_spider(self):
-        with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                ModernWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_universal_mw_deprecated_mw_modern_spider(self):
-        with (
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-            ),
-            pytest.raises(
-                ValueError, match=r"only compatible with \(deprecated\) spiders"
-            ),
-        ):
-            await self._test_douple_wrap(
-                UniversalWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                ModernWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_modern_mw_universal_mw_universal_spider(self):
-        with warnings.catch_warnings():
-            warnings.simplefilter("error")
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                UniversalWrapSpiderMiddleware,
-                UniversalWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_modern_mw_deprecated_mw_universal_spider(self):
-        with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                UniversalWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_universal_mw_deprecated_mw_universal_spider(self):
-        with pytest.warns(
-            ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-        ):
-            await self._test_douple_wrap(
-                UniversalWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                UniversalWrapSpider,
-                [ITEM_A, ITEM_A, ITEM_D, ITEM_C, ITEM_C],
-            )
-
-    @coroutine_test
-    async def test_modern_mw_universal_mw_deprecated_spider(self):
-        with pytest.warns(
-            ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
-        ):
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                UniversalWrapSpiderMiddleware,
-                DeprecatedWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_modern_mw_deprecated_mw_deprecated_spider(self):
-        with pytest.raises(ValueError, match=r"trying to combine spider middlewares"):
-            await self._test_douple_wrap(
-                ModernWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                DeprecatedWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_universal_mw_deprecated_mw_deprecated_spider(self):
-        with (
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated process_start_requests\(\)"
-            ),
-            pytest.warns(
-                ScrapyDeprecationWarning, match=r"deprecated start_requests\(\)"
-            ),
-        ):
-            await self._test_douple_wrap(
-                UniversalWrapSpiderMiddleware,
-                DeprecatedWrapSpiderMiddleware,
-                DeprecatedWrapSpider,
-            )
-
-    @coroutine_test
-    async def test_universal_mw_uses_process_start(self):
-        """Test that process_start_requests() isn't used when process_start() exists."""
-        await self._test([UniversalSpiderMiddleware], ModernWrapSpider, [ITEM_B])
-
     async def _test_sleep(self, spider_middlewares):
         class TestSpider(Spider):
             name = "test"
diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py
index 76976d962..26397d37c 100644
--- a/tests/test_spidermiddleware_start.py
+++ b/tests/test_spidermiddleware_start.py
@@ -23,20 +23,3 @@ class TestMiddleware:
             async for request in mw.process_start(start())
         ]
         assert result == [True, True, False, "foo"]
-
-    @coroutine_test
-    async def test_sync(self):
-        crawler = get_crawler(Spider)
-        mw = build_from_crawler(StartSpiderMiddleware, crawler)
-
-        def start():
-            yield Request("data:,1")
-            yield Request("data:,2", meta={"is_start_request": True})
-            yield Request("data:,2", meta={"is_start_request": False})
-            yield Request("data:,2", meta={"is_start_request": "foo"})
-
-        result = [
-            request.meta["is_start_request"]
-            for request in mw.process_start_requests(start(), Spider("test"))
-        ]
-        assert result == [True, True, False, "foo"]

From 8d69a7c8659ff207420a8c5d725fe0cafa2b866c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Mon, 4 May 2026 19:38:29 +0200
Subject: [PATCH 12/36] Deprecate FormRequest in favor of form2request (#6438)

* WIP

* Add docs

* Remove FormRequest from tests

* Silence mypy issue

* Address docs-tests issues

---------

Co-authored-by: Adrian Chaves <adrian@zyte.com>
Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/conf.py                     |   1 +
 docs/topics/dynamic-content.rst  |   2 +-
 docs/topics/leaks.rst            |   2 +-
 docs/topics/request-response.rst | 225 ++++++++++---------------------
 scrapy/http/__init__.py          |  17 ++-
 scrapy/http/request/form.py      |   8 ++
 tests/test_contracts.py          |  32 +++--
 tests/test_http_request_form.py  |   2 +-
 tests/test_request_dict.py       |   8 +-
 tox.ini                          |   1 +
 10 files changed, 125 insertions(+), 173 deletions(-)

diff --git a/docs/conf.py b/docs/conf.py
index e3e618860..99d5df7da 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -154,6 +154,7 @@ scrapy_intersphinx_enable = [
     "coverage",
     "cryptography",
     "cssselect",
+    "form2request",
     "itemloaders",
     "parsel",
     "pytest",
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 090caa6e0..5a399e094 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -83,7 +83,7 @@ request with Scrapy.
 
 It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
 method and URL. However, you may also need to reproduce the body, headers and
-form parameters (see :class:`~scrapy.FormRequest`) of that request.
+form parameters (see :ref:`form`) of that request.
 
 As all major browsers allow to export the requests in curl_ format, Scrapy
 incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index ff05cd284..0913a3310 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -82,7 +82,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:
     ExampleSpider                       1   oldest: 15s ago
     HtmlResponse                       10   oldest: 1s ago
     Selector                            2   oldest: 0s ago
-    FormRequest                       878   oldest: 7s ago
+    Request                           878   oldest: 7s ago
 
 As you can see, that report also shows the "age" of the oldest object in each
 class. If you're running multiple spiders per process chances are you can
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f23c7611c..ba7b3ee3c 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -246,6 +246,78 @@ Request objects
     .. automethod:: to_dict
 
 
+.. _form:
+
+Creating requests that submit HTML forms
+----------------------------------------
+
+Use :doc:`form2request <form2request:index>` to build request data from an HTML
+``<form>`` element and convert it to a :class:`~scrapy.Request`.
+
+Install it with pip:
+
+.. code-block:: bash
+
+    pip install form2request
+
+Select the desired form with CSS or XPath, then build and convert request
+data:
+
+.. code-block:: python
+
+    from form2request import form2request
+
+
+    def parse(self, response):
+        form = response.css("form#search")
+        request_data = form2request(form, data={"q": "scrapy"})
+        yield request_data.to_scrapy(callback=self.parse_results)
+
+Use ``data`` to override field values. To drop a field from the resulting
+request, set its value to ``None``.
+
+By default, form2request simulates clicking the first submit button. To submit
+without clicking any button, pass ``click=False``. To click a specific submit
+button, pass its element:
+
+.. code-block:: python
+
+    def parse(self, response):
+        form = response.css("form#checkout")
+        submit = form.css('button[name="pay"]')
+        request_data = form2request(form, click=submit)
+
+.. _topics-request-response-ref-request-userlogin:
+
+Using form2request to simulate a user login
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is usual for web sites to provide pre-populated form fields through ``<input
+type="hidden">`` elements, such as session related data or authentication
+tokens (for login pages). Build the request from the form and only override the
+credentials:
+
+.. code-block:: python
+
+    import scrapy
+    from form2request import form2request
+
+
+    class LoginSpider(scrapy.Spider):
+        name = "example.com"
+        start_urls = ["http://www.example.com/users/login.php"]
+
+        def parse(self, response):
+            form = response.css("form")
+            request_data = form2request(
+                form,
+                data={"username": "john", "password": "secret"},
+            )
+            yield request_data.to_scrapy(callback=self.after_login)
+
+        def after_login(self, response): ...
+
+
 Other functions related to requests
 -----------------------------------
 
@@ -771,159 +843,6 @@ Request subclasses
 Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
 it to implement your own custom functionality.
 
-FormRequest objects
--------------------
-
-The FormRequest class extends the base :class:`~scrapy.Request` with functionality for
-dealing with HTML forms. It uses `lxml.html forms`_  to pre-populate form
-fields with form data from :class:`Response` objects.
-
-.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
-
-.. currentmodule:: None
-
-.. class:: scrapy.FormRequest(url, [formdata, ...])
-    :canonical: scrapy.http.request.form.FormRequest
-
-    The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The
-    remaining arguments are the same as for the :class:`~scrapy.Request` class and are
-    not documented here.
-
-    :param formdata: is a dictionary (or iterable of (key, value) tuples)
-       containing HTML Form data which will be url-encoded and assigned to the
-       body of the request.
-    :type formdata: dict or collections.abc.Iterable
-
-    The :class:`~scrapy.FormRequest` objects support the following class method in
-    addition to the standard :class:`~scrapy.Request` methods:
-
-    .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
-
-       Returns a new :class:`~scrapy.FormRequest` object with its form field values
-       pre-populated with those found in the HTML ``<form>`` element contained
-       in the given response. For an example see
-       :ref:`topics-request-response-ref-request-userlogin`.
-
-       The policy is to automatically simulate a click, by default, on any form
-       control that looks clickable, like a ``<input type="submit">``.  Even
-       though this is quite convenient, and often the desired behaviour,
-       sometimes it can cause problems which could be hard to debug. For
-       example, when working with forms that are filled and/or submitted using
-       javascript, the default :meth:`from_response` behaviour may not be the
-       most appropriate. To disable this behaviour you can set the
-       ``dont_click`` argument to ``True``. Also, if you want to change the
-       control clicked (instead of disabling it) you can also use the
-       ``clickdata`` argument.
-
-       .. caution:: Using this method with select elements which have leading
-          or trailing whitespace in the option values will not work due to a
-          `bug in lxml`_, which should be fixed in lxml 3.8 and above.
-
-       :param response: the response containing a HTML form which will be used
-          to pre-populate the form fields
-       :type response: :class:`~scrapy.http.Response` object
-
-       :param formname: if given, the form with name attribute set to this value will be used.
-       :type formname: str
-
-       :param formid: if given, the form with id attribute set to this value will be used.
-       :type formid: str
-
-       :param formxpath: if given, the first form that matches the xpath will be used.
-       :type formxpath: str
-
-       :param formcss: if given, the first form that matches the css selector will be used.
-       :type formcss: str
-
-       :param formnumber: the number of form to use, when the response contains
-          multiple forms. The first one (and also the default) is ``0``.
-       :type formnumber: int
-
-       :param formdata: fields to override in the form data. If a field was
-          already present in the response ``<form>`` element, its value is
-          overridden by the one passed in this parameter. If a value passed in
-          this parameter is ``None``, the field will not be included in the
-          request, even if it was present in the response ``<form>`` element.
-       :type formdata: dict
-
-       :param clickdata: attributes to lookup the control clicked. If it's not
-         given, the form data will be submitted simulating a click on the
-         first clickable element. In addition to html attributes, the control
-         can be identified by its zero-based index relative to other
-         submittable inputs inside the form, via the ``nr`` attribute.
-       :type clickdata: dict
-
-       :param dont_click: If True, the form data will be submitted without
-         clicking in any element.
-       :type dont_click: bool
-
-       The other parameters of this class method are passed directly to the
-       :class:`~scrapy.FormRequest` ``__init__()`` method.
-
-.. currentmodule:: scrapy.http
-
-Request usage examples
-----------------------
-
-Using FormRequest to send data via HTTP POST
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you want to simulate a HTML Form POST in your spider and send a couple of
-key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your
-spider) like this:
-
-.. skip: next
-.. code-block:: python
-
-   return [
-       FormRequest(
-           url="http://www.example.com/post/action",
-           formdata={"name": "John Doe", "age": "27"},
-           callback=self.after_post,
-       )
-   ]
-
-.. _topics-request-response-ref-request-userlogin:
-
-Using FormRequest.from_response() to simulate a user login
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-It is usual for web sites to provide pre-populated form fields through ``<input
-type="hidden">`` elements, such as session related data or authentication
-tokens (for login pages). When scraping, you'll want these fields to be
-automatically pre-populated and only override a couple of them, such as the
-user name and password. You can use the :meth:`.FormRequest.from_response`
-method for this job. Here's an example spider which uses it:
-
-.. code-block:: python
-
-    import scrapy
-
-
-    def authentication_failed(response):
-        # TODO: Check the contents of the response and return True if it failed
-        # or False if it succeeded.
-        pass
-
-
-    class LoginSpider(scrapy.Spider):
-        name = "example.com"
-        start_urls = ["http://www.example.com/users/login.php"]
-
-        def parse(self, response):
-            return scrapy.FormRequest.from_response(
-                response,
-                formdata={"username": "john", "password": "secret"},
-                callback=self.after_login,
-            )
-
-        def after_login(self, response):
-            if authentication_failed(response):
-                self.logger.error("Login failed")
-                return
-
-            # continue scraping with authenticated session...
-
 JsonRequest
 -----------
 
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index 0e5c2b53b..e20e894ae 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers,
 Request and Response outside this module.
 """
 
+from warnings import catch_warnings, filterwarnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http.headers import Headers
 from scrapy.http.request import Request
-from scrapy.http.request.form import FormRequest
 from scrapy.http.request.json_request import JsonRequest
 from scrapy.http.request.rpc import XmlRpcRequest
 from scrapy.http.response import Response
@@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse
 from scrapy.http.response.json import JsonResponse
 from scrapy.http.response.text import TextResponse
 from scrapy.http.response.xml import XmlResponse
+from scrapy.utils.deprecate import create_deprecated_class
+
+with catch_warnings():
+    filterwarnings("ignore", category=ScrapyDeprecationWarning)
+
+    from scrapy.http.request.form import FormRequest as _FormRequest
+
+    FormRequest = create_deprecated_class(
+        name="FormRequest",
+        new_class=_FormRequest,
+        subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
+        instance_warn_message="{cls} is deprecated, use the form2request library instead.",
+    )
 
 __all__ = [
     "FormRequest",
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index affa14499..4da595b22 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -10,10 +10,12 @@ from __future__ import annotations
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
+from warnings import warn
 
 from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http.request import Request
 from scrapy.utils.python import is_listlike, to_bytes
 
@@ -30,6 +32,12 @@ if TYPE_CHECKING:
 
     from scrapy.http.response.text import TextResponse
 
+warn(
+    "The entire scrapy.http.request.form module is deprecated. Use the "
+    "form2request library instead.",
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
 
 FormdataVType: TypeAlias = str | Iterable[str]
 FormdataKVType: TypeAlias = tuple[str, FormdataVType]
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index a35ef7010..008e326ec 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -3,7 +3,6 @@ from unittest import TextTestResult
 import pytest
 from twisted.python import failure
 
-from scrapy import FormRequest
 from scrapy.contracts import Contract, ContractsManager
 from scrapy.contracts.default import (
     CallbackKeywordArgumentsContract,
@@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock):
     meta = None
 
 
+class TaggedRequest(Request):
+    def __init__(self, url, contract_tag=None, **kwargs):
+        super().__init__(url, **kwargs)
+        self.contract_tag = contract_tag
+
+
 class CustomSuccessContract(Contract):
     name = "custom_success_contract"
 
@@ -49,12 +54,13 @@ class CustomFailContract(Contract):
         raise TypeError("Error in adjust_request_args")
 
 
-class CustomFormContract(Contract):
-    name = "custom_form"
-    request_cls = FormRequest
+class CustomTaggedRequestContract(Contract):
+    name = "custom_tagged_request"
+    request_cls = TaggedRequest
 
     def adjust_request_args(self, args):
-        args["formdata"] = {"name": "scrapy"}
+        args["contract_tag"] = "custom"
+        args["method"] = "POST"
         return args
 
 
@@ -179,10 +185,10 @@ class DemoSpider(Spider):
         @returns items 1 1
         """
 
-    def custom_form(self, response):
+    def custom_tagged_request(self, response):
         """
         @url http://scrapy.org
-        @custom_form
+        @custom_tagged_request
         """
 
     def invalid_regex(self, response):
@@ -253,7 +259,7 @@ class TestContractsManager:
         MetadataContract,
         ReturnsContract,
         ScrapesContract,
-        CustomFormContract,
+        CustomTaggedRequestContract,
         CustomSuccessContract,
         CustomFailContract,
     ]
@@ -533,17 +539,21 @@ class TestContractsManager:
 
         assert crawler.spider.visited == 2
 
-    def test_form_contract(self):
+    def test_custom_tagged_request_contract(self):
         spider = DemoSpider()
-        request = self.conman.from_method(spider.custom_form, self.results)
+        request = self.conman.from_method(spider.custom_tagged_request, self.results)
         assert request.method == "POST"
-        assert isinstance(request, FormRequest)
+        assert isinstance(request, TaggedRequest)
+        assert request.contract_tag == "custom"
 
     def test_inherited_contracts(self):
         spider = InheritsDemoSpider()
 
         requests = self.conman.from_spider(spider, self.results)
         assert requests
+        assert any(
+            isinstance(request, TaggedRequest) for request in requests if request
+        )
 
 
 class CustomFailContractPreProcess(Contract):
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index 92f2f3fac..8ba9f3422 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -27,7 +27,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
 
 
 class TestFormRequest(TestRequest):
-    request_class = FormRequest
+    request_class = FormRequest  # type: ignore[assignment]
 
     def assertQueryEqual(self, first, second, msg=None):
         first = to_unicode(first).split("&")
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index ea7018541..78ff18b15 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -1,7 +1,7 @@
 import pytest
 
 from scrapy import Request, Spider
-from scrapy.http import FormRequest, JsonRequest
+from scrapy.http import JsonRequest
 from scrapy.utils.request import request_from_dict
 
 
@@ -67,12 +67,10 @@ class TestRequestSerialization:
             assert r1.dumps_kwargs == r2.dumps_kwargs
 
     def test_request_class(self):
-        r1 = FormRequest("http://www.example.com")
+        r1 = CustomRequest("http://www.example.com")
         self._assert_serializes_ok(r1, spider=self.spider)
-        r2 = CustomRequest("http://www.example.com")
+        r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
         self._assert_serializes_ok(r2, spider=self.spider)
-        r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
-        self._assert_serializes_ok(r3, spider=self.spider)
 
     def test_callback_serialization(self):
         r = Request(
diff --git a/tox.ini b/tox.ini
index 219a2b616..f1b004ab8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -245,6 +245,7 @@ changedir = docs
 deps =
     {[test-requirements]deps}
     -rdocs/requirements.txt
+    form2request
 commands =
     pytest
 

From 9776a72a6a7301b3c715363be6505c2f2a5ea23e Mon Sep 17 00:00:00 2001
From: "Albert Eduardovich N." <anamaev263@gmail.com>
Date: Mon, 4 May 2026 21:32:46 +0300
Subject: [PATCH 13/36] chore: simplify some code, get rid of nested fns where
 it's makes sence + pylint (#7401)

* chore: simplify some code, get rid of nested fns where it's makes sence

reasoning: https://wemake-python-styleguide.readthedocs.io/en/latest/pages/usage/violations/best_practices.html#wemake_python_styleguide.violations.best_practices.NestedFunctionViolation

* chore: address some pylint rules

* remove `pylint: disable=deprecated-class` from `reactor.py`

* fix typo

* fix another typo :)

* revert backward incompatible arg renames

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 extras/qps-bench-server.py                |   2 +-
 pyproject.toml                            |  13 +-
 scrapy/contracts/__init__.py              |   2 +-
 scrapy/core/downloader/handlers/http11.py |  20 +--
 scrapy/core/downloader/middleware.py      | 151 +++++++++++-----------
 scrapy/core/engine.py                     |  40 +++---
 scrapy/core/http2/stream.py               |  18 +--
 scrapy/core/spidermw.py                   |  98 +++++++-------
 scrapy/crawler.py                         |  46 ++++---
 scrapy/extensions/feedexport.py           |  16 ++-
 scrapy/pipelines/__init__.py              |  69 +++++-----
 scrapy/pipelines/files.py                 | 134 ++++++++++---------
 scrapy/settings/__init__.py               |   8 +-
 scrapy/utils/benchserver.py               |   2 +-
 scrapy/utils/defer.py                     |   6 +-
 scrapy/utils/misc.py                      |  13 +-
 scrapy/utils/python.py                    |  21 +--
 tests/mockserver/http.py                  |   2 +-
 tests/mockserver/simple_https.py          |   2 +-
 tox.ini                                   |   3 +-
 20 files changed, 350 insertions(+), 316 deletions(-)

diff --git a/extras/qps-bench-server.py b/extras/qps-bench-server.py
index 734614aa5..569438574 100755
--- a/extras/qps-bench-server.py
+++ b/extras/qps-bench-server.py
@@ -18,7 +18,7 @@ class Root(Resource):
         self.tail.clear()
         self.start = self.lastmark = self.lasttime = time()
 
-    def getChild(self, request, name):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):
diff --git a/pyproject.toml b/pyproject.toml
index 2132c5c50..a68222b36 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -185,6 +185,7 @@ jobs = 1          # >1 hides results
 extension-pkg-allow-list=[
     "lxml",
 ]
+load-plugins = ["pylint_per_file_ignores"]
 
 [tool.pylint."MESSAGES CONTROL"]
 enable = [
@@ -244,15 +245,13 @@ disable = [
     "unused-import",
 
     # Ones that we may want to address (fix, ignore per-line or move to "don't want to fix")
-    "abstract-method",
     "arguments-differ",
-    "arguments-renamed",
-    "dangerous-default-value",
     "keyword-arg-before-vararg",
-    "pointless-statement",
-    "raise-missing-from",
-    "unnecessary-dunder-call",
-    "used-before-assignment",
+]
+# requires `pylint_per_file_ignores` plugin
+per-file-ignores = [
+    # Extended list of ones that we may want to address, only for tests
+    "./tests/*:abstract-method,arguments-renamed,dangerous-default-value,pointless-statement,raise-missing-from,unnecessary-dunder-call,used-before-assignment",
 ]
 
 [tool.pytest.ini_options]
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index f84bbe0d7..24f56b7a6 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -125,7 +125,7 @@ class ContractsManager:
     def from_spider(self, spider: Spider, results: TestResult) -> list[Request | None]:
         requests: list[Request | None] = []
         for method in self.tested_methods_from_spidercls(type(spider)):
-            bound_method = spider.__getattribute__(method)
+            bound_method = getattr(spider, method)
             try:
                 requests.append(self.from_method(bound_method, results))
             except Exception:
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 439638d33..975078803 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -6,6 +6,7 @@ import ipaddress
 import logging
 import re
 from contextlib import suppress
+from functools import partial
 from io import BytesIO
 from time import monotonic
 from typing import TYPE_CHECKING, Any, TypedDict, TypeVar, cast
@@ -547,11 +548,7 @@ class ScrapyAgent:
                 get_warnsize_msg(expected_size, warnsize, request, expected=True)
             )
 
-        def _cancel(_: Any) -> None:
-            # Abort connection immediately.
-            txresponse._transport._producer.abortConnection()
-
-        d: Deferred[_ResultT] = Deferred(_cancel)
+        d: Deferred[_ResultT] = Deferred(partial(self._cancel, txresponse=txresponse))
         txresponse.deliverBody(
             _ResponseReader(
                 finished=d,
@@ -570,6 +567,11 @@ class ScrapyAgent:
 
         return d
 
+    @staticmethod
+    def _cancel(_: Any, txresponse: TxResponse) -> None:
+        # Abort connection immediately.
+        txresponse._transport._producer.abortConnection()
+
     def _cb_bodydone(self, result: _ResultT, url: str) -> Response:
         headers = self._headers_from_twisted_response(result["txresponse"])
         try:
@@ -667,17 +669,17 @@ class _ResponseReader(Protocol):
             assert hostname is not None
             _log_ssl_conn_debug_info(hostname, connection)
 
-    def dataReceived(self, bodyBytes: bytes) -> None:
+    def dataReceived(self, data: bytes) -> None:
         # This maybe called several times after cancel was called with buffered data.
         if self._finished.called:
             return
 
         assert self.transport
-        self._bodybuf.write(bodyBytes)
-        self._bytes_received += len(bodyBytes)
+        self._bodybuf.write(data)
+        self._bytes_received += len(data)
 
         if stop_download := check_stop_download(
-            signals.bytes_received, self._crawler, self._request, data=bodyBytes
+            signals.bytes_received, self._crawler, self._request, data=data
         ):
             self.transport.stopProducing()
             self.transport.loseConnection()
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index d08c6f8e3..6685d90af 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -75,87 +75,82 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         download_func: Callable[[Request], Coroutine[Any, Any, Response]],
         request: Request,
     ) -> Response | Request:
-        async def process_request(request: Request) -> Response | Request:
-            for method in self.methods["process_request"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(request=request, spider=self._spider),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request), _warn=global_object_name(method)
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return None, Response or "
-                        f"Request, got {response.__class__.__name__}"
-                    )
-                if response:
-                    return response
-            return await download_func(request)
-
-        async def process_response(response: Response | Request) -> Response | Request:
-            if response is None:
-                raise TypeError("Received None in process_response")
-            if isinstance(response, Request):
-                return response
-
-            for method in self.methods["process_response"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(request=request, response=response, spider=self._spider),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request, response=response),
-                        _warn=global_object_name(method),
-                    )
-                if not isinstance(response, (Response, Request)):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return Response or Request, "
-                        f"got {type(response)}"
-                    )
-                if isinstance(response, Request):
-                    return response
-            return response
-
-        async def process_exception(exception: Exception) -> Response | Request:
-            for method in self.methods["process_exception"]:
-                method = cast("Callable", method)
-                if method in self._mw_methods_requiring_spider:
-                    response = await ensure_awaitable(
-                        method(
-                            request=request, exception=exception, spider=self._spider
-                        ),
-                        _warn=global_object_name(method),
-                    )
-                else:
-                    response = await ensure_awaitable(
-                        method(request=request, exception=exception),
-                        _warn=global_object_name(method),
-                    )
-                if response is not None and not isinstance(
-                    response, (Response, Request)
-                ):
-                    raise _InvalidOutput(
-                        f"Middleware {method.__qualname__} must return None, Response or "
-                        f"Request, got {type(response)}"
-                    )
-                if response:
-                    return response
-            raise exception
 
         try:
-            result: Response | Request = await process_request(request)
+            result: Response | Request = await self._process_request(
+                request, download_func
+            )
         except Exception as ex:
             await _defer_sleep_async()
             # either returns a request or response (which we pass to process_response())
             # or reraises the exception
-            result = await process_exception(ex)
-        return await process_response(result)
+            result = await self._process_exception(ex, request)
+        return await self._process_response(result, request)
+
+    def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+        if method in self._mw_methods_requiring_spider:
+            kwargs["spider"] = self._spider
+
+        return method(**kwargs)
+
+    async def _process_request(
+        self,
+        request: Request,
+        download_func: Callable[[Request], Coroutine[Any, Any, Response]],
+    ) -> Response | Request:
+        for method in self.methods["process_request"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request),
+                _warn=global_object_name(method),
+            )
+            if response is not None and not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return None, Response or "
+                    f"Request, got {response.__class__.__name__}"
+                )
+            if response:
+                return response
+        return await download_func(request)
+
+    async def _process_response(
+        self, response: Response | Request, request: Request
+    ) -> Response | Request:
+        if response is None:
+            raise TypeError("Received None in process_response")
+        if isinstance(response, Request):
+            return response
+
+        for method in self.methods["process_response"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request, response=response),
+                _warn=global_object_name(method),
+            )
+
+            if not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return Response or Request, "
+                    f"got {type(response)}"
+                )
+            if isinstance(response, Request):
+                return response
+        return response
+
+    async def _process_exception(
+        self, exception: Exception, request: Request | Response
+    ) -> Response | Request:
+        for method in self.methods["process_exception"]:
+            method = cast("Callable", method)
+            response = await ensure_awaitable(
+                self._handle_mw_method(method, request=request, exception=exception),
+                _warn=global_object_name(method),
+            )
+            if response is not None and not isinstance(response, (Response, Request)):
+                raise _InvalidOutput(
+                    f"Middleware {method.__qualname__} must return None, Response or "
+                    f"Request, got {type(response)}"
+                )
+            if response:
+                return response
+        raise exception
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 94920e840..1033e874f 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -11,6 +11,7 @@ import asyncio
 import contextlib
 import logging
 import warnings
+from functools import partial
 from time import time
 from traceback import format_exc
 from typing import TYPE_CHECKING, Any
@@ -273,7 +274,7 @@ class ExecutionEngine:
         """
         assert self._start is not None
         try:
-            item_or_request = await self._start.__anext__()
+            item_or_request = await anext(self._start)
         except StopAsyncIteration:
             self._start = None
         except Exception as exception:
@@ -352,6 +353,10 @@ class ExecutionEngine:
             or self.scraper.slot.needs_backout()
         )
 
+    def _remove_request(self, _: Any, request: Request) -> None:
+        assert self._slot
+        self._slot.remove_request(request)
+
     def _start_scheduled_request(self) -> bool:
         assert self._slot is not None  # typing
         assert self.spider is not None  # typing
@@ -371,11 +376,7 @@ class ExecutionEngine:
             )
         )
 
-        def _remove_request(_: Any) -> None:
-            assert self._slot
-            self._slot.remove_request(request)
-
-        d2: Deferred[None] = d.addBoth(_remove_request)
+        d2: Deferred[None] = d.addBoth(partial(self._remove_request, request=request))
         d2.addErrback(
             lambda f: logger.info(
                 "Error while removing request from slot",
@@ -612,30 +613,33 @@ class ExecutionEngine:
             "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
         )
 
-        def log_failure(msg: str) -> None:
-            logger.error(msg, exc_info=True, extra={"spider": spider})  # noqa: LOG014
-
         try:
             await self._slot.close()
         except Exception:
-            log_failure("Slot close failure")
+            logger.error("Slot close failure", exc_info=True, extra={"spider": spider})
 
         try:
             self.downloader.close()
         except Exception:
-            log_failure("Downloader close failure")
+            logger.error(
+                "Downloader close failure", exc_info=True, extra={"spider": spider}
+            )
 
         try:
             await self.scraper.close_spider_async()
         except Exception:
-            log_failure("Scraper close failure")
+            logger.error(
+                "Scraper close failure", exc_info=True, extra={"spider": spider}
+            )
 
         if hasattr(self._slot.scheduler, "close"):
             try:
                 if (d := self._slot.scheduler.close(reason)) is not None:
                     await maybe_deferred_to_future(d)
             except Exception:
-                log_failure("Scheduler close failure")
+                logger.error(
+                    "Scheduler close failure", exc_info=True, extra={"spider": spider}
+                )
 
         try:
             await self.signals.send_catch_log_async(
@@ -644,7 +648,11 @@ class ExecutionEngine:
                 reason=reason,
             )
         except Exception:
-            log_failure("Error while sending spider_close signal")
+            logger.error(
+                "Error while sending spider_close signal",
+                exc_info=True,
+                extra={"spider": spider},
+            )
 
         assert self.crawler.stats
         try:
@@ -661,7 +669,7 @@ class ExecutionEngine:
             else:
                 self.crawler.stats.close_spider(reason=reason)
         except Exception:
-            log_failure("Stats close failure")
+            logger.error("Stats close failure")
 
         logger.info(
             "Spider closed (%(reason)s)",
@@ -675,4 +683,4 @@ class ExecutionEngine:
         try:
             await ensure_awaitable(self._spider_closed_callback(spider))
         except Exception:
-            log_failure("Error running spider_closed_callback")
+            logger.error("Error running spider_closed_callback")
diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.py
index 71569b217..4d072c555 100644
--- a/scrapy/core/http2/stream.py
+++ b/scrapy/core/http2/stream.py
@@ -155,16 +155,16 @@ class Stream:
             "status": None,
         }
 
-        def _cancel(_: Any) -> None:
-            # Close this stream as gracefully as possible
-            # If the associated request is initiated we reset this stream
-            # else we directly call close() method
-            if self.metadata["request_sent"]:
-                self.reset_stream(StreamCloseReason.CANCELLED)
-            else:
-                self.close(StreamCloseReason.CANCELLED)
+        self._deferred_response: Deferred[Response] = Deferred(self._cancel)
 
-        self._deferred_response: Deferred[Response] = Deferred(_cancel)
+    def _cancel(self, _: Any) -> None:
+        # Close this stream as gracefully as possible
+        # If the associated request is initiated we reset this stream
+        # else we directly call close() method
+        if self.metadata["request_sent"]:
+            self.reset_stream(StreamCloseReason.CANCELLED)
+        else:
+            self.close(StreamCloseReason.CANCELLED)
 
     def __repr__(self) -> str:
         return f"Stream(id={self.stream_id!r})"
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 790317357..d48a65967 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -112,40 +112,59 @@ class SpiderMiddlewareManager(MiddlewareManager):
         exception_processor_index: int,
         recover_to: MutableChain[_T] | MutableAsyncChain[_T],
     ) -> Iterable[_T] | AsyncIterator[_T]:
-        def process_sync(iterable: Iterable[_T]) -> Iterable[_T]:
-            try:
-                yield from iterable
-            except Exception as ex:
-                exception_result = cast(
-                    "Failure | MutableChain[_T]",
-                    self._process_spider_exception(
-                        response, ex, exception_processor_index
-                    ),
-                )
-                if isinstance(exception_result, Failure):
-                    raise
-                assert isinstance(recover_to, MutableChain)
-                recover_to.extend(exception_result)
-
-        async def process_async(iterable: AsyncIterator[_T]) -> AsyncIterator[_T]:
-            try:
-                async for r in iterable:
-                    yield r
-            except Exception as ex:
-                exception_result = cast(
-                    "Failure | MutableAsyncChain[_T]",
-                    self._process_spider_exception(
-                        response, ex, exception_processor_index
-                    ),
-                )
-                if isinstance(exception_result, Failure):
-                    raise
-                assert isinstance(recover_to, MutableAsyncChain)
-                recover_to.extend(exception_result)
 
         if isinstance(iterable, AsyncIterator):
-            return process_async(iterable)
-        return process_sync(iterable)
+            return self._process_async(
+                response,
+                iterable,
+                exception_processor_index,
+                cast("MutableAsyncChain[_T]", recover_to),
+            )
+        return self._process_sync(
+            response,
+            iterable,
+            exception_processor_index,
+            cast("MutableChain[_T]", recover_to),
+        )
+
+    def _process_sync(
+        self,
+        response: Response,
+        iterable: Iterable[_T],
+        exception_processor_index: int,
+        recover_to: MutableChain[_T],
+    ) -> Iterable[_T]:
+        try:
+            yield from iterable
+        except Exception as ex:
+            exception_result = cast(
+                "Failure | MutableChain[_T]",
+                self._process_spider_exception(response, ex, exception_processor_index),
+            )
+            if isinstance(exception_result, Failure):
+                raise
+            assert isinstance(recover_to, MutableChain)
+            recover_to.extend(exception_result)
+
+    async def _process_async(
+        self,
+        response: Response,
+        iterable: AsyncIterator[_T],
+        exception_processor_index: int,
+        recover_to: MutableAsyncChain[_T],
+    ) -> AsyncIterator[_T]:
+        try:
+            async for r in iterable:
+                yield r
+        except Exception as ex:
+            exception_result = cast(
+                "Failure | MutableAsyncChain[_T]",
+                self._process_spider_exception(response, ex, exception_processor_index),
+            )
+            if isinstance(exception_result, Failure):
+                raise
+            assert isinstance(recover_to, MutableAsyncChain)
+            recover_to.extend(exception_result)
 
     def _process_spider_exception(
         self,
@@ -350,25 +369,14 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 "scrape_response_async() called on a SpiderMiddlewareManager"
                 " instance created without a crawler."
             )
-
-        async def process_callback_output(
-            result: Iterable[_T] | AsyncIterator[_T],
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return await self._process_callback_output(response, result)
-
-        def process_spider_exception(
-            exception: Exception,
-        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return self._process_spider_exception(response, exception)
-
         try:
             it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
                 scrape_func, response, request
             )
-            return await process_callback_output(it)
+            return await self._process_callback_output(response, it)
         except Exception as ex:
             await _defer_sleep_async()
-            return process_spider_exception(ex)
+            return self._process_spider_exception(response, ex)
 
     async def process_start(
         self, spider: Spider | None = None
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index 0a19e9985..8e3ae7879 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -7,6 +7,7 @@ import pprint
 import signal
 import warnings
 from abc import ABC, abstractmethod
+from functools import partial
 from typing import TYPE_CHECKING, Any, TypeVar
 
 from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks
@@ -568,28 +569,30 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         crawler = self.create_crawler(crawler_or_spidercls)
         return self._crawl(crawler, *args, **kwargs)
 
+    async def _crawl_and_track(
+        self, crawler: Crawler, *args: Any, **kwargs: Any
+    ) -> None:
+        try:
+            await crawler.crawl_async(*args, **kwargs)
+        except Exception:
+            self.bootstrap_failed = True
+            raise  # re-raise so asyncio still logs it to stderr naturally
+
+    def _done(self, task: asyncio.Task[None], crawler: Crawler) -> None:
+        self._active.discard(task)
+        self.crawlers.discard(crawler)
+        self.bootstrap_failed |= not getattr(crawler, "spider", None)
+
     def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
         # At this point the asyncio loop has been installed either by the user
         # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
         loop = asyncio.get_event_loop()
         self.crawlers.add(crawler)
 
-        async def _crawl_and_track() -> None:
-            try:
-                await crawler.crawl_async(*args, **kwargs)
-            except Exception:
-                self.bootstrap_failed = True
-                raise  # re-raise so asyncio still logs it to stderr naturally
-
-        task = loop.create_task(_crawl_and_track())
+        task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
         self._active.add(task)
+        task.add_done_callback(partial(self._done, crawler=crawler))
 
-        def _done(_: asyncio.Task[None]) -> None:
-            self.crawlers.discard(crawler)
-            self._active.discard(task)
-            self.bootstrap_failed |= not getattr(crawler, "spider", None)
-
-        task.add_done_callback(_done)
         return task
 
     async def stop(self) -> None:
@@ -1007,14 +1010,15 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
         if (loop := self._reactorless_loop) is None:
             return
 
-        def _create_shutdown_task() -> None:
-            coro = self._shutdown_graceful_reactorless()
-            try:
-                loop.create_task(coro)
-            except RuntimeError:
-                coro.close()
+        loop.call_soon_threadsafe(self._create_shutdown_task)
 
-        loop.call_soon_threadsafe(_create_shutdown_task)
+    def _create_shutdown_task(self) -> None:
+        assert self._reactorless_loop
+        coro = self._shutdown_graceful_reactorless()
+        try:
+            self._reactorless_loop.create_task(coro)
+        except RuntimeError:
+            coro.close()
 
     async def _shutdown_graceful_reactorless(self) -> None:
         await self.stop()
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 8f70dbc2e..d0430d0db 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -534,13 +534,15 @@ class FeedExporter:
         # Send FEED_EXPORTER_CLOSED signal
         await self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)
 
+    @staticmethod
+    def _get_file(slot_: FeedSlot) -> IO[bytes]:
+        assert slot_.file
+        if isinstance(slot_.file, PostProcessingManager):
+            slot_.file.close()
+            return slot_.file.file
+        return slot_.file
+
     async def _close_slot(self, slot: FeedSlot, spider: Spider) -> None:
-        def get_file(slot_: FeedSlot) -> IO[bytes]:
-            assert slot_.file
-            if isinstance(slot_.file, PostProcessingManager):
-                slot_.file.close()
-                return slot_.file.file
-            return slot_.file
 
         if slot.itemcount:
             # Normal case
@@ -557,7 +559,7 @@ class FeedExporter:
         slot_type = type(slot.storage).__name__
         assert self.crawler.stats
         try:
-            await ensure_awaitable(slot.storage.store(get_file(slot)))
+            await ensure_awaitable(slot.storage.store(self._get_file(slot)))
         except Exception:
             logger.error(
                 "Error storing %s",
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 18473c534..84fb5f85e 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -37,16 +37,16 @@ class ItemPipelineManager(MiddlewareManager):
             settings.get_component_priority_dict_with_base("ITEM_PIPELINES")
         )
 
-    def _add_middleware(self, pipe: Any) -> None:
-        if hasattr(pipe, "open_spider"):
-            self.methods["open_spider"].append(pipe.open_spider)
-            self._check_mw_method_spider_arg(pipe.open_spider)
-        if hasattr(pipe, "close_spider"):
-            self.methods["close_spider"].appendleft(pipe.close_spider)
-            self._check_mw_method_spider_arg(pipe.close_spider)
-        if hasattr(pipe, "process_item"):
-            self.methods["process_item"].append(pipe.process_item)
-            self._check_mw_method_spider_arg(pipe.process_item)
+    def _add_middleware(self, mw: Any) -> None:
+        if hasattr(mw, "open_spider"):
+            self.methods["open_spider"].append(mw.open_spider)
+            self._check_mw_method_spider_arg(mw.open_spider)
+        if hasattr(mw, "close_spider"):
+            self.methods["close_spider"].appendleft(mw.close_spider)
+            self._check_mw_method_spider_arg(mw.close_spider)
+        if hasattr(mw, "process_item"):
+            self.methods["process_item"].append(mw.process_item)
+            self._check_mw_method_spider_arg(mw.process_item)
 
     def process_item(self, item: Any, spider: Spider) -> Deferred[Any]:
         warnings.warn(
@@ -62,32 +62,44 @@ class ItemPipelineManager(MiddlewareManager):
             "process_item", item, add_spider=True, warn_deferred=True
         )
 
+    def _get_dfd(
+        self,
+        method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
+    ) -> Deferred[None]:
+        if method in self._mw_methods_requiring_spider:
+            return _maybeDeferred_coro(method, True, self._spider)
+        return _maybeDeferred_coro(method, True)
+
+    @staticmethod
+    def _eb(failure: Failure) -> Failure:
+        assert isinstance(failure.value, FirstError)
+        return failure.value.subFailure
+
     def _process_parallel_dfd(self, methodname: str) -> Deferred[list[None]]:
         methods = cast(
             "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
             self.methods[methodname],
         )
 
-        def get_dfd(
-            method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
-        ) -> Deferred[None]:
-            if method in self._mw_methods_requiring_spider:
-                return _maybeDeferred_coro(method, True, self._spider)
-            return _maybeDeferred_coro(method, True)
-
-        dfds = [get_dfd(m) for m in methods]
+        dfds = [self._get_dfd(m) for m in methods]
         d: Deferred[list[tuple[bool, None]]] = DeferredList(
             dfds, fireOnOneErrback=True, consumeErrors=True
         )
         d2: Deferred[list[None]] = d.addCallback(lambda r: [x[1] for x in r])
 
-        def eb(failure: Failure) -> Failure:
-            assert isinstance(failure.value, FirstError)
-            return failure.value.subFailure
-
-        d2.addErrback(eb)
+        d2.addErrback(self._eb)
         return d2
 
+    def get_awaitable(
+        self,
+        method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
+    ) -> Awaitable[None]:
+        if method in self._mw_methods_requiring_spider:
+            result = method(self._spider)
+        else:
+            result = method()
+        return ensure_awaitable(result, _warn=global_object_name(method))
+
     async def _process_parallel_asyncio(self, methodname: str) -> list[None]:
         methods = cast(
             "Iterable[Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None]]",
@@ -96,16 +108,7 @@ class ItemPipelineManager(MiddlewareManager):
         if not methods:
             return []
 
-        def get_awaitable(
-            method: Callable[..., Coroutine[Any, Any, None] | Deferred[None] | None],
-        ) -> Awaitable[None]:
-            if method in self._mw_methods_requiring_spider:
-                result = method(self._spider)
-            else:
-                result = method()
-            return ensure_awaitable(result, _warn=global_object_name(method))
-
-        awaitables = [get_awaitable(m) for m in methods]
+        awaitables = [self.get_awaitable(m) for m in methods]
         await asyncio.gather(*awaitables)
         return [None for _ in methods]
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 1eb17c3d3..ffed96446 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -186,16 +186,18 @@ class S3FilesStore:
             raise ValueError(f"Incorrect URI scheme in {uri}, expected 's3'")
         self.bucket, self.prefix = uri[5:].split("/", 1)
 
+    @staticmethod
+    def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
+        checksum = boto_key["ETag"].strip('"')
+        last_modified = boto_key["LastModified"]
+        modified_stamp = time.mktime(last_modified.timetuple())
+        return {"checksum": checksum, "last_modified": modified_stamp}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _onsuccess(boto_key: dict[str, Any]) -> StatInfo:
-            checksum = boto_key["ETag"].strip('"')
-            last_modified = boto_key["LastModified"]
-            modified_stamp = time.mktime(last_modified.timetuple())
-            return {"checksum": checksum, "last_modified": modified_stamp}
 
-        return self._get_boto_key(path).addCallback(_onsuccess)
+        return self._get_boto_key(path).addCallback(self._onsuccess)
 
     def _get_boto_key(self, path: str) -> Deferred[dict[str, Any]]:
         key_name = f"{self.prefix}{path}"
@@ -308,20 +310,22 @@ class GCSFilesStore:
                 {"bucket": bucket},
             )
 
+    @staticmethod
+    def _onsuccess(blob: Any) -> StatInfo:
+        if blob:
+            checksum = base64.b64decode(blob.md5_hash).hex()
+            last_modified = time.mktime(blob.updated.timetuple())
+            return {"checksum": checksum, "last_modified": last_modified}
+        return {}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _onsuccess(blob: Any) -> StatInfo:
-            if blob:
-                checksum = base64.b64decode(blob.md5_hash).hex()
-                last_modified = time.mktime(blob.updated.timetuple())
-                return {"checksum": checksum, "last_modified": last_modified}
-            return {}
 
         blob_path = self._get_blob_path(path)
         return deferred_from_coro(
             run_in_thread(self.bucket.get_blob, blob_path)
-        ).addCallback(_onsuccess)
+        ).addCallback(self._onsuccess)
 
     def _get_content_type(self, headers: dict[str, str] | None) -> str:
         if headers and "Content-Type" in headers:
@@ -395,26 +399,26 @@ class FTPFilesStore:
             )
         )
 
+    def _stat_file(self, path: str) -> StatInfo:
+        try:
+            with FTP() as ftp:
+                ftp.connect(self.host, self.port)
+                ftp.login(self.username, self.password)
+                if self.USE_ACTIVE_MODE:
+                    ftp.set_pasv(False)
+                file_path = f"{self.basedir}/{path}"
+                last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
+                m = hashlib.md5()  # noqa: S324
+                ftp.retrbinary(f"RETR {file_path}", m.update)
+            return {"last_modified": last_modified, "checksum": m.hexdigest()}
+        # The file doesn't exist
+        except Exception:
+            return {}
+
     def stat_file(
         self, path: str, info: MediaPipeline.SpiderInfo
     ) -> Deferred[StatInfo]:
-        def _stat_file(path: str) -> StatInfo:
-            try:
-                with FTP() as ftp:
-                    ftp.connect(self.host, self.port)
-                    ftp.login(self.username, self.password)
-                    if self.USE_ACTIVE_MODE:
-                        ftp.set_pasv(False)
-                    file_path = f"{self.basedir}/{path}"
-                    last_modified = float(ftp.voidcmd(f"MDTM {file_path}")[4:].strip())
-                    m = hashlib.md5()  # noqa: S324
-                    ftp.retrbinary(f"RETR {file_path}", m.update)
-                return {"last_modified": last_modified, "checksum": m.hexdigest()}
-            # The file doesn't exist
-            except Exception:
-                return {}
-
-        return deferred_from_coro(run_in_thread(_stat_file, path))
+        return deferred_from_coro(run_in_thread(self._stat_file, path))
 
 
 class FilesPipeline(MediaPipeline):
@@ -534,43 +538,51 @@ class FilesPipeline(MediaPipeline):
         store_cls = self.STORE_SCHEMES[scheme]
         return store_cls(uri)
 
+    def _onsuccess(
+        self,
+        result: StatInfo,
+        request: Request,
+        info: MediaPipeline.SpiderInfo,
+        path: str,
+    ) -> FileInfo | None:
+        if not result:
+            return None  # returning None force download
+
+        last_modified = result.get("last_modified", None)
+        if not last_modified:
+            return None  # returning None force download
+
+        age_seconds = time.time() - last_modified
+        age_days = age_seconds / 60 / 60 / 24
+        if age_days > self.expires:
+            return None  # returning None force download
+
+        referer = referer_str(request)
+        logger.debug(
+            "File (uptodate): Downloaded %(medianame)s from %(request)s "
+            "referred in <%(referer)s>",
+            {"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
+            extra={"spider": info.spider},
+        )
+        self.inc_stats("uptodate")
+
+        checksum = result.get("checksum", None)
+        return {
+            "url": request.url,
+            "path": path,
+            "checksum": checksum,
+            "status": "uptodate",
+        }
+
     def media_to_download(
         self, request: Request, info: MediaPipeline.SpiderInfo, *, item: Any = None
     ) -> Deferred[FileInfo | None] | None:
-        def _onsuccess(result: StatInfo) -> FileInfo | None:
-            if not result:
-                return None  # returning None force download
-
-            last_modified = result.get("last_modified", None)
-            if not last_modified:
-                return None  # returning None force download
-
-            age_seconds = time.time() - last_modified
-            age_days = age_seconds / 60 / 60 / 24
-            if age_days > self.expires:
-                return None  # returning None force download
-
-            referer = referer_str(request)
-            logger.debug(
-                "File (uptodate): Downloaded %(medianame)s from %(request)s "
-                "referred in <%(referer)s>",
-                {"medianame": self.MEDIA_NAME, "request": request, "referer": referer},
-                extra={"spider": info.spider},
-            )
-            self.inc_stats("uptodate")
-
-            checksum = result.get("checksum", None)
-            return {
-                "url": request.url,
-                "path": path,
-                "checksum": checksum,
-                "status": "uptodate",
-            }
-
         path = self.file_path(request, info=info, item=item)
         # maybeDeferred() overloads don't seem to support a Union[_T, Deferred[_T]] return type
         dfd: Deferred[StatInfo] = maybeDeferred(self.store.stat_file, path, info)  # type: ignore[call-overload]
-        dfd2: Deferred[FileInfo | None] = dfd.addCallback(_onsuccess)
+        dfd2: Deferred[FileInfo | None] = dfd.addCallback(
+            functools.partial(self._onsuccess, request=request, info=info, path=path)
+        )
         dfd2.addErrback(lambda _: None)
         dfd2.addErrback(
             lambda f: logger.error(
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 2a1b1932c..d4671fd35 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -143,7 +143,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
             raise ValueError(f"{item!r} not found in the {name} setting ({value!r}).")
         self.set(name, [v for v in value if v != item], self.getpriority(name) or 0)
 
-    def get(self, name: _SettingsKey, default: Any = None) -> Any:
+    def get(self, name: _SettingsKey, default: Any = None) -> Any:  # pylint: disable=arguments-renamed
         """
         Get a setting value without affecting its original type.
 
@@ -510,7 +510,7 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         component_priority_dict[cls] = priority
         self.set(name, component_priority_dict, self.getpriority(name) or 0)
 
-    def setdefault(
+    def setdefault(  # pylint: disable=arguments-renamed
         self,
         name: _SettingsKey,
         default: Any = None,
@@ -691,14 +691,14 @@ class BaseSettings(MutableMapping[_SettingsKey, Any]):
         else:
             p.text(pformat(self.copy_to_dict()))
 
-    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:
+    def pop(self, name: _SettingsKey, default: Any = __default) -> Any:  # pylint: disable=arguments-renamed
         try:
             value = self.attributes[name].value
         except KeyError:
             if default is self.__default:
                 raise
             return default
-        self.__delitem__(name)
+        del self[name]
         return value
 
 
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index 59f04b8ad..c4c96ef00 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -9,7 +9,7 @@ from twisted.web.server import Request, Site
 class Root(Resource):
     isLeaf = True
 
-    def getChild(self, name: str, request: Request) -> Resource:
+    def getChild(self, path: str, request: Request) -> Resource:
         return self
 
     def render(self, request: Request) -> bytes:
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 7d7636fe6..90fd04777 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -262,7 +262,7 @@ class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
     def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
-        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred = deferred_from_coro(anext(self.aiterator))
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
     def __next__(self) -> Deferred[Any]:
@@ -370,10 +370,10 @@ async def aiter_errback(
     """Wrap an async iterable calling an errback if an error is caught while
     iterating it. Similar to :func:`scrapy.utils.defer.iter_errback`.
     """
-    it = aiterable.__aiter__()
+    it = aiter(aiterable)
     while True:
         try:
-            yield await it.__anext__()
+            yield await anext(it)
         except StopAsyncIteration:
             break
         except Exception:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index fba80484c..757756e65 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -255,6 +255,11 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
 _generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
 
 
+def _returns_none(return_node: ast.Return) -> bool:
+    value = return_node.value
+    return value is None or (isinstance(value, ast.Constant) and value.value is None)
+
+
 def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noqa: A002
     """
     Returns True if a callable is a generator function which includes a
@@ -263,12 +268,6 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
     if callable in _generator_callbacks_cache:
         return bool(_generator_callbacks_cache[callable])
 
-    def returns_none(return_node: ast.Return) -> bool:
-        value = return_node.value
-        return value is None or (
-            isinstance(value, ast.Constant) and value.value is None
-        )
-
     if inspect.isgeneratorfunction(callable):
         func = callable
         while isinstance(func, partial):
@@ -284,7 +283,7 @@ def is_generator_with_return_value(callable: Callable[..., Any]) -> bool:  # noq
 
         tree = ast.parse(code)
         for node in walk_callable(tree):
-            if isinstance(node, ast.Return) and not returns_none(node):
+            if isinstance(node, ast.Return) and not _returns_none(node):
                 _generator_callbacks_cache[callable] = True
                 return bool(_generator_callbacks_cache[callable])
 
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index bf70fa7fc..9cb695111 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -99,6 +99,16 @@ def to_bytes(
     return text.encode(encoding, errors)
 
 
+def _chunk_iter(text: str, chunk_size: int) -> Iterable[tuple[str, int]]:
+    offset = len(text)
+    while True:
+        offset -= chunk_size * 1024
+        if offset <= 0:
+            break
+        yield (text[offset:], offset)
+    yield (text, 0)
+
+
 def re_rsearch(
     pattern: str | Pattern[str], text: str, chunk_size: int = 1024
 ) -> tuple[int, int] | None:
@@ -115,19 +125,10 @@ def re_rsearch(
     the start position of the match, and the ending (regarding the entire text).
     """
 
-    def _chunk_iter() -> Iterable[tuple[str, int]]:
-        offset = len(text)
-        while True:
-            offset -= chunk_size * 1024
-            if offset <= 0:
-                break
-            yield (text[offset:], offset)
-        yield (text, 0)
-
     if isinstance(pattern, str):
         pattern = re.compile(pattern)
 
-    for chunk, offset in _chunk_iter():
+    for chunk, offset in _chunk_iter(text, chunk_size):
         matches = list(pattern.finditer(chunk))
         if matches:
             start, end = matches[-1].span()
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index 6cf0046c0..622324a10 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -85,7 +85,7 @@ class Root(resource.Resource):
         self.putChild(b"response-headers", ResponseHeadersResource())
         self.putChild(b"set-cookie", SetCookie())
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
     def render(self, request):
diff --git a/tests/mockserver/simple_https.py b/tests/mockserver/simple_https.py
index 5f23dd2c4..a8f483ee1 100644
--- a/tests/mockserver/simple_https.py
+++ b/tests/mockserver/simple_https.py
@@ -13,7 +13,7 @@ class Root(resource.Resource):
         resource.Resource.__init__(self)
         self.putChild(b"file", Data(b"0123456789", "text/plain"))
 
-    def getChild(self, name, request):
+    def getChild(self, path, request):
         return self
 
 
diff --git a/tox.ini b/tox.ini
index f1b004ab8..07583a11e 100644
--- a/tox.ini
+++ b/tox.ini
@@ -89,8 +89,9 @@ basepython = python3
 deps =
     {[testenv:extra-deps]deps}
     pylint==4.0.2
+    pylint-per-file-ignores # https://github.com/pylint-dev/pylint/issues/3767#issuecomment-1319916278
 commands =
-    pylint conftest.py docs extras scrapy tests
+    pylint {posargs:conftest.py docs extras scrapy tests}
 
 [testenv:twinecheck]
 basepython = python3

From 33452f3aeb850c54ea13c74194cd633daaa31698 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 5 May 2026 01:20:21 +0500
Subject: [PATCH 14/36] Silence deprecation warnings in TestFormRequest.
 (#7491)

---
 tests/test_http_request_form.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index 8ba9f3422..a4f87d50d 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -26,6 +26,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
     return parse_qs(uqs, True)
 
 
+@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
 class TestFormRequest(TestRequest):
     request_class = FormRequest  # type: ignore[assignment]
 

From fc14a0ce5989e32794636a287c6914c0af396ed9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 5 May 2026 01:24:47 +0500
Subject: [PATCH 15/36] Enable mypy warn_return_any. (#7492)

---
 pyproject.toml                               |  1 -
 scrapy/commands/parse.py                     | 11 ++++-----
 scrapy/core/downloader/__init__.py           |  8 ++++---
 scrapy/core/downloader/contextfactory.py     |  8 +++----
 scrapy/core/downloader/handlers/s3.py        |  2 +-
 scrapy/core/http2/agent.py                   |  4 ++--
 scrapy/core/scheduler.py                     |  4 ++--
 scrapy/http/cookies.py                       |  2 +-
 scrapy/pipelines/files.py                    |  5 ++--
 scrapy/pipelines/media.py                    |  6 ++---
 scrapy/robotstxt.py                          |  4 ++--
 scrapy/utils/benchserver.py                  |  2 +-
 scrapy/utils/datatypes.py                    |  4 ++--
 scrapy/utils/decorators.py                   |  4 ++--
 scrapy/utils/deprecate.py                    |  6 ++---
 scrapy/utils/misc.py                         |  4 ++--
 scrapy/utils/reactor.py                      |  6 ++---
 scrapy/utils/signal.py                       |  8 +++++--
 scrapy/utils/spider.py                       |  7 +++---
 tests/mockserver/http_resources.py           |  2 +-
 tests/mockserver/utils.py                    | 10 +++++---
 tests/test_downloadermiddleware_robotstxt.py |  4 ++--
 tests/test_http2_client_protocol.py          |  2 +-
 tests/test_scheduler.py                      |  8 +++----
 tests/test_spidermiddleware.py               | 13 +++++------
 tests/utils/decorators.py                    |  3 ++-
 tests_typing/test_spiders.mypy-testing       |  2 +-
 tox.ini                                      | 24 +++++++++++---------
 28 files changed, 87 insertions(+), 77 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index a68222b36..11d36d0ac 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,7 +91,6 @@ extra_checks = false # weird addErrback() errors
 untyped_calls_exclude = [
     "twisted",
 ]
-warn_return_any = false # 37 errors
 
 [[tool.mypy.overrides]]
 module = "tests.*"
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index 632186a7d..2ac65bf3f 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -144,17 +144,16 @@ class Command(BaseRunSpiderCommand):
     def iterate_spider_output(self, result: _T) -> Iterable[Any]: ...
 
     def iterate_spider_output(self, result: Any) -> Iterable[Any] | Deferred[Any]:
+        d: Deferred[Any]
         if inspect.isasyncgen(result):
             d = deferred_from_coro(
                 collect_asyncgen(aiter_errback(result, self.handle_exception))
             )
-            d.addCallback(self.iterate_spider_output)
-            return d
+            return d.addCallback(self.iterate_spider_output)
+        d = deferred_from_coro(result)
         if inspect.iscoroutine(result):
-            d = deferred_from_coro(result)
-            d.addCallback(self.iterate_spider_output)
-            return d
-        return arg_to_iter(deferred_from_coro(result))
+            return d.addCallback(self.iterate_spider_output)
+        return arg_to_iter(d)
 
     def add_items(self, lvl: int, new_items: list[Any]) -> None:
         old_items = self.items.get(lvl, [])
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 01d5d42b0..7c0ee0eec 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -128,11 +128,12 @@ class Downloader:
     ) -> Generator[Deferred[Any], Any, Response | Request]:
         self.active.add(request)
         try:
-            return (
-                yield deferred_from_coro(
+            result: Response | Request = yield (
+                deferred_from_coro(
                     self.middleware.download_async(self._enqueue_request, request)
                 )
             )
+            return result
         finally:
             self.active.remove(request)
 
@@ -163,7 +164,8 @@ class Downloader:
         return key, self.slots[key]
 
     def get_slot_key(self, request: Request) -> str:
-        if (meta_slot := request.meta.get(self.DOWNLOAD_SLOT)) is not None:
+        meta_slot: str | None = request.meta.get(self.DOWNLOAD_SLOT)
+        if meta_slot is not None:
             return meta_slot
 
         key = urlparse_cached(request).hostname or ""
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index bbda7efcd..6575d59c1 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -108,7 +108,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     def _get_cert_options(self) -> CertificateOptions:
         with _filter_method_warning():
-            return CertificateOptions(
+            return CertificateOptions(  # type: ignore[no-any-return]
                 method=self._ssl_method,
                 fixBrokenPeers=True,
                 acceptableCiphers=self.tls_ciphers,
@@ -122,7 +122,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     def _get_context(self) -> SSL.Context:
         cert_options = self._get_cert_options()
-        ctx = cert_options.getContext()
+        ctx: SSL.Context = cert_options.getContext()
         ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
         return ctx
 
@@ -135,7 +135,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         # Otherwise use the normal Twisted function.
         # Note that this doesn't use self._get_context().
         with _filter_method_warning():
-            return optionsForClientTLS(
+            return optionsForClientTLS(  # type: ignore[no-any-return]
                 hostname=hostname.decode("ascii"),
                 extraCertificateOptions={
                     "method": self._ssl_method,
@@ -186,7 +186,7 @@ class BrowserLikeContextFactory(_ScrapyClientContextFactory):
 
     def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         with _filter_method_warning():
-            return optionsForClientTLS(
+            return optionsForClientTLS(  # type: ignore[no-any-return]
                 hostname=hostname.decode("ascii"),
                 extraCertificateOptions={"method": self._ssl_method},
             )
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index a609cedb3..19a1e8503 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -39,7 +39,7 @@ class S3DownloadHandler(BaseDownloadHandler):
                 )
             )
 
-        _http_handler = build_from_crawler(
+        _http_handler: BaseDownloadHandler = build_from_crawler(
             load_object(crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"]),
             crawler,
         )
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 540b9aa74..822dffc4b 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -143,7 +143,7 @@ class H2Agent:
         )
 
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
-        return self.endpoint_factory.endpointForURI(uri)
+        return self.endpoint_factory.endpointForURI(uri)  # type: ignore[no-any-return]
 
     def get_key(self, uri: URI) -> ConnectionKeyT:
         """
@@ -187,7 +187,7 @@ class ScrapyProxyH2Agent(H2Agent):
         self._proxy_uri = proxy_uri
 
     def get_endpoint(self, uri: URI) -> HostnameEndpoint:
-        return self.endpoint_factory.endpointForURI(self._proxy_uri)
+        return self.endpoint_factory.endpointForURI(self._proxy_uri)  # type: ignore[no-any-return]
 
     def get_key(self, uri: URI) -> ConnectionKeyT:
         """We use the proxy uri instead of uri obtained from request url"""
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index 78329460e..7217da942 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -4,7 +4,7 @@ import json
 import logging
 from abc import abstractmethod
 from pathlib import Path
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
 # working around https://github.com/sphinx-doc/sphinx/issues/10400
 from twisted.internet.defer import Deferred  # noqa: TC002
@@ -334,7 +334,7 @@ class Scheduler(BaseScheduler):
         cls = crawler.settings[f"SCHEDULER_START_{queue}_QUEUE"]
         if not cls:
             return None
-        return load_object(cls)
+        return cast("type[BaseQueue]", load_object(cls))
 
     def has_pending_requests(self) -> bool:
         return len(self) > 0
diff --git a/scrapy/http/cookies.py b/scrapy/http/cookies.py
index 09286606d..599f20947 100644
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@@ -74,7 +74,7 @@ class CookieJar:
 
     @property
     def _cookies(self) -> dict[str, dict[str, dict[str, Cookie]]]:
-        return self.jar._cookies  # type: ignore[attr-defined]
+        return self.jar._cookies  # type: ignore[attr-defined,no-any-return]
 
     def clear_session_cookies(self) -> None:
         return self.jar.clear_session_cookies()
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ffed96446..0066fd38f 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -323,9 +323,10 @@ class GCSFilesStore:
     ) -> Deferred[StatInfo]:
 
         blob_path = self._get_blob_path(path)
-        return deferred_from_coro(
+        d: Deferred[Any] = deferred_from_coro(
             run_in_thread(self.bucket.get_blob, blob_path)
-        ).addCallback(self._onsuccess)
+        )
+        return d.addCallback(self._onsuccess)
 
     def _get_content_type(self, headers: dict[str, str] | None) -> str:
         if headers and "Content-Type" in headers:
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 1043da332..5b5d2dcb2 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -29,9 +29,9 @@ from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
-    # typing.Self requires Python 3.11
-    from collections.abc import Awaitable
+    from collections.abc import Awaitable, Callable
 
+    # typing.Self requires Python 3.11
     from typing_extensions import Self
 
     from scrapy import Spider
@@ -153,7 +153,7 @@ class MediaPipeline(ABC):
     ) -> FileInfo:
         fp = self._fingerprinter.fingerprint(request)
 
-        eb = request.errback
+        eb: Callable[[Failure], FileInfo] | None = request.errback
         request.callback = NO_CALLBACK
         request.errback = None
 
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index 18b622546..0c64ea5a5 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import logging
 import sys
 from abc import ABCMeta, abstractmethod
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 from urllib.robotparser import RobotFileParser
 
 from protego import Protego
@@ -103,7 +103,7 @@ class RerpRobotParser(RobotParser):
     def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
         user_agent = to_unicode(user_agent)
         url = to_unicode(url)
-        return self.rp.is_allowed(user_agent, url)
+        return cast("bool", self.rp.is_allowed(user_agent, url))
 
 
 class ProtegoRobotParser(RobotParser):
diff --git a/scrapy/utils/benchserver.py b/scrapy/utils/benchserver.py
index c4c96ef00..403cd54a8 100644
--- a/scrapy/utils/benchserver.py
+++ b/scrapy/utils/benchserver.py
@@ -30,7 +30,7 @@ class Root(Resource):
 def _getarg(
     request: Request, name: bytes, default: Any = None, type_: type = str
 ) -> Any:
-    return type_(request.args[name][0]) if name in request.args else default  # type: ignore[index,operator]
+    return type_(request.args[name][0]) if name in request.args else default
 
 
 if __name__ == "__main__":
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 64d2cde1b..146e3ae56 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -13,7 +13,7 @@ import warnings
 import weakref
 from collections import OrderedDict
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, TypeVar
+from typing import TYPE_CHECKING, Any, AnyStr, TypeVar, cast
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 
@@ -181,7 +181,7 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 
     def __getitem__(self, key: _KT) -> _VT | None:
         try:
-            return super().__getitem__(key)
+            return cast("_VT", super().__getitem__(key))
         except (TypeError, KeyError):
             return None  # key is either not weak-referenceable or not cached
 
diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.py
index aea2557d2..a5bb6fa24 100644
--- a/scrapy/utils/decorators.py
+++ b/scrapy/utils/decorators.py
@@ -3,7 +3,7 @@ from __future__ import annotations
 import inspect
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, overload
+from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload
 
 from twisted.internet.defer import Deferred, maybeDeferred
 
@@ -115,7 +115,7 @@ def _warn_spider_arg(
         @wraps(func)
         async def async_inner(*args: _P.args, **kwargs: _P.kwargs) -> _T:
             check_args(*args, **kwargs)
-            return await func(*args, **kwargs)
+            return cast("_T", await func(*args, **kwargs))
 
         return async_inner
 
diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.py
index da1838030..359f819d7 100644
--- a/scrapy/utils/deprecate.py
+++ b/scrapy/utils/deprecate.py
@@ -4,7 +4,7 @@ from __future__ import annotations
 
 import inspect
 import warnings
-from typing import TYPE_CHECKING, Any, overload
+from typing import TYPE_CHECKING, Any, cast, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.python import get_func_args_dict
@@ -68,7 +68,7 @@ def create_deprecated_class(
         def __new__(  # pylint: disable=bad-classmethod-argument
             metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]
         ) -> type:
-            cls = super().__new__(metacls, name, bases, clsdict_)
+            cls: type = super().__new__(metacls, name, bases, clsdict_)
             if metacls.deprecated_class is None:
                 metacls.deprecated_class = cls
             return cls
@@ -100,7 +100,7 @@ def create_deprecated_class(
                 # is the deprecated class itself - subclasses of the
                 # deprecated class should not use custom `__subclasscheck__`
                 # method.
-                return super().__subclasscheck__(sub)
+                return cast("bool", super().__subclasscheck__(sub))
 
             if not inspect.isclass(sub):
                 raise TypeError("issubclass() arg 1 must be a class")
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 757756e65..3baa02e2b 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -13,7 +13,7 @@ from contextlib import contextmanager
 from functools import partial
 from importlib import import_module
 from pkgutil import iter_modules
-from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, overload
+from typing import IO, TYPE_CHECKING, Any, ParamSpec, Protocol, TypeVar, cast, overload
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.item import Item
@@ -51,7 +51,7 @@ def arg_to_iter(arg: Any) -> Iterable[Any]:
     if arg is None:
         return ()
     if not isinstance(arg, _ITERABLE_SINGLE_VALUES) and hasattr(arg, "__iter__"):
-        return arg
+        return cast("Iterable[Any]", arg)
     return [arg]
 
 
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index b41b8af38..ddebd8ba2 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -33,12 +33,12 @@ def listen_tcp(portrange: list[int], host: str, factory: ServerFactory) -> Port:
     if len(portrange) > 2:
         raise ValueError(f"invalid portrange: {portrange}")
     if not portrange:
-        return reactor.listenTCP(0, factory, interface=host)
+        return reactor.listenTCP(0, factory, interface=host)  # type: ignore[no-any-return]
     if len(portrange) == 1:
-        return reactor.listenTCP(portrange[0], factory, interface=host)
+        return reactor.listenTCP(portrange[0], factory, interface=host)  # type: ignore[no-any-return]
     for x in range(portrange[0], portrange[1] + 1):
         try:
-            return reactor.listenTCP(x, factory, interface=host)
+            return reactor.listenTCP(x, factory, interface=host)  # type: ignore[no-any-return]
         except error.CannotListenError:
             if x == portrange[1]:
                 raise
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index 997e8eba8..d9a72273a 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -7,6 +7,7 @@ import logging
 import warnings
 from collections.abc import Awaitable, Callable, Generator, Sequence
 from typing import Any as TypingAny
+from typing import cast
 
 from pydispatch.dispatcher import (
     Anonymous,
@@ -185,7 +186,7 @@ async def _send_catch_log_asyncio(
     handlers: list[Awaitable[TypingAny]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
 
-        async def handler(receiver: Callable) -> TypingAny:
+        async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
             result: TypingAny
             try:
                 result = await ensure_awaitable(
@@ -208,7 +209,10 @@ async def _send_catch_log_asyncio(
 
         handlers.append(handler(receiver))
 
-    return await asyncio.gather(*handlers, return_exceptions=True)
+    return cast(
+        "list[tuple[TypingAny, TypingAny]]",
+        await asyncio.gather(*handlers, return_exceptions=True),
+    )
 
 
 def disconnect_all(signal: TypingAny = Any, sender: TypingAny = Any) -> None:
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 75d6c9bb0..9f43df1ce 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -41,11 +41,10 @@ def iterate_spider_output(
 ) -> Iterable[Any] | AsyncGenerator[_T] | Deferred[_T]:
     if inspect.isasyncgen(result):
         return result
+    d: Deferred[_T] = deferred_from_coro(result)
     if inspect.iscoroutine(result):
-        d = deferred_from_coro(result)
-        d.addCallback(iterate_spider_output)
-        return d
-    return arg_to_iter(deferred_from_coro(result))
+        return d.addCallback(iterate_spider_output)
+    return arg_to_iter(d)
 
 
 def iter_spider_classes(module: ModuleType) -> Iterable[type[Spider]]:
diff --git a/tests/mockserver/http_resources.py b/tests/mockserver/http_resources.py
index 7f52f0092..98ac6cf6a 100644
--- a/tests/mockserver/http_resources.py
+++ b/tests/mockserver/http_resources.py
@@ -239,7 +239,7 @@ class ArbitraryLengthPayloadResource(LeafResource):
 
 class NoMetaRefreshRedirect(Redirect):
     def render(self, request: server.Request) -> bytes:
-        content = Redirect.render(self, request)
+        content: bytes = Redirect.render(self, request)
         return content.replace(
             b'http-equiv="refresh"', b'http-no-equiv="do-not-refresh-me"'
         )
diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py
index f900ee693..188eef61d 100644
--- a/tests/mockserver/utils.py
+++ b/tests/mockserver/utils.py
@@ -1,22 +1,26 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from cryptography.x509 import load_pem_x509_certificate
 from OpenSSL import SSL
 from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
-from twisted.internet.ssl import CertificateOptions, ContextFactory
+from twisted.internet.ssl import CertificateOptions
 
 from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
 from scrapy.utils.python import to_bytes
 
+if TYPE_CHECKING:
+    from twisted.internet.interfaces import IOpenSSLContextFactory
+
 
 def ssl_context_factory(
     keyfile: str = "keys/localhost.key",
     certfile: str = "keys/localhost.crt",
     cipher_string: str | None = None,
-) -> ContextFactory:
+) -> IOpenSSLContextFactory:
     keyfile_path = Path(__file__).parent.parent / keyfile
     certfile_path = Path(__file__).parent.parent / certfile
 
@@ -27,7 +31,7 @@ def ssl_context_factory(
         cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes())  # type: ignore[assignment]
         key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes())  # type: ignore[assignment]
 
-    factory = CertificateOptions(
+    factory: CertificateOptions = CertificateOptions(
         privateKey=key,
         certificate=cert,
     )
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index 15c68779d..082fc743e 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -23,8 +23,8 @@ if TYPE_CHECKING:
 
 
 class TestRobotsTxtMiddleware:
-    def setup_method(self):
-        self.crawler = mock.MagicMock()
+    def setup_method(self) -> None:
+        self.crawler: mock.MagicMock = mock.MagicMock()
         self.crawler.settings = Settings()
         self.crawler.engine.download_async = mock.AsyncMock()
 
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 65da89472..6072e2f6d 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -234,7 +234,7 @@ class TestHttps2ClientProtocol:
         pem = self.key_file.read_text(
             encoding="utf-8"
         ) + self.certificate_file.read_text(encoding="utf-8")
-        return PrivateCertificate.loadPEM(pem)
+        return PrivateCertificate.loadPEM(pem)  # type: ignore[no-any-return]
 
     @async_yield_fixture  # type: ignore[untyped-decorator]
     async def client(
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 7fec479f3..eb88baf01 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -4,7 +4,7 @@ import warnings
 from abc import ABC, abstractmethod
 from collections import deque
 from contextlib import AbstractAsyncContextManager, asynccontextmanager
-from typing import TYPE_CHECKING, Any, NamedTuple
+from typing import TYPE_CHECKING, Any, NamedTuple, cast
 from unittest.mock import Mock
 
 import pytest
@@ -29,9 +29,9 @@ if TYPE_CHECKING:
 class MemoryScheduler(BaseScheduler):
     paused = False
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(*args, **kwargs)
-        self.queue = deque(
+        self.queue: deque[Request] = deque(
             Request(value) if isinstance(value, str) else value
             for value in getattr(self, "queue", [])
         )
@@ -68,7 +68,7 @@ class MockDownloader:
 
     def get_slot_key(self, request: Request) -> str:
         if Downloader.DOWNLOAD_SLOT in request.meta:
-            return request.meta[Downloader.DOWNLOAD_SLOT]
+            return cast("str", request.meta[Downloader.DOWNLOAD_SLOT])
 
         return urlparse_cached(request).hostname or ""
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 9651e3f8b..bfa60ac9c 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from collections.abc import AsyncIterator, Iterable
 from inspect import isasyncgen
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 from unittest import mock
 
 import pytest
@@ -27,7 +27,7 @@ if TYPE_CHECKING:
 
 
 class TestSpiderMiddleware:
-    def setup_method(self):
+    def setup_method(self) -> None:
         self.request = Request("http://example.com/index.html")
         self.response = Response(self.request.url, request=self.request)
         self.crawler = get_crawler(Spider, {"SPIDER_MIDDLEWARES_BASE": {}})
@@ -39,11 +39,10 @@ class TestSpiderMiddleware:
         Raise exception in case of failure.
         """
 
-        def scrape_func(
+        async def scrape_func(
             response: Response | Failure, request: Request
-        ) -> defer.Deferred[Iterable[Any]]:
-            it = mock.MagicMock()
-            return defer.succeed(it)
+        ) -> Iterable[Any]:
+            return mock.MagicMock()
 
         return await self.mwman.scrape_response_async(
             scrape_func, self.response, self.request
@@ -141,7 +140,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
     async def _scrape_func(
         self, response: Response | Failure, request: Request
     ) -> Iterable[Any] | AsyncIterator[Any]:
-        return self._callback()
+        return cast("Iterable[Any] | AsyncIterator[Any]", self._callback())
 
     async def _get_middleware_result(
         self, *mw_classes: type[Any], start_index: int | None = None
diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py
index 4750158e5..0e8988a0a 100644
--- a/tests/utils/decorators.py
+++ b/tests/utils/decorators.py
@@ -35,7 +35,8 @@ def inline_callbacks_test(
         async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None:
             await deferred_to_future(inlineCallbacks(f)(*args, **kwargs))
 
-        return wrapper_coro
+        # Likely https://github.com/python/mypy/issues/17171
+        return wrapper_coro  # type: ignore[no-any-return]
 
     @wraps(f)
     @inlineCallbacks
diff --git a/tests_typing/test_spiders.mypy-testing b/tests_typing/test_spiders.mypy-testing
index 162e31d0c..dfdd2ba4d 100644
--- a/tests_typing/test_spiders.mypy-testing
+++ b/tests_typing/test_spiders.mypy-testing
@@ -59,7 +59,7 @@ def test_spider_parse_override_no_kwargs() -> None:
 @pytest.mark.mypy_testing
 def test_spider_parse_override_specific_kwargs() -> None:
     spider = SpecificKwargsSpider()
-    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: builtins.int) -> Any
+    reveal_type(spider.parse)  # R: def (response: scrapy.http.response.Response, page: int) -> Any
 
 
 @pytest.mark.mypy_testing
diff --git a/tox.ini b/tox.ini
index 07583a11e..a3b0cd91d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,26 +44,28 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.19.1
+    mypy==1.20.2
     typing-extensions==4.15.0
-    Pillow==12.1.1
+    Pillow==12.2.0
     Protego==0.6.0
-    attrs==25.4.0
-    boto3-stubs[s3]==1.42.59
+    Twisted==25.5.0
+    attrs==26.1.0
+    boto3-stubs[s3]==1.43.2
     botocore-stubs==1.42.41
     h2==4.3.0
     httpx==0.28.1
     itemadapter==0.13.1
     ptpython==3.0.32
-    ipython
-    pyOpenSSL==25.3.0
-    pytest==9.0.2
-    types-Pygments==2.19.0.20251121
-    types-defusedxml==0.7.0.20250822
+    # newer ones require newer Python
+    ipython==8.39.0
+    pyOpenSSL==26.1.0
+    pytest==9.0.3
+    types-Pygments==2.20.0.20260408
+    types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
-    types-pexpect==4.9.0.20260127
+    types-pexpect==4.9.0.20260408
     uvloop==0.22.1
-    w3lib==2.4.0
+    w3lib==2.4.1
     zstandard==0.25.0
 commands =
     mypy {posargs:scrapy tests}

From 5223dbe3fdf801a8a3e7877e271f54519ff73f0a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 6 May 2026 13:42:24 +0500
Subject: [PATCH 16/36] Pin pyOpenSSL to the versions allowing mutable
 contexts. (#7494)

---
 pyproject.toml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index 11d36d0ac..f13e67020 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -18,7 +18,8 @@ dependencies = [
     "packaging",
     "parsel>=1.5.0",
     "protego>=0.1.15",
-    "pyOpenSSL>=22.0.0",
+    # pyOpenSSL pinned until Twisted with immutable contexts is released and supported in Scrapy
+    "pyOpenSSL>=22.0.0,<26.2.0",
     "queuelib>=1.4.2",
     "service_identity>=18.1.0",
     "tldextract",

From 7f15ca92fc5c486d3ba121dfafa5c061ac63eb71 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Tue, 12 May 2026 09:49:23 +0200
Subject: [PATCH 17/36] =?UTF-8?q?sphinx-scrapy:=200.8.5=20=E2=86=92=200.8.?=
 =?UTF-8?q?6=20(#7507)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .pre-commit-config.yaml | 2 +-
 docs/requirements.in    | 2 +-
 docs/requirements.txt   | 2 +-
 tox.ini                 | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index bbf24dd63..6b9ef3c04 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -27,6 +27,6 @@ repos:
   hooks:
     - id: sphinx-lint
 - repo: https://github.com/scrapy/sphinx-scrapy
-  rev: 0.8.5
+  rev: 0.8.6
   hooks:
     - id: sphinx-scrapy
diff --git a/docs/requirements.in b/docs/requirements.in
index 6320721cd..140791641 100644
--- a/docs/requirements.in
+++ b/docs/requirements.in
@@ -5,4 +5,4 @@ sphinx
 sphinx-notfound-page
 sphinx-rtd-theme
 sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.5
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
diff --git a/docs/requirements.txt b/docs/requirements.txt
index 53cb82638..9c93dacd0 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -153,7 +153,7 @@ sphinx-rtd-theme==3.1.0
     # via
     #   -r docs/requirements.in
     #   sphinx-rtd-dark-mode
-sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@96826815002921f27a2e369b12c0c25af7a1f8b2
+sphinx-scrapy @ git+https://github.com/scrapy/sphinx-scrapy.git@b1d55db4d16a5425fc68576d63519bbfe26dd9c0
     # via -r docs/requirements.in
 sphinx-sitemap==2.9.0
     # via sphinx-scrapy
diff --git a/tox.ini b/tox.ini
index a3b0cd91d..e0e95c194 100644
--- a/tox.ini
+++ b/tox.ini
@@ -5,7 +5,7 @@
 
 [tox]
 requires =
-    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.5
+    sphinx-scrapy[tox] @ git+https://github.com/scrapy/sphinx-scrapy.git@0.8.6
 envlist = pre-commit,pylint,typing,py,docs
 minversion = 1.7.0
 

From 7fc84d372aee8def89f427a5190bb837ff3bfa55 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 12 May 2026 17:45:54 +0500
Subject: [PATCH 18/36] Cleanup/clarifications of proxy support and
 improvements for proxy tests (#7496)

* Run mitmproxy-based tests for every handler and improve them.

* Fixes for H2DownloadHandler.

* Revert "Fixes for H2DownloadHandler."

This reverts commit bcdbd097cde54048a4b85e325937ef747a09e68a.

* Add TestHttpsProxy for HTTP11DownloadHandler.

* Use the configured context factory in ScrapyProxyAgent.

* Reword.

* Reword.

* pragma: no cover for H2DownloadHandler proxy code.

* Raise an exception for HTTPS proxies instead of using HTTP.

* Remove non-working proxy support and explicitly forbid HTTP.

* pragma: no cover

* Add a docs note about HTTPS proxies.

* Rename ScrapyProxyAgent.
---
 conftest.py                                   |  27 ++++
 docs/news.rst                                 |  11 ++
 docs/topics/download-handlers.rst             |   2 +
 docs/topics/downloader-middleware.rst         |  11 ++
 scrapy/core/downloader/handlers/http11.py     |  12 +-
 scrapy/core/downloader/handlers/http2.py      |  34 ++---
 scrapy/core/http2/agent.py                    |  27 ----
 tests/mockserver/mitm_proxy.py                |  64 +++++++++
 tests/mockserver/mitm_proxy_addon.py          |   5 +
 tests/test_downloader_handler_httpx.py        |  27 ++--
 .../test_downloader_handler_twisted_http11.py |  33 +++--
 .../test_downloader_handler_twisted_http2.py  |  51 +++----
 tests/test_downloader_handlers_http_base.py   | 123 +++++++++++++++++
 tests/test_proxy_connect.py                   | 125 ------------------
 14 files changed, 323 insertions(+), 229 deletions(-)
 create mode 100644 tests/mockserver/mitm_proxy.py
 create mode 100644 tests/mockserver/mitm_proxy_addon.py
 delete mode 100644 tests/test_proxy_connect.py

diff --git a/conftest.py b/conftest.py
index 4f8d32e1b..fcbf59426 100644
--- a/conftest.py
+++ b/conftest.py
@@ -11,6 +11,7 @@ from scrapy.utils.reactor import set_asyncio_event_loop_policy
 from scrapy.utils.reactorless import install_reactor_import_hook
 from tests.keys import generate_keys
 from tests.mockserver.http import MockServer
+from tests.mockserver.mitm_proxy import MitmProxy
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -72,6 +73,32 @@ def mockserver() -> Generator[MockServer]:
         yield mockserver
 
 
+@pytest.fixture  # function scope because it modifies os.environ
+def mitm_proxy_server(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
+    proxy = MitmProxy()
+    url = proxy.start()
+    monkeypatch.setenv("http_proxy", url)
+    monkeypatch.setenv("https_proxy", url)
+
+    try:
+        yield proxy
+    finally:
+        proxy.stop()
+
+
+@pytest.fixture  # function scope because it modifies os.environ
+def mitm_proxy_server_https(monkeypatch: pytest.MonkeyPatch) -> Generator[MitmProxy]:
+    proxy = MitmProxy()
+    url = proxy.start().replace("http://", "https://")
+    monkeypatch.setenv("http_proxy", url)
+    monkeypatch.setenv("https_proxy", url)
+
+    try:
+        yield proxy
+    finally:
+        proxy.stop()
+
+
 @pytest.fixture(scope="session")
 def reactor_pytest(request) -> str:
     return request.config.getoption("--reactor")
diff --git a/docs/news.rst b/docs/news.rst
index 676e0cbbf..7f3952eaf 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,17 @@
 Release notes
 =============
 
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-   ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made
+    private as it's an implementation detail of
+    :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
+    (:issue:`7496`)
+
 .. _release-2.15.2:
 
 Scrapy 2.15.2 (2026-04-28)
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 96cf46c35..1e95864c6 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -206,6 +206,8 @@ If you want to use this handler you need to replace the default one for the
 
     Known limitations of the HTTP/2 implementation in this handler include:
 
+    -   No support for proxies.
+
     -   No support for HTTP/2 Cleartext (h2c), since no major browser supports
         HTTP/2 unencrypted (refer `http2 faq`_).
 
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 75e48be41..873c32c13 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -745,8 +745,19 @@ HttpProxyMiddleware
     Handling of this meta key needs to be implemented inside the :ref:`download
     handler <topics-download-handlers>`, so it's not guaranteed to be supported
     by all 3rd-party handlers. It's currently unsupported by
+    :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` and
     :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`.
 
+.. note::
+
+    Usually a proxy URL uses the ``http://`` scheme. More rarely, it uses the
+    ``https://`` one. While both kinds of proxy URLs can be used with both HTTP
+    and HTTPS destination URLs, the specifics of the network exchange are
+    different for all 4 cases and it's possible that HTTPS proxies are fully or
+    partially unsupported by a given download handler. Currently,
+    :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+    supports HTTPS proxies only for HTTP destinations.
+
 HttpProxyMiddleware settings
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 975078803..1127c533f 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -338,17 +338,19 @@ class TunnelingAgent(Agent):
         )
 
 
-class ScrapyProxyAgent(Agent):
+class _ScrapyProxyAgent(Agent):
     def __init__(
         self,
         reactor: ReactorBase,
         proxyURI: bytes,
+        contextFactory: IPolicyForHTTPS,
         connectTimeout: float | None = None,
         bindAddress: tuple[str, int] | None = None,
         pool: HTTPConnectionPool | None = None,
     ):
         super().__init__(  # type: ignore[no-untyped-call]
             reactor=reactor,
+            contextFactory=contextFactory,
             connectTimeout=connectTimeout,
             bindAddress=bindAddress,
             pool=pool,
@@ -380,7 +382,6 @@ class ScrapyProxyAgent(Agent):
 
 class ScrapyAgent:
     _Agent = Agent
-    _ProxyAgent = ScrapyProxyAgent
     _TunnelingAgent = TunnelingAgent
 
     def __init__(
@@ -421,6 +422,10 @@ class ScrapyAgent:
             if not proxy_port:
                 proxy_port = 443 if proxy_parsed.scheme == "https" else 80
             if urlparse_cached(request).scheme == "https":
+                if proxy_parsed.scheme == "https":  # pragma: no cover
+                    raise NotImplementedError(
+                        "HTTPS proxies for HTTPS destinations are not supported"
+                    )
                 assert proxy_host is not None
                 proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                 proxyConf = (proxy_host, proxy_port, proxyAuth)
@@ -432,9 +437,10 @@ class ScrapyAgent:
                     bindAddress=bindaddress,
                     pool=self._pool,
                 )
-            return self._ProxyAgent(
+            return _ScrapyProxyAgent(
                 reactor=reactor,
                 proxyURI=to_bytes(proxy, encoding="ascii"),
+                contextFactory=self._contextFactory,
                 connectTimeout=timeout,
                 bindAddress=bindaddress,
                 pool=self._pool,
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index a4f786363..b28bcfd89 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -4,19 +4,20 @@ from time import monotonic
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag
 
-from twisted.web.client import URI
-
 from scrapy.core.downloader.contextfactory import _load_context_factory_from_settings
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
-from scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent
-from scrapy.exceptions import DownloadTimeoutError, NotConfigured
+from scrapy.core.http2.agent import H2Agent, H2ConnectionPool
+from scrapy.exceptions import (
+    DownloadTimeoutError,
+    NotConfigured,
+    UnsupportedURLSchemeError,
+)
 from scrapy.utils._download_handlers import (
     normalize_bind_address,
     wrap_twisted_exceptions,
 )
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.python import to_bytes
 
 if TYPE_CHECKING:
     from twisted.internet.base import DelayedCall
@@ -44,6 +45,10 @@ class H2DownloadHandler(BaseDownloadHandler):
         self._bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
 
     async def download_request(self, request: Request) -> Response:
+        if urlparse_cached(request).scheme == "http":  # pragma: no cover
+            raise UnsupportedURLSchemeError(
+                f"{type(self).__name__} doesn't support plain HTTP."
+            )
         agent = ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
@@ -62,7 +67,6 @@ class H2DownloadHandler(BaseDownloadHandler):
 
 class ScrapyH2Agent:
     _Agent = H2Agent
-    _ProxyAgent = ScrapyProxyH2Agent
 
     def __init__(
         self,
@@ -81,24 +85,10 @@ class ScrapyH2Agent:
     def _get_agent(self, request: Request, timeout: float | None) -> H2Agent:
         from twisted.internet import reactor
 
+        if request.meta.get("proxy"):  # pragma: no cover
+            raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
         bind_address = request.meta.get("bindaddress") or self._bind_address
         bind_address = normalize_bind_address(bind_address)
-        proxy = request.meta.get("proxy")
-        if proxy:
-            if urlparse_cached(request).scheme == "https":
-                # ToDo
-                raise NotImplementedError(
-                    "Tunneling via CONNECT method using HTTP/2.0 is not yet supported"
-                )
-            return self._ProxyAgent(
-                reactor=reactor,
-                context_factory=self._context_factory,
-                proxy_uri=URI.fromBytes(to_bytes(proxy, encoding="ascii")),
-                connect_timeout=timeout,
-                bind_address=bind_address,
-                pool=self._pool,
-            )
-
         return self._Agent(
             reactor=reactor,
             context_factory=self._context_factory,
diff --git a/scrapy/core/http2/agent.py b/scrapy/core/http2/agent.py
index 822dffc4b..7137a0f2b 100644
--- a/scrapy/core/http2/agent.py
+++ b/scrapy/core/http2/agent.py
@@ -165,30 +165,3 @@ class H2Agent:
             lambda conn: conn.request(request, spider)
         )
         return d2
-
-
-class ScrapyProxyH2Agent(H2Agent):
-    def __init__(
-        self,
-        reactor: ReactorBase,
-        proxy_uri: URI,
-        pool: H2ConnectionPool,
-        context_factory: BrowserLikePolicyForHTTPS = BrowserLikePolicyForHTTPS(),  # noqa: B008
-        connect_timeout: float | None = None,
-        bind_address: tuple[str, int] | None = None,
-    ) -> None:
-        super().__init__(
-            reactor=reactor,
-            pool=pool,
-            context_factory=context_factory,
-            connect_timeout=connect_timeout,
-            bind_address=bind_address,
-        )
-        self._proxy_uri = proxy_uri
-
-    def get_endpoint(self, uri: URI) -> HostnameEndpoint:
-        return self.endpoint_factory.endpointForURI(self._proxy_uri)  # type: ignore[no-any-return]
-
-    def get_key(self, uri: URI) -> ConnectionKeyT:
-        """We use the proxy uri instead of uri obtained from request url"""
-        return b"http-proxy", self._proxy_uri.host, self._proxy_uri.port
diff --git a/tests/mockserver/mitm_proxy.py b/tests/mockserver/mitm_proxy.py
new file mode 100644
index 000000000..1ec5a79b5
--- /dev/null
+++ b/tests/mockserver/mitm_proxy.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+from subprocess import PIPE, Popen
+from urllib.parse import urlsplit, urlunsplit
+
+
+class MitmProxy:
+    auth_user = "scrapy"
+    auth_pass = "scrapy"
+
+    def start(self) -> str:
+        script = """
+import sys
+from mitmproxy.tools.main import mitmdump
+sys.argv[0] = "mitmdump"
+sys.exit(mitmdump())
+        """
+        cert_path = Path(__file__).parent.parent.resolve() / "keys"
+        args = [
+            "--listen-host",
+            "127.0.0.1",
+            "--listen-port",
+            "0",
+            "--proxyauth",
+            f"{self.auth_user}:{self.auth_pass}",
+            "--set",
+            f"confdir={cert_path}",
+            "--ssl-insecure",
+            "-s",
+            str(Path(__file__).with_name("mitm_proxy_addon.py")),
+        ]
+        self.proc: Popen[str] = Popen(
+            [
+                sys.executable,
+                "-u",
+                "-c",
+                script,
+                *args,
+            ],
+            stdout=PIPE,
+            text=True,
+        )
+        assert self.proc.stdout is not None
+        line = ""
+        for line in self.proc.stdout:
+            m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
+            if m:
+                host_port = m.group(1)
+                return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
+        self.stop()
+        raise RuntimeError(f"Failed to parse mitmdump output: {line}")
+
+    def stop(self) -> None:
+        self.proc.kill()
+        self.proc.communicate()
+
+
+def wrong_credentials(proxy_url: str) -> str:
+    bad_auth_proxy = list(urlsplit(proxy_url))
+    bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
+    return urlunsplit(bad_auth_proxy)
diff --git a/tests/mockserver/mitm_proxy_addon.py b/tests/mockserver/mitm_proxy_addon.py
new file mode 100644
index 000000000..04ba2e4a1
--- /dev/null
+++ b/tests/mockserver/mitm_proxy_addon.py
@@ -0,0 +1,5 @@
+def response(flow) -> None:
+    # add custom headers to be able to check that the request went through the proxy
+    flow.response.headers["X-Via-Mitmproxy"] = "1"
+    if flow.client_conn.tls_established:
+        flow.response.headers["X-Via-Mitmproxy-TLS"] = "1"
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index a0e3fccb3..d4dfe8396 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -17,6 +17,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsInvalidDNSPatternBase,
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
+    TestMitmProxyBase,
     TestSimpleHttpsBase,
 )
 from tests.utils.decorators import coroutine_test
@@ -41,6 +42,15 @@ class HttpxDownloadHandlerMixin:
 
         return HttpxDownloadHandler
 
+    @property
+    def settings_dict(self) -> dict[str, Any] | None:
+        return {
+            "DOWNLOAD_HANDLERS": {
+                "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+                "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
+            }
+        }
+
 
 class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
     handler_supports_bindaddress_meta = False
@@ -108,15 +118,8 @@ class TestHttpsCustomCiphers(HttpxDownloadHandlerMixin, TestHttpsCustomCiphersBa
     pass
 
 
-class TestHttpWithCrawler(TestHttpWithCrawlerBase):
-    @property
-    def settings_dict(self) -> dict[str, Any] | None:
-        return {
-            "DOWNLOAD_HANDLERS": {
-                "http": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
-                "https": "scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler",
-            }
-        }
+class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
+    pass
 
 
 class TestHttpsWithCrawler(TestHttpWithCrawler):
@@ -136,3 +139,9 @@ class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
 @pytest.mark.skip(reason="Proxy support is not implemented yet")
 class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
     is_secure = True
+
+
+@pytest.mark.skip(reason="Proxy support is not implemented yet")
+@pytest.mark.requires_mitmproxy
+class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
+    pass
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 01fb26d85..89cf58c96 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -19,6 +19,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsInvalidDNSPatternBase,
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
+    TestMitmProxyBase,
     TestSimpleHttpsBase,
 )
 
@@ -34,6 +35,15 @@ class HTTP11DownloadHandlerMixin:
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
         return HTTP11DownloadHandler
 
+    @property
+    def settings_dict(self) -> dict[str, Any] | None:
+        return {
+            "DOWNLOAD_HANDLERS": {
+                "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
+                "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
+            }
+        }
+
 
 def test_not_configured_without_reactor() -> None:
     crawler = Crawler(Spider, {"TWISTED_REACTOR_ENABLED": False})
@@ -71,15 +81,8 @@ class TestHttpsCustomCiphers(HTTP11DownloadHandlerMixin, TestHttpsCustomCiphersB
     pass
 
 
-class TestHttpWithCrawler(TestHttpWithCrawlerBase):
-    @property
-    def settings_dict(self) -> dict[str, Any] | None:
-        return {
-            "DOWNLOAD_HANDLERS": {
-                "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
-                "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
-            }
-        }
+class TestHttpWithCrawler(HTTP11DownloadHandlerMixin, TestHttpWithCrawlerBase):
+    pass
 
 
 class TestHttpsWithCrawler(TestHttpWithCrawler):
@@ -88,3 +91,15 @@ class TestHttpsWithCrawler(TestHttpWithCrawler):
 
 class TestHttpProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
     pass
+
+
+class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
+    is_secure = True
+    # not implemented
+    handler_supports_tls_in_tls = False
+
+
+@pytest.mark.requires_mitmproxy
+class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
+    # not implemented
+    handler_supports_tls_in_tls = False
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 3273a264e..60b276166 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -10,13 +10,8 @@ from twisted.web.http import H2_ENABLED
 
 from scrapy import Spider
 from scrapy.crawler import Crawler
-from scrapy.exceptions import (
-    DownloadFailedError,
-    NotConfigured,
-    UnsupportedURLSchemeError,
-)
+from scrapy.exceptions import DownloadFailedError, NotConfigured
 from scrapy.http import Request
-from scrapy.utils.defer import maybe_deferred_to_future
 from tests.test_downloader_handlers_http_base import (
     TestHttpProxyBase,
     TestHttpsBase,
@@ -25,13 +20,13 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsInvalidDNSPatternBase,
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
+    TestMitmProxyBase,
 )
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from scrapy.core.downloader.handlers import DownloadHandlerProtocol
     from tests.mockserver.http import MockServer
-    from tests.mockserver.proxy_echo import ProxyEchoMockServer
 
 
 pytestmark = [
@@ -52,6 +47,15 @@ class H2DownloadHandlerMixin:
 
         return H2DownloadHandler
 
+    @property
+    def settings_dict(self) -> dict[str, Any] | None:
+        return {
+            "DOWNLOAD_HANDLERS": {
+                "http": None,
+                "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
+            }
+        }
+
 
 def test_not_configured_without_reactor() -> None:
     from scrapy.core.downloader.handlers.http2 import H2DownloadHandler  # noqa: PLC0415
@@ -167,16 +171,7 @@ class TestHttp2CustomCiphers(H2DownloadHandlerMixin, TestHttpsCustomCiphersBase)
     pass
 
 
-class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
-    @property
-    def settings_dict(self) -> dict[str, Any] | None:
-        return {
-            "DOWNLOAD_HANDLERS": {
-                "http": None,
-                "https": "scrapy.core.downloader.handlers.http2.H2DownloadHandler",
-            }
-        }
-
+class TestHttp2WithCrawler(H2DownloadHandlerMixin, TestHttpWithCrawlerBase):
     is_secure = True
 
     def test_bytes_received_stop_download_callback(self) -> None:  # type: ignore[override]
@@ -192,24 +187,12 @@ class TestHttp2WithCrawler(TestHttpWithCrawlerBase):
         pytest.skip("headers_received support is not implemented")
 
 
+@pytest.mark.skip(reason="Proxy support is not implemented yet")
 class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
     is_secure = True
-    expected_http_proxy_request_body = b"/"
 
-    @coroutine_test
-    async def test_download_with_proxy_https_timeout(
-        self, proxy_mockserver: ProxyEchoMockServer
-    ) -> None:
-        with pytest.raises(NotImplementedError):
-            await maybe_deferred_to_future(
-                super().test_download_with_proxy_https_timeout(proxy_mockserver)  # type: ignore[arg-type]
-            )
 
-    @coroutine_test
-    async def test_download_with_proxy_without_http_scheme(
-        self, proxy_mockserver: ProxyEchoMockServer
-    ) -> None:
-        with pytest.raises(UnsupportedURLSchemeError):
-            await maybe_deferred_to_future(
-                super().test_download_with_proxy_without_http_scheme(proxy_mockserver)  # type: ignore[arg-type]
-            )
+@pytest.mark.skip(reason="Proxy support is not implemented yet")
+@pytest.mark.requires_mitmproxy
+class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase):
+    pass
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 2a781dace..138ddd944 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -4,6 +4,8 @@ from __future__ import annotations
 
 import gzip
 import json
+import logging
+import os
 import platform
 import re
 import sys
@@ -36,6 +38,7 @@ from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests import NON_EXISTING_RESOLVABLE
+from tests.mockserver.mitm_proxy import MitmProxy, wrong_credentials
 from tests.mockserver.proxy_echo import ProxyEchoMockServer
 from tests.mockserver.simple_https import SimpleMockServer
 from tests.spiders import (
@@ -43,6 +46,7 @@ from tests.spiders import (
     BytesReceivedErrbackSpider,
     HeadersReceivedCallbackSpider,
     HeadersReceivedErrbackSpider,
+    SimpleSpider,
     SingleRequestSpider,
 )
 from tests.utils.decorators import coroutine_test
@@ -1072,6 +1076,8 @@ class TestHttpWithCrawlerBase(ABC):
 class TestHttpProxyBase(ABC):
     is_secure = False
     expected_http_proxy_request_body = b"http://example.com"
+    # whether the handler supports HTTPS proxies with HTTPS destinations
+    handler_supports_tls_in_tls: bool = True
 
     @property
     @abstractmethod
@@ -1124,6 +1130,8 @@ class TestHttpProxyBase(ABC):
     ) -> None:
         if NON_EXISTING_RESOLVABLE:
             pytest.skip("Non-existing hosts are resolvable")
+        if self.is_secure and not self.handler_supports_tls_in_tls:
+            pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
         http_proxy = proxy_mockserver.url("", is_secure=self.is_secure)
         domain = "https://no-such-domain.nosuch"
         request = Request(domain, meta={"proxy": http_proxy, "download_timeout": 0.2})
@@ -1143,3 +1151,118 @@ class TestHttpProxyBase(ABC):
         assert response.status == 200
         assert response.url == request.url
         assert response.body == self.expected_http_proxy_request_body
+
+
+class TestMitmProxyBase(ABC):
+    # whether the handler supports HTTPS proxies with HTTPS destinations
+    handler_supports_tls_in_tls: bool = True
+
+    @property
+    @abstractmethod
+    def settings_dict(self) -> dict[str, Any] | None:
+        raise NotImplementedError
+
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @coroutine_test
+    async def test_http_proxy(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        mockserver: MockServer,
+        mitm_proxy_server: MitmProxy,
+        https_dest: bool,
+    ) -> None:
+        """HTTP proxy, HTTP or HTTPS destination."""
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=200", is_secure=https_dest)
+            )
+        assert isinstance(crawler.spider, SingleRequestSpider)
+        self._assert_got_response_code(200, caplog.text)
+        self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @coroutine_test
+    async def test_https_proxy(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        mockserver: MockServer,
+        mitm_proxy_server_https: MitmProxy,
+        https_dest: bool,
+    ) -> None:
+        """HTTPS proxy, HTTP or HTTPS destination."""
+        if https_dest and not self.handler_supports_tls_in_tls:
+            pytest.skip("HTTPS proxies for HTTPS destinations are not supported")
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=200", is_secure=https_dest)
+            )
+        assert isinstance(crawler.spider, SingleRequestSpider)
+        self._assert_got_response_code(200, caplog.text)
+        self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @coroutine_test
+    async def test_http_proxy_auth_error(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        monkeypatch: pytest.MonkeyPatch,
+        mockserver: MockServer,
+        mitm_proxy_server: MitmProxy,
+        https_dest: bool,
+    ) -> None:
+        """HTTP proxy, HTTP or HTTPS destination, wrong proxy creds."""
+        envvar = "https_proxy" if https_dest else "http_proxy"
+        monkeypatch.setenv(envvar, wrong_credentials(os.environ[envvar]))
+        crawler = get_crawler(SimpleSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200", is_secure=https_dest)
+            )
+        # The proxy returns a 407 error code but it does not reach the client;
+        # it just sees an exception.
+        self._assert_got_auth_exception(caplog.text)
+
+    @pytest.mark.parametrize(
+        "https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]
+    )
+    @coroutine_test
+    async def test_dont_leak_proxy_authorization_header(
+        self,
+        caplog: pytest.LogCaptureFixture,
+        mockserver: MockServer,
+        mitm_proxy_server: MitmProxy,
+        https_dest: bool,
+    ) -> None:
+        """HTTP proxy, HTTP or HTTPS destination. Check that the auth header
+        is not sent to the destination."""
+        request = Request(mockserver.url("/echo", is_secure=https_dest))
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(seed=request)
+        assert isinstance(crawler.spider, SingleRequestSpider)
+        self._assert_got_response_code(200, caplog.text)
+        self._assert_headers(crawler.spider.meta["responses"][0].headers, https_dest)
+        echo = json.loads(crawler.spider.meta["responses"][0].text)
+        assert "Proxy-Authorization" not in echo["headers"]
+
+    @staticmethod
+    def _assert_headers(headers: Headers, https_dest: bool) -> None:
+        assert b"X-Via-Mitmproxy" in headers
+        if https_dest:
+            assert b"X-Via-Mitmproxy-TLS" in headers
+
+    @staticmethod
+    def _assert_got_response_code(code: int, log: str) -> None:
+        assert str(log).count(f"Crawled ({code})") == 1
+
+    @staticmethod
+    def _assert_got_auth_exception(log: str) -> None:
+        assert "Proxy Authentication Required" in log or "407" in log
diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py
deleted file mode 100644
index c89b51ad3..000000000
--- a/tests/test_proxy_connect.py
+++ /dev/null
@@ -1,125 +0,0 @@
-import json
-import os
-import re
-import sys
-from pathlib import Path
-from subprocess import PIPE, Popen
-from urllib.parse import urlsplit, urlunsplit
-
-import pytest
-from testfixtures import LogCapture
-
-from scrapy.http import Request
-from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
-from tests.spiders import SimpleSpider, SingleRequestSpider
-from tests.utils.decorators import inline_callbacks_test
-
-
-class MitmProxy:
-    auth_user = "scrapy"
-    auth_pass = "scrapy"
-
-    def start(self) -> str:
-        script = """
-import sys
-from mitmproxy.tools.main import mitmdump
-sys.argv[0] = "mitmdump"
-sys.exit(mitmdump())
-        """
-        cert_path = Path(__file__).parent.resolve() / "keys"
-        args = [
-            "--listen-host",
-            "127.0.0.1",
-            "--listen-port",
-            "0",
-            "--proxyauth",
-            f"{self.auth_user}:{self.auth_pass}",
-            "--set",
-            f"confdir={cert_path}",
-            "--ssl-insecure",
-        ]
-        self.proc = Popen(
-            [
-                sys.executable,
-                "-u",
-                "-c",
-                script,
-                *args,
-            ],
-            stdout=PIPE,
-            text=True,
-        )
-        assert self.proc.stdout is not None
-        line = self.proc.stdout.readline()
-        m = re.search(r"listening at (?:http://)?([^:]+:\d+)", line)
-        if not m:
-            raise RuntimeError(f"Failed to parse mitmdump output: {line}")
-        host_port = m.group(1)
-        return f"http://{self.auth_user}:{self.auth_pass}@{host_port}"
-
-    def stop(self) -> None:
-        self.proc.kill()
-        self.proc.communicate()
-
-
-def _wrong_credentials(proxy_url: str) -> str:
-    bad_auth_proxy = list(urlsplit(proxy_url))
-    bad_auth_proxy[1] = bad_auth_proxy[1].replace("scrapy:scrapy@", "wrong:wronger@")
-    return urlunsplit(bad_auth_proxy)
-
-
-@pytest.mark.requires_mitmproxy
-class TestProxyConnect:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    def setup_method(self):
-        self._oldenv = os.environ.copy()
-        self._proxy = MitmProxy()
-        proxy_url = self._proxy.start()
-        os.environ["https_proxy"] = proxy_url
-        os.environ["http_proxy"] = proxy_url
-
-    def teardown_method(self):
-        self._proxy.stop()
-        os.environ = self._oldenv
-
-    @inline_callbacks_test
-    def test_https_connect_tunnel(self):
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        self._assert_got_response_code(200, log)
-
-    @inline_callbacks_test
-    def test_https_tunnel_auth_error(self):
-        os.environ["https_proxy"] = _wrong_credentials(os.environ["https_proxy"])
-        crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(self.mockserver.url("/status?n=200", is_secure=True))
-        # The proxy returns a 407 error code but it does not reach the client;
-        # he just sees a TunnelError.
-        self._assert_got_tunnel_error(log)
-
-    @inline_callbacks_test
-    def test_https_tunnel_without_leak_proxy_authorization_header(self):
-        request = Request(self.mockserver.url("/echo", is_secure=True))
-        crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(seed=request)
-        self._assert_got_response_code(200, log)
-        echo = json.loads(crawler.spider.meta["responses"][0].text)
-        assert "Proxy-Authorization" not in echo["headers"]
-
-    def _assert_got_response_code(self, code, log):
-        assert str(log).count(f"Crawled ({code})") == 1
-
-    def _assert_got_tunnel_error(self, log):
-        assert "TunnelError" in str(log)

From f7db039d1ca2bcfee003453ddbb2210578ed9b62 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 12 May 2026 18:22:31 +0500
Subject: [PATCH 19/36] Update the TLS code to Twisted 26.4.0 (#7347)

* Unpin Twisted, add a twisted-trunk tox env (without http2 for now).

* Hide the _setAcceptableProtocols import.

* Silence ScrapyClientContextFactory HTTP/1.0 warnings.

* Update ScrapyClientTLSOptions for unreleased Twisted.

* Add twisted-trunk to CI.

* Update for new changes in Twisted trunk.

* Xfail a test failing with newer Twisted.

* Test Twisted trunk with extra-deps, update the ALPN code.

* Cleanup.

* Fix typing.

* Update relevant type hints for Twisted trunk.

* Update test_no_context_sharing().

* Silence a weird mypy error.

* Update Twisted versions, unpin pyOpenSSL.

* Update a comment.

* Silence pylint.

* Make a factory fixture.

* Improve the _setAcceptableProtocols comment.

* Set OP_LEGACY_SERVER_CONNECT on new Twisted too.

* Add Twisted[http2] to extra-deps-pinned.
---
 pyproject.toml                            |  8 +-
 scrapy/core/downloader/contextfactory.py  | 59 +++++++++++---
 scrapy/core/downloader/handlers/http11.py |  3 +-
 scrapy/core/downloader/tls.py             | 99 ++++++++++++++++++++++-
 scrapy/shell.py                           |  5 +-
 scrapy/utils/_deps_compat.py              |  4 +
 tests/mockserver/utils.py                 | 11 +--
 tests/test_core_downloader.py             | 81 ++++++++++++++-----
 tests/test_crawler_subprocess.py          |  4 +
 tox.ini                                   |  7 +-
 10 files changed, 232 insertions(+), 49 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index f13e67020..45e18ccc9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,8 +7,7 @@ name = "Scrapy"
 dynamic = ["version"]
 description = "A high-level Web Crawling and Web Scraping framework"
 dependencies = [
-    # Twisted pinned until Scrapy is updated for its internal TLS API changes
-    "Twisted>=21.7.0,<=25.5.0",
+    "Twisted>=21.7.0",
     "cryptography>=37.0.0",
     "cssselect>=0.9.1",
     "defusedxml>=0.7.1",
@@ -18,10 +17,9 @@ dependencies = [
     "packaging",
     "parsel>=1.5.0",
     "protego>=0.1.15",
-    # pyOpenSSL pinned until Twisted with immutable contexts is released and supported in Scrapy
-    "pyOpenSSL>=22.0.0,<26.2.0",
+    "pyOpenSSL>=22.0.0",
     "queuelib>=1.4.2",
-    "service_identity>=18.1.0",
+    "service_identity>=23.1.0",
     "tldextract",
     "w3lib>=1.17.0",
     "zope.interface>=5.1.0",
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 6575d59c1..7f7d34c21 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -5,7 +5,6 @@ from contextlib import contextmanager
 from typing import TYPE_CHECKING, Any, cast
 
 from OpenSSL import SSL
-from twisted.internet._sslverify import _setAcceptableProtocols
 from twisted.internet.ssl import (
     AcceptableCiphers,
     CertificateOptions,
@@ -19,9 +18,11 @@ from zope.interface.verify import verifyObject
 from scrapy.core.downloader.tls import (
     DEFAULT_CIPHERS,
     _ScrapyClientTLSOptions,
+    _ScrapyClientTLSOptions26,
     openssl_methods,
 )
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils._deps_compat import TWISTED_TLS_NEW_IMPL
 from scrapy.utils.deprecate import create_deprecated_class
 from scrapy.utils.misc import build_from_crawler, load_object
 
@@ -108,7 +109,7 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
 
     def _get_cert_options(self) -> CertificateOptions:
         with _filter_method_warning():
-            return CertificateOptions(  # type: ignore[no-any-return]
+            return _ScrapyCertificateOptions(
                 method=self._ssl_method,
                 fixBrokenPeers=True,
                 acceptableCiphers=self.tls_ciphers,
@@ -121,17 +122,21 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         return self._get_context()
 
     def _get_context(self) -> SSL.Context:
-        cert_options = self._get_cert_options()
-        ctx: SSL.Context = cert_options.getContext()
-        ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
-        return ctx
+        return self._get_cert_options().getContext()
 
     def creatorForNetloc(self, hostname: bytes, port: int) -> ClientTLSOptions:
         if not self._verify_certificates:
-            # _ScrapyClientTLSOptions is needed to skip verification errors
+            # Our options class is needed to skip verification errors
+            if TWISTED_TLS_NEW_IMPL:
+                return _ScrapyClientTLSOptions26(
+                    self._get_cert_options()._makeTLSConnection,
+                    hostname.decode("ascii"),
+                    verbose_logging=self.tls_verbose_logging,
+                )
             return _ScrapyClientTLSOptions(
-                hostname.decode("ascii"), self._get_context()
-            )  # type: ignore[no-untyped-call]
+                hostname.decode("ascii"),  # type: ignore[arg-type]
+                self._get_context(),  # type: ignore[arg-type]
+            )
         # Otherwise use the normal Twisted function.
         # Note that this doesn't use self._get_context().
         with _filter_method_warning():
@@ -202,8 +207,25 @@ class _AcceptableProtocolsContextFactory:
     the acceptable protocols on the :class:`.ClientTLSOptions` instance
     returned by it. It's only needed because we support custom factories via
     :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
+
+    It's a no-op on Twisted 26.4.0+, though using it with custom
+    factories on those Twisted versions may be not enough for HTTP/2 support.
     """
 
+    # Something needs to call set_alpn_protos() for ALPN to work.
+    #
+    # Twisted < 26.4.0 does it in OpenSSLCertificateOptions._makeContext()
+    # (requires passing acceptableProtocols from the factory to
+    # OpenSSLCertificateOptions) and in TLSMemoryBIOFactory._createConnection()
+    # based on H2ClientFactory.acceptableProtocols (too late, it seems).
+    #
+    # Newer Twisted does it in OpenSSLCertificateOptions._makeContext() as
+    # well, and in OpenSSLCertificateOptions._makeTLSConnection() based on
+    # H2ClientFactory.acceptableProtocols (which now works).
+    #
+    # When we drop DOWNLOADER_CLIENTCONTEXTFACTORY it looks like we can replace
+    # all of this with _ScrapyClientContextFactory.acceptableProtocols.
+
     def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
         verifyObject(IPolicyForHTTPS, context_factory)
         self._wrapped_context_factory: Any = context_factory
@@ -213,7 +235,12 @@ class _AcceptableProtocolsContextFactory:
         options: ClientTLSOptions = self._wrapped_context_factory.creatorForNetloc(
             hostname, port
         )
-        _setAcceptableProtocols(options._ctx, self._acceptable_protocols)
+        if not TWISTED_TLS_NEW_IMPL:
+            from twisted.internet._sslverify import (  # type: ignore[attr-defined]  # noqa: PLC0415  # pylint: disable=no-name-in-module
+                _setAcceptableProtocols,
+            )
+
+            _setAcceptableProtocols(options._ctx, self._acceptable_protocols)  # type: ignore[attr-defined]
         return options
 
 
@@ -225,6 +252,18 @@ AcceptableProtocolsContextFactory = create_deprecated_class(
 )
 
 
+class _ScrapyCertificateOptions(CertificateOptions):
+    """A wrapper needed to add flags to the SSL context before it's used."""
+
+    def _makeContext(self, skipCiphers: bool = False) -> SSL.Context:
+        if TWISTED_TLS_NEW_IMPL:
+            ctx = super()._makeContext(skipCiphers)
+        else:
+            ctx = super()._makeContext()
+        ctx.set_options(0x4)  # OP_LEGACY_SERVER_CONNECT
+        return ctx
+
+
 def _load_context_factory_from_settings(crawler: Crawler) -> IPolicyForHTTPS:
     """Create an instance of :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`.
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 1127c533f..e6451cdd7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -225,7 +225,8 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         if respm and int(respm.group("status")) == 200:
             # set proper Server Name Indication extension
             sslOptions = self._contextFactory.creatorForNetloc(  # type: ignore[call-arg,misc]
-                self._tunneledHost, self._tunneledPort
+                self._tunneledHost,  # type: ignore[arg-type]
+                self._tunneledPort,
             )
             self._protocol.transport.startTLS(sslOptions, self._protocolFactory)
             self._tunnelReadyDeferred.callback(self._protocol)
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 9ec5b8c48..779c05daa 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -1,15 +1,34 @@
+from __future__ import annotations
+
 import logging
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from OpenSSL import SSL
 from service_identity import VerificationError
 from service_identity.exceptions import CertificateError
-from service_identity.pyopenssl import verify_hostname, verify_ip_address
+from service_identity.hazmat import (
+    DNS_ID,
+    IPAddress_ID,
+    ServiceID,
+    verify_service_identity,
+)
+from service_identity.pyopenssl import (
+    extract_patterns,
+    verify_hostname,
+    verify_ip_address,
+)
 from twisted.internet._sslverify import ClientTLSOptions
 from twisted.internet.ssl import AcceptableCiphers
 
 from scrapy.utils.deprecate import create_deprecated_class
 
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from OpenSSL.crypto import X509
+    from twisted.protocols.tls import TLSMemoryBIOProtocol
+
+
 logger = logging.getLogger(__name__)
 
 
@@ -39,6 +58,8 @@ class _ScrapyClientTLSOptions(ClientTLSOptions):
 
     Instances of this class are returned from
     :class:`._ScrapyClientContextFactory`.
+
+    This class is used on Twisted older than 26.4.0.
     """
 
     def _identityVerifyingInfoCallback(
@@ -64,7 +85,7 @@ class _ScrapyClientTLSOptions(ClientTLSOptions):
                     e,
                 )
         else:
-            super()._identityVerifyingInfoCallback(connection, where, ret)  # type: ignore[no-untyped-call]
+            super()._identityVerifyingInfoCallback(connection, where, ret)  # type: ignore[misc]
 
 
 ScrapyClientTLSOptions = create_deprecated_class(
@@ -75,6 +96,78 @@ ScrapyClientTLSOptions = create_deprecated_class(
 )
 
 
+class _ScrapyClientTLSOptions26(ClientTLSOptions):
+    """
+    SSL Client connection creator ignoring certificate verification errors
+    (for genuinely invalid certificates or bugs in verification code).
+
+    Same as Twisted's private _sslverify.ClientTLSOptions,
+    except that VerificationError, CertificateError and ValueError
+    exceptions are caught, so that the connection is not closed, only
+    logging warnings.
+
+    Instances of this class are returned from
+    :class:`.ScrapyClientContextFactory`.
+
+    This class is used on Twisted 26.4.0 and newer.
+    """
+
+    def __init__(
+        self,
+        createConnection: Callable[[TLSMemoryBIOProtocol], SSL.Connection],
+        hostname: str,
+        verbose_logging: bool = False,
+    ):
+        super().__init__(createConnection, hostname)
+        self.verbose_logging: bool = verbose_logging
+
+    def clientConnectionForTLS(
+        self, tlsProtocol: TLSMemoryBIOProtocol
+    ) -> SSL.Connection:
+        """This method is needed to override the verify callback."""
+        conn = super().clientConnectionForTLS(tlsProtocol)
+        callback = self._verifyCB(
+            self._hostnameIsDnsName, self._hostnameASCII, self.verbose_logging
+        )
+        conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback)
+        return conn
+
+    @staticmethod
+    def _verifyCB(
+        hostIsDNS: bool, hostnameASCII: str, verbose_logging: bool
+    ) -> Callable[[SSL.Connection, X509, int, int, int], bool]:
+        svcid: ServiceID = (
+            DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII)
+        )
+
+        def verifyCallback(
+            conn: SSL.Connection, cert: X509, err: int, depth: int, ok: int
+        ) -> bool:
+            if depth != 0:
+                # We are only verifying the leaf certificate.
+                return bool(ok)
+
+            try:
+                verify_service_identity(extract_patterns(cert), [svcid], [])
+            except (CertificateError, VerificationError) as e:
+                logger.warning(
+                    'Remote certificate is not valid for hostname "%s"; %s',
+                    hostnameASCII,
+                    e,
+                )
+            except ValueError as e:
+                logger.warning(
+                    "Ignoring error while verifying certificate "
+                    'from host "%s" (exception: %r)',
+                    hostnameASCII,
+                    e,
+                )
+
+            return True
+
+        return verifyCallback
+
+
 DEFAULT_CIPHERS: AcceptableCiphers = AcceptableCiphers.fromOpenSSLCipherString(
     "DEFAULT"
 )
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 0966d9f55..44e542470 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -241,7 +241,10 @@ class Shell:
 
             with contextlib.suppress(IgnoreRequest):
                 response = threads.blockingCallFromThread(
-                    reactor, deferred_f_from_coro_f(self._schedule), request, spider
+                    reactor,
+                    deferred_f_from_coro_f(self._schedule),  # type: ignore[arg-type]
+                    request,
+                    spider,
                 )
         else:
             assert self._loop
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index 2ca6f657c..b7086bc98 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -4,6 +4,10 @@ from twisted import version as TWISTED_VERSION
 from twisted.python.versions import Version as TxVersion
 
 TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
+# changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
+TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
+# AsyncioSelectorReactor no longer calls get_event_loop(), https://github.com/twisted/twisted/pull/12508
+TWISTED_LOOP_314_CHANGES = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
 
 PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
 # SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object
diff --git a/tests/mockserver/utils.py b/tests/mockserver/utils.py
index 188eef61d..17d78ecdd 100644
--- a/tests/mockserver/utils.py
+++ b/tests/mockserver/utils.py
@@ -1,13 +1,13 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, cast
 
 from cryptography.hazmat.primitives.serialization import load_pem_private_key
 from cryptography.x509 import load_pem_x509_certificate
 from OpenSSL import SSL
 from OpenSSL.crypto import FILETYPE_PEM, load_certificate, load_privatekey
-from twisted.internet.ssl import CertificateOptions
+from twisted.internet.ssl import CertificateOptions, ContextFactory
 
 from scrapy.utils._deps_compat import PYOPENSSL_WANTS_X509_PKEY
 from scrapy.utils.python import to_bytes
@@ -31,13 +31,14 @@ def ssl_context_factory(
         cert = load_certificate(FILETYPE_PEM, certfile_path.read_bytes())  # type: ignore[assignment]
         key = load_privatekey(FILETYPE_PEM, keyfile_path.read_bytes())  # type: ignore[assignment]
 
+    # https://github.com/twisted/twisted/issues/12638
     factory: CertificateOptions = CertificateOptions(
-        privateKey=key,
-        certificate=cert,
+        privateKey=key,  # type: ignore[arg-type]
+        certificate=cert,  # type: ignore[arg-type]
     )
     if cipher_string:
         ctx = factory.getContext()
         # disabling TLS1.3 because it unconditionally enables some strong ciphers
         ctx.set_options(SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_TLSv1_3)
         ctx.set_cipher_list(to_bytes(cipher_string))
-    return factory
+    return cast("ContextFactory", factory)
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index f9c5abf8a..43edf1774 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -6,6 +6,10 @@ from typing import TYPE_CHECKING, cast
 import OpenSSL.SSL
 import pytest
 from pytest_twisted import async_yield_fixture
+from twisted.internet.protocol import Factory
+from twisted.internet.protocol import Protocol as TxProtocol
+from twisted.internet.ssl import optionsForClientTLS
+from twisted.protocols.tls import TLSMemoryBIOFactory, TLSMemoryBIOProtocol
 from twisted.web import server, static
 from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
 from twisted.web.client import Response as TxResponse
@@ -17,7 +21,10 @@ from scrapy.core.downloader.contextfactory import (
 )
 from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils._deps_compat import PYOPENSSL_SET_CIPHER_LIST_TMP_CONN
+from scrapy.utils._deps_compat import (
+    PYOPENSSL_SET_CIPHER_LIST_TMP_CONN,
+    TWISTED_TLS_NEW_IMPL,
+)
 from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes
@@ -98,50 +105,82 @@ class TestContextFactoryBase:
 
 
 class TestContextFactory(TestContextFactoryBase):
-    @coroutine_test
-    async def test_payload(self, server_url: str) -> None:
-        s = "0123456789" * 10
+    @pytest.fixture
+    def factory(self) -> _ScrapyClientContextFactory:
         crawler = get_crawler()
-        client_context_factory = _load_context_factory_from_settings(crawler)
-        body = await self.get_page(
-            server_url + "payload", client_context_factory, body=s
+        return _load_context_factory_from_settings(crawler)
+
+    @staticmethod
+    def _get_dummy_protocol() -> TLSMemoryBIOProtocol:
+        # from Twisted src/twisted/web/test/test_agent.py::dummyTLSProtocol()
+        factory = TLSMemoryBIOFactory(
+            optionsForClientTLS("example.com"), True, Factory.forProtocol(TxProtocol)
         )
+        return factory.buildProtocol(None)
+
+    @coroutine_test
+    async def test_payload(
+        self, factory: _ScrapyClientContextFactory, server_url: str
+    ) -> None:
+        s = "0123456789" * 10
+        body = await self.get_page(server_url + "payload", factory, body=s)
         assert body == to_bytes(s)
 
-    def test_no_context_sharing(self) -> None:
+    @pytest.mark.skipif(
+        TWISTED_TLS_NEW_IMPL,
+        reason="The context is not stored on this Twisted version",
+    )
+    def test_no_context_sharing(self, factory: _ScrapyClientContextFactory) -> None:
         """Every call to creatorForNetloc() should give a fresh context."""
-        crawler = get_crawler()
-        client_context_factory: _ScrapyClientContextFactory = (
-            _load_context_factory_from_settings(crawler)
-        )
-        creator1 = client_context_factory.creatorForNetloc(b"website1.tld", 443)
+        creator1 = factory.creatorForNetloc(b"website1.tld", 443)
         assert creator1._hostnameBytes == b"website1.tld"
-        creator2 = client_context_factory.creatorForNetloc(b"website2.tld", 443)
+        creator2 = factory.creatorForNetloc(b"website2.tld", 443)
         assert creator2._hostnameBytes == b"website2.tld"
-        assert creator1._ctx is not creator2._ctx
+        assert creator1._ctx is not creator2._ctx  # type: ignore[attr-defined]
+
+    def test_no_context_sharing_with_conn(
+        self, factory: _ScrapyClientContextFactory
+    ) -> None:
+        """Like test_no_context_sharing() but get the context from a connection."""
+        creator1 = factory.creatorForNetloc(b"website1.tld", 443)
+        assert creator1._hostnameBytes == b"website1.tld"
+        conn1 = creator1.clientConnectionForTLS(self._get_dummy_protocol())
+
+        creator2 = factory.creatorForNetloc(b"website2.tld", 443)
+        assert creator2._hostnameBytes == b"website2.tld"
+        conn2 = creator2.clientConnectionForTLS(self._get_dummy_protocol())
+
+        assert conn1.get_context() is not conn2.get_context()
 
     @pytest.mark.skipif(
         PYOPENSSL_SET_CIPHER_LIST_TMP_CONN,
         reason="Fails or doesn't make sense on this pyOpenSSL version",
     )
-    def test_no_immutable_ctx_warning(self) -> None:
+    def test_no_immutable_ctx_warning(
+        self, factory: _ScrapyClientContextFactory
+    ) -> None:
         """There should be no pyOpenSSL context modification warning.
 
         pyOpenSSL < 25.1.0 doesn't produce this warning, and on 25.1.0 it's
         always produced due to
         https://github.com/scrapy/scrapy/issues/6859#issuecomment-4294917851.
         """
-        crawler = get_crawler()
-        client_context_factory: _ScrapyClientContextFactory = (
-            _load_context_factory_from_settings(crawler)
-        )
         with warnings.catch_warnings():
             warnings.filterwarnings(
                 "error",
                 category=DeprecationWarning,
                 message="Attempting to mutate a Context after a Connection was created",
             )
-            client_context_factory.creatorForNetloc(b"website.tld", 443)
+            factory.creatorForNetloc(b"website.tld", 443)
+
+    def test_ctx_flags(self, factory: _ScrapyClientContextFactory) -> None:
+        """The context should have the expected flags set."""
+        creator = factory.creatorForNetloc(b"website.tld", 443)
+        conn = creator.clientConnectionForTLS(self._get_dummy_protocol())
+        ctx = conn.get_context()
+        # fragile but pyOpenSSL doesn't have Context.get_options()
+        options = OpenSSL.SSL._lib.SSL_CTX_get_options(ctx._context)  # type: ignore[attr-defined]
+        assert options & 0x4  # OP_LEGACY_SERVER_CONNECT
 
 
 class TestContextFactoryTLSMethod(TestContextFactoryBase):
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index beae4f277..dc2fdda4c 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -14,6 +14,7 @@ from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from w3lib import __version__ as w3lib_version
 
+from scrapy.utils._deps_compat import TWISTED_LOOP_314_CHANGES
 from tests.utils import async_sleep, get_script_run_env
 from tests.utils.decorators import coroutine_test
 
@@ -171,6 +172,9 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
         assert "async pipeline opened!" in log
 
+    @pytest.mark.xfail(
+        TWISTED_LOOP_314_CHANGES, reason="Breaks with Twisted changes for Python 3.14"
+    )
     @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
diff --git a/tox.ini b/tox.ini
index e0e95c194..72686d3d0 100644
--- a/tox.ini
+++ b/tox.ini
@@ -48,7 +48,7 @@ deps =
     typing-extensions==4.15.0
     Pillow==12.2.0
     Protego==0.6.0
-    Twisted==25.5.0
+    Twisted==26.4.0
     attrs==26.1.0
     boto3-stubs[s3]==1.43.2
     botocore-stubs==1.42.41
@@ -119,7 +119,7 @@ deps =
     parsel==1.5.0
     pyOpenSSL==22.0.0
     queuelib==1.4.2
-    service_identity==18.1.0
+    service_identity==23.1.0
     w3lib==1.17.0
     zope.interface==5.1.0
     {[test-requirements]deps}
@@ -159,6 +159,7 @@ basepython = {[pinned]basepython}
 deps =
     {[pinned]deps}
     Pillow==8.3.2
+    Twisted[http2]==21.7.0
     boto3==1.20.0
     bpython==0.7.1
     brotli==1.2.0; implementation_name != "pypy"
@@ -234,7 +235,7 @@ deps =
     parsel==1.5.0
     pyOpenSSL==24.3.0
     queuelib==1.4.2
-    service_identity==18.1.0
+    service_identity==23.1.0
     w3lib==1.20.0
     zope.interface==5.1.0
 commands =

From 3b34ab88c0712edcc43a53b53bc2517d5d031a2a Mon Sep 17 00:00:00 2001
From: Abhinav W <wabhinow223@gmail.com>
Date: Tue, 12 May 2026 20:04:51 +0530
Subject: [PATCH 20/36] docs: document daily log file rotation (#7501)

* docs: document daily log file rotation

* Skip the new snippet in doc tests

---------

Co-authored-by: Adrian <adrian@zyte.com>
---
 docs/topics/logging.rst | 42 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/docs/topics/logging.rst b/docs/topics/logging.rst
index a398d6c83..bbb5d0458 100644
--- a/docs/topics/logging.rst
+++ b/docs/topics/logging.rst
@@ -194,6 +194,48 @@ If :setting:`LOG_SHORT_NAMES` is set, then the logs will not display the Scrapy
 component that prints the log. It is unset by default, hence logs contain the
 Scrapy component responsible for that log output.
 
+Rotating log files
+------------------
+
+Scrapy's :setting:`LOG_FILE` setting writes logs to a single file. It does not
+rotate log files automatically, but you can use Python's standard
+:mod:`logging.handlers` module when running Scrapy from a script.
+
+For example, to rotate the log file every day:
+
+.. skip: next
+
+.. code-block:: python
+
+    import logging
+    from logging.handlers import TimedRotatingFileHandler
+
+    from scrapy.crawler import CrawlerProcess
+    from scrapy.utils.project import get_project_settings
+
+    from myproject.spiders.myspider import MySpider
+
+    settings = get_project_settings()
+    process = CrawlerProcess(settings, install_root_handler=False)
+
+    handler = TimedRotatingFileHandler(
+        "scrapy.log",
+        when="midnight",
+        backupCount=7,
+        encoding=settings.get("LOG_ENCODING"),
+    )
+    handler.setFormatter(
+        logging.Formatter(settings.get("LOG_FORMAT"), settings.get("LOG_DATEFORMAT"))
+    )
+
+    root_logger = logging.getLogger()
+    root_logger.setLevel(settings.get("LOG_LEVEL"))
+    root_logger.addHandler(handler)
+
+    process.crawl(MySpider)
+    process.start()
+
+
 Command-line options
 --------------------
 

From 2798c03bb006b2041734d71a0046252e1239d631 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 12 May 2026 23:49:37 +0500
Subject: [PATCH 21/36] Add Python 3.14 to CI. (#6604)

* Add Python 3.14 (alpha3) to CI.

* Disable mitmproxy on 3.14 for now.

* 3.14.0-alpha.4.

* 3.14.0-alpha.5

* 3.14.0-beta2.

* 3.14 release.

* Fix test_non_pickable_object.

* Fix handling of file:/path feed URIs.

* Better mocking of streams for TextTestResult.

* Do not use .php in test_file_path() as it's now a known extension.

* Fix the URL in TestFeedExporterSignals.

* Fix typing.

* Bump more envs to 3.14.

* Silence pylint.

* Fix another test for .php handling change.

* Remove test_install_asyncio_reactor.

* More bumps to 3.14.

* Revert docs-tests to use 3.13.

* Debug options for Windows.

* Re-enable xdist.

* Revert Windows PYTEST_ADDOPTS.

* Silence loop policy deprecation warnings.

* Restore a lost pylint suppression.

* Update asyncio_enabled_reactor_same_loop.py to new Twisted.

* Fix RobotFileParser tests for Python 3.14.5.
---
 .github/workflows/checks.yml                  |  6 ++---
 .github/workflows/publish.yml                 |  2 +-
 .github/workflows/tests-macos.yml             |  4 +--
 .github/workflows/tests-ubuntu.yml            | 15 ++++++-----
 .github/workflows/tests-windows.yml           |  9 ++++---
 .readthedocs.yml                              |  2 +-
 docs/topics/downloader-middleware.rst         |  4 +--
 pyproject.toml                                |  3 ++-
 scrapy/utils/_deps_compat.py                  |  2 --
 scrapy/utils/reactor.py                       | 24 ++++++++++-------
 .../asyncio_enabled_reactor_same_loop.py      |  5 ++--
 tests/test_crawler_subprocess.py              |  4 ---
 tests/test_robotstxt_interface.py             | 26 ++++++++++++++++---
 13 files changed, 65 insertions(+), 41 deletions(-)

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index cb05784fa..49ea3277a 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -17,7 +17,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: pylint
         - python-version: "3.10"
@@ -27,13 +27,13 @@ jobs:
           env:
             TOXENV: typing-tests
         # Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: docs
         - python-version: "3.13"
           env:
             TOXENV: docs-tests
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: twinecheck
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index ad327e465..7779bbb6b 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -21,7 +21,7 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.13"
+          python-version: "3.14"
       - run: |
           python -m pip install --upgrade build
           python -m build
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 2a1c62833..0409b3ef2 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -18,11 +18,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
         env:
           - TOXENV: py
         include:
-          - python-version: '3.13'
+          - python-version: '3.14'
             env:
               TOXENV: no-reactor
 
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index a193ca05f..524c79cdb 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -31,10 +31,13 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: py
-        - python-version: "3.13"
+        - python-version: "3.14"
+          env:
+            TOXENV: py
+        - python-version: "3.14"
           env:
             TOXENV: default-reactor
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor
         # pinned due to https://github.com/pypy/pypy/issues/5388
@@ -63,20 +66,20 @@ jobs:
           env:
             TOXENV: botocore-pinned
 
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: extra-deps
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor-extra-deps
         # pinned due to https://github.com/pypy/pypy/issues/5388
         - python-version: pypy3.11-7.3.20
           env:
             TOXENV: pypy3-extra-deps
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: botocore
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: mitmproxy
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 48aa56e15..840c7f68e 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -31,10 +31,13 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: py
-        - python-version: "3.13"
+        - python-version: "3.14"
+          env:
+            TOXENV: py
+        - python-version: "3.14"
           env:
             TOXENV: default-reactor
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: no-reactor
 
@@ -46,7 +49,7 @@ jobs:
           env:
             TOXENV: extra-deps-pinned
 
-        - python-version: "3.13"
+        - python-version: "3.14"
           env:
             TOXENV: extra-deps
 
diff --git a/.readthedocs.yml b/.readthedocs.yml
index 6d1aeb507..a2773dcf2 100644
--- a/.readthedocs.yml
+++ b/.readthedocs.yml
@@ -2,7 +2,7 @@ version: 2
 build:
   os: ubuntu-24.04
   tools:
-    python: "3.13"
+    python: "3.14"
   commands:
     - pip install tox
     - tox -e docs
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 873c32c13..a51e431d2 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -1152,9 +1152,9 @@ Based on :class:`~urllib.robotparser.RobotFileParser`:
 * is compliant with `Martijn Koster's 1996 draft specification
   <https://www.robotstxt.org/norobots-rfc.txt>`_
 
-* lacks support for wildcard matching
+* lacks support for wildcard matching (before Python 3.14.5)
 
-* doesn't use the length based rule
+* doesn't use the length based rule (before Python 3.14.5)
 
 It is faster than Protego and backward-compatible with versions of Scrapy before 1.8.0.
 
diff --git a/pyproject.toml b/pyproject.toml
index 45e18ccc9..6c9ff3100 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
     "Topic :: Internet :: WWW/HTTP",
@@ -448,4 +449,4 @@ split-on-trailing-comma = false
 convention = "pep257"
 
 [tool.sphinx-scrapy]
-python-version = "3.13"  # Keep in sync with .github/workflows/checks.yml.
+python-version = "3.14"  # Keep in sync with .github/workflows/checks.yml.
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index b7086bc98..cb5424476 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -6,8 +6,6 @@ from twisted.python.versions import Version as TxVersion
 TWISTED_FAILURE_HAS_STACK = TWISTED_VERSION < TxVersion("twisted", 24, 10, 0)
 # changes to private _sslverify code, https://github.com/twisted/twisted/pull/12506
 TWISTED_TLS_NEW_IMPL = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
-# AsyncioSelectorReactor no longer calls get_event_loop(), https://github.com/twisted/twisted/pull/12508
-TWISTED_LOOP_314_CHANGES = TWISTED_VERSION >= TxVersion("twisted", 26, 4, 0)
 
 PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
 # SSL.Context.use_certificate() wants an X509 object, SSL.Context.use_privatekey() wants a PKey object
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index ddebd8ba2..37db6c25f 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import asyncio
 import sys
+import warnings
 from contextlib import suppress
 from typing import TYPE_CHECKING, Any, Generic, ParamSpec, TypeVar
 from warnings import catch_warnings, filterwarnings
@@ -93,16 +94,19 @@ _asyncio_reactor_path = "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
 
 
 def set_asyncio_event_loop_policy() -> None:
-    """The policy functions from asyncio often behave unexpectedly,
-    so we restrict their use to the absolutely essential case.
-    This should only be used to install the reactor.
-    """
-    policy = asyncio.get_event_loop_policy()
-    if sys.platform == "win32" and not isinstance(
-        policy, asyncio.WindowsSelectorEventLoopPolicy
-    ):
-        policy = asyncio.WindowsSelectorEventLoopPolicy()
-        asyncio.set_event_loop_policy(policy)
+    """Needed due to https://github.com/twisted/twisted/issues/12527."""
+    if sys.platform != "win32":
+        return
+    with warnings.catch_warnings():
+        warnings.filterwarnings(
+            "ignore",
+            message=r"'asyncio\.(get_event_loop_policy|WindowsSelectorEventLoopPolicy)' is deprecated",
+            category=DeprecationWarning,
+        )
+        policy = asyncio.get_event_loop_policy()
+        if not isinstance(policy, asyncio.WindowsSelectorEventLoopPolicy):
+            policy = asyncio.WindowsSelectorEventLoopPolicy()  # pylint: disable=deprecated-class
+            asyncio.set_event_loop_policy(policy)
 
 
 def install_reactor(reactor_path: str, event_loop_path: str | None = None) -> None:
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
index 578e0029d..a2e63a0d0 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@@ -9,8 +9,9 @@ from scrapy.crawler import CrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncio.set_event_loop(Loop())
-asyncioreactor.install()
+loop = Loop()
+asyncio.set_event_loop(loop)
+asyncioreactor.install(loop)
 
 
 class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index dc2fdda4c..beae4f277 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -14,7 +14,6 @@ from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from w3lib import __version__ as w3lib_version
 
-from scrapy.utils._deps_compat import TWISTED_LOOP_314_CHANGES
 from tests.utils import async_sleep, get_script_run_env
 from tests.utils.decorators import coroutine_test
 
@@ -172,9 +171,6 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
         assert "async pipeline opened!" in log
 
-    @pytest.mark.xfail(
-        TWISTED_LOOP_314_CHANGES, reason="Breaks with Twisted changes for Python 3.14"
-    )
     @pytest.mark.requires_uvloop
     def test_asyncio_enabled_reactor_same_loop(self):
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index 6c0dd9d2c..29b23496a 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -1,3 +1,5 @@
+import sys
+
 import pytest
 
 from scrapy.robotstxt import (
@@ -137,16 +139,32 @@ class TestDecodeRobotsTxt:
 
 
 class TestPythonRobotParser(BaseRobotParserTest):
+    # https://github.com/python/cpython/pull/149374 improves it
+    IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5)
+
     def setup_method(self):
         super()._setUp(PythonRobotParser)
 
+    @pytest.mark.skipif(
+        not IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support length based directives precedence.",
+    )
     def test_length_based_precedence(self):
-        pytest.skip(
-            "RobotFileParser does not support length based directives precedence."
-        )
+        super().test_length_based_precedence()
 
+    @pytest.mark.skipif(
+        IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support order based directives precedence.",
+    )
+    def test_order_based_precedence(self):
+        super().test_order_based_precedence()
+
+    @pytest.mark.skipif(
+        not IMPROVED_ROBOTFILEPARSER,
+        reason="RobotFileParser from this Python version does not support wildcards.",
+    )
     def test_allowed_wildcards(self):
-        pytest.skip("RobotFileParser does not support wildcards.")
+        super().test_allowed_wildcards()
 
 
 @pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")

From 2d007bc4508423bbd57ee9017da22c16bbd576e1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 13 May 2026 01:03:14 +0500
Subject: [PATCH 22/36] Make more of the internal handler helpers private.
 (#7510)

---
 docs/news.rst                             | 22 ++++++++++++---
 scrapy/core/downloader/handlers/http11.py | 33 +++++++++++------------
 scrapy/core/downloader/handlers/http2.py  |  8 +++---
 3 files changed, 36 insertions(+), 27 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 7f3952eaf..5eef10d84 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,10 +9,24 @@ Scrapy VERSION (unreleased)
 Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
--   ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent`` has been made
-    private as it's an implementation detail of
-    :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`.
-    (:issue:`7496`)
+-   The following classes and functions, intended for internal use by
+    :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+    and :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler`, have
+    been made private:
+
+    - ``scrapy.core.downloader.handlers.http11.ScrapyAgent``
+
+    - ``scrapy.core.downloader.handlers.http11.ScrapyProxyAgent``
+
+    - ``scrapy.core.downloader.handlers.http11.TunnelingAgent``
+
+    - ``scrapy.core.downloader.handlers.http11.TunnelingTCP4ClientEndpoint``
+
+    - ``scrapy.core.downloader.handlers.http11.tunnel_request_data()``
+
+    - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent``
+
+    (:issue:`7496`, #TBD)
 
 .. _release-2.15.2:
 
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index e6451cdd7..fbe24e52e 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -111,7 +111,7 @@ class HTTP11DownloadHandler(BaseHttpDownloadHandler):
                 "download_warnsize", "DOWNLOAD_WARNSIZE"
             )
 
-        agent = ScrapyAgent(
+        agent = _ScrapyAgent(
             contextFactory=self._contextFactory,
             bindAddress=self._bind_address,
             pool=self._pool,
@@ -161,7 +161,7 @@ class TunnelError(Exception):
     """An HTTP CONNECT tunnel could not be established by the proxy."""
 
 
-class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
+class _TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     """An endpoint that tunnels through proxies to allow HTTPS downloads. To
     accomplish that, this endpoint sends an HTTP CONNECT to the proxy.
     The HTTP CONNECT is always sent when using this endpoint, I think this could
@@ -197,7 +197,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
     def requestTunnel(self, protocol: Protocol) -> Protocol:
         """Asks the proxy to open a tunnel."""
         assert protocol.transport
-        tunnelReq = tunnel_request_data(
+        tunnelReq = _tunnel_request_data(
             self._tunneledHost, self._tunneledPort, self._proxyAuthHeader
         )
         protocol.transport.write(tunnelReq)
@@ -221,7 +221,7 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         if b"\r\n\r\n" not in self._connectBuffer:
             return
         self._protocol.dataReceived = self._protocolDataReceived  # type: ignore[method-assign]
-        respm = TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
+        respm = _TunnelingTCP4ClientEndpoint._responseMatcher.match(self._connectBuffer)
         if respm and int(respm.group("status")) == 200:
             # set proper Server Name Indication extension
             sslOptions = self._contextFactory.creatorForNetloc(  # type: ignore[call-arg,misc]
@@ -258,18 +258,18 @@ class TunnelingTCP4ClientEndpoint(TCP4ClientEndpoint):
         return self._tunnelReadyDeferred
 
 
-def tunnel_request_data(
+def _tunnel_request_data(
     host: str, port: int, proxy_auth_header: bytes | None = None
 ) -> bytes:
     r"""
     Return binary content of a CONNECT request.
 
     >>> from scrapy.utils.python import to_unicode as s
-    >>> s(tunnel_request_data("example.com", 8080))
+    >>> s(_tunnel_request_data("example.com", 8080))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\n\r\n'
-    >>> s(tunnel_request_data("example.com", 8080, b"123"))
+    >>> s(_tunnel_request_data("example.com", 8080, b"123"))
     'CONNECT example.com:8080 HTTP/1.1\r\nHost: example.com:8080\r\nProxy-Authorization: 123\r\n\r\n'
-    >>> s(tunnel_request_data(b"example.com", "8090"))
+    >>> s(_tunnel_request_data(b"example.com", "8090"))
     'CONNECT example.com:8090 HTTP/1.1\r\nHost: example.com:8090\r\n\r\n'
     """
     host_value = to_bytes(host, encoding="ascii") + b":" + to_bytes(str(port))
@@ -281,7 +281,7 @@ def tunnel_request_data(
     return tunnel_req
 
 
-class TunnelingAgent(Agent):
+class _TunnelingAgent(Agent):
     """An agent that uses a L{TunnelingTCP4ClientEndpoint} to make HTTPS
     downloads. It may look strange that we have chosen to subclass Agent and not
     ProxyAgent but consider that after the tunnel is opened the proxy is
@@ -303,8 +303,8 @@ class TunnelingAgent(Agent):
         self._proxyConf: tuple[str, int, bytes | None] = proxyConf
         self._contextFactory: IPolicyForHTTPS = contextFactory
 
-    def _getEndpoint(self, uri: URI) -> TunnelingTCP4ClientEndpoint:
-        return TunnelingTCP4ClientEndpoint(
+    def _getEndpoint(self, uri: URI) -> _TunnelingTCP4ClientEndpoint:
+        return _TunnelingTCP4ClientEndpoint(
             reactor=self._reactor,
             host=uri.host,
             port=uri.port,
@@ -381,10 +381,7 @@ class _ScrapyProxyAgent(Agent):
         )
 
 
-class ScrapyAgent:
-    _Agent = Agent
-    _TunnelingAgent = TunnelingAgent
-
+class _ScrapyAgent:
     def __init__(
         self,
         *,
@@ -430,7 +427,7 @@ class ScrapyAgent:
                 assert proxy_host is not None
                 proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                 proxyConf = (proxy_host, proxy_port, proxyAuth)
-                return self._TunnelingAgent(
+                return _TunnelingAgent(
                     reactor=reactor,
                     proxyConf=proxyConf,
                     contextFactory=self._contextFactory,
@@ -447,7 +444,7 @@ class ScrapyAgent:
                 pool=self._pool,
             )
 
-        return self._Agent(  # type: ignore[no-untyped-call]
+        return Agent(
             reactor=reactor,
             contextFactory=self._contextFactory,
             connectTimeout=timeout,
@@ -465,7 +462,7 @@ class ScrapyAgent:
         url = urldefrag(request.url)[0]
         method = to_bytes(request.method)
         headers = TxHeaders(request.headers)
-        if isinstance(agent, self._TunnelingAgent):
+        if isinstance(agent, _TunnelingAgent):
             headers.removeHeader(b"Proxy-Authorization")
         bodyproducer = _RequestBodyProducer(request.body) if request.body else None
         start_time = monotonic()
diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.py
index b28bcfd89..f60c58d1b 100644
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@@ -49,7 +49,7 @@ class H2DownloadHandler(BaseDownloadHandler):
             raise UnsupportedURLSchemeError(
                 f"{type(self).__name__} doesn't support plain HTTP."
             )
-        agent = ScrapyH2Agent(
+        agent = _ScrapyH2Agent(
             context_factory=self._context_factory,
             pool=self._pool,
             bind_address=self._bind_address,
@@ -65,9 +65,7 @@ class H2DownloadHandler(BaseDownloadHandler):
         self._pool.close_connections()
 
 
-class ScrapyH2Agent:
-    _Agent = H2Agent
-
+class _ScrapyH2Agent:
     def __init__(
         self,
         context_factory: IPolicyForHTTPS,
@@ -89,7 +87,7 @@ class ScrapyH2Agent:
             raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
         bind_address = request.meta.get("bindaddress") or self._bind_address
         bind_address = normalize_bind_address(bind_address)
-        return self._Agent(
+        return H2Agent(
             reactor=reactor,
             context_factory=self._context_factory,
             connect_timeout=timeout,

From 4cfe7a08cdbe502cea477f67958bdc51c60a13c4 Mon Sep 17 00:00:00 2001
From: Adrian <adrian@zyte.com>
Date: Thu, 14 May 2026 13:07:13 +0200
Subject: [PATCH 23/36] Add CITATION.cff (#7519)

---
 CITATION.cff | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 CITATION.cff

diff --git a/CITATION.cff b/CITATION.cff
new file mode 100644
index 000000000..24a426d36
--- /dev/null
+++ b/CITATION.cff
@@ -0,0 +1,6 @@
+cff-version: 1.2.0
+message: If you use Scrapy in published research, please cite it as below.
+title: Scrapy
+authors:
+  - name: Scrapy contributors
+url: https://scrapy.org

From ae4a8e39e10818e0c13b9e9c32173cee4a0fa00d Mon Sep 17 00:00:00 2001
From: Olivia Choi <choio@oregonstate.edu>
Date: Thu, 14 May 2026 04:27:43 -0700
Subject: [PATCH 24/36] Document TLS method setting in avoiding bans guide
 (#7518)

* Document TLS method setting in avoiding bans guide

Mention DOWNLOADER_CLIENT_TLS_METHOD in the avoiding getting banned section.

* Update practices.rst

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/practices.rst | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docs/topics/practices.rst b/docs/topics/practices.rst
index 4f036db29..2a8f5b4c1 100644
--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@@ -409,6 +409,10 @@ Here are some tips to keep in mind when dealing with these kinds of sites:
 * use a pool of rotating IPs. For example, the free `Tor project`_ or paid
   services like `ProxyMesh`_. An open source alternative is `scrapoxy`_, a
   super proxy that you can attach your own proxies to.
+* for HTTPS websites, if blocking appears related to TLS behavior, consider
+  adjusting the :setting:`DOWNLOADER_CLIENT_TLS_METHOD` setting, since some
+  websites may respond differently depending on the TLS method used by the
+  client.
 * use a ban avoidance service, such as `Zyte API`_, which provides a `Scrapy
   plugin <https://github.com/scrapy-plugins/scrapy-zyte-api>`__ and additional
   features, like `AI web scraping <https://www.zyte.com/ai-web-scraping/>`__

From 85c616c5c71d4de01746f9c190274fe3d2ac9ed0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 14 May 2026 22:56:05 +0500
Subject: [PATCH 25/36] Fix certificate issuer verification on new Twisted.
 (#7520)

---
 pyproject.toml                                |  1 +
 scrapy/core/downloader/contextfactory.py      |  1 -
 scrapy/core/downloader/tls.py                 | 19 ++---
 tests/test_downloader_handler_httpx.py        |  6 ++
 .../test_downloader_handler_twisted_http11.py |  9 +++
 .../test_downloader_handler_twisted_http2.py  |  9 +++
 tests/test_downloader_handlers_http_base.py   | 69 ++++++++++++++++++-
 7 files changed, 96 insertions(+), 18 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 6c9ff3100..76b8bbead 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -267,6 +267,7 @@ markers = [
     "requires_botocore: marks tests that need botocore (but not boto3)",
     "requires_boto3: marks tests that need botocore and boto3",
     "requires_mitmproxy: marks tests that need mitmproxy",
+    "requires_internet: marks tests that need real Internet access",
 ]
 filterwarnings = [
     "ignore::DeprecationWarning:twisted.web.static",
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index 7f7d34c21..d2626fd9d 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -131,7 +131,6 @@ class _ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
                 return _ScrapyClientTLSOptions26(
                     self._get_cert_options()._makeTLSConnection,
                     hostname.decode("ascii"),
-                    verbose_logging=self.tls_verbose_logging,
                 )
             return _ScrapyClientTLSOptions(
                 hostname.decode("ascii"),  # type: ignore[arg-type]
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 779c05daa..cb06f81df 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -107,34 +107,23 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
     logging warnings.
 
     Instances of this class are returned from
-    :class:`.ScrapyClientContextFactory`.
+    :class:`._ScrapyClientContextFactory`.
 
     This class is used on Twisted 26.4.0 and newer.
     """
 
-    def __init__(
-        self,
-        createConnection: Callable[[TLSMemoryBIOProtocol], SSL.Connection],
-        hostname: str,
-        verbose_logging: bool = False,
-    ):
-        super().__init__(createConnection, hostname)
-        self.verbose_logging: bool = verbose_logging
-
     def clientConnectionForTLS(
         self, tlsProtocol: TLSMemoryBIOProtocol
     ) -> SSL.Connection:
         """This method is needed to override the verify callback."""
         conn = super().clientConnectionForTLS(tlsProtocol)
-        callback = self._verifyCB(
-            self._hostnameIsDnsName, self._hostnameASCII, self.verbose_logging
-        )
+        callback = self._verifyCB(self._hostnameIsDnsName, self._hostnameASCII)
         conn.set_verify(SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, callback)
         return conn
 
     @staticmethod
     def _verifyCB(
-        hostIsDNS: bool, hostnameASCII: str, verbose_logging: bool
+        hostIsDNS: bool, hostnameASCII: str
     ) -> Callable[[SSL.Connection, X509, int, int, int], bool]:
         svcid: ServiceID = (
             DNS_ID(hostnameASCII) if hostIsDNS else IPAddress_ID(hostnameASCII)
@@ -145,7 +134,7 @@ class _ScrapyClientTLSOptions26(ClientTLSOptions):
         ) -> bool:
             if depth != 0:
                 # We are only verifying the leaf certificate.
-                return bool(ok)
+                return True
 
             try:
                 verify_service_identity(extract_patterns(cert), [svcid], [])
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index d4dfe8396..bc1acf9f9 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -18,6 +18,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
     TestMitmProxyBase,
+    TestRealWebsiteBase,
     TestSimpleHttpsBase,
 )
 from tests.utils.decorators import coroutine_test
@@ -145,3 +146,8 @@ class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
 @pytest.mark.requires_mitmproxy
 class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
     pass
+
+
+@pytest.mark.requires_internet
+class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
+    pass
diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py
index 89cf58c96..fb3305945 100644
--- a/tests/test_downloader_handler_twisted_http11.py
+++ b/tests/test_downloader_handler_twisted_http11.py
@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import sys
 from typing import TYPE_CHECKING, Any
 
 import pytest
@@ -20,6 +21,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
     TestMitmProxyBase,
+    TestRealWebsiteBase,
     TestSimpleHttpsBase,
 )
 
@@ -103,3 +105,10 @@ class TestHttpsProxy(HTTP11DownloadHandlerMixin, TestHttpProxyBase):
 class TestMitmProxy(HTTP11DownloadHandlerMixin, TestMitmProxyBase):
     # not implemented
     handler_supports_tls_in_tls = False
+
+
+@pytest.mark.requires_internet
+class TestRealWebsite(HTTP11DownloadHandlerMixin, TestRealWebsiteBase):
+    @property
+    def platform_cert_store_works(self) -> bool:
+        return sys.platform != "win32"
diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py
index 60b276166..24cb9b1aa 100644
--- a/tests/test_downloader_handler_twisted_http2.py
+++ b/tests/test_downloader_handler_twisted_http2.py
@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import sys
 from typing import TYPE_CHECKING, Any
 
 import pytest
@@ -21,6 +22,7 @@ from tests.test_downloader_handlers_http_base import (
     TestHttpsWrongHostnameBase,
     TestHttpWithCrawlerBase,
     TestMitmProxyBase,
+    TestRealWebsiteBase,
 )
 from tests.utils.decorators import coroutine_test
 
@@ -196,3 +198,10 @@ class TestHttp2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase):
 @pytest.mark.requires_mitmproxy
 class TestMitmProxy(H2DownloadHandlerMixin, TestMitmProxyBase):
     pass
+
+
+@pytest.mark.requires_internet
+class TestRealWebsite(H2DownloadHandlerMixin, TestRealWebsiteBase):
+    @property
+    def platform_cert_store_works(self) -> bool:
+        return sys.platform != "win32"
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 138ddd944..a8d002d30 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -6,7 +6,6 @@ import gzip
 import json
 import logging
 import os
-import platform
 import re
 import sys
 from abc import ABC, abstractmethod
@@ -524,7 +523,7 @@ class TestHttpBase(ABC):
             await download_handler.download_request(request)
         assert "download_latency" in request.meta
         latency = request.meta["download_latency"]
-        if sys.version_info < (3, 13) and platform.system() == "Windows":
+        if sys.version_info < (3, 13) and sys.platform == "win32":
             # time.monotonic() resolution is too low here:
             # https://docs.python.org/3/whatsnew/3.13.html#time
             assert latency >= 0
@@ -1266,3 +1265,69 @@ class TestMitmProxyBase(ABC):
     @staticmethod
     def _assert_got_auth_exception(log: str) -> None:
         assert "Proxy Authentication Required" in log or "407" in log
+
+
+class TestRealWebsiteBase(ABC):
+    @property
+    @abstractmethod
+    def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def settings_dict(self) -> dict[str, Any] | None:
+        raise NotImplementedError
+
+    @property
+    @abstractmethod
+    def platform_cert_store_works(self) -> bool:
+        """Whether valid certificates can be verified.
+
+        Twisted on Windows cannot do that out of the box, see e.g.
+        https://github.com/twisted/twisted/issues/6371.
+        """
+        return True
+
+    @asynccontextmanager
+    async def get_dh(
+        self, settings_dict: dict[str, Any] | None = None
+    ) -> AsyncGenerator[DownloadHandlerProtocol]:
+        crawler = get_crawler(DefaultSpider, settings_dict)
+        crawler.spider = crawler._create_spider()
+        dh = build_from_crawler(self.download_handler_cls, crawler)
+        try:
+            yield dh
+        finally:
+            await dh.close()
+
+    @coroutine_test
+    async def test_download(self) -> None:
+        request = Request("https://books.toscrape.com/")
+        async with self.get_dh() as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.status == 200
+        assert "All products | Books to Scrape - Sandbox" in response.text
+
+    @coroutine_test
+    async def test_download_with_spider(self) -> None:
+        crawler = get_crawler(SingleRequestSpider, self.settings_dict)
+        await maybe_deferred_to_future(
+            crawler.crawl(seed=Request("https://books.toscrape.com/"))
+        )
+        assert isinstance(crawler.spider, SingleRequestSpider)
+        failure = crawler.spider.meta.get("failure")
+        assert failure is None
+        reason = crawler.spider.meta["close_reason"]
+        assert reason == "finished"
+
+    @coroutine_test
+    async def test_verify_certs(self) -> None:
+        if not self.platform_cert_store_works:
+            pytest.skip("Cannot verify certificates")
+        request = Request("https://books.toscrape.com/")
+        async with self.get_dh(
+            {"DOWNLOAD_VERIFY_CERTIFICATES": True}
+        ) as download_handler:
+            response = await download_handler.download_request(request)
+        assert response.status == 200
+        assert "All products | Books to Scrape - Sandbox" in response.text

From a8a8f20d9c49ae7f611cbdc90676b64f6349dba2 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 15 May 2026 16:36:05 +0500
Subject: [PATCH 26/36] Remove support for sync process_spider_output().
 (#7504)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Remove support for sync process_spider_output().

* Slight check fix.

* Don't expect exceptions from calling process_spider_output().

* Remove dead code.

* Fix test_deprecated_mw_spider_arg() to run all methods.

* Fix typing.

* Typos.

* Remove references to the removed section, move the universal section to the spider middleware docs and write it from a different perspective

* Fix indentation issue

* Minor rewordings

* older versions → lower versions

* Update news.rst

---------

Co-authored-by: Adrian Chaves <adrian@zyte.com>
---
 docs/news.rst                               |  21 +-
 docs/topics/coroutines.rst                  | 146 +----------
 docs/topics/spider-middleware.rst           |  63 +++--
 scrapy/core/spidermw.py                     | 253 ++++----------------
 scrapy/middleware.py                        |   5 +-
 scrapy/utils/python.py                      |  13 +-
 tests/test_request_cb_kwargs.py             |   4 +-
 tests/test_spidermiddleware.py              | 244 ++++---------------
 tests/test_spidermiddleware_output_chain.py | 130 +---------
 tests/test_utils_python.py                  |  11 -
 10 files changed, 170 insertions(+), 720 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 5eef10d84..3173dfbc1 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -1380,8 +1380,8 @@ Highlights:
 
 -   Added the :reqmeta:`allow_offsite` request meta key
 
--   :ref:`Spider middlewares that don't support asynchronous spider output
-    <sync-async-spider-middleware>` are deprecated
+-   Spider middlewares that don't support asynchronous spider output are
+    deprecated
 
 -   Added a base class for :ref:`universal spider middlewares
     <universal-spider-middleware>`
@@ -1519,13 +1519,11 @@ Deprecations
     ``start_queue_cls`` parameter.
     (:issue:`6752`)
 
--   :ref:`Spider middlewares that don't support asynchronous spider output
-    <sync-async-spider-middleware>` are deprecated. The async iterable
-    downgrading feature, needed for using such middlewares with asynchronous
-    callbacks and with other spider middlewares that produce asynchronous
-    iterables, is also deprecated. Please update all such middlewares to
-    support asynchronous spider output.
-    (:issue:`6664`)
+-   Spider middlewares that don't support asynchronous spider output are
+    deprecated. The async iterable downgrading feature, needed for using such
+    middlewares with asynchronous callbacks and with other spider middlewares
+    that produce asynchronous iterables, is also deprecated. Please update all
+    such middlewares to support asynchronous spider output. (:issue:`6664`)
 
 -   Functions that were imported from :mod:`w3lib.url` and re-exported in
     :mod:`scrapy.utils.url` are now deprecated, you should import them from
@@ -1784,9 +1782,8 @@ Documentation
 -   Documented the setting values set in the default project template.
     (:issue:`6762`, :issue:`6775`)
 
--   Improved the :ref:`docs <sync-async-spider-middleware>` about asynchronous
-    iterable support in spider middlewares.
-    (:issue:`6688`)
+-   Improved the docs about asynchronous iterable support in spider
+    middlewares. (:issue:`6688`)
 
 -   Improved the :ref:`docs <coroutine-deferred-apis>` about using
     :class:`~twisted.internet.defer.Deferred`-based APIs in coroutine-based
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index ea3a60b4c..ba68f0dbc 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -23,9 +23,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.Request` callbacks.
 
-    If you are using any custom or third-party :ref:`spider middleware
-    <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
-
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
 
@@ -39,13 +36,9 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   The
     :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-    method of :ref:`spider middlewares <topics-spider-middleware>`.
-
-    If defined as a coroutine, it must be an :term:`asynchronous generator`.
-    The input ``result`` parameter is an :term:`asynchronous iterable`.
-
-    See also :ref:`sync-async-spider-middleware` and
-    :ref:`universal-spider-middleware`.
+    method of :ref:`spider middlewares <topics-spider-middleware>`, which
+    *must* be defined as an :term:`asynchronous generator` except in
+    :ref:`universal spider middlewares <universal-spider-middleware>`.
 
 -   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` method
     of :ref:`spider middlewares <custom-spider-middleware>`, which *must* be
@@ -277,136 +270,3 @@ You can also send multiple requests in parallel:
                 "price": responses[0][1].css(".price::text").get(),
                 "price2": responses[1][1].css(".color::text").get(),
             }
-
-
-.. _sync-async-spider-middleware:
-
-Mixing synchronous and asynchronous spider middlewares
-======================================================
-
-The output of a :class:`~scrapy.Request` callback is passed as the ``result``
-parameter to the
-:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
-of the first :ref:`spider middleware <topics-spider-middleware>` from the
-:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
-Then the output of that ``process_spider_output`` method is passed to the
-``process_spider_output`` method of the next spider middleware, and so on for
-every active spider middleware.
-
-Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
-in this chain of calls.
-
-However, if any of the ``process_spider_output`` methods is defined as a
-synchronous method, and the previous ``Request`` callback or
-``process_spider_output`` method is a coroutine, there are some drawbacks to
-the asynchronous-to-synchronous conversion that Scrapy does so that the
-synchronous ``process_spider_output`` method gets a synchronous iterable as its
-``result`` parameter:
-
--   The whole output of the previous ``Request`` callback or
-    ``process_spider_output`` method is awaited at this point.
-
--   If an exception raises while awaiting the output of the previous
-    ``Request`` callback or ``process_spider_output`` method, none of that
-    output will be processed.
-
-    This contrasts with the regular behavior, where all items yielded before
-    an exception raises are processed.
-
-Asynchronous-to-synchronous conversions are supported for backward
-compatibility, but they are deprecated and will stop working in a future
-version of Scrapy.
-
-To avoid asynchronous-to-synchronous conversions, when defining ``Request``
-callbacks as coroutine methods or when using spider middlewares whose
-``process_spider_output`` method is an :term:`asynchronous generator`, all
-active spider middlewares must either have their ``process_spider_output``
-method defined as an asynchronous generator or :ref:`define a
-process_spider_output_async method <universal-spider-middleware>`.
-
-.. _sync-async-spider-middleware-users:
-
-For middleware users
---------------------
-
-If you have asynchronous callbacks or use asynchronous-only spider middlewares
-you should make sure the asynchronous-to-synchronous conversions
-:ref:`described above <sync-async-spider-middleware>` don't happen. To do this,
-make sure all spider middlewares you use support asynchronous spider output.
-Even if you don't have asynchronous callbacks and don't use asynchronous-only
-spider middlewares in your project, it's still a good idea to make sure all
-middlewares you use support asynchronous spider output, so that it will be easy
-to start using asynchronous callbacks in the future. Because of this, Scrapy
-logs a warning when it detects a synchronous-only spider middleware.
-
-If you want to update middlewares you wrote, see the :ref:`following section
-<sync-async-spider-middleware-authors>`. If you have 3rd-party middlewares that
-aren't yet updated by their authors, you can :ref:`subclass <tut-inheritance>`
-them to make them :ref:`universal <universal-spider-middleware>` and use the
-subclasses in your projects.
-
-.. _sync-async-spider-middleware-authors:
-
-For middleware authors
-----------------------
-
-If you have a spider middleware that defines a synchronous
-``process_spider_output`` method, you should update it to support asynchronous
-spider output for :ref:`better compatibility <sync-async-spider-middleware>`,
-even if you don't yet use it with asynchronous callbacks, especially if you
-publish this middleware for other people to use. You have two options for this:
-
-1. Make the middleware asynchronous, by making the ``process_spider_output``
-   method an :term:`asynchronous generator`.
-2. Make the middleware universal, as described in the :ref:`next section
-   <universal-spider-middleware>`.
-
-If your middleware won't be used in projects with synchronous-only middlewares,
-e.g. because it's an internal middleware and you know that all other
-middlewares in your projects are already updated, it's safe to choose the first
-option. Otherwise, it's better to choose the second option.
-
-.. _universal-spider-middleware:
-
-Universal spider middlewares
-----------------------------
-
-To allow writing a spider middleware that supports asynchronous execution of
-its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
-:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
-while maintaining support for older Scrapy versions, you may define
-``process_spider_output`` as a synchronous method and define an
-:term:`asynchronous generator` version of that method with an alternative name:
-``process_spider_output_async``.
-
-For example:
-
-.. code-block:: python
-
-    class UniversalSpiderMiddleware:
-        def process_spider_output(self, response, result):
-            for r in result:
-                # ... do something with r
-                yield r
-
-        async def process_spider_output_async(self, response, result):
-            async for r in result:
-                # ... do something with r
-                yield r
-
-.. note:: This is an interim measure to allow, for a time, to write code that
-          works in Scrapy 2.7 and later without requiring
-          asynchronous-to-synchronous conversions, and works in earlier Scrapy
-          versions as well.
-
-          In some future version of Scrapy, however, this feature will be
-          deprecated and, eventually, in a later version of Scrapy, this
-          feature will be removed, and all spider middlewares will be expected
-          to define their ``process_spider_output`` method as an asynchronous
-          generator.
-
-Since 2.13.0, Scrapy provides a base class,
-:class:`~scrapy.spidermiddlewares.base.BaseSpiderMiddleware`, which implements
-the ``process_spider_output()`` and ``process_spider_output_async()`` methods,
-so instead of duplicating the processing code you can override the
-``get_processed_request()`` and/or the ``get_processed_item()`` method.
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 8b000697a..820d5910c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -117,36 +117,28 @@ one or more of these methods:
         :type response: :class:`~scrapy.http.Response` object
 
     .. method:: process_spider_output(response, result)
+        :async:
 
-        This method is called with the results returned from the Spider, after
-        it has processed the response.
+        This method is an :term:`asynchronous generator` called with the
+        results from the spider after the spider has processed the response.
 
-        :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.Request` objects and :ref:`item objects
-        <topics-items>`.
-
-        Consider defining this method as an :term:`asynchronous generator`,
-        which will be a requirement in a future version of Scrapy. However, if
-        you plan on sharing your spider middleware with other people, consider
-        either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
-        as a minimum requirement of your spider middleware, or :ref:`making
-        your spider middleware universal <universal-spider-middleware>` so that
-        it works with Scrapy versions earlier than Scrapy 2.7.
+        .. seealso:: :ref:`universal-spider-middleware`.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
-        :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.Request` objects and
-          :ref:`item objects <topics-items>`
+        :param result: the results from the spider
+        :type result: an :term:`asynchronous iterable` of
+          :class:`~scrapy.Request` objects and :ref:`item objects
+          <topics-items>`
 
     .. method:: process_spider_output_async(response, result)
         :async:
 
-        If defined, this method must be an :term:`asynchronous generator`,
-        which will be called instead of :meth:`process_spider_output` if
-        ``result`` is an :term:`asynchronous iterable`.
+        Alternative name for :meth:`process_spider_output` used when
+        implementing a :ref:`universal spider middleware
+        <universal-spider-middleware>`.
 
     .. method:: process_spider_exception(response, exception)
 
@@ -174,13 +166,40 @@ one or more of these methods:
         :type exception: :exc:`Exception` object
 
 
+.. _universal-spider-middleware:
+
+Universal spider middlewares
+----------------------------
+
+In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous*
+generator.
+
+To support those versions and higher Scrapy versions in the same middleware,
+rename your asynchronous :method:`~SpiderMiddleware.process_spider_output()`
+method to :method:`~SpiderMiddleware.process_spider_output_async()`, and define
+a synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower
+versions.
+
+For example:
+
+.. code-block:: python
+
+    class UniversalSpiderMiddleware:
+        async def process_spider_output_async(self, response, result):
+            async for r in result:
+                # ... do something with r
+                yield r
+
+        def process_spider_output(self, response, result):
+            for r in result:
+                # ... do something with r
+                yield r
+
 Base class for custom spider middlewares
 ----------------------------------------
 
 Scrapy provides a base class for custom spider middlewares. It's not required
-to use it but it can help with simplifying middleware implementations and
-reducing the amount of boilerplate code in :ref:`universal middlewares
-<universal-spider-middleware>`.
+to use it but it can help with simplifying middleware implementations.
 
 .. module:: scrapy.spidermiddlewares.base
 
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index d48a65967..bcf38bb75 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -9,29 +9,28 @@ from __future__ import annotations
 import logging
 from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
-from inspect import isasyncgenfunction, iscoroutine
+from inspect import isasyncgenfunction
 from itertools import islice
 from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
 from warnings import warn
 
-from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy import Request, Spider
 from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import (
     _defer_sleep_async,
     deferred_from_coro,
     maybe_deferred_to_future,
 )
-from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name
+from scrapy.utils.python import MutableAsyncChain, global_object_name
 
 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from twisted.internet.defer import Deferred
 
     from scrapy.settings import BaseSettings
 
@@ -46,10 +45,6 @@ ScrapeFunc: TypeAlias = Callable[
 ]
 
 
-def _isiterable(o: Any) -> bool:
-    return isinstance(o, (Iterable, AsyncIterator))
-
-
 class SpiderMiddlewareManager(MiddlewareManager):
     component_name = "spider middleware"
 
@@ -67,13 +62,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
         if hasattr(mw, "process_start"):
             self.methods["process_start"].appendleft(mw.process_start)
 
-        process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
+        process_spider_output = self._get_process_spider_output(mw)
         self.methods["process_spider_output"].appendleft(process_spider_output)
-        if callable(process_spider_output):
+        if process_spider_output is not None:
             self._check_mw_method_spider_arg(process_spider_output)
-        elif isinstance(process_spider_output, tuple):
-            for m in process_spider_output:
-                self._check_mw_method_spider_arg(m)
 
         process_spider_exception = getattr(mw, "process_spider_exception", None)
         self.methods["process_spider_exception"].appendleft(process_spider_exception)
@@ -105,48 +97,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return await scrape_func(Failure(), request)
         return await scrape_func(response, request)
 
-    def _evaluate_iterable(
-        self,
-        response: Response,
-        iterable: Iterable[_T] | AsyncIterator[_T],
-        exception_processor_index: int,
-        recover_to: MutableChain[_T] | MutableAsyncChain[_T],
-    ) -> Iterable[_T] | AsyncIterator[_T]:
-
-        if isinstance(iterable, AsyncIterator):
-            return self._process_async(
-                response,
-                iterable,
-                exception_processor_index,
-                cast("MutableAsyncChain[_T]", recover_to),
-            )
-        return self._process_sync(
-            response,
-            iterable,
-            exception_processor_index,
-            cast("MutableChain[_T]", recover_to),
-        )
-
-    def _process_sync(
-        self,
-        response: Response,
-        iterable: Iterable[_T],
-        exception_processor_index: int,
-        recover_to: MutableChain[_T],
-    ) -> Iterable[_T]:
-        try:
-            yield from iterable
-        except Exception as ex:
-            exception_result = cast(
-                "Failure | MutableChain[_T]",
-                self._process_spider_exception(response, ex, exception_processor_index),
-            )
-            if isinstance(exception_result, Failure):
-                raise
-            assert isinstance(recover_to, MutableChain)
-            recover_to.extend(exception_result)
-
-    async def _process_async(
+    async def _evaluate_iterable(
         self,
         response: Response,
         iterable: AsyncIterator[_T],
@@ -157,13 +108,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
             async for r in iterable:
                 yield r
         except Exception as ex:
-            exception_result = cast(
-                "Failure | MutableAsyncChain[_T]",
-                self._process_spider_exception(response, ex, exception_processor_index),
+            exception_result: MutableAsyncChain[_T] = self._process_spider_exception(
+                response, ex, exception_processor_index
             )
-            if isinstance(exception_result, Failure):
-                raise
-            assert isinstance(recover_to, MutableAsyncChain)
             recover_to.extend(exception_result)
 
     def _process_spider_exception(
@@ -171,7 +118,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         response: Response,
         exception: Exception,
         start_index: int = 0,
-    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
+    ) -> MutableAsyncChain[_T]:
         # don't handle _InvalidOutput exception
         if isinstance(exception, _InvalidOutput):
             raise exception
@@ -181,28 +128,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
-            method = cast("Callable", method)
             if method in self._mw_methods_requiring_spider:
                 result = method(
                     response=response, exception=exception, spider=self._spider
                 )
             else:
                 result = method(response=response, exception=exception)
-            if _isiterable(result):
+            if isinstance(result, (Iterable, AsyncIterator)):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = (
-                    self._process_spider_output(response, result, method_index + 1)
-                )
-                # _process_spider_output() returns a Deferred only because of downgrading so this can be
-                # simplified when downgrading is removed.
-                if dfd.called:
-                    # the result is available immediately if _process_spider_output didn't do downgrading
-                    return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result)
-                # we forbid waiting here because otherwise we would need to return a deferred from
-                # _process_spider_exception too, which complicates the architecture
-                msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded"
-                raise _InvalidOutput(msg)
+                if isinstance(result, Iterable):
+                    result = as_async_generator(result)
+                return self._process_spider_output(response, result, method_index + 1)
             if result is None:
                 continue
             msg = (
@@ -212,124 +149,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
             raise _InvalidOutput(msg)
         raise exception
 
-    # This method cannot be made async def, as _process_spider_exception relies on the Deferred result
-    # being available immediately which doesn't work when it's a wrapped coroutine.
-    # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
-    @inlineCallbacks
-    def _process_spider_output(  # noqa: PLR0912
+    def _process_spider_output(
         self,
         response: Response,
-        result: Iterable[_T] | AsyncIterator[_T],
+        result: AsyncIterator[_T],
         start_index: int = 0,
-    ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]:
+    ) -> MutableAsyncChain[_T]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        recovered: MutableChain[_T] | MutableAsyncChain[_T]
-        last_result_is_async = isinstance(result, AsyncIterator)
-        recovered = MutableAsyncChain() if last_result_is_async else MutableChain()
-
-        # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
-        # 1. def foo. Sync iterables are passed as is, async ones are downgraded.
-        # 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
-        # 3. def foo + async def foo_async. Iterables are passed to the respective method.
-        # Storing methods and method tuples in the same list is weird but we should be able to roll this back
-        # when we drop this compatibility feature.
-
+        recovered: MutableAsyncChain[_T] = MutableAsyncChain()
         method_list = islice(self.methods["process_spider_output"], start_index, None)
-        for method_index, method_pair in enumerate(method_list, start=start_index):
-            if method_pair is None:
+        for method_index, method in enumerate(method_list, start=start_index):
+            if method is None:
                 continue
-            need_upgrade = need_downgrade = False
-            if isinstance(method_pair, tuple):
-                # This tuple handling is only needed until _async compatibility methods are removed.
-                method_sync, method_async = method_pair
-                method = method_async if last_result_is_async else method_sync
+            if method in self._mw_methods_requiring_spider:
+                result = method(response=response, result=result, spider=self._spider)
             else:
-                method = method_pair
-                if not last_result_is_async and isasyncgenfunction(method):
-                    need_upgrade = True
-                elif last_result_is_async and not isasyncgenfunction(method):
-                    need_downgrade = True
-            try:
-                if need_upgrade:
-                    # Iterable -> AsyncIterator
-                    result = as_async_generator(result)
-                elif need_downgrade:
-                    logger.warning(
-                        f"Async iterable passed to {global_object_name(method)} was"
-                        f" downgraded to a non-async one. This is deprecated and will"
-                        f" stop working in a future version of Scrapy. Please see"
-                        f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users"
-                        f" for more information."
-                    )
-                    assert isinstance(result, AsyncIterator)
-                    # AsyncIterator -> Iterable
-                    result = yield deferred_from_coro(collect_asyncgen(result))
-                    if isinstance(recovered, AsyncIterator):
-                        recovered_collected = yield deferred_from_coro(
-                            collect_asyncgen(recovered)
-                        )
-                        recovered = MutableChain(recovered_collected)
-                # might fail directly if the output value is not a generator
-                if method in self._mw_methods_requiring_spider:
-                    result = method(
-                        response=response, result=result, spider=self._spider
-                    )
-                else:
-                    result = method(response=response, result=result)
-            except Exception as ex:
-                exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
-                    self._process_spider_exception(response, ex, method_index + 1)
-                )
-                if isinstance(exception_result, Failure):
-                    raise
-                return exception_result
-            if _isiterable(result):
-                result = self._evaluate_iterable(
-                    response, result, method_index + 1, recovered
-                )
-            else:
-                if iscoroutine(result):
-                    result.close()  # Silence warning about not awaiting
-                    msg = (
-                        f"{global_object_name(method)} must be an asynchronous "
-                        f"generator (i.e. use yield)"
-                    )
-                else:
-                    msg = (
-                        f"{global_object_name(method)} must return an iterable, got "
-                        f"{type(result)}"
-                    )
-                raise _InvalidOutput(msg)
-            last_result_is_async = isinstance(result, AsyncIterator)
-
-        if last_result_is_async:
-            return MutableAsyncChain(result, recovered)
-        return MutableChain(result, recovered)  # type: ignore[arg-type]
+                result = method(response=response, result=result)
+            result = self._evaluate_iterable(
+                response, result, method_index + 1, recovered
+            )
+        return MutableAsyncChain(result, recovered)
 
     async def _process_callback_output(
-        self,
-        response: Response,
-        result: Iterable[_T] | AsyncIterator[_T],
-    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-        recovered: MutableChain[_T] | MutableAsyncChain[_T]
-        if isinstance(result, AsyncIterator):
-            recovered = MutableAsyncChain()
-        else:
-            recovered = MutableChain()
+        self, response: Response, result: AsyncIterator[_T]
+    ) -> MutableAsyncChain[_T]:
+        recovered: MutableAsyncChain[_T] = MutableAsyncChain()
         result = self._evaluate_iterable(response, result, 0, recovered)
-        result = await maybe_deferred_to_future(
-            cast(
-                "Deferred[Iterable[_T] | AsyncIterator[_T]]",
-                self._process_spider_output(response, result),
-            )
-        )
-        if isinstance(result, AsyncIterator):
-            return MutableAsyncChain(result, recovered)
-        if isinstance(recovered, AsyncIterator):
-            recovered_collected = await collect_asyncgen(recovered)
-            recovered = MutableChain(recovered_collected)
-        return MutableChain(result, recovered)
+        result = self._process_spider_output(response, result)
+        return MutableAsyncChain(result, recovered)
 
     def scrape_response(
         self,
@@ -340,7 +188,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         response: Response,
         request: Request,
         spider: Spider,
-    ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]:  # pragma: no cover
+    ) -> Deferred[MutableAsyncChain[_T]]:  # pragma: no cover
         warn(
             "SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead",
             ScrapyDeprecationWarning,
@@ -363,7 +211,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         scrape_func: ScrapeFunc[_T],
         response: Response,
         request: Request,
-    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
+    ) -> MutableAsyncChain[_T]:
         if not self.crawler:
             raise RuntimeError(
                 "scrape_response_async() called on a SpiderMiddlewareManager"
@@ -373,7 +221,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
             it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
                 scrape_func, response, request
             )
-            return await self._process_callback_output(response, it)
+            ait = it if isinstance(it, AsyncIterator) else as_async_generator(it)
+            return await self._process_callback_output(response, ait)
         except Exception as ex:
             await _defer_sleep_async()
             return self._process_spider_exception(response, ex)
@@ -399,27 +248,23 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
-    def _get_async_method_pair(
-        mw: Any, methodname: str
-    ) -> Callable | tuple[Callable, Callable] | None:
-        normal_method: Callable | None = getattr(mw, methodname, None)
-        methodname_async = methodname + "_async"
-        async_method: Callable | None = getattr(mw, methodname_async, None)
+    def _get_process_spider_output(mw: Any) -> Callable | None:
+        normal_method: Callable | None = getattr(mw, "process_spider_output", None)
+        async_method: Callable | None = getattr(mw, "process_spider_output_async", None)
         if not async_method:
             if normal_method and not isasyncgenfunction(normal_method):
-                logger.warning(
+                raise TypeError(
                     f"Middleware {global_object_name(mw.__class__)} doesn't support"
-                    f" asynchronous spider output, this is deprecated and will stop"
-                    f" working in a future version of Scrapy. The middleware should"
-                    f" be updated to support it. Please see"
-                    f" https://docs.scrapy.org/en/latest/topics/coroutines.html#for-middleware-users"
-                    f" for more information."
+                    f" asynchronous spider output. Its process_spider_output() method"
+                    f" should be an async generator function or it should additionally"
+                    f" define a process_spider_output_async() method."
                 )
             return normal_method
         if not normal_method:
             logger.error(
-                f"Middleware {global_object_name(mw.__class__)} has {methodname_async} "
-                f"without {methodname}, skipping this method."
+                f"Middleware {global_object_name(mw.__class__)} has"
+                f" process_spider_output_async() without process_spider_output(),"
+                f" skipping this method. Please rename it to process_spider_output()."
             )
             return None
         if not isasyncgenfunction(async_method):
@@ -431,8 +276,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
         if isasyncgenfunction(normal_method):
             logger.error(
                 f"{global_object_name(normal_method)} is an async "
-                f"generator function while {methodname_async} exists, "
-                f"skipping both methods."
+                f"generator function while process_spider_output_async() exists, "
+                f"skipping both methods. Please remove process_spider_output_async()."
             )
             return None
-        return normal_method, async_method
+        return async_method
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index c4e0ce16e..14b591a5f 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -50,10 +50,7 @@ class MiddlewareManager(ABC):
             )
         self.middlewares: tuple[Any, ...] = middlewares
         # Only process_spider_output and process_spider_exception can be None.
-        # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
-        self.methods: dict[str, deque[Callable | tuple[Callable, Callable] | None]] = (
-            defaultdict(deque)
-        )
+        self.methods: dict[str, deque[Callable | None]] = defaultdict(deque)
         self._mw_methods_requiring_spider: set[Callable] = set()
         for mw in middlewares:
             self._add_middleware(mw)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 9cb695111..2d40e8555 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,12 +8,14 @@ import gc
 import inspect
 import re
 import sys
+import warnings
 import weakref
 from collections.abc import AsyncIterator, Iterable, Mapping
 from functools import partial, wraps
 from itertools import chain
 from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar, overload
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
 
 if TYPE_CHECKING:
@@ -294,12 +296,13 @@ else:
         gc.collect()
 
 
-class MutableChain(Iterable[_T]):
-    """
-    Thin wrapper around itertools.chain, allowing to add iterables "in-place"
-    """
-
+class MutableChain(Iterable[_T]):  # pragma: no cover
     def __init__(self, *args: Iterable[_T]):
+        warnings.warn(
+            "MutableChain is deprecated and will be removed in a future Scrapy version.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
         self.data: Iterator[_T] = chain.from_iterable(args)
 
     def extend(self, *iterables: Iterable[_T]) -> None:
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 8d3977452..0d96e1d88 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -38,8 +38,8 @@ class InjectArgumentsSpiderMiddleware:
         if request.callback.__name__ == "parse_spider_mw":
             request.cb_kwargs["from_process_spider_input"] = True
 
-    def process_spider_output(self, response, result):
-        for element in result:
+    async def process_spider_output(self, response, result):
+        async for element in result:
             if (
                 isinstance(element, Request)
                 and element.callback.__name__ == "parse_spider_mw_2"
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index bfa60ac9c..2e0436ae1 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Any, cast
 from unittest import mock
 
 import pytest
-from testfixtures import LogCapture
 from twisted.internet import defer
 
 from scrapy.core.spidermw import SpiderMiddlewareManager
@@ -63,20 +62,6 @@ class TestProcessSpiderInputInvalidOutput(TestSpiderMiddleware):
             await self._scrape_response()
 
 
-class TestProcessSpiderOutputInvalidOutput(TestSpiderMiddleware):
-    """Invalid return value for process_spider_output method"""
-
-    @coroutine_test
-    async def test_invalid_process_spider_output(self):
-        class InvalidProcessSpiderOutputMiddleware:
-            def process_spider_output(self, response, result):
-                return 1
-
-        self.mwman._add_middleware(InvalidProcessSpiderOutputMiddleware())
-        with pytest.raises(_InvalidOutput):
-            await self._scrape_response()
-
-
 class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
     """Invalid return value for process_spider_exception method"""
 
@@ -87,13 +72,15 @@ class TestProcessSpiderExceptionInvalidOutput(TestSpiderMiddleware):
                 return 1
 
         class RaiseExceptionProcessSpiderOutputMiddleware:
-            def process_spider_output(self, response, result):
+            async def process_spider_output(self, response, result):
                 raise RuntimeError
+                yield  # pylint: disable=unreachable
 
         self.mwman._add_middleware(InvalidProcessSpiderOutputExceptionMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+        it = await self._scrape_response()
         with pytest.raises(_InvalidOutput):
-            await self._scrape_response()
+            await collect_asyncgen(it)
 
 
 class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
@@ -106,13 +93,15 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):
                 return None
 
         class RaiseExceptionProcessSpiderOutputMiddleware:
-            def process_spider_output(self, response, result):
+            async def process_spider_output(self, response, result):
                 1 / 0
+                yield
 
         self.mwman._add_middleware(ProcessSpiderExceptionReturnNoneMiddleware())
         self.mwman._add_middleware(RaiseExceptionProcessSpiderOutputMiddleware())
+        it = await self._scrape_response()
         with pytest.raises(ZeroDivisionError):
-            await self._scrape_response()
+            await collect_asyncgen(it)
 
 
 class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
@@ -155,43 +144,19 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
             self._scrape_func, self.response, self.request
         )
 
-    async def _test_simple_base(
-        self,
-        *mw_classes: type[Any],
-        downgrade: bool = False,
-        start_index: int | None = None,
-    ) -> None:
-        with LogCapture() as log:
-            result = await self._get_middleware_result(
-                *mw_classes, start_index=start_index
-            )
-        assert isinstance(result, Iterable)
-        result_list = list(result)
-        assert len(result_list) == self.RESULT_COUNT
-        assert isinstance(result_list[0], self.ITEM_TYPE)
-        assert ("downgraded to a non-async" in str(log)) == downgrade
-        assert ("doesn't support asynchronous spider output" in str(log)) == (
-            ProcessSpiderOutputSimpleMiddleware in mw_classes
-        )
-
     async def _test_asyncgen_base(
         self,
         *mw_classes: type[Any],
-        downgrade: bool = False,
         start_index: int | None = None,
     ) -> None:
-        with LogCapture() as log:
-            result = await self._get_middleware_result(
-                *mw_classes, start_index=start_index
-            )
+        result = await self._get_middleware_result(*mw_classes, start_index=start_index)
         assert isinstance(result, AsyncIterator)
         result_list = await collect_asyncgen(result)
         assert len(result_list) == self.RESULT_COUNT
         assert isinstance(result_list[0], self.ITEM_TYPE)
-        assert ("downgraded to a non-async" in str(log)) == downgrade
 
 
-class ProcessSpiderOutputSimpleMiddleware:
+class ProcessSpiderOutputSyncMiddleware:
     def process_spider_output(self, response, result):
         yield from result
 
@@ -232,53 +197,36 @@ class TestProcessSpiderOutputSimple(TestBaseAsyncSpiderMiddleware):
     """process_spider_output tests for simple callbacks"""
 
     ITEM_TYPE = dict
-    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_SYNC = ProcessSpiderOutputSyncMiddleware
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
     @coroutine_test
-    async def test_simple(self):
-        """Simple mw"""
-        await self._test_simple_base(self.MW_SIMPLE)
+    async def test_sync(self):
+        """Unsupported sync mw"""
+        with pytest.raises(
+            TypeError, match=r"doesn't support asynchronous spider output"
+        ):
+            await self._get_middleware_result(self.MW_SYNC)
 
     @coroutine_test
     async def test_asyncgen(self):
-        """Asyncgen mw; upgrade"""
+        """Asyncgen mw"""
         await self._test_asyncgen_base(self.MW_ASYNCGEN)
 
-    @coroutine_test
-    async def test_simple_asyncgen(self):
-        """Simple mw -> asyncgen mw; upgrade"""
-        await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE)
-
-    @coroutine_test
-    async def test_asyncgen_simple(self):
-        """Asyncgen mw -> simple mw; upgrade then downgrade"""
-        await self._test_simple_base(self.MW_SIMPLE, self.MW_ASYNCGEN, downgrade=True)
-
     @coroutine_test
     async def test_universal(self):
         """Universal mw"""
-        await self._test_simple_base(self.MW_UNIVERSAL)
-
-    @coroutine_test
-    async def test_universal_simple(self):
-        """Universal mw -> simple mw"""
-        await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL)
-
-    @coroutine_test
-    async def test_simple_universal(self):
-        """Simple mw -> universal mw"""
-        await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE)
+        await self._test_asyncgen_base(self.MW_UNIVERSAL)
 
     @coroutine_test
     async def test_universal_asyncgen(self):
-        """Universal mw -> asyncgen mw; upgrade"""
+        """Universal mw -> asyncgen mw"""
         await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_UNIVERSAL)
 
     @coroutine_test
     async def test_asyncgen_universal(self):
-        """Asyncgen mw -> universal mw; upgrade"""
+        """Asyncgen mw -> universal mw"""
         await self._test_asyncgen_base(self.MW_UNIVERSAL, self.MW_ASYNCGEN)
 
 
@@ -289,31 +237,6 @@ class TestProcessSpiderOutputAsyncGen(TestProcessSpiderOutputSimple):
         for item in super()._callback():
             yield item
 
-    @coroutine_test
-    async def test_simple(self):
-        """Simple mw; downgrade"""
-        await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
-
-    @coroutine_test
-    async def test_simple_asyncgen(self):
-        """Simple mw -> asyncgen mw; downgrade then upgrade"""
-        await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_SIMPLE, downgrade=True)
-
-    @coroutine_test
-    async def test_universal(self):
-        """Universal mw"""
-        await self._test_asyncgen_base(self.MW_UNIVERSAL)
-
-    @coroutine_test
-    async def test_universal_simple(self):
-        """Universal mw -> simple mw; downgrade"""
-        await self._test_simple_base(self.MW_SIMPLE, self.MW_UNIVERSAL, downgrade=True)
-
-    @coroutine_test
-    async def test_simple_universal(self):
-        """Simple mw -> universal mw; downgrade"""
-        await self._test_simple_base(self.MW_UNIVERSAL, self.MW_SIMPLE, downgrade=True)
-
 
 class ProcessSpiderOutputNonIterableMiddleware:
     def process_spider_output(self, response, result):
@@ -325,24 +248,6 @@ class ProcessSpiderOutputCoroutineMiddleware:
         return result
 
 
-class TestProcessSpiderOutputInvalidResult(TestBaseAsyncSpiderMiddleware):
-    @coroutine_test
-    async def test_non_iterable(self):
-        with pytest.raises(
-            _InvalidOutput,
-            match=r"\.process_spider_output must return an iterable, got <class 'NoneType'>",
-        ):
-            await self._get_middleware_result(ProcessSpiderOutputNonIterableMiddleware)
-
-    @coroutine_test
-    async def test_coroutine(self):
-        with pytest.raises(
-            _InvalidOutput,
-            match=r"\.process_spider_output must be an asynchronous generator",
-        ):
-            await self._get_middleware_result(ProcessSpiderOutputCoroutineMiddleware)
-
-
 class ProcessStartSimpleMiddleware:
     async def process_start(self, start):
         async for item_or_request in start:
@@ -415,11 +320,11 @@ class TestUniversalMiddlewareManager:
         return SpiderMiddlewareManager.from_crawler(crawler)
 
     def test_simple_mw(self, mwman: SpiderMiddlewareManager) -> None:
-        mw = ProcessSpiderOutputSimpleMiddleware()
-        mwman._add_middleware(mw)
-        assert (
-            mwman.methods["process_spider_output"][0] == mw.process_spider_output  # pylint: disable=comparison-with-callable
-        )
+        mw = ProcessSpiderOutputSyncMiddleware()
+        with pytest.raises(
+            TypeError, match=r"doesn't support asynchronous spider output"
+        ):
+            mwman._add_middleware(mw)
 
     def test_async_mw(self, mwman: SpiderMiddlewareManager) -> None:
         mw = ProcessSpiderOutputAsyncGenMiddleware()
@@ -431,9 +336,8 @@ class TestUniversalMiddlewareManager:
     def test_universal_mw(self, mwman: SpiderMiddlewareManager) -> None:
         mw = ProcessSpiderOutputUniversalMiddleware()
         mwman._add_middleware(mw)
-        assert mwman.methods["process_spider_output"][0] == (
-            mw.process_spider_output,
-            mw.process_spider_output_async,
+        assert (
+            mwman.methods["process_spider_output"][0] == mw.process_spider_output_async  # pylint: disable=comparison-with-callable
         )
 
     def test_universal_mw_no_sync(
@@ -441,7 +345,7 @@ class TestUniversalMiddlewareManager:
     ) -> None:
         mwman._add_middleware(UniversalMiddlewareNoSync())
         assert (
-            "UniversalMiddlewareNoSync has process_spider_output_async"
+            "UniversalMiddlewareNoSync has process_spider_output_async()"
             " without process_spider_output" in caplog.text
         )
         assert mwman.methods["process_spider_output"][0] is None
@@ -465,7 +369,7 @@ class TestUniversalMiddlewareManager:
         mwman._add_middleware(UniversalMiddlewareBothAsync())
         assert (
             "UniversalMiddlewareBothAsync.process_spider_output "
-            "is an async generator function while process_spider_output_async exists"
+            "is an async generator function while process_spider_output_async() exists"
             in caplog.text
         )
         assert mwman.methods["process_spider_output"][0] is None
@@ -473,7 +377,6 @@ class TestUniversalMiddlewareManager:
 
 class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
     ITEM_TYPE = dict
-    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
@@ -488,66 +391,22 @@ class TestBuiltinMiddlewareSimple(TestBaseAsyncSpiderMiddleware):
             self._scrape_func, self.response, self.request
         )
 
-    @coroutine_test
-    async def test_just_builtin(self):
-        await self._test_simple_base()
-
-    @coroutine_test
-    async def test_builtin_simple(self):
-        await self._test_simple_base(self.MW_SIMPLE, start_index=1000)
-
-    @coroutine_test
-    async def test_builtin_async(self):
-        """Upgrade"""
-        await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
-
-    @coroutine_test
-    async def test_builtin_universal(self):
-        await self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
-
-    @coroutine_test
-    async def test_simple_builtin(self):
-        await self._test_simple_base(self.MW_SIMPLE)
-
-    @coroutine_test
-    async def test_async_builtin(self):
-        """Upgrade"""
-        await self._test_asyncgen_base(self.MW_ASYNCGEN)
-
-    @coroutine_test
-    async def test_universal_builtin(self):
-        await self._test_simple_base(self.MW_UNIVERSAL)
-
-
-class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
-    async def _callback(self) -> Any:
-        for item in super()._callback():
-            yield item
-
     @coroutine_test
     async def test_just_builtin(self):
         await self._test_asyncgen_base()
 
-    @coroutine_test
-    async def test_builtin_simple(self):
-        """Downgrade"""
-        await self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
-
     @coroutine_test
     async def test_builtin_async(self):
+        """Upgrade"""
         await self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
 
     @coroutine_test
     async def test_builtin_universal(self):
         await self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
 
-    @coroutine_test
-    async def test_simple_builtin(self):
-        """Downgrade"""
-        await self._test_simple_base(self.MW_SIMPLE, downgrade=True)
-
     @coroutine_test
     async def test_async_builtin(self):
+        """Upgrade"""
         await self._test_asyncgen_base(self.MW_ASYNCGEN)
 
     @coroutine_test
@@ -555,9 +414,14 @@ class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
         await self._test_asyncgen_base(self.MW_UNIVERSAL)
 
 
+class TestBuiltinMiddlewareAsyncGen(TestBuiltinMiddlewareSimple):
+    async def _callback(self) -> Any:
+        for item in super()._callback():
+            yield item
+
+
 class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
     ITEM_TYPE = dict
-    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
     MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
@@ -576,18 +440,13 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
     @coroutine_test
     async def test_exc_simple(self):
         """Simple exc mw"""
-        await self._test_simple_base(self.MW_EXC_SIMPLE)
+        await self._test_asyncgen_base(self.MW_EXC_SIMPLE)
 
     @coroutine_test
     async def test_exc_async(self):
         """Async exc mw"""
         await self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
 
-    @coroutine_test
-    async def test_exc_simple_simple(self):
-        """Simple exc mw -> simple output mw"""
-        await self._test_simple_base(self.MW_SIMPLE, self.MW_EXC_SIMPLE)
-
     @coroutine_test
     async def test_exc_async_async(self):
         """Async exc mw -> async output mw"""
@@ -598,25 +457,27 @@ class TestProcessSpiderException(TestBaseAsyncSpiderMiddleware):
         """Simple exc mw -> async output mw; upgrade"""
         await self._test_asyncgen_base(self.MW_ASYNCGEN, self.MW_EXC_SIMPLE)
 
-    @coroutine_test
-    async def test_exc_async_simple(self):
-        """Async exc mw -> simple output mw; cannot work as downgrading is not supported"""
-        await self._test_asyncgen_nodowngrade(self.MW_SIMPLE, self.MW_EXC_ASYNCGEN)
-
 
 class TestDeprecatedSpiderArg(TestSpiderMiddleware):
     @coroutine_test
     async def test_deprecated_mw_spider_arg(self):
-        class DeprecatedSpiderArgMiddleware:
+        class DeprecatedSpiderArgMiddleware1:
             def process_spider_input(self, response, spider):
                 return None
 
-            def process_spider_output(self, response, result, spider):
+            async def process_spider_output(self, response, result, spider):
                 1 / 0
+                yield
 
+        class DeprecatedSpiderArgMiddleware2:
             def process_spider_exception(self, response, exception, spider):
                 return []
 
+        with pytest.warns(
+            ScrapyDeprecationWarning,
+            match=r"process_spider_exception\(\) requires a spider argument",
+        ):
+            self.mwman._add_middleware(DeprecatedSpiderArgMiddleware2())
         with (
             pytest.warns(
                 ScrapyDeprecationWarning,
@@ -626,13 +487,10 @@ class TestDeprecatedSpiderArg(TestSpiderMiddleware):
                 ScrapyDeprecationWarning,
                 match=r"process_spider_output\(\) requires a spider argument",
             ),
-            pytest.warns(
-                ScrapyDeprecationWarning,
-                match=r"process_spider_exception\(\) requires a spider argument",
-            ),
         ):
-            self.mwman._add_middleware(DeprecatedSpiderArgMiddleware())
-        await self._scrape_response()
+            self.mwman._add_middleware(DeprecatedSpiderArgMiddleware1())
+        it = await self._scrape_response()
+        await collect_asyncgen(it)
 
     @coroutine_test
     async def test_deprecated_mwman_spider_arg(self):
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 8cf08dd94..afdfaa322 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -169,8 +169,8 @@ class NotGeneratorCallbackSpiderMiddlewareRightAfterSpider(NotGeneratorCallbackS
 # ================================================================================
 # (4) exceptions from a middleware process_spider_output method (generator)
 class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        for r in result:
+    async def process_spider_output(self, response, result):
+        async for r in result:
             r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
 
@@ -182,8 +182,8 @@ class _GeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
 
 
 class GeneratorFailMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        for r in result:
+    async def process_spider_output(self, response, result):
+        async for r in result:
             r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
             raise LookupError
@@ -201,8 +201,8 @@ class GeneratorDoNothingAfterFailureMiddleware(_GeneratorDoNothingMiddleware):
 
 
 class GeneratorRecoverMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        for r in result:
+    async def process_spider_output(self, response, result):
+        async for r in result:
             r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
             yield r
 
@@ -237,86 +237,6 @@ class GeneratorOutputChainSpider(Spider):
         yield {"processed": ["parse-second-item"]}
 
 
-# ================================================================================
-# (5) exceptions from a middleware process_spider_output method (not generator)
-
-
-class _NotGeneratorDoNothingMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        out = []
-        for r in result:
-            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
-            out.append(r)
-        return out
-
-    def process_spider_exception(self, response, exception):
-        method = f"{self.__class__.__name__}.process_spider_exception"
-        self.crawler.spider.logger.info(
-            "%s: %s caught", method, exception.__class__.__name__
-        )
-
-
-class NotGeneratorFailMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        out = []
-        for r in result:
-            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
-            out.append(r)
-        raise ReferenceError
-
-    def process_spider_exception(self, response, exception):
-        method = f"{self.__class__.__name__}.process_spider_exception"
-        self.crawler.spider.logger.info(
-            "%s: %s caught", method, exception.__class__.__name__
-        )
-        return [{"processed": [method]}]
-
-
-class NotGeneratorDoNothingAfterFailureMiddleware(_NotGeneratorDoNothingMiddleware):
-    pass
-
-
-class NotGeneratorRecoverMiddleware(_BaseSpiderMiddleware):
-    def process_spider_output(self, response, result):
-        out = []
-        for r in result:
-            r["processed"].append(f"{self.__class__.__name__}.process_spider_output")
-            out.append(r)
-        return out
-
-    def process_spider_exception(self, response, exception):
-        method = f"{self.__class__.__name__}.process_spider_exception"
-        self.crawler.spider.logger.info(
-            "%s: %s caught", method, exception.__class__.__name__
-        )
-        return [{"processed": [method]}]
-
-
-class NotGeneratorDoNothingAfterRecoveryMiddleware(_NotGeneratorDoNothingMiddleware):
-    pass
-
-
-class NotGeneratorOutputChainSpider(Spider):
-    name = "NotGeneratorOutputChainSpider"
-    custom_settings = {
-        "SPIDER_MIDDLEWARES": {
-            NotGeneratorFailMiddleware: 10,
-            NotGeneratorDoNothingAfterFailureMiddleware: 8,
-            NotGeneratorRecoverMiddleware: 5,
-            NotGeneratorDoNothingAfterRecoveryMiddleware: 3,
-        },
-    }
-
-    async def start(self):
-        yield Request(self.mockserver.url("/status?n=200"))
-
-    def parse(self, response):
-        return [
-            {"processed": ["parse-first-item"]},
-            {"processed": ["parse-second-item"]},
-        ]
-
-
 # ================================================================================
 class TestSpiderMiddleware:
     mockserver: MockServer
@@ -481,41 +401,3 @@ class TestSpiderMiddleware:
         assert str(item_from_callback) in str(log4)
         assert str(item_recovered) in str(log4)
         assert "parse-second-item" not in str(log4)
-
-    @coroutine_test
-    async def test_not_a_generator_output_chain(self):
-        """
-        (5) An exception from a middleware's process_spider_output method should be sent
-        to the process_spider_exception method from the next middleware in the chain.
-        The result of the recovery by the process_spider_exception method should be handled
-        by the process_spider_output method from the next middleware.
-        The final item count should be 1 (from the process_spider_exception chain, the items
-        from the spider callback are lost)
-        """
-        log5 = await self.crawl_log(NotGeneratorOutputChainSpider)
-        assert "'item_scraped_count': 1" in str(log5)
-        assert (
-            "GeneratorRecoverMiddleware.process_spider_exception: ReferenceError caught"
-            in str(log5)
-        )
-        assert (
-            "GeneratorDoNothingAfterFailureMiddleware.process_spider_exception: ReferenceError caught"
-            in str(log5)
-        )
-        assert (
-            "GeneratorFailMiddleware.process_spider_exception: ReferenceError caught"
-            not in str(log5)
-        )
-        assert (
-            "GeneratorDoNothingAfterRecoveryMiddleware.process_spider_exception: ReferenceError caught"
-            not in str(log5)
-        )
-        item_recovered = {
-            "processed": [
-                "NotGeneratorRecoverMiddleware.process_spider_exception",
-                "NotGeneratorDoNothingAfterRecoveryMiddleware.process_spider_output",
-            ]
-        }
-        assert str(item_recovered) in str(log5)
-        assert "parse-first-item" not in str(log5)
-        assert "parse-second-item" not in str(log5)
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 8dfac5f02..e8fe45749 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -12,7 +12,6 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import aiter_errback
 from scrapy.utils.python import (
     MutableAsyncChain,
-    MutableChain,
     binary_is_text,
     get_func_args,
     memoizemethod_noargs,
@@ -30,16 +29,6 @@ _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
 
 
-def test_mutablechain():
-    m = MutableChain(range(2), [2, 3], (4, 5))
-    m.extend(range(6, 7))
-    m.extend([7, 8])
-    m.extend([9, 10], (11, 12))
-    assert next(m) == 0
-    assert m.__next__() == 1
-    assert list(m) == list(range(2, 13))
-
-
 class TestMutableAsyncChain:
     @staticmethod
     async def g1():

From f875af4a86b294eb168cb64daba0f60b63d389f4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 18 May 2026 18:52:13 +0500
Subject: [PATCH 27/36] Add BaseStreamingDownloadHandler and migrate
 HttpxDownloadHandler to it. (#7524)

---
 docs/topics/download-handlers.rst             |   5 +-
 docs/topics/request-response.rst              |   6 +-
 scrapy/core/downloader/handlers/_base_http.py |  25 ++
 .../downloader/handlers/_base_streaming.py    | 307 +++++++++++++++
 scrapy/core/downloader/handlers/_httpx.py     | 358 ++++++------------
 scrapy/core/downloader/handlers/http11.py     |   3 +-
 scrapy/http/response/__init__.py              |   5 +-
 scrapy/utils/_download_handlers.py            |  30 +-
 scrapy/utils/ssl.py                           |  19 +-
 tests/test_crawl.py                           |  17 +-
 tests/test_downloader_handler_httpx.py        |  29 +-
 tests/test_downloader_handlers_http_base.py   |  33 +-
 tox.ini                                       |   4 +-
 13 files changed, 533 insertions(+), 308 deletions(-)
 create mode 100644 scrapy/core/downloader/handlers/_base_http.py
 create mode 100644 scrapy/core/downloader/handlers/_base_streaming.py

diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index 1e95864c6..272ff0dcd 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -279,14 +279,11 @@ If you want to use this handler you need to replace the default ones for the
     some websites may be different. Additionally, these are the Scrapy features
     that are explicitly not supported when using it:
 
-    - Proxy support (the :reqmeta:`proxy` meta key).
-
     - Per-request bind address support (the :reqmeta:`bindaddress` meta key).
       The global :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported but the
       port number, if specified, will be ignored.
 
-    - The :setting:`DOWNLOADER_CLIENT_TLS_CIPHERS` and
-      :setting:`DOWNLOADER_CLIENT_TLS_METHOD` settings.
+    - The :setting:`DOWNLOADER_CLIENT_TLS_METHOD` setting.
 
     - Settings specific to the Twisted networking or HTTP implementation, like
       :setting:`DNS_RESOLVER`.
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index ba7b3ee3c..d1e4851d9 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -917,7 +917,7 @@ Response objects
     :type request: scrapy.Request
 
     :param certificate: an object representing the server's SSL certificate.
-    :type certificate: twisted.internet.ssl.Certificate
+    :type certificate: typing.Any
 
     :param ip_address: The IP address of the server from which the Response originated.
     :type ip_address: :class:`ipaddress.IPv4Address` or :class:`ipaddress.IPv6Address`
@@ -1009,8 +1009,8 @@ Response objects
 
     .. attribute:: Response.certificate
 
-        A :class:`twisted.internet.ssl.Certificate` object representing
-        the server's SSL certificate.
+        An object representing the server's SSL certificate. Its type and
+        contents depend on the download handler that produced the response.
 
         Only populated for ``https`` responses, ``None`` otherwise.
 
diff --git a/scrapy/core/downloader/handlers/_base_http.py b/scrapy/core/downloader/handlers/_base_http.py
new file mode 100644
index 000000000..83d130462
--- /dev/null
+++ b/scrapy/core/downloader/handlers/_base_http.py
@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from abc import ABC
+from typing import TYPE_CHECKING
+
+from .base import BaseDownloadHandler
+
+if TYPE_CHECKING:
+    from scrapy.crawler import Crawler
+
+
+class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
+    """Base class for built-in HTTP download handlers."""
+
+    def __init__(self, crawler: Crawler):
+        super().__init__(crawler)
+        self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
+        self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")
+        self._fail_on_dataloss: bool = crawler.settings.getbool(
+            "DOWNLOAD_FAIL_ON_DATALOSS"
+        )
+        self._tls_verbose_logging: bool = crawler.settings.getbool(
+            "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
+        )
+        self._fail_on_dataloss_warned: bool = False
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
new file mode 100644
index 000000000..16b655716
--- /dev/null
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+import base64
+import logging
+import time
+from abc import ABC, abstractmethod
+from io import BytesIO
+from typing import TYPE_CHECKING, Any, ClassVar, Generic, NoReturn, TypedDict, TypeVar
+from urllib.parse import quote, urlsplit
+
+from scrapy import Request, signals
+from scrapy.exceptions import (
+    DownloadCancelledError,
+    NotConfigured,
+    ResponseDataLossError,
+)
+from scrapy.utils._download_handlers import (
+    check_stop_download,
+    get_dataloss_msg,
+    get_maxsize_msg,
+    get_warnsize_msg,
+    make_response,
+    normalize_bind_address,
+)
+from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.url import add_http_if_no_scheme
+
+from ._base_http import BaseHttpDownloadHandler
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterable
+    from contextlib import AbstractAsyncContextManager
+    from ipaddress import IPv4Address, IPv6Address
+
+    from _typeshed import SizedBuffer
+
+    # typing.NotRequired requires Python 3.11
+    from typing_extensions import NotRequired
+
+    from scrapy.crawler import Crawler
+    from scrapy.http import Headers, Response
+
+
+logger = logging.getLogger(__name__)
+
+_ResponseT = TypeVar("_ResponseT")
+
+
+class _BaseResponseArgs(TypedDict):
+    status: int
+    url: str
+    headers: Headers
+    certificate: NotRequired[Any]
+    ip_address: NotRequired[IPv4Address | IPv6Address | None]
+    protocol: str | None
+
+
+class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_ResponseT]):
+    """A base class for HTTP download handlers that follow the streaming logic flow."""
+
+    _DEFAULT_CONNECT_TIMEOUT: ClassVar[float] = 10
+    experimental: ClassVar[bool] = False
+    requires_asyncio: ClassVar[bool] = True
+    # require subclasses to disable proxies explicitly with an explanation
+    supports_proxies: ClassVar[bool] = True
+    supports_per_request_bindaddress: ClassVar[bool] = False
+
+    def __init__(self, crawler: Crawler):
+        if self.requires_asyncio and not is_asyncio_available():  # pragma: no cover
+            raise NotConfigured(
+                f"{type(self).__name__} requires the asyncio support. Make"
+                f" sure that you have either enabled the asyncio Twisted"
+                f" reactor in the TWISTED_REACTOR setting or disabled the"
+                f" TWISTED_REACTOR_ENABLED setting. See the asyncio documentation"
+                f" of Scrapy for more information."
+            )
+        self._check_deps_installed()
+        super().__init__(crawler)
+        if self.experimental:
+            logger.warning(
+                f"{type(self).__name__} is experimental and is not recommended for production use."
+            )
+        self._bind_address = normalize_bind_address(
+            crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
+        )
+        self._proxy_auth_encoding: str = crawler.settings.get("HTTPPROXY_AUTH_ENCODING")
+        # these are useful for many handlers but used in different ways by them
+        self._pool_size_total: int = crawler.settings.getint("CONCURRENT_REQUESTS")
+        self._pool_size_per_host: int = crawler.settings.getint(
+            "CONCURRENT_REQUESTS_PER_DOMAIN"
+        )
+
+    @staticmethod
+    @abstractmethod
+    def _check_deps_installed() -> None:
+        """Raise NotConfigured if the required deps are not installed."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _make_request(
+        self, request: Request, timeout: float
+    ) -> AbstractAsyncContextManager[_ResponseT]:
+        """Return an async context manager yielding the library-specific response.
+
+        Exceptions raised by the library should be reraised as Scrapy-specific ones.
+        """
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def _extract_headers(response: _ResponseT) -> Headers:
+        """Convert library-specific response headers to a
+        :class:`~scrapy.http.headers.Headers` object."""
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def _build_base_response_args(
+        response: _ResponseT, request: Request, headers: Headers
+    ) -> _BaseResponseArgs:
+        """Build kwargs for :func:`scrapy.utils._download_handlers.make_response`."""
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def _iter_body_chunks(response: _ResponseT) -> AsyncIterable[SizedBuffer]:
+        """Return an async iterable yielding body chunks from the response."""
+        raise NotImplementedError
+
+    @staticmethod
+    @abstractmethod
+    def _is_dataloss_exception(exc: Exception) -> bool:
+        """Return True if ``exc`` represents dataloss."""
+        raise NotImplementedError
+
+    def _log_tls_info(self, response: _ResponseT, request: Request) -> None:
+        """Log TLS connection details, if possible."""
+
+    async def download_request(self, request: Request) -> Response:
+        if not self.supports_proxies and request.meta.get("proxy"):
+            raise NotImplementedError(f"{type(self).__name__} doesn't support proxies.")
+        if not self.supports_per_request_bindaddress and request.meta.get(
+            "bindaddress"
+        ):
+            logger.error(
+                f"The 'bindaddress' request meta key is not supported by"
+                f" {type(self).__name__} and will be ignored."
+            )
+        timeout: float = request.meta.get(
+            "download_timeout", self._DEFAULT_CONNECT_TIMEOUT
+        )
+        start_time = time.monotonic()
+        async with self._make_request(request, timeout) as response:
+            request.meta["download_latency"] = time.monotonic() - start_time
+            return await self._read_response(response, request)
+
+    async def _read_response(self, response: _ResponseT, request: Request) -> Response:
+        maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
+        warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
+
+        headers = self._extract_headers(response)
+        content_length = headers.get("Content-Length")
+        expected_size = int(content_length) if content_length is not None else None
+        if maxsize and expected_size and expected_size > maxsize:
+            self._cancel_maxsize(expected_size, maxsize, request, expected=True)
+
+        reached_warnsize = False
+        if warnsize and expected_size and expected_size > warnsize:
+            reached_warnsize = True
+            logger.warning(
+                get_warnsize_msg(expected_size, warnsize, request, expected=True)
+            )
+
+        make_response_base_args = self._build_base_response_args(
+            response, request, headers
+        )
+
+        if self._tls_verbose_logging:
+            self._log_tls_info(response, request)
+
+        if stop_download := check_stop_download(
+            signals.headers_received,
+            self.crawler,
+            request,
+            headers=headers,
+            body_length=expected_size,
+        ):
+            return make_response(
+                **make_response_base_args,
+                stop_download=stop_download,
+            )
+
+        response_body = BytesIO()
+        bytes_received = 0
+        try:
+            async for chunk in self._iter_body_chunks(response):
+                response_body.write(chunk)
+                bytes_received += len(chunk)
+
+                if stop_download := check_stop_download(
+                    signals.bytes_received, self.crawler, request, data=chunk
+                ):
+                    return make_response(
+                        **make_response_base_args,
+                        body=response_body.getvalue(),
+                        stop_download=stop_download,
+                    )
+
+                if maxsize and bytes_received > maxsize:
+                    response_body.truncate(0)
+                    self._cancel_maxsize(
+                        bytes_received, maxsize, request, expected=False
+                    )
+
+                if warnsize and bytes_received > warnsize and not reached_warnsize:
+                    reached_warnsize = True
+                    logger.warning(
+                        get_warnsize_msg(
+                            bytes_received, warnsize, request, expected=False
+                        )
+                    )
+        except Exception as e:
+            if not self._is_dataloss_exception(e):
+                raise
+            fail_on_dataloss: bool = request.meta.get(
+                "download_fail_on_dataloss", self._fail_on_dataloss
+            )
+            if not fail_on_dataloss:
+                return make_response(
+                    **make_response_base_args,
+                    body=response_body.getvalue(),
+                    flags=["dataloss"],
+                )
+            if not self._fail_on_dataloss_warned:
+                logger.warning(get_dataloss_msg(request.url))
+                self._fail_on_dataloss_warned = True
+            raise ResponseDataLossError(str(e)) from e
+
+        return make_response(
+            **make_response_base_args,
+            body=response_body.getvalue(),
+        )
+
+    def _get_bind_address_host(self) -> str | None:
+        """Return the host portion of the bind address.
+
+        Needed for handlers that don't support the bind port.
+        """
+        if self._bind_address is None:
+            return None
+        host, port = self._bind_address
+        if port != 0:
+            logger.warning(
+                "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not "
+                "support binding to a specific local port. Ignoring the port "
+                "and binding only to %r.",
+                port,
+                type(self).__name__,
+                host,
+            )
+        return host
+
+    @staticmethod
+    def _cancel_maxsize(
+        size: int, limit: int, request: Request, *, expected: bool
+    ) -> NoReturn:
+        warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
+        logger.warning(warning_msg)
+        raise DownloadCancelledError(warning_msg)
+
+    @staticmethod
+    def _extract_proxy(request: Request) -> tuple[str | None, str | None]:
+        """Return a tuple of the proxy URL with a scheme and the value of the
+        Proxy-Authorization header.
+
+        This is useful for handlers that take the proxy headers separately.
+        """
+        proxy: str | None = request.meta.get("proxy")
+        if not proxy:
+            return None, None
+        proxy = add_http_if_no_scheme(proxy)
+        auth_header: list[bytes] | None = request.headers.pop(
+            b"Proxy-Authorization", None
+        )
+        return proxy, auth_header[0].decode("ascii") if auth_header else None
+
+    def _extract_proxy_url_with_creds(self, request: Request) -> str | None:
+        """Return the proxy URL with the userinfo added based on the
+        Proxy-Authorization header.
+
+        This is useful for handlers that cannot take the proxy headers
+        separately.
+        """
+        proxy_url, auth_header = self._extract_proxy(request)
+        if proxy_url is None or auth_header is None:
+            return proxy_url
+        scheme, token = auth_header.split(" ", 1)
+        if scheme != "Basic":
+            raise ValueError(
+                f"Expected Basic auth in Proxy-Authorization, got {scheme}"
+            )
+        user, password = (
+            base64.b64decode(token).decode(self._proxy_auth_encoding).split(":", 1)
+        )
+        parts = urlsplit(proxy_url)
+        netloc = f"{quote(user)}:{quote(password)}@{parts.netloc}"
+        return parts._replace(netloc=netloc).geturl()
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index 313fb1c4f..43e1cd965 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -3,45 +3,34 @@
 from __future__ import annotations
 
 import ipaddress
-import logging
 import ssl
-import time
-from http.cookiejar import Cookie, CookieJar
-from io import BytesIO
-from typing import TYPE_CHECKING, Any, NoReturn, TypedDict
+from contextlib import asynccontextmanager
+from typing import TYPE_CHECKING, ClassVar
 
-from scrapy import Request, signals
 from scrapy.exceptions import (
     CannotResolveHostError,
-    DownloadCancelledError,
     DownloadConnectionRefusedError,
     DownloadFailedError,
     DownloadTimeoutError,
     NotConfigured,
-    ResponseDataLossError,
     UnsupportedURLSchemeError,
 )
-from scrapy.http import Headers, Response
-from scrapy.utils._download_handlers import (
-    BaseHttpDownloadHandler,
-    check_stop_download,
-    get_dataloss_msg,
-    get_maxsize_msg,
-    get_warnsize_msg,
-    make_response,
-    normalize_bind_address,
+from scrapy.http import Headers
+from scrapy.utils._download_handlers import NullCookieJar
+from scrapy.utils.ssl import (
+    _log_sslobj_debug_info,
+    _make_insecure_ssl_ctx,
+    _make_ssl_context,
 )
-from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.ssl import _log_sslobj_debug_info, _make_ssl_context
+
+from ._base_streaming import BaseStreamingDownloadHandler, _BaseResponseArgs
 
 if TYPE_CHECKING:
-    from contextlib import AbstractAsyncContextManager
-    from http.client import HTTPResponse
-    from ipaddress import IPv4Address, IPv6Address
-    from urllib.request import Request as ULRequest
+    from collections.abc import AsyncIterator
 
     from httpcore import AsyncNetworkStream
 
+    from scrapy import Request
     from scrapy.crawler import Crawler
 
 
@@ -50,89 +39,90 @@ try:
 except ImportError:
     httpx = None  # type: ignore[assignment]
 
-logger = logging.getLogger(__name__)
+
+if TYPE_CHECKING:
+    _Base = BaseStreamingDownloadHandler[httpx.Response]
+else:
+    _Base = BaseStreamingDownloadHandler
 
 
-class _BaseResponseArgs(TypedDict):
-    status: int
-    url: str
-    headers: Headers
-    ip_address: IPv4Address | IPv6Address
-    protocol: str
-
-
-# workaround for (and from) https://github.com/encode/httpx/issues/2992
-class _NullCookieJar(CookieJar):  # pragma: no cover
-    """A CookieJar that rejects all cookies."""
-
-    def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
-        pass
-
-    def set_cookie(self, cookie: Cookie) -> None:
-        pass
-
-
-class HttpxDownloadHandler(BaseHttpDownloadHandler):
-    _DEFAULT_CONNECT_TIMEOUT = 10
+class HttpxDownloadHandler(_Base):
+    experimental: ClassVar[bool] = True
 
     def __init__(self, crawler: Crawler):
-        # we skip HttpxDownloadHandler tests with the non-asyncio reactor
-        if not is_asyncio_available():  # pragma: no cover
-            raise NotConfigured(
-                f"{type(self).__name__} requires the asyncio support. Make"
-                f" sure that you have either enabled the asyncio Twisted"
-                f" reactor in the TWISTED_REACTOR setting or disabled the"
-                f" TWISTED_REACTOR_ENABLED setting. See the asyncio"
-                f" documentation of Scrapy for more information."
-            )
+        super().__init__(crawler)
+        self._verify_certificates: bool = crawler.settings.getbool(
+            "DOWNLOAD_VERIFY_CERTIFICATES"
+        )
+        self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
+        self._bind_host: str | None = self._get_bind_address_host()
+        self._limits: httpx.Limits = httpx.Limits(
+            # hard limit on simultaneous connections
+            max_connections=self._pool_size_total,
+            # total number of idle connections in the pool (extra ones are closed)
+            max_keepalive_connections=self._pool_size_total,
+        )
+
+        self._default_client: httpx.AsyncClient = self._make_client()
+        # httpx doesn't support per-request proxies: https://github.com/encode/httpx/discussions/3183,
+        # so we keep a pool of clients per proxy URL. LRU eviction can be added here if needed.
+        self._proxy_clients: dict[str, httpx.AsyncClient] = {}
+
+    @staticmethod
+    def _check_deps_installed() -> None:
         if httpx is None:  # pragma: no cover
             raise NotConfigured(
-                f"{type(self).__name__} requires the httpx library to be installed."
+                "HttpxDownloadHandler requires the httpx library to be installed."
             )
-        super().__init__(crawler)
-        logger.warning(
-            "HttpxDownloadHandler is experimental and is not recommended for production use."
-        )
-        bind_address = crawler.settings.get("DOWNLOAD_BIND_ADDRESS")
-        bind_address = normalize_bind_address(bind_address)
 
-        self._bind_address: str | None = None
+    def _make_client(self, proxy_url: str | None = None) -> httpx.AsyncClient:
+        if proxy_url:
+            if proxy_url.startswith("https:") and not self._verify_certificates:
+                proxy_ssl_context = _make_insecure_ssl_ctx()
+            else:
+                proxy_ssl_context = None
+            proxy = httpx.Proxy(proxy_url, ssl_context=proxy_ssl_context)
+        else:
+            proxy = None
 
-        if bind_address is not None:
-            host, port = bind_address
-            if port != 0:
-                logger.warning(
-                    "DOWNLOAD_BIND_ADDRESS specifies a port (%s), but %s does not "
-                    "support binding to a specific local port. Ignoring the port "
-                    "and binding only to %r.",
-                    port,
-                    type(self).__name__,
-                    host,
-                )
-            self._bind_address = host
-
-        self._client = httpx.AsyncClient(
-            cookies=_NullCookieJar(),
+        client = httpx.AsyncClient(
+            cookies=NullCookieJar(),
             transport=httpx.AsyncHTTPTransport(
-                verify=_make_ssl_context(crawler.settings),
-                local_address=self._bind_address,
+                verify=self._ssl_context,
+                local_address=self._bind_host,
+                limits=self._limits,
+                trust_env=False,
+                proxy=proxy,
             ),
         )
         # https://github.com/encode/httpx/discussions/1566
         for header_name in ("accept", "accept-encoding", "user-agent"):
-            self._client.headers.pop(header_name, None)
+            client.headers.pop(header_name, None)
+        return client
 
-    async def download_request(self, request: Request) -> Response:
-        self._warn_unsupported_meta(request.meta)
+    def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
+        if proxy_url is None:
+            return self._default_client
+        if cached := self._proxy_clients.get(proxy_url):
+            return cached
+        client = self._make_client(proxy_url)
+        self._proxy_clients[proxy_url] = client
+        return client
 
-        timeout: float = request.meta.get(
-            "download_timeout", self._DEFAULT_CONNECT_TIMEOUT
-        )
-        start_time = time.monotonic()
+    @asynccontextmanager
+    async def _make_request(
+        self, request: Request, timeout: float
+    ) -> AsyncIterator[httpx.Response]:
+        client = self._get_client(self._extract_proxy_url_with_creds(request))
         try:
-            async with self._get_httpx_response(request, timeout) as httpx_response:
-                request.meta["download_latency"] = time.monotonic() - start_time
-                return await self._read_response(httpx_response, request)
+            async with client.stream(
+                request.method,
+                request.url,
+                content=request.body,
+                headers=request.headers.to_tuple_list(),
+                timeout=timeout,
+            ) as response:
+                yield response
         except httpx.TimeoutException as e:
             raise DownloadTimeoutError(
                 f"Getting {request.url} took longer than {timeout} seconds."
@@ -149,159 +139,55 @@ class HttpxDownloadHandler(BaseHttpDownloadHandler):
             ):
                 raise CannotResolveHostError(error_message) from e
             raise DownloadConnectionRefusedError(str(e)) from e
-        except httpx.NetworkError as e:
+        except httpx.ProxyError as e:
+            raise DownloadConnectionRefusedError(str(e)) from e
+        except (httpx.NetworkError, httpx.RemoteProtocolError) as e:
             raise DownloadFailedError(str(e)) from e
-        except httpx.RemoteProtocolError as e:
-            raise DownloadFailedError(str(e)) from e
-
-    def _warn_unsupported_meta(self, meta: dict[str, Any]) -> None:
-        if meta.get("bindaddress"):
-            # configurable only per-client:
-            # https://github.com/encode/httpx/issues/755#issuecomment-2746121794
-            logger.error(
-                f"The 'bindaddress' request meta key is not supported by"
-                f" {type(self).__name__} and will be ignored."
-            )
-        if meta.get("proxy"):
-            # configurable only per-client:
-            # https://github.com/encode/httpx/issues/486
-            logger.error(
-                f"The 'proxy' request meta key is not supported by"
-                f" {type(self).__name__} and will be ignored."
-            )
-
-    def _get_httpx_response(
-        self, request: Request, timeout: float
-    ) -> AbstractAsyncContextManager[httpx.Response]:
-        return self._client.stream(
-            request.method,
-            request.url,
-            content=request.body,
-            headers=request.headers.to_tuple_list(),
-            timeout=timeout,
-        )
-
-    async def _read_response(
-        self, httpx_response: httpx.Response, request: Request
-    ) -> Response:
-        maxsize: int = request.meta.get("download_maxsize", self._default_maxsize)
-        warnsize: int = request.meta.get("download_warnsize", self._default_warnsize)
-
-        content_length = httpx_response.headers.get("Content-Length")
-        expected_size = int(content_length) if content_length is not None else None
-        if maxsize and expected_size and expected_size > maxsize:
-            self._cancel_maxsize(expected_size, maxsize, request, expected=True)
-
-        reached_warnsize = False
-        if warnsize and expected_size and expected_size > warnsize:
-            reached_warnsize = True
-            logger.warning(
-                get_warnsize_msg(expected_size, warnsize, request, expected=True)
-            )
-
-        headers = Headers(httpx_response.headers.multi_items())
-        network_stream: AsyncNetworkStream = httpx_response.extensions["network_stream"]
-
-        make_response_base_args: _BaseResponseArgs = {
-            "status": httpx_response.status_code,
-            "url": request.url,
-            "headers": headers,
-            "ip_address": self._get_server_ip(network_stream),
-            "protocol": httpx_response.http_version,
-        }
-
-        self._log_tls_info(network_stream)
-
-        if stop_download := check_stop_download(
-            signals.headers_received,
-            self.crawler,
-            request,
-            headers=headers,
-            body_length=expected_size,
-        ):
-            return make_response(
-                **make_response_base_args,
-                stop_download=stop_download,
-            )
-
-        response_body = BytesIO()
-        bytes_received = 0
-        try:
-            async for chunk in httpx_response.aiter_raw():
-                response_body.write(chunk)
-                bytes_received += len(chunk)
-
-                if stop_download := check_stop_download(
-                    signals.bytes_received, self.crawler, request, data=chunk
-                ):
-                    return make_response(
-                        **make_response_base_args,
-                        body=response_body.getvalue(),
-                        stop_download=stop_download,
-                    )
-
-                if maxsize and bytes_received > maxsize:
-                    response_body.truncate(0)
-                    self._cancel_maxsize(
-                        bytes_received, maxsize, request, expected=False
-                    )
-
-                if warnsize and bytes_received > warnsize and not reached_warnsize:
-                    reached_warnsize = True
-                    logger.warning(
-                        get_warnsize_msg(
-                            bytes_received, warnsize, request, expected=False
-                        )
-                    )
-        except httpx.RemoteProtocolError as e:
-            # special handling of the dataloss case
-            if (
-                "peer closed connection without sending complete message body"
-                not in str(e)
-            ):
-                raise
-            fail_on_dataloss: bool = request.meta.get(
-                "download_fail_on_dataloss", self._fail_on_dataloss
-            )
-            if not fail_on_dataloss:
-                return make_response(
-                    **make_response_base_args,
-                    body=response_body.getvalue(),
-                    flags=["dataloss"],
-                )
-            self._log_dataloss_warning(request.url)
-            raise ResponseDataLossError(str(e)) from e
-
-        return make_response(
-            **make_response_base_args,
-            body=response_body.getvalue(),
-        )
 
     @staticmethod
-    def _get_server_ip(network_stream: AsyncNetworkStream) -> IPv4Address | IPv6Address:
-        extra_server_addr = network_stream.get_extra_info("server_addr")
-        return ipaddress.ip_address(extra_server_addr[0])
+    def _extract_headers(response: httpx.Response) -> Headers:
+        return Headers(response.headers.multi_items())
 
-    def _log_tls_info(self, network_stream: AsyncNetworkStream) -> None:
-        if not self._tls_verbose_logging:
-            return
+    @staticmethod
+    def _build_base_response_args(
+        response: httpx.Response,
+        request: Request,
+        headers: Headers,
+    ) -> _BaseResponseArgs:
+        network_stream: AsyncNetworkStream = response.extensions["network_stream"]
+        server_addr = network_stream.get_extra_info("server_addr")
+        ip_address = ipaddress.ip_address(server_addr[0])
+        ssl_object = network_stream.get_extra_info("ssl_object")
+        if isinstance(ssl_object, ssl.SSLObject):
+            cert = ssl_object.getpeercert(binary_form=True)
+        else:
+            cert = None
+        return {
+            "status": response.status_code,
+            "url": request.url,
+            "headers": headers,
+            "certificate": cert,
+            "ip_address": ip_address,
+            "protocol": response.http_version,
+        }
+
+    @staticmethod
+    def _iter_body_chunks(response: httpx.Response) -> AsyncIterator[bytes]:
+        return response.aiter_raw()
+
+    @staticmethod
+    def _is_dataloss_exception(exc: Exception) -> bool:
+        return isinstance(
+            exc, httpx.RemoteProtocolError
+        ) and "peer closed connection without sending complete message body" in str(exc)
+
+    def _log_tls_info(self, response: httpx.Response, request: Request) -> None:
+        network_stream: AsyncNetworkStream = response.extensions["network_stream"]
         extra_ssl_object = network_stream.get_extra_info("ssl_object")
         if isinstance(extra_ssl_object, ssl.SSLObject):
             _log_sslobj_debug_info(extra_ssl_object)
 
-    def _log_dataloss_warning(self, url: str) -> None:
-        if self._fail_on_dataloss_warned:
-            return
-        logger.warning(get_dataloss_msg(url))
-        self._fail_on_dataloss_warned = True
-
-    @staticmethod
-    def _cancel_maxsize(
-        size: int, limit: int, request: Request, *, expected: bool
-    ) -> NoReturn:
-        warning_msg = get_maxsize_msg(size, limit, request, expected=expected)
-        logger.warning(warning_msg)
-        raise DownloadCancelledError(warning_msg)
-
     async def close(self) -> None:
-        await self._client.aclose()
+        await self._default_client.aclose()
+        for client in self._proxy_clients.values():
+            await client.aclose()
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index fbe24e52e..85a5ae0d7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -41,7 +41,6 @@ from scrapy.exceptions import (
 )
 from scrapy.http import Headers, Response
 from scrapy.utils._download_handlers import (
-    BaseHttpDownloadHandler,
     check_stop_download,
     get_dataloss_msg,
     get_maxsize_msg,
@@ -57,6 +56,8 @@ from scrapy.utils.python import to_bytes, to_unicode
 from scrapy.utils.ssl import _log_ssl_conn_debug_info
 from scrapy.utils.url import add_http_if_no_scheme
 
+from ._base_http import BaseHttpDownloadHandler
+
 if TYPE_CHECKING:
     from twisted.internet.base import ReactorBase
     from twisted.internet.interfaces import IConsumer
diff --git a/scrapy/http/response/__init__.py b/scrapy/http/response/__init__.py
index a80ea3da8..09b1c8b32 100644
--- a/scrapy/http/response/__init__.py
+++ b/scrapy/http/response/__init__.py
@@ -20,7 +20,6 @@ if TYPE_CHECKING:
     from collections.abc import Callable, Iterable, Mapping
     from ipaddress import IPv4Address, IPv6Address
 
-    from twisted.internet.ssl import Certificate
     from twisted.python.failure import Failure
 
     # typing.Self requires Python 3.11
@@ -77,7 +76,7 @@ class Response(object_ref):
         body: bytes = b"",
         flags: list[str] | None = None,
         request: Request | None = None,
-        certificate: Certificate | None = None,
+        certificate: Any = None,
         ip_address: IPv4Address | IPv6Address | None = None,
         protocol: str | None = None,
     ):
@@ -87,7 +86,7 @@ class Response(object_ref):
         self._set_url(url)
         self.request: Request | None = request
         self._flags: list[str] | None = list(flags) if flags else None
-        self.certificate: Certificate | None = certificate
+        self.certificate: Any = certificate
         self.ip_address: IPv4Address | IPv6Address | None = ip_address
         self.protocol: str | None = protocol
 
diff --git a/scrapy/utils/_download_handlers.py b/scrapy/utils/_download_handlers.py
index 9538dd81e..23b715ac6 100644
--- a/scrapy/utils/_download_handlers.py
+++ b/scrapy/utils/_download_handlers.py
@@ -2,8 +2,8 @@
 
 from __future__ import annotations
 
-from abc import ABC
 from contextlib import contextmanager
+from http.cookiejar import CookieJar
 from typing import TYPE_CHECKING, Any
 
 from twisted.internet.defer import CancelledError
@@ -15,7 +15,6 @@ from twisted.web.client import ResponseFailed
 from twisted.web.error import SchemeNotSupported
 
 from scrapy import responsetypes
-from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.exceptions import (
     CannotResolveHostError,
     DownloadCancelledError,
@@ -29,29 +28,24 @@ from scrapy.utils.log import logger
 
 if TYPE_CHECKING:
     from collections.abc import Iterator
+    from http.client import HTTPResponse
+    from http.cookiejar import Cookie
     from ipaddress import IPv4Address, IPv6Address
-
-    from twisted.internet.ssl import Certificate
+    from urllib.request import Request as ULRequest
 
     from scrapy import Request
     from scrapy.crawler import Crawler
     from scrapy.http import Headers, Response
 
 
-class BaseHttpDownloadHandler(BaseDownloadHandler, ABC):
-    """Base class for built-in HTTP download handlers."""
+class NullCookieJar(CookieJar):  # pragma: no cover
+    """A CookieJar that rejects all cookies."""
 
-    def __init__(self, crawler: Crawler):
-        super().__init__(crawler)
-        self._default_maxsize: int = crawler.settings.getint("DOWNLOAD_MAXSIZE")
-        self._default_warnsize: int = crawler.settings.getint("DOWNLOAD_WARNSIZE")
-        self._fail_on_dataloss: bool = crawler.settings.getbool(
-            "DOWNLOAD_FAIL_ON_DATALOSS"
-        )
-        self._tls_verbose_logging: bool = crawler.settings.getbool(
-            "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
-        )
-        self._fail_on_dataloss_warned: bool = False
+    def extract_cookies(self, response: HTTPResponse, request: ULRequest) -> None:
+        pass
+
+    def set_cookie(self, cookie: Cookie) -> None:
+        pass
 
 
 @contextmanager
@@ -103,7 +97,7 @@ def make_response(
     headers: Headers,
     body: bytes = b"",
     flags: list[str] | None = None,
-    certificate: Certificate | None = None,
+    certificate: Any = None,
     ip_address: IPv4Address | IPv6Address | None = None,
     protocol: str | None = None,
     stop_download: StopDownload | None = None,
diff --git a/scrapy/utils/ssl.py b/scrapy/utils/ssl.py
index a8b9cd225..3fa2c77ba 100644
--- a/scrapy/utils/ssl.py
+++ b/scrapy/utils/ssl.py
@@ -54,6 +54,18 @@ def _make_ssl_context(settings: BaseSettings) -> ssl.SSLContext:
     return ctx
 
 
+def _make_insecure_ssl_ctx() -> ssl.SSLContext:
+    """Create an SSL context that doesn't verify certificates.
+
+    Compared to :func:`~scrapy.utils.ssl._make_ssl_context` this is much more
+    simple.
+    """
+    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    return ctx
+
+
 def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
     cipher = sslobj.cipher()
     logger.debug(
@@ -61,8 +73,11 @@ def _log_sslobj_debug_info(sslobj: ssl.SSLObject) -> None:
         f" using protocol {sslobj.version()},"
         f" cipher {cipher[0] if cipher else None}"
     )
-    # The peer certificate is unavailable on SSLObject unless peer
-    # certificate verification is enabled, which we don't want.
+    if cert := sslobj.getpeercert():
+        # Not available without certificate verification
+        logger.debug(
+            f'SSL connection certificate: issuer "{cert["issuer"]}", subject "{cert["subject"]}"'
+        )
 
 
 # pyOpenSSL utils
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 811ce1d18..52abb1ccc 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode, urlparse
 
 import pytest
+from cryptography.x509 import load_der_x509_certificate
 from testfixtures import LogCapture
 from twisted.internet.defer import succeed
 from twisted.internet.ssl import Certificate
@@ -641,11 +642,6 @@ class TestCrawlSpider:
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         assert crawler.spider.meta["responses"][0].certificate is None
 
-    @pytest.mark.xfail(
-        'config.getoption("--reactor") == "none"',
-        reason="Not implemented in HttpxDownloadHandler",
-        strict=True,
-    )
     @pytest.mark.parametrize(
         "url",
         [
@@ -669,9 +665,14 @@ class TestCrawlSpider:
         await crawler.crawl_async(seed=url, mockserver=mockserver)
         assert isinstance(crawler.spider, SingleRequestSpider)
         cert = crawler.spider.meta["responses"][0].certificate
-        assert isinstance(cert, Certificate)
-        assert cert.getSubject().commonName == b"localhost"
-        assert cert.getIssuer().commonName == b"localhost"
+        assert cert is not None
+        if isinstance(cert, Certificate):  # Twisted
+            assert cert.getSubject().commonName == b"localhost"
+            assert cert.getIssuer().commonName == b"localhost"
+        elif isinstance(cert, bytes):  # DER bytes
+            cert_x509 = load_der_x509_certificate(cert)
+            assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
+            assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
 
     @pytest.mark.parametrize(
         "url",
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index bc1acf9f9..f26a17f02 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -36,7 +36,6 @@ pytest.importorskip("httpx")
 class HttpxDownloadHandlerMixin:
     @property
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
-        # the import will fail if httpx is not installed
         from scrapy.core.downloader.handlers._httpx import (  # noqa: PLC0415
             HttpxDownloadHandler,
         )
@@ -73,27 +72,13 @@ class TestHttp(HttpxDownloadHandlerMixin, TestHttpBase):
         assert "DOWNLOAD_BIND_ADDRESS specifies a port (12345)" in caplog.text
         assert "Ignoring the port" in caplog.text
 
-    @coroutine_test
-    async def test_unsupported_proxy(
-        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
-    ) -> None:
-        meta = {"proxy": "127.0.0.2"}
-        request = Request(mockserver.url("/text"), meta=meta)
-        async with self.get_dh() as download_handler:
-            response = await download_handler.download_request(request)
-        assert response.body == b"Works"
-        assert (
-            "The 'proxy' request meta key is not supported by HttpxDownloadHandler"
-            in caplog.text
-        )
-
 
 class TestHttps(HttpxDownloadHandlerMixin, TestHttpsBase):
     handler_supports_bindaddress_meta = False
     tls_log_message = "SSL connection to 127.0.0.1 using protocol TLSv1.3, cipher"
 
     @pytest.mark.skip(reason="The check is Twisted-specific")
-    def test_verify_certs_deprecated(self):
+    def test_verify_certs_deprecated(self) -> None:  # type: ignore[override]
         pass
 
 
@@ -126,23 +111,15 @@ class TestHttpWithCrawler(HttpxDownloadHandlerMixin, TestHttpWithCrawlerBase):
 class TestHttpsWithCrawler(TestHttpWithCrawler):
     is_secure = True
 
-    @pytest.mark.skip(reason="response.certificate is not implemented")
-    @coroutine_test
-    async def test_response_ssl_certificate(self, mockserver: MockServer) -> None:
-        pass
 
-
-@pytest.mark.skip(reason="Proxy support is not implemented yet")
 class TestHttpProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
-    pass
+    expected_http_proxy_request_body = b"http://example.com/"
 
 
-@pytest.mark.skip(reason="Proxy support is not implemented yet")
-class TestHttpsProxy(HttpxDownloadHandlerMixin, TestHttpProxyBase):
+class TestHttpsProxy(TestHttpProxy):
     is_secure = True
 
 
-@pytest.mark.skip(reason="Proxy support is not implemented yet")
 @pytest.mark.requires_mitmproxy
 class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
     pass
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index a8d002d30..51da8b38f 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -17,6 +17,7 @@ from typing import TYPE_CHECKING, Any, ClassVar
 from urllib.parse import urlparse
 
 import pytest
+from cryptography.x509 import load_der_x509_certificate
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
@@ -981,15 +982,19 @@ class TestHttpWithCrawlerBase(ABC):
         if not self.is_secure:
             pytest.skip("Only applies to HTTPS")
         # copy of TestCrawl.test_response_ssl_certificate()
-        # the current test implementation can only work for Twisted-based download handlers
         crawler = get_crawler(SingleRequestSpider, self.settings_dict)
         url = mockserver.url("/echo?body=test", is_secure=self.is_secure)
         await crawler.crawl_async(seed=url, mockserver=mockserver)
         assert isinstance(crawler.spider, SingleRequestSpider)
         cert = crawler.spider.meta["responses"][0].certificate
-        assert isinstance(cert, Certificate)
-        assert cert.getSubject().commonName == b"localhost"
-        assert cert.getIssuer().commonName == b"localhost"
+        assert cert is not None
+        if isinstance(cert, Certificate):  # Twisted
+            assert cert.getSubject().commonName == b"localhost"
+            assert cert.getIssuer().commonName == b"localhost"
+        elif isinstance(cert, bytes):  # DER bytes
+            cert_x509 = load_der_x509_certificate(cert)
+            assert cert_x509.subject.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
+            assert cert_x509.issuer.rfc4514_string() == "CN=localhost,O=Scrapy,C=IE"
 
     @coroutine_test
     async def test_response_ip_address(self, mockserver: MockServer) -> None:
@@ -1279,7 +1284,6 @@ class TestRealWebsiteBase(ABC):
         raise NotImplementedError
 
     @property
-    @abstractmethod
     def platform_cert_store_works(self) -> bool:
         """Whether valid certificates can be verified.
 
@@ -1331,3 +1335,22 @@ class TestRealWebsiteBase(ABC):
             response = await download_handler.download_request(request)
         assert response.status == 200
         assert "All products | Books to Scrape - Sandbox" in response.text
+
+    @pytest.mark.parametrize("verify_certs", [True, False])
+    @coroutine_test
+    async def test_tls_logging(
+        self, caplog: pytest.LogCaptureFixture, verify_certs: bool
+    ) -> None:
+        if verify_certs and not self.platform_cert_store_works:
+            pytest.skip("Cannot verify certificates")
+        request = Request("https://books.toscrape.com/")
+        async with self.get_dh(
+            {
+                "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING": True,
+                "DOWNLOAD_VERIFY_CERTIFICATES": verify_certs,
+            }
+        ) as download_handler:
+            with caplog.at_level("DEBUG"):
+                response = await download_handler.download_request(request)
+        assert response.status == 200
+        assert "SSL connection to books.toscrape.com using protocol" in caplog.text
diff --git a/tox.ini b/tox.ini
index 72686d3d0..da5a8a9a6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -113,7 +113,7 @@ deps =
     Twisted==21.7.0
     cryptography==37.0.0
     cssselect==0.9.1
-    httpx==0.26.0
+    httpx==0.27.1
     itemadapter==0.1.0
     lxml==4.6.4
     parsel==1.5.0
@@ -165,7 +165,7 @@ deps =
     brotli==1.2.0; implementation_name != "pypy"
     brotlicffi==1.2.0.0; implementation_name == "pypy"
     google-cloud-storage==1.29.0
-    httpx==0.26.0
+    httpx==0.27.1
     ipython==7.1.0
     robotexclusionrulesparser==1.6.2
     uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"

From 55c17a89858e72c2e85391b1db3bbb2ba6c77360 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 18 May 2026 18:54:06 +0500
Subject: [PATCH 28/36] Disable mypy allow_any_generics. (#7532)

---
 pyproject.toml                                |  1 -
 scrapy/commands/genspider.py                  |  2 +-
 scrapy/contracts/__init__.py                  | 12 +++++-----
 scrapy/core/downloader/handlers/s3.py         |  6 +++--
 scrapy/core/downloader/middleware.py          | 10 ++++-----
 scrapy/core/spidermw.py                       | 14 +++++++-----
 scrapy/extensions/httpcache.py                |  4 +++-
 scrapy/http/response/text.py                  |  2 +-
 scrapy/middleware.py                          |  6 ++---
 scrapy/spiders/crawl.py                       |  4 +++-
 scrapy/squeues.py                             |  2 +-
 scrapy/utils/asyncio.py                       |  2 +-
 scrapy/utils/conf.py                          |  4 ++--
 scrapy/utils/datatypes.py                     | 22 +++++++++----------
 scrapy/utils/defer.py                         |  2 +-
 scrapy/utils/log.py                           |  3 ++-
 scrapy/utils/misc.py                          |  6 +++--
 scrapy/utils/reactor.py                       |  2 +-
 scrapy/utils/signal.py                        |  4 +++-
 scrapy/utils/template.py                      |  2 +-
 scrapy/utils/trackref.py                      |  4 +++-
 .../reactorless_custom_settings.py            |  2 +-
 .../twisted_reactor_custom_settings_select.py |  2 +-
 tests/mocks/dummydbm.py                       |  2 +-
 tests/spiders.py                              | 13 ++++++-----
 tests/test_command_genspider.py               |  2 +-
 tests/test_command_shell.py                   |  5 ++---
 tests/test_command_startproject.py            |  2 +-
 tests/test_feedexport.py                      |  6 +++--
 tests/test_feedexport_batch.py                |  2 +-
 tests/test_http2_client_protocol.py           |  2 +-
 tests/test_loader.py                          |  2 +-
 tests/test_pipeline_files.py                  |  8 +++----
 tests/test_pipeline_images.py                 |  8 +++----
 tests/test_scheduler_base.py                  |  4 ++--
 tests/test_spidermiddleware.py                |  2 +-
 tests/test_utils_datatypes.py                 |  5 +++--
 tests/test_utils_log.py                       |  4 +++-
 tests/test_utils_request.py                   |  5 +++--
 tox.ini                                       | 10 ++++-----
 40 files changed, 112 insertions(+), 88 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 76b8bbead..2f07dcd80 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -86,7 +86,6 @@ pattern = "^(?P<version>.+)$"
 
 [tool.mypy]
 strict = true
-allow_any_generics = true # 67 errors
 extra_checks = false # weird addErrback() errors
 untyped_calls_exclude = [
     "twisted",
diff --git a/scrapy/commands/genspider.py b/scrapy/commands/genspider.py
index 17bcf19b0..cc8624fa1 100644
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@@ -147,7 +147,7 @@ class Command(ScrapyCommand):
         name: str,
         url: str,
         template_name: str,
-        template_file: str | os.PathLike,
+        template_file: str | os.PathLike[str],
     ) -> None:
         """Generate the spider module, based on the given template"""
         assert self.settings is not None
diff --git a/scrapy/contracts/__init__.py b/scrapy/contracts/__init__.py
index 24f56b7a6..c0da7dfa4 100644
--- a/scrapy/contracts/__init__.py
+++ b/scrapy/contracts/__init__.py
@@ -27,7 +27,7 @@ class Contract:
     request_cls: type[Request] | None = None
     name: str
 
-    def __init__(self, method: Callable, *args: Any):
+    def __init__(self, method: Callable[..., Any], *args: Any):
         self.testcase_pre = _create_testcase(method, f"@{self.name} pre-hook")
         self.testcase_post = _create_testcase(method, f"@{self.name} post-hook")
         self.args: tuple[Any, ...] = args
@@ -105,7 +105,7 @@ class ContractsManager:
 
         return methods
 
-    def extract_contracts(self, method: Callable) -> list[Contract]:
+    def extract_contracts(self, method: Callable[..., Any]) -> list[Contract]:
         contracts: list[Contract] = []
         assert method.__doc__ is not None
         for line_ in method.__doc__.split("\n"):
@@ -134,7 +134,9 @@ class ContractsManager:
 
         return requests
 
-    def from_method(self, method: Callable, results: TestResult) -> Request | None:
+    def from_method(
+        self, method: Callable[..., Any], results: TestResult
+    ) -> Request | None:
         contracts = self.extract_contracts(method)
         if contracts:
             request_cls = Request
@@ -170,7 +172,7 @@ class ContractsManager:
         return None
 
     def _clean_req(
-        self, request: Request, method: Callable, results: TestResult
+        self, request: Request, method: Callable[..., Any], results: TestResult
     ) -> None:
         """stop the request from returning objects and records any errors"""
 
@@ -195,7 +197,7 @@ class ContractsManager:
         request.errback = eb_wrapper
 
 
-def _create_testcase(method: Callable, desc: str) -> TestCase:
+def _create_testcase(method: Callable[..., Any], desc: str) -> TestCase:
     spider = method.__self__.name  # type: ignore[attr-defined]
 
     class ContractTestCase(TestCase):
diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.py
index 19a1e8503..c6660ff48 100644
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, cast
 
 from scrapy.core.downloader.handlers.base import BaseDownloadHandler
 from scrapy.exceptions import NotConfigured
@@ -9,6 +9,8 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping
+
     from scrapy import Request
     from scrapy.crawler import Crawler
     from scrapy.http import Response
@@ -59,7 +61,7 @@ class S3DownloadHandler(BaseDownloadHandler):
             awsrequest = botocore.awsrequest.AWSRequest(
                 method=request.method,
                 url=f"{scheme}://s3.amazonaws.com/{bucket}{path}",
-                headers=request.headers.to_unicode_dict(),
+                headers=cast("Mapping[str, Any]", request.headers.to_unicode_dict()),
                 data=request.body,
             )
             assert self._signer
diff --git a/scrapy/core/downloader/middleware.py b/scrapy/core/downloader/middleware.py
index 6685d90af..ab74e22a4 100644
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@@ -8,7 +8,7 @@ from __future__ import annotations
 
 import warnings
 from functools import wraps
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any
 
 from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
 from scrapy.http import Request, Response
@@ -87,7 +87,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             result = await self._process_exception(ex, request)
         return await self._process_response(result, request)
 
-    def _handle_mw_method(self, method: Callable, **kwargs: Any) -> Any:
+    def _handle_mw_method(self, method: Callable[..., Any], **kwargs: Any) -> Any:
         if method in self._mw_methods_requiring_spider:
             kwargs["spider"] = self._spider
 
@@ -99,7 +99,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         download_func: Callable[[Request], Coroutine[Any, Any, Response]],
     ) -> Response | Request:
         for method in self.methods["process_request"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request),
                 _warn=global_object_name(method),
@@ -122,7 +122,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
             return response
 
         for method in self.methods["process_response"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request, response=response),
                 _warn=global_object_name(method),
@@ -141,7 +141,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
         self, exception: Exception, request: Request | Response
     ) -> Response | Request:
         for method in self.methods["process_exception"]:
-            method = cast("Callable", method)
+            assert method is not None
             response = await ensure_awaitable(
                 self._handle_mw_method(method, request=request, exception=exception),
                 _warn=global_object_name(method),
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index bcf38bb75..fbc6f2530 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -11,7 +11,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from functools import wraps
 from inspect import isasyncgenfunction
 from itertools import islice
-from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar
 from warnings import warn
 
 from twisted.python.failure import Failure
@@ -79,7 +79,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
         request: Request,
     ) -> Iterable[_T] | AsyncIterator[_T]:
         for method in self.methods["process_spider_input"]:
-            method = cast("Callable", method)
+            assert method is not None
             try:
                 if method in self._mw_methods_requiring_spider:
                     result = method(response=response, spider=self._spider)
@@ -248,9 +248,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     # This method is only needed until _async compatibility methods are removed.
     @staticmethod
-    def _get_process_spider_output(mw: Any) -> Callable | None:
-        normal_method: Callable | None = getattr(mw, "process_spider_output", None)
-        async_method: Callable | None = getattr(mw, "process_spider_output_async", None)
+    def _get_process_spider_output(mw: Any) -> Callable[..., Any] | None:
+        normal_method: Callable[..., Any] | None = getattr(
+            mw, "process_spider_output", None
+        )
+        async_method: Callable[..., Any] | None = getattr(
+            mw, "process_spider_output_async", None
+        )
         if not async_method:
             if normal_method and not isasyncgenfunction(normal_method):
                 raise TypeError(
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index 86b066a38..dbb79b02d 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -313,7 +313,9 @@ class FilesystemCacheStorage:
         self.expiration_secs: int = settings.getint("HTTPCACHE_EXPIRATION_SECS")
         self.use_gzip: bool = settings.getbool("HTTPCACHE_GZIP")
         # https://github.com/python/mypy/issues/10740
-        self._open: Callable[Concatenate[str | os.PathLike, str, ...], IO[bytes]] = (
+        self._open: Callable[
+            Concatenate[str | os.PathLike[str], str, ...], IO[bytes]
+        ] = (
             gzip.open if self.use_gzip else open  # type: ignore[assignment]
         )
 
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 13853f64d..6876e35e8 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -220,7 +220,7 @@ class TextResponse(Response):
 
     def follow_all(
         self,
-        urls: Iterable[str | Link] | parsel.SelectorList | None = None,
+        urls: Iterable[str | Link] | parsel.SelectorList[Any] | None = None,
         callback: CallbackT | None = None,
         method: str = "GET",
         headers: Mapping[AnyStr, Any] | Iterable[tuple[AnyStr, Any]] | None = None,
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 14b591a5f..37ec0f583 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -50,8 +50,8 @@ class MiddlewareManager(ABC):
             )
         self.middlewares: tuple[Any, ...] = middlewares
         # Only process_spider_output and process_spider_exception can be None.
-        self.methods: dict[str, deque[Callable | None]] = defaultdict(deque)
-        self._mw_methods_requiring_spider: set[Callable] = set()
+        self.methods: dict[str, deque[Callable[..., Any] | None]] = defaultdict(deque)
+        self._mw_methods_requiring_spider: set[Callable[..., Any]] = set()
         for mw in middlewares:
             self._add_middleware(mw)
 
@@ -116,7 +116,7 @@ class MiddlewareManager(ABC):
     def _add_middleware(self, mw: Any) -> None:  # noqa: B027
         pass
 
-    def _check_mw_method_spider_arg(self, method: Callable) -> None:
+    def _check_mw_method_spider_arg(self, method: Callable[..., Any]) -> None:
         if argument_is_required(method, "spider"):
             warnings.warn(
                 f"{method.__qualname__}() requires a spider argument,"
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index f0d093c6e..e14279b64 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -46,7 +46,9 @@ def _identity_process_request(request: Request, response: Response) -> Request |
     return request
 
 
-def _get_method(method: Callable | str | None, spider: Spider) -> Callable | None:
+def _get_method(
+    method: Callable[..., Any] | str | None, spider: Spider
+) -> Callable[..., Any] | None:
     if callable(method):
         return method
     if isinstance(method, str):
diff --git a/scrapy/squeues.py b/scrapy/squeues.py
index 7007cd4b8..e06e38e23 100644
--- a/scrapy/squeues.py
+++ b/scrapy/squeues.py
@@ -26,7 +26,7 @@ if TYPE_CHECKING:
 
 def _with_mkdir(queue_class: type[queue.BaseQueue]) -> type[queue.BaseQueue]:
     class DirectoriesCreated(queue_class):  # type: ignore[valid-type,misc]
-        def __init__(self, path: str | PathLike, *args: Any, **kwargs: Any):
+        def __init__(self, path: str | PathLike[str], *args: Any, **kwargs: Any):
             dirname = Path(path).parent
             if not dirname.exists():
                 dirname.mkdir(parents=True, exist_ok=True)
diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py
index ecb4c1492..44604c0fe 100644
--- a/scrapy/utils/asyncio.py
+++ b/scrapy/utils/asyncio.py
@@ -148,7 +148,7 @@ class AsyncioLoopingCall:
         self._func: Callable[_P, _T] = func
         self._args: tuple[Any, ...] = args
         self._kwargs: dict[str, Any] = kwargs
-        self._task: asyncio.Task | None = None
+        self._task: asyncio.Task[None] | None = None
         self.interval: float | None = None
         self._start_time: float | None = None
 
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5869cf52e..4850b370b 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -71,8 +71,8 @@ def arglist_to_dict(arglist: list[str]) -> dict[str, str]:
 
 
 def closest_scrapy_cfg(
-    path: str | os.PathLike = ".",
-    prevpath: str | os.PathLike | None = None,
+    path: str | os.PathLike[str] = ".",
+    prevpath: str | os.PathLike[str] | None = None,
 ) -> str:
     """Return the path to the closest scrapy.cfg file by traversing the current
     directory and its parents
diff --git a/scrapy/utils/datatypes.py b/scrapy/utils/datatypes.py
index 146e3ae56..4e65c062e 100644
--- a/scrapy/utils/datatypes.py
+++ b/scrapy/utils/datatypes.py
@@ -28,7 +28,7 @@ _KT = TypeVar("_KT")
 _VT = TypeVar("_VT")
 
 
-class CaselessDict(dict):
+class CaselessDict(dict):  # type: ignore[type-arg]
     __slots__ = ()
 
     def __new__(cls, *args: Any, **kwargs: Any) -> Self:
@@ -99,20 +99,20 @@ class CaselessDict(dict):
         return dict.pop(self, self.normkey(key), *args)
 
 
-class CaseInsensitiveDict(collections.UserDict):
+class CaseInsensitiveDict(collections.UserDict[str | bytes, Any]):
     """A dict-like structure that accepts strings or bytes
     as keys and allows case-insensitive lookups.
     """
 
     def __init__(self, *args: Any, **kwargs: Any) -> None:
-        self._keys: dict = {}
+        self._keys: dict[str | bytes, Any] = {}
         super().__init__(*args, **kwargs)
 
-    def __getitem__(self, key: AnyStr) -> Any:
+    def __getitem__(self, key: str | bytes) -> Any:
         normalized_key = self._normkey(key)
         return super().__getitem__(self._keys[normalized_key.lower()])
 
-    def __setitem__(self, key: AnyStr, value: Any) -> None:
+    def __setitem__(self, key: str | bytes, value: Any) -> None:
         normalized_key = self._normkey(key)
         try:
             lower_key = self._keys[normalized_key.lower()]
@@ -122,19 +122,19 @@ class CaseInsensitiveDict(collections.UserDict):
         super().__setitem__(normalized_key, self._normvalue(value))
         self._keys[normalized_key.lower()] = normalized_key
 
-    def __delitem__(self, key: AnyStr) -> None:
+    def __delitem__(self, key: str | bytes) -> None:
         normalized_key = self._normkey(key)
         stored_key = self._keys.pop(normalized_key.lower())
         super().__delitem__(stored_key)
 
-    def __contains__(self, key: AnyStr) -> bool:  # type: ignore[override]
+    def __contains__(self, key: str | bytes) -> bool:  # type: ignore[override]
         normalized_key = self._normkey(key)
         return normalized_key.lower() in self._keys
 
     def __repr__(self) -> str:
         return f"<{self.__class__.__name__}: {super().__repr__()}>"
 
-    def _normkey(self, key: AnyStr) -> AnyStr:
+    def _normkey(self, key: str | bytes) -> str | bytes:
         return key
 
     def _normvalue(self, value: Any) -> Any:
@@ -158,7 +158,7 @@ class LocalCache(OrderedDict[_KT, _VT]):
         super().__setitem__(key, value)
 
 
-class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
+class LocalWeakReferencedCache(weakref.WeakKeyDictionary[_KT, _VT | None]):
     """
     A weakref.WeakKeyDictionary implementation that uses LocalCache as its
     underlying data structure, making it ordered and capable of being size-limited.
@@ -172,9 +172,9 @@ class LocalWeakReferencedCache(weakref.WeakKeyDictionary):
 
     def __init__(self, limit: int | None = None):
         super().__init__()
-        self.data: LocalCache = LocalCache(limit=limit)
+        self.data: LocalCache[_KT, _VT] = LocalCache(limit=limit)
 
-    def __setitem__(self, key: _KT, value: _VT) -> None:
+    def __setitem__(self, key: _KT, value: _VT | None) -> None:
         # if raised, key is not weak-referenceable, skip caching
         with contextlib.suppress(TypeError):
             super().__setitem__(key, value)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 90fd04777..29a34d4ef 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -173,7 +173,7 @@ def parallel(
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-class _AsyncCooperatorAdapter(Iterator, Generic[_T]):
+class _AsyncCooperatorAdapter(Iterator[Deferred[Any]], Generic[_T]):
     """A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more
diff --git a/scrapy/utils/log.py b/scrapy/utils/log.py
index 102362506..ee65d4155 100644
--- a/scrapy/utils/log.py
+++ b/scrapy/utils/log.py
@@ -260,7 +260,8 @@ def logformatter_adapter(
     return (level, message, args)
 
 
-class SpiderLoggerAdapter(logging.LoggerAdapter):
+# LoggerAdapter is only parameterized since Python 3.11
+class SpiderLoggerAdapter(logging.LoggerAdapter):  # type: ignore[type-arg]
     def process(
         self, msg: str, kwargs: MutableMapping[str, Any]
     ) -> tuple[str, MutableMapping[str, Any]]:
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 3baa02e2b..0b67eaa34 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
 
 
 _ITERABLE_SINGLE_VALUES = dict, Item, str, bytes
-_ITER_T = TypeVar("_ITER_T", bound=dict | Item | str | bytes)
+_ITER_T = TypeVar("_ITER_T", bound=dict[Any, Any] | Item | str | bytes)
 _T = TypeVar("_T")
 _T_co = TypeVar("_T_co", covariant=True)
 _P = ParamSpec("_P")
@@ -252,7 +252,9 @@ def walk_callable(node: ast.AST) -> Iterable[ast.AST]:
         yield node
 
 
-_generator_callbacks_cache = LocalWeakReferencedCache(limit=128)
+_generator_callbacks_cache: LocalWeakReferencedCache[Callable[..., Any], bool] = (
+    LocalWeakReferencedCache(limit=128)
+)
 
 
 def _returns_none(return_node: ast.Return) -> bool:
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 37db6c25f..7ab58093a 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -55,7 +55,7 @@ class CallLaterOnce(Generic[_T]):
         self._a: tuple[Any, ...] = a
         self._kw: dict[str, Any] = kw
         self._call: CallLaterResult | None = None
-        self._deferreds: list[Deferred] = []
+        self._deferreds: list[Deferred[None]] = []
 
     def schedule(self, delay: float = 0) -> None:
         # circular import
diff --git a/scrapy/utils/signal.py b/scrapy/utils/signal.py
index d9a72273a..ee391a49e 100644
--- a/scrapy/utils/signal.py
+++ b/scrapy/utils/signal.py
@@ -186,7 +186,9 @@ async def _send_catch_log_asyncio(
     handlers: list[Awaitable[TypingAny]] = []
     for receiver in liveReceivers(getAllReceivers(sender, signal)):
 
-        async def handler(receiver: Callable) -> tuple[Callable, TypingAny]:
+        async def handler(
+            receiver: Callable[..., Any],
+        ) -> tuple[Callable[..., Any], TypingAny]:
             result: TypingAny
             try:
                 result = await ensure_awaitable(
diff --git a/scrapy/utils/template.py b/scrapy/utils/template.py
index 3e4dae5c8..977d5a42a 100644
--- a/scrapy/utils/template.py
+++ b/scrapy/utils/template.py
@@ -11,7 +11,7 @@ if TYPE_CHECKING:
     from os import PathLike
 
 
-def render_templatefile(path: str | PathLike, **kwargs: Any) -> None:
+def render_templatefile(path: str | PathLike[str], **kwargs: Any) -> None:
     path_obj = Path(path)
     raw = path_obj.read_text("utf8")
 
diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.py
index 082aca4b1..87df10a02 100644
--- a/scrapy/utils/trackref.py
+++ b/scrapy/utils/trackref.py
@@ -30,7 +30,9 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
 
-live_refs: defaultdict[type, WeakKeyDictionary] = defaultdict(WeakKeyDictionary)
+live_refs: defaultdict[type, WeakKeyDictionary[object, float]] = defaultdict(
+    WeakKeyDictionary
+)
 
 
 class object_ref:
diff --git a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
index 93259f696..f49741441 100644
--- a/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
+++ b/tests/AsyncCrawlerProcess/reactorless_custom_settings.py
@@ -23,7 +23,7 @@ class NoRequestsSpider(scrapy.Spider):
         yield
 
 
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
     try:
         task.result()
     except Exception:
diff --git a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
index f6fead718..b85f60f87 100644
--- a/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
+++ b/tests/AsyncCrawlerProcess/twisted_reactor_custom_settings_select.py
@@ -17,7 +17,7 @@ class AsyncioReactorSpider(scrapy.Spider):
     }
 
 
-def log_task_exception(task: Task) -> None:
+def log_task_exception(task: Task[None]) -> None:
     try:
         task.result()
     except Exception:
diff --git a/tests/mocks/dummydbm.py b/tests/mocks/dummydbm.py
index e358eaca4..98fe58860 100644
--- a/tests/mocks/dummydbm.py
+++ b/tests/mocks/dummydbm.py
@@ -4,7 +4,7 @@ from collections import defaultdict
 from typing import Any
 
 
-class DummyDB(dict):
+class DummyDB(dict):  # type: ignore[type-arg]
     """Provide dummy DBM-like interface."""
 
     def close(self):
diff --git a/tests/spiders.py b/tests/spiders.py
index 373dc00bd..065a0e6d7 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -22,6 +22,7 @@ from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
 from scrapy.utils.test import get_from_asyncio_queue
 
 if TYPE_CHECKING:
+    from scrapy.settings import _SettingsKey
     from tests.mockserver.http import MockServer
 
 
@@ -94,19 +95,19 @@ class DelaySpider(MetaSpider):
 class LogSpider(MetaSpider):
     name = "log_spider"
 
-    def log_debug(self, message: str, extra: dict | None = None):
+    def log_debug(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.debug(message, extra=extra)
 
-    def log_info(self, message: str, extra: dict | None = None):
+    def log_info(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.info(message, extra=extra)
 
-    def log_warning(self, message: str, extra: dict | None = None):
+    def log_warning(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.warning(message, extra=extra)
 
-    def log_error(self, message: str, extra: dict | None = None):
+    def log_error(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.error(message, extra=extra)
 
-    def log_critical(self, message: str, extra: dict | None = None):
+    def log_critical(self, message: str, extra: dict[str, Any] | None = None):
         self.logger.critical(message, extra=extra)
 
     def parse(self, response):
@@ -417,7 +418,7 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
     """
 
     name = "crawl_spider_with_parse_method"
-    custom_settings: dict = {
+    custom_settings: dict[_SettingsKey, Any] = {
         "RETRY_HTTP_CODES": [],  # no need to retry
     }
     rules = (Rule(LinkExtractor(), callback="parse", follow=True),)
diff --git a/tests/test_command_genspider.py b/tests/test_command_genspider.py
index fd9505060..67e3eb50a 100644
--- a/tests/test_command_genspider.py
+++ b/tests/test_command_genspider.py
@@ -9,7 +9,7 @@ from tests.test_commands import TestProjectBase
 from tests.utils.cmdline import call, proc
 
 
-def find_in_file(filename: Path, regex: str) -> re.Match | None:
+def find_in_file(filename: Path, regex: str) -> re.Match[str] | None:
     """Find first pattern occurrence in file"""
     pattern = re.compile(regex)
     with filename.open("r", encoding="utf-8") as f:
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index b407211dd..1585835cc 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -4,7 +4,7 @@ import os
 import sys
 from io import BytesIO
 from pathlib import Path
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 import pytest
 from pexpect.popen_spawn import PopenSpawn
@@ -151,8 +151,7 @@ class TestInteractiveShell:
         env = os.environ.copy()
         env["SCRAPY_PYTHON_SHELL"] = "python"
         logfile = BytesIO()
-        # https://github.com/python/typeshed/issues/14915
-        p = PopenSpawn(args, env=cast("os._Environ", env), timeout=5)
+        p = PopenSpawn(args, env=env, timeout=5)
         p.logfile_read = logfile
         p.expect_exact("Available Scrapy objects")
         p.sendline(f"fetch('{mockserver.url('/')}')")
diff --git a/tests/test_command_startproject.py b/tests/test_command_startproject.py
index 4eed09fcd..2a9d0ed57 100644
--- a/tests/test_command_startproject.py
+++ b/tests/test_command_startproject.py
@@ -79,7 +79,7 @@ class TestStartprojectCommand:
 
 
 def get_permissions_dict(
-    path: str | os.PathLike, renamings=None, ignore=None
+    path: str | os.PathLike[str], renamings=None, ignore=None
 ) -> dict[str, str]:
     def get_permissions(path: Path) -> str:
         return oct(path.stat().st_mode)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index cbf568524..27e0c6445 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -41,7 +41,7 @@ from tests.spiders import ItemSpider
 from tests.utils.decorators import coroutine_test, inline_callbacks_test
 
 if TYPE_CHECKING:
-    from collections.abc import Callable, Iterable
+    from collections.abc import Awaitable, Callable, Iterable
 
 
 def path_to_url(path: str | Path) -> str:
@@ -1312,7 +1312,9 @@ class TestFeedExporterSignals:
         self.feed_slot_closed_received = True
 
     async def run_signaled_feed_exporter(
-        self, feed_exporter_signal_handler: Callable, feed_slot_signal_handler: Callable
+        self,
+        feed_exporter_signal_handler: Callable[[], Awaitable[None] | None],
+        feed_slot_signal_handler: Callable[[Any], Awaitable[None] | None],
     ) -> None:
         crawler = get_crawler(settings_dict=self.settings)
         feed_exporter = FeedExporter.from_crawler(crawler)
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 3b70e896c..d855d0f74 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -29,7 +29,7 @@ if TYPE_CHECKING:
     from os import PathLike
 
 
-def build_url(path: str | PathLike) -> str:
+def build_url(path: str | PathLike[str]) -> str:
     path_str = str(path)
     if path_str[0] != "/":
         path_str = "/" + path_str
diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py
index 6072e2f6d..cec5d728b 100644
--- a/tests/test_http2_client_protocol.py
+++ b/tests/test_http2_client_protocol.py
@@ -61,7 +61,7 @@ def make_html_body(val: str) -> bytes:
 
 class DummySpider(Spider):
     name = "dummy"
-    start_urls: list = []
+    start_urls = []
 
     def parse(self, response):
         print(response)
diff --git a/tests/test_loader.py b/tests/test_loader.py
index 7cfc7ed26..c094d25d8 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -39,7 +39,7 @@ class AttrsNameItem:
 
 @dataclasses.dataclass
 class NameDataClass:
-    name: list = dataclasses.field(default_factory=list)
+    name: list[str] = dataclasses.field(default_factory=list)
 
 
 # test item loaders
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index dbe8d85a5..c6a41fa6e 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -383,11 +383,11 @@ class TestFilesPipelineFieldsItem(TestFilesPipelineFieldsMixin):
 class FilesPipelineTestDataClass:
     name: str
     # default fields
-    file_urls: list = dataclasses.field(default_factory=list)
-    files: list = dataclasses.field(default_factory=list)
+    file_urls: list[str] = dataclasses.field(default_factory=list)
+    files: list[dict[str, str]] = dataclasses.field(default_factory=list)
     # overridden fields
-    custom_file_urls: list = dataclasses.field(default_factory=list)
-    custom_files: list = dataclasses.field(default_factory=list)
+    custom_file_urls: list[str] = dataclasses.field(default_factory=list)
+    custom_files: list[dict[str, str]] = dataclasses.field(default_factory=list)
 
 
 class TestFilesPipelineFieldsDataClass(TestFilesPipelineFieldsMixin):
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 199ec5afa..38662348f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -314,11 +314,11 @@ class TestImagesPipelineFieldsItem(TestImagesPipelineFieldsMixin):
 class ImagesPipelineTestDataClass:
     name: str
     # default fields
-    image_urls: list = dataclasses.field(default_factory=list)
-    images: list = dataclasses.field(default_factory=list)
+    image_urls: list[str] = dataclasses.field(default_factory=list)
+    images: list[dict[str, str]] = dataclasses.field(default_factory=list)
     # overridden fields
-    custom_image_urls: list = dataclasses.field(default_factory=list)
-    custom_images: list = dataclasses.field(default_factory=list)
+    custom_image_urls: list[str] = dataclasses.field(default_factory=list)
+    custom_images: list[dict[str, str]] = dataclasses.field(default_factory=list)
 
 
 class TestImagesPipelineFieldsDataClass(TestImagesPipelineFieldsMixin):
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index 176fefbbf..db023e1f8 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -41,10 +41,10 @@ class MinimalScheduler:
 
 
 class SimpleScheduler(MinimalScheduler):
-    def open(self, spider: Spider) -> defer.Deferred:
+    def open(self, spider: Spider) -> defer.Deferred[str]:
         return defer.succeed("open")
 
-    def close(self, reason: str) -> defer.Deferred:
+    def close(self, reason: str) -> defer.Deferred[str]:
         return defer.succeed("close")
 
     def __len__(self) -> int:
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2e0436ae1..a0d296553 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -110,7 +110,7 @@ class TestBaseAsyncSpiderMiddleware(TestSpiderMiddleware):
     Should work for process_spider_output and, when it's supported, process_start.
     """
 
-    ITEM_TYPE: type | tuple
+    ITEM_TYPE: type | tuple[type, ...]
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
     @staticmethod
diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.py
index c58b02ca5..f573993a1 100644
--- a/tests/test_utils_datatypes.py
+++ b/tests/test_utils_datatypes.py
@@ -2,6 +2,7 @@ import copy
 import warnings
 from abc import ABC, abstractmethod
 from collections.abc import Iterator, Mapping, MutableMapping
+from typing import Any
 
 import pytest
 
@@ -20,7 +21,7 @@ from scrapy.utils.python import garbage_collect
 class TestCaseInsensitiveDictBase(ABC):
     @property
     @abstractmethod
-    def dict_class(self) -> type[MutableMapping]:
+    def dict_class(self) -> type[MutableMapping[str, Any]]:
         raise NotImplementedError
 
     def test_init_dict(self):
@@ -206,7 +207,7 @@ class TestCaseInsensitiveDictBase(ABC):
 
 
 class TestCaseInsensitiveDict(TestCaseInsensitiveDictBase):
-    dict_class = CaseInsensitiveDict
+    dict_class = CaseInsensitiveDict  # type: ignore[assignment]
 
     def test_repr(self):
         d1 = self.dict_class({"foo": "bar"})
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index f40e424ff..318370d40 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -144,7 +144,9 @@ class TestStreamLogger:
     ],
 )
 def test_spider_logger_adapter_process(
-    base_extra: Mapping[str, Any], log_extra: MutableMapping, expected_extra: dict
+    base_extra: Mapping[str, Any],
+    log_extra: MutableMapping[str, Any],
+    expected_extra: dict[str, Any],
 ) -> None:
     logger = logging.getLogger("test")
     spider_logger_adapter = SpiderLoggerAdapter(logger, base_extra)
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index b25129c87..70800dcec 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 from hashlib import sha1
+from typing import Any
 from weakref import WeakKeyDictionary
 
 import pytest
@@ -56,13 +57,13 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
 
 
 class TestFingerprint:
-    function: staticmethod = staticmethod(fingerprint)
+    function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
     cache: (
         WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
         | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
     ) = _fingerprint_cache
     default_cache_key = (None, False)
-    known_hashes: tuple[tuple[Request, bytes | str, dict], ...] = (
+    known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
         (
             Request("http://example.org"),
             b"xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD",
diff --git a/tox.ini b/tox.ini
index da5a8a9a6..5fef37e5b 100644
--- a/tox.ini
+++ b/tox.ini
@@ -44,13 +44,13 @@ commands =
 [testenv:typing]
 basepython = python3.10
 deps =
-    mypy==1.20.2
+    mypy==2.1.0
     typing-extensions==4.15.0
     Pillow==12.2.0
     Protego==0.6.0
     Twisted==26.4.0
     attrs==26.1.0
-    boto3-stubs[s3]==1.43.2
+    boto3-stubs[s3]==1.43.9
     botocore-stubs==1.42.41
     h2==4.3.0
     httpx==0.28.1
@@ -58,12 +58,12 @@ deps =
     ptpython==3.0.32
     # newer ones require newer Python
     ipython==8.39.0
-    pyOpenSSL==26.1.0
+    pyOpenSSL==26.2.0
     pytest==9.0.3
-    types-Pygments==2.20.0.20260408
+    types-Pygments==2.20.0.20260508
     types-defusedxml==0.7.0.20260504
     types-lxml==2026.2.16
-    types-pexpect==4.9.0.20260408
+    types-pexpect==4.9.0.20260508
     uvloop==0.22.1
     w3lib==2.4.1
     zstandard==0.25.0

From a84b7850fc11b13345d5658ff7aef8a042d0e622 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 19 May 2026 17:26:21 +0500
Subject: [PATCH 29/36] Release notes for Scrapy 2.16.0. (#7536)

---
 docs/news.rst                     | 259 +++++++++++++++++++++++++++++-
 docs/topics/spider-middleware.rst |   6 +-
 2 files changed, 259 insertions(+), 6 deletions(-)

diff --git a/docs/news.rst b/docs/news.rst
index 3173dfbc1..429ea09bf 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,8 +3,31 @@
 Release notes
 =============
 
-Scrapy VERSION (unreleased)
----------------------------
+.. _release-2.16.0:
+
+Scrapy 2.16.0 (unreleased)
+--------------------------
+
+Highlights:
+
+-   Official support for Python 3.14
+
+-   Support for Twisted 26.4.0+
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+-   Increased the minimum versions of the following dependencies:
+
+    - service_identity_: 18.1.0 → 23.1.0
+
+    (:issue:`7347`)
+
+-   Added support for Twisted 26.4.0+.
+    (:issue:`7347`, :issue:`7505`, :issue:`7520`)
+
+-   Added support for Python 3.14.
+    (:issue:`6604`, :issue:`7460`)
 
 Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -26,7 +49,237 @@ Backward-incompatible changes
 
     - ``scrapy.core.downloader.handlers.http2.ScrapyH2Agent``
 
-    (:issue:`7496`, #TBD)
+    (:issue:`7496`, :issue:`7510`)
+
+Deprecations
+~~~~~~~~~~~~
+
+-   ``scrapy.FormRequest`` is deprecated. You can use the :doc:`form2request
+    <form2request:index>` library instead, see :ref:`form`.
+    (:issue:`6438`)
+
+-   ``scrapy.utils.python.MutableChain`` is deprecated.
+    (:issue:`7504`)
+
+Deprecation removals
+~~~~~~~~~~~~~~~~~~~~
+
+-   The ``start_requests()`` method of :class:`~scrapy.Spider`, deprecated in
+    2.13.0, is removed and no longer called. Use :meth:`~scrapy.Spider.start`
+    instead, or both to maintain support for lower Scrapy versions.
+    (:issue:`7490`)
+
+-   Support for ``process_start_requests()`` methods of :ref:`spider middlewares
+    <topics-spider-middleware>`, deprecated in 2.13.0, is removed. Use
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start` instead,
+    or both to maintain support for lower Scrapy versions.
+    (:issue:`7490`)
+
+-   Support for synchronous ``process_spider_output()`` methods of spider
+    middlewares, deprecated in Scrapy 2.13.0, is removed. You should upgrade
+    the affected middlewares to have asynchronous ``process_spider_output()``
+    methods.
+    (:issue:`7504`)
+
+-   The ``spider`` arguments of the following methods of
+    :class:`~scrapy.core.scraper.Scraper`, deprecated in Scrapy 2.13.0, are
+    removed:
+
+    - ``close_spider()``
+
+    - ``enqueue_scrape()``
+
+    - ``handle_spider_error()``
+
+    - ``handle_spider_output()``
+
+    (:issue:`7487`)
+
+-   HTTP/1.0 support code, deprecated in Scrapy 2.13.0, is removed. This
+    includes:
+
+    - ``scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler``
+
+    - The ``scrapy.core.downloader.webclient`` module.
+
+    - The ``DOWNLOADER_HTTPCLIENTFACTORY`` setting.
+
+    (:issue:`7486`)
+
+-   The following functions, deprecated in Scrapy 2.13.0, are removed, you
+    should import them from :mod:`w3lib.url` directly instead:
+
+    - ``scrapy.utils.url.add_or_replace_parameter()``
+
+    - ``scrapy.utils.url.add_or_replace_parameters()``
+
+    - ``scrapy.utils.url.any_to_uri()``
+
+    - ``scrapy.utils.url.canonicalize_url()``
+
+    - ``scrapy.utils.url.file_uri_to_path()``
+
+    - ``scrapy.utils.url.is_url()``
+
+    - ``scrapy.utils.url.parse_data_uri()``
+
+    - ``scrapy.utils.url.parse_url()``
+
+    - ``scrapy.utils.url.path_to_file_uri()``
+
+    - ``scrapy.utils.url.safe_download_url()``
+
+    - ``scrapy.utils.url.safe_url_string()``
+
+    - ``scrapy.utils.url.url_query_cleaner()``
+
+    - ``scrapy.utils.url.url_query_parameter()``
+
+    (:issue:`7487`)
+
+-   The following test-related code, deprecated in Scrapy 2.13.0, is removed:
+
+    - the ``scrapy.utils.testproc`` module
+
+    - the ``scrapy.utils.testsite`` module
+
+    - ``scrapy.utils.test.assert_gcs_environ()``
+
+    - ``scrapy.utils.test.get_ftp_content_and_delete()``
+
+    - ``scrapy.utils.test.get_gcs_content_and_delete()``
+
+    - ``scrapy.utils.test.mock_google_cloud_storage()``
+
+    - ``scrapy.utils.test.skip_if_no_boto()``
+
+    - ``scrapy.utils.test.TestSpider``
+
+    (:issue:`7487`)
+
+-   ``scrapy.utils.versions.scrapy_components_versions()``, deprecated in
+    Scrapy 2.13.0, is removed, you can use
+    :func:`scrapy.utils.versions.get_versions` instead.
+    (:issue:`7487`)
+
+-   ``scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware`` and
+    ``scrapy.utils.url.escape_ajax()``, deprecated in Scrapy 2.13.0, are
+    removed.
+    (:issue:`7487`)
+
+-   The ``__init__()`` method of priority queue classes (see
+    :setting:`SCHEDULER_PRIORITY_QUEUE`) now needs to support a keyword-only
+    ``start_queue_cls`` parameter, not supporting it was deprecated in Scrapy
+    2.13.0.
+    (:issue:`7487`)
+
+-   ``scrapy.spiders.init.InitSpider``, deprecated in Scrapy 2.13.0, is
+    removed.
+    (:issue:`7487`)
+
+New features
+~~~~~~~~~~~~
+
+-   New features and improvements for
+    :class:`~scrapy.core.downloader.handlers._httpx.HttpxDownloadHandler`:
+
+    - Support for proxies.
+
+    - Support for the :reqmeta:`download_latency` meta key.
+
+    - Support for :attr:`Response.certificate
+      <scrapy.http.Response.certificate>`.
+
+    - Default headers set by the ``httpx`` library are no longer added to
+      requests.
+
+    (:issue:`7441`, :issue:`7524`)
+
+-   :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
+    skips HTTPS proxy certificate verification when the
+    :setting:`DOWNLOAD_VERIFY_CERTIFICATES` setting is set to ``False``.
+    (:issue:`7496`)
+
+Improvements
+~~~~~~~~~~~~
+
+-   :func:`time.monotonic` is used instead of :func:`time.time` to calculate
+    elapsed time in various places.
+    (:issue:`7377`)
+
+-   Improved extraction of the file extension from the URL in
+    :class:`~scrapy.pipelines.files.FilesPipeline`.
+    (:issue:`4225`, :issue:`7414`)
+
+-   Other code refactoring and improvements.
+    (:issue:`7401`)
+
+Bug fixes
+~~~~~~~~~
+
+-   :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler` now
+    raises an exception when a request has an ``https://`` destination and an
+    ``https://`` proxy, which is not supported by this handler. Previously it
+    tried to connect to the proxy via HTTP in this case.
+    (:issue:`7496`)
+
+-   :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` now
+    raises an exception for requests with ``http://`` URLs instead of trying to
+    connect, which is not supported by this handler.
+    (:issue:`7496`)
+
+-   :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` no longer
+    adds the ``:status`` pseudo-header to :attr:`Response.headers
+    <scrapy.http.Response.headers>`.
+    (:issue:`7441`)
+
+-   Fixed :func:`scrapy.utils.response.open_in_browser` removing the ``<head>``
+    tag when adding the ``<base>`` tag.
+    (:issue:`7459`)
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Documented that
+    :class:`~scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler`
+    doesn't support HTTPS proxies for HTTPS destinations and that
+    :class:`~scrapy.core.downloader.handlers.http2.H2DownloadHandler` doesn't
+    support proxies at all.
+    (:issue:`7496`)
+
+-   Added an example of using
+    :class:`logging.handlers.TimedRotatingFileHandler` to rotate Scrapy logs.
+    (:issue:`3628`, :issue:`7501`)
+
+-   Added a ``CITATION.cff`` file.
+    (:issue:`7502`, :issue:`7519`)
+
+-   Mentioned :setting:`DOWNLOADER_CLIENT_TLS_METHOD` in :ref:`bans`.
+    (:issue:`5232`, :issue:`7518`)
+
+-   Other documentation improvements and fixes.
+    (:issue:`7417`,
+    :issue:`7463`,
+    :issue:`7472`,
+    :issue:`7480`,
+    :issue:`7489`,
+    :issue:`7503`,
+    :issue:`7507`)
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Added tests that connect to https://books.toscrape.com/ to test the
+    behavior with a real website. These tests are marked with the
+    ``requires_internet`` pytest mark and can be skipped with e.g.
+    ``-m 'not requires_internet'`` if you cannot or don't want to run them.
+    (:issue:`7520`)
+
+-   Type hints improvements and fixes.
+    (:issue:`7492`, :issue:`7532`)
+
+-   CI and test improvements and fixes.
+    (:issue:`7441`, :issue:`7466`, :issue:`7491`, :issue:`7496`)
 
 .. _release-2.15.2:
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 820d5910c..99bbdf292 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -175,9 +175,9 @@ In Scrapy 2.6.3 and lower, ``process_spider_output()`` must be a *synchronous*
 generator.
 
 To support those versions and higher Scrapy versions in the same middleware,
-rename your asynchronous :method:`~SpiderMiddleware.process_spider_output()`
-method to :method:`~SpiderMiddleware.process_spider_output_async()`, and define
-a synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower
+rename your asynchronous :meth:`~SpiderMiddleware.process_spider_output`
+method to :meth:`~SpiderMiddleware.process_spider_output_async`, and define a
+synchronous ``process_spider_output()`` method to be used by 2.6.3 and lower
 versions.
 
 For example:

From abe9c638414e4a39a338763c85a3a0d84696b9b3 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 19 May 2026 17:27:16 +0500
Subject: [PATCH 30/36] =?UTF-8?q?Bump=20version:=202.15.2=20=E2=86=92=202.?=
 =?UTF-8?q?16.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 SECURITY.md    | 4 ++--
 docs/news.rst  | 2 +-
 pyproject.toml | 2 +-
 scrapy/VERSION | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 752d6318a..49cafdf9e 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -4,8 +4,8 @@
 
 | Version | Supported          |
 | ------- | ------------------ |
-| 2.15.x     | :white_check_mark: |
-| < 2.15.x   | :x:                |
+| 2.16.x     | :white_check_mark: |
+| < 2.16.x   | :x:                |
 
 ## Reporting a Vulnerability
 
diff --git a/docs/news.rst b/docs/news.rst
index 429ea09bf..16aa02b25 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -5,7 +5,7 @@ Release notes
 
 .. _release-2.16.0:
 
-Scrapy 2.16.0 (unreleased)
+Scrapy 2.16.0 (2026-05-19)
 --------------------------
 
 Highlights:
diff --git a/pyproject.toml b/pyproject.toml
index 2f07dcd80..888f95574 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -137,7 +137,7 @@ module = [
 ignore_missing_imports = true
 
 [tool.bumpversion]
-current_version = "2.15.2"
+current_version = "2.16.0"
 commit = true
 tag = true
 tag_name = "{new_version}"
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 07d875c2d..752490696 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.15.2
+2.16.0

From 4a165508591da164393c448eebc86c9aa48081da Mon Sep 17 00:00:00 2001
From: SpiliosDmk <150211937+SpiliosDimakopoulos@users.noreply.github.com>
Date: Wed, 20 May 2026 11:27:24 +0300
Subject: [PATCH 31/36] DOC -> Add missing documentation for CloseSpider &
 CoreStats (#7421)

* DOC -> Add missing documentation for CloseSpider & CoreStats

* Apply the suggestion from the PR review.

---------

Co-authored-by: Andrey Rakhmatullin <wrar@wrar.name>
---
 docs/topics/extensions.rst | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index f0a8bb5ed..735e46c29 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -136,6 +136,19 @@ Core Stats extension
 Enable the collection of core statistics, provided the stats collection is
 enabled (see :ref:`topics-stats`).
 
+The following stats are collected:
+
+* ``start_time``: start date/time of the crawl (:class:`~datetime.datetime`).
+* ``finish_time``: end date/time of the crawl (:class:`~datetime.datetime`).
+* ``elapsed_time_seconds``: total crawl duration in seconds (:class:`float`).
+* ``finish_reason``: the closing reason string (e.g. ``"finished"``,
+  ``"closespider_timeout"``).
+* ``item_scraped_count``: total number of items that passed all pipelines.
+* ``item_dropped_count``: total number of items dropped by a pipeline.
+* ``item_dropped_reasons_count/<ExceptionName>``: per-exception drop count
+  (e.g. ``item_dropped_reasons_count/DropItem``).
+* ``response_received_count``: total number of HTTP responses received.
+
 .. _topics-extensions-ref-telnetconsole:
 
 Log Count extension
@@ -247,6 +260,7 @@ settings:
 * :setting:`CLOSESPIDER_TIMEOUT_NO_ITEM`
 * :setting:`CLOSESPIDER_ITEMCOUNT`
 * :setting:`CLOSESPIDER_PAGECOUNT`
+* :setting:`CLOSESPIDER_PAGECOUNT_NO_ITEM`
 * :setting:`CLOSESPIDER_ERRORCOUNT`
 
 .. note::
@@ -260,12 +274,11 @@ settings:
 CLOSESPIDER_TIMEOUT
 """""""""""""""""""
 
-Default: ``0``
+Default: ``0.0``
 
-An integer which specifies a number of seconds. If the spider remains open for
-more than that number of seconds, it will be automatically closed with the
-reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
-timeout.
+If the spider remains open for more than this number of seconds, it will be
+automatically closed with the reason ``closespider_timeout``. If zero (or non
+set), spiders won't be closed by timeout.
 
 .. setting:: CLOSESPIDER_TIMEOUT_NO_ITEM
 

From 44406806f819b0e5230ed0dbe619a31defeb7afc Mon Sep 17 00:00:00 2001
From: Fardin Alizadeh <fardinalizadeh3007@gmail.com>
Date: Tue, 2 Jun 2026 12:34:38 +0330
Subject: [PATCH 32/36] fix typos (#7564)

---
 docs/_templates/layout.html                | 2 +-
 docs/news.rst                              | 4 ++--
 tests/test_downloadermiddleware_cookies.py | 2 +-
 tests/test_engine_loop.py                  | 2 +-
 tests/test_zz_resources.py                 | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/_templates/layout.html b/docs/_templates/layout.html
index 6ec565e24..29394799b 100644
--- a/docs/_templates/layout.html
+++ b/docs/_templates/layout.html
@@ -1,6 +1,6 @@
 {% extends "!layout.html" %}
 
-{# Overriden to include a link to scrapy.org, not just to the docs root #}
+{# Overridden to include a link to scrapy.org, not just to the docs root #}
 {%- block sidebartitle %}
 
 {# the logo helper function was removed in Sphinx 6 and deprecated since Sphinx 4 #}
diff --git a/docs/news.rst b/docs/news.rst
index 16aa02b25..fd9786433 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -2200,7 +2200,7 @@ Backward-incompatible changes
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 -   User-defined cookies for HTTPS requests will have the ``secure`` flag set
-    to ``True`` unless it's set to ``False`` explictly. This is important when
+    to ``True`` unless it's set to ``False`` explicitly. This is important when
     these cookies are reused in HTTP requests, e.g. after a redirect to an HTTP
     URL.
     (:issue:`6357`)
@@ -2235,7 +2235,7 @@ Backward-incompatible changes
       ``crawler.settings`` instead. When they call ``__init__()`` of the base
       class they should pass the ``crawler`` argument to it too.
     - A ``from_settings()`` method shouldn't be defined. Class-specific
-      initialization code should go into either an overriden ``from_crawler()``
+      initialization code should go into either an overridden ``from_crawler()``
       method or into ``__init__()``.
     - It's now possible to override ``from_crawler()`` and it's not necessary
       to call ``MediaPipeline.from_crawler()`` in it if other recommendations
diff --git a/tests/test_downloadermiddleware_cookies.py b/tests/test_downloadermiddleware_cookies.py
index 49215329e..225562644 100644
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@@ -42,7 +42,7 @@ def _cookie_to_set_cookie_value(cookie):
 def _cookies_to_set_cookie_list(cookies):
     """Given a group of cookie defined either as a dictionary or as a list of
     dictionaries (i.e. in a format supported by the cookies parameter of
-    Request), return the equivalen list of strings that can be associated to a
+    Request), return the equivalent list of strings that can be associated to a
     ``Set-Cookie`` header."""
     if not cookies:
         return []
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 8b5705487..6cc8c0650 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -263,7 +263,7 @@ class TestRequestSendOrder:
     @coroutine_test
     async def test_shared_queues(self):
         """If SCHEDULER_START_*_QUEUE is falsy, start requests and other
-        requests share the same queue, i.e. start requests are not priorized
+        requests share the same queue, i.e. start requests are not prioritized
         over other requests if their priority matches."""
         nums = list(range(1, 14))
         response_seconds = 0
diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py
index 1faa2487a..a8292745d 100644
--- a/tests/test_zz_resources.py
+++ b/tests/test_zz_resources.py
@@ -26,7 +26,7 @@ def test_stderr_log_handler() -> None:
 
     It's added in ``configure_logging()``, called by ``{Async,}CrawlerProcess``
     (without ``install_root_handler=False``). It can be removed with
-    ``_uninstall_scrapy_root_handler()`` if installing it was really neeeded.
+    ``_uninstall_scrapy_root_handler()`` if installing it was really needed.
     """
     c = sum(1 for h in logging.root.handlers if type(h) is logging.StreamHandler)  # pylint: disable=unidiomatic-typecheck
     assert c == 0

From 90deebe75e49e7217592072e4cb30b56750093c8 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 3 Jun 2026 12:16:24 +0500
Subject: [PATCH 33/36] Convert tests that fail with testfixtures 12.0.0
 (#7545)

---
 tests/test_spidermiddleware_httperror.py | 77 ++++++++++++------------
 tests/test_utils_log.py                  | 36 +++++------
 2 files changed, 56 insertions(+), 57 deletions(-)

diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py
index d5ad59346..6a054ded5 100644
--- a/tests/test_spidermiddleware_httperror.py
+++ b/tests/test_spidermiddleware_httperror.py
@@ -1,17 +1,19 @@
 from __future__ import annotations
 
 import logging
+from typing import TYPE_CHECKING
 
 import pytest
-from testfixtures import LogCapture
 
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
-from tests.mockserver.http import MockServer
 from tests.spiders import MockServerSpider
-from tests.utils.decorators import inline_callbacks_test
+from tests.utils.decorators import coroutine_test
+
+if TYPE_CHECKING:
+    from tests.mockserver.http import MockServer
 
 
 class _HttpErrorSpider(MockServerSpider):
@@ -192,65 +194,66 @@ class TestHttpErrorMiddlewareHandleAll:
 
 
 class TestHttpErrorMiddlewareIntegrational:
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    @inline_callbacks_test
-    def test_middleware_works(self):
+    @coroutine_test
+    async def test_middleware_works(self, mockserver: MockServer) -> None:
         crawler = get_crawler(_HttpErrorSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, _HttpErrorSpider)
         assert not crawler.spider.skipped
         assert crawler.spider.parsed == {"200"}
         assert crawler.spider.failed == {"404", "402", "500"}
 
+        assert crawler.stats
         get_value = crawler.stats.get_value
         assert get_value("httperror/response_ignored_count") == 3
         assert get_value("httperror/response_ignored_status_count/404") == 1
         assert get_value("httperror/response_ignored_status_count/402") == 1
         assert get_value("httperror/response_ignored_status_count/500") == 1
 
-    @inline_callbacks_test
-    def test_logging(self):
+    @coroutine_test
+    async def test_logging(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(_HttpErrorSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver, bypass_status_codes={402})
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver, bypass_status_codes={402})
+        assert isinstance(crawler.spider, _HttpErrorSpider)
         assert crawler.spider.parsed == {"200", "402"}
         assert crawler.spider.skipped == {"402"}
         assert crawler.spider.failed == {"404", "500"}
 
-        assert "Ignoring response <404" in str(log)
-        assert "Ignoring response <500" in str(log)
-        assert "Ignoring response <200" not in str(log)
-        assert "Ignoring response <402" not in str(log)
+        assert "Ignoring response <404" in caplog.text
+        assert "Ignoring response <500" in caplog.text
+        assert "Ignoring response <200" not in caplog.text
+        assert "Ignoring response <402" not in caplog.text
 
-    @inline_callbacks_test
-    def test_logging_level(self):
+    @coroutine_test
+    async def test_logging_level(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         # HttpError logs ignored responses with level INFO
         crawler = get_crawler(_HttpErrorSpider)
-        with LogCapture(level=logging.INFO) as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, _HttpErrorSpider)
         assert crawler.spider.parsed == {"200"}
         assert crawler.spider.failed == {"404", "402", "500"}
 
-        assert "Ignoring response <402" in str(log)
-        assert "Ignoring response <404" in str(log)
-        assert "Ignoring response <500" in str(log)
-        assert "Ignoring response <200" not in str(log)
+        assert "Ignoring response <402" in caplog.text
+        assert "Ignoring response <404" in caplog.text
+        assert "Ignoring response <500" in caplog.text
+        assert "Ignoring response <200" not in caplog.text
 
         # with level WARNING, we shouldn't capture anything from HttpError
+        caplog.clear()
         crawler = get_crawler(_HttpErrorSpider)
-        with LogCapture(level=logging.WARNING) as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.WARNING):
+            await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, _HttpErrorSpider)
         assert crawler.spider.parsed == {"200"}
         assert crawler.spider.failed == {"404", "402", "500"}
 
-        assert "Ignoring response <402" not in str(log)
-        assert "Ignoring response <404" not in str(log)
-        assert "Ignoring response <500" not in str(log)
-        assert "Ignoring response <200" not in str(log)
+        assert "Ignoring response <402" not in caplog.text
+        assert "Ignoring response <404" not in caplog.text
+        assert "Ignoring response <500" not in caplog.text
+        assert "Ignoring response <200" not in caplog.text
diff --git a/tests/test_utils_log.py b/tests/test_utils_log.py
index 318370d40..8e5020022 100644
--- a/tests/test_utils_log.py
+++ b/tests/test_utils_log.py
@@ -42,33 +42,29 @@ class TestFailureToExcInfo:
 
 
 class TestTopLevelFormatter:
-    def setup_method(self):
-        self.handler = LogCapture()
-        self.handler.addFilter(TopLevelFormatter(["test"]))
-
-    def test_top_level_logger(self):
+    def test_top_level_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+        caplog.handler.addFilter(TopLevelFormatter(["test"]))
         logger = logging.getLogger("test")
-        with self.handler as log:
-            logger.warning("test log msg")
-        log.check(("test", "WARNING", "test log msg"))
+        logger.warning("test log msg")
+        assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples
 
-    def test_children_logger(self):
+    def test_children_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+        caplog.handler.addFilter(TopLevelFormatter(["test"]))
         logger = logging.getLogger("test.test1")
-        with self.handler as log:
-            logger.warning("test log msg")
-        log.check(("test", "WARNING", "test log msg"))
+        logger.warning("test log msg")
+        assert ("test", logging.WARNING, "test log msg") in caplog.record_tuples
 
-    def test_overlapping_name_logger(self):
+    def test_overlapping_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+        caplog.handler.addFilter(TopLevelFormatter(["test"]))
         logger = logging.getLogger("test2")
-        with self.handler as log:
-            logger.warning("test log msg")
-        log.check(("test2", "WARNING", "test log msg"))
+        logger.warning("test log msg")
+        assert ("test2", logging.WARNING, "test log msg") in caplog.record_tuples
 
-    def test_different_name_logger(self):
+    def test_different_name_logger(self, caplog: pytest.LogCaptureFixture) -> None:
+        caplog.handler.addFilter(TopLevelFormatter(["test"]))
         logger = logging.getLogger("different")
-        with self.handler as log:
-            logger.warning("test log msg")
-        log.check(("different", "WARNING", "test log msg"))
+        logger.warning("test log msg")
+        assert ("different", logging.WARNING, "test log msg") in caplog.record_tuples
 
 
 class TestLogCounterHandler:

From fed75a6c76ab846d0d00b8f100ba947302380a01 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 3 Jun 2026 17:48:12 +0500
Subject: [PATCH 34/36] Refactor test_crawl.py and test_crawler.py. (#7566)

---
 pyproject.toml                   |   1 +
 tests/test_crawl.py              | 773 ++++++++++++++++++-------------
 tests/test_crawler.py            | 266 ++++++-----
 tests/test_crawler_subprocess.py | 100 ++--
 4 files changed, 636 insertions(+), 504 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index 888f95574..e027ea74d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -225,6 +225,7 @@ disable = [
     "too-many-positional-arguments",
     "too-many-public-methods",
     "too-many-return-statements",
+    "undefined-variable",
     "unused-argument",
     "unused-variable",
     "useless-import-alias",  # used as a hint to mypy
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 52abb1ccc..ada4c31ce 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -9,7 +9,6 @@ from urllib.parse import urlencode, urlparse
 
 import pytest
 from cryptography.x509 import load_der_x509_certificate
-from testfixtures import LogCapture
 from twisted.internet.defer import succeed
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
@@ -24,7 +23,6 @@ from scrapy.utils.engine import format_engine_status, get_engine_status
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import NON_EXISTING_RESOLVABLE
-from tests.mockserver.http import MockServer
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
     AsyncDefAsyncioGenExcSpider,
@@ -56,44 +54,36 @@ from tests.spiders import (
     StartGoodAndBadOutput,
     StartItemSpider,
 )
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
     from scrapy.statscollectors import StatsCollector
+    from tests.mockserver.http import MockServer
 
 
 class TestCrawl:
-    mockserver: MockServer
-
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
-    @inline_callbacks_test
-    def test_follow_all(self):
+    @coroutine_test
+    async def test_follow_all(self, mockserver: MockServer) -> None:
         crawler = get_crawler(FollowAllSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, FollowAllSpider)
         assert len(crawler.spider.urls_visited) == 11  # 10 + start_url
 
     @coroutine_test
-    async def test_fixed_delay(self):
-        await self._test_delay(total=3, delay=0.2)
+    async def test_fixed_delay(self, mockserver: MockServer) -> None:
+        await self._test_delay(mockserver, total=3, delay=0.2)
 
     @coroutine_test
-    async def test_randomized_delay(self):
-        await self._test_delay(total=3, delay=0.1, randomize=True)
+    async def test_randomized_delay(self, mockserver: MockServer) -> None:
+        await self._test_delay(mockserver, total=3, delay=0.1, randomize=True)
 
+    @staticmethod
     async def _test_delay(
-        self, total: int, delay: float, randomize: bool = False
+        mockserver: MockServer, total: int, delay: float, randomize: bool = False
     ) -> None:
         crawl_kwargs = {
             "maxlatency": delay * 2,
-            "mockserver": self.mockserver,
+            "mockserver": mockserver,
             "total": total,
         }
         tolerance = 1 - (0.6 if randomize else 0.2)
@@ -122,18 +112,20 @@ class TestCrawl:
         average = total_time / (len(times) - 1)
         assert average <= delay / tolerance, "test total or delay values are too small"
 
-    @inline_callbacks_test
-    def test_timeout_success(self):
+    @coroutine_test
+    async def test_timeout_success(self, mockserver: MockServer) -> None:
         crawler = get_crawler(DelaySpider)
-        yield crawler.crawl(n=0.5, mockserver=self.mockserver)
+        await crawler.crawl_async(n=0.5, mockserver=mockserver)
+        assert isinstance(crawler.spider, DelaySpider)
         assert crawler.spider.t1 > 0
         assert crawler.spider.t2 > 0
         assert crawler.spider.t2 > crawler.spider.t1
 
-    @inline_callbacks_test
-    def test_timeout_failure(self):
+    @coroutine_test
+    async def test_timeout_failure(self, mockserver: MockServer) -> None:
         crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
-        yield crawler.crawl(n=0.5, mockserver=self.mockserver)
+        await crawler.crawl_async(n=0.5, mockserver=mockserver)
+        assert isinstance(crawler.spider, DelaySpider)
         assert crawler.spider.t1 > 0
         assert crawler.spider.t2 == 0
         assert crawler.spider.t2_err > 0
@@ -141,81 +133,96 @@ class TestCrawl:
 
         # server hangs after receiving response headers
         crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
-        yield crawler.crawl(n=0.5, b=1, mockserver=self.mockserver)
+        await crawler.crawl_async(n=0.5, b=1, mockserver=mockserver)
+        assert isinstance(crawler.spider, DelaySpider)
         assert crawler.spider.t1 > 0
         assert crawler.spider.t2 == 0
         assert crawler.spider.t2_err > 0
         assert crawler.spider.t2_err > crawler.spider.t1
 
-    @inline_callbacks_test
-    def test_retry_503(self):
+    @coroutine_test
+    async def test_retry_503(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/status?n=503"), mockserver=self.mockserver
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=503"), mockserver=mockserver
             )
-        self._assert_retried(log)
+        self._assert_retried(caplog.text)
 
-    @inline_callbacks_test
-    def test_retry_conn_failed(self):
+    @coroutine_test
+    async def test_retry_conn_failed(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                "http://localhost:65432/status?n=503", mockserver=self.mockserver
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                "http://localhost:65432/status?n=503", mockserver=mockserver
             )
-        self._assert_retried(log)
+        self._assert_retried(caplog.text)
 
-    @inline_callbacks_test
-    def test_retry_dns_error(self):
+    @coroutine_test
+    async def test_retry_dns_error(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         if NON_EXISTING_RESOLVABLE:
             pytest.skip("Non-existing hosts are resolvable")
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
+        with caplog.at_level(logging.DEBUG):
             # try to fetch the homepage of a nonexistent domain
-            yield crawler.crawl(
-                "http://dns.resolution.invalid./", mockserver=self.mockserver
+            await crawler.crawl_async(
+                "http://dns.resolution.invalid./", mockserver=mockserver
             )
-        self._assert_retried(log)
+        self._assert_retried(caplog.text)
 
-    @inline_callbacks_test
-    def test_start_bug_before_yield(self):
-        with LogCapture("scrapy", level=logging.ERROR) as log:
+    @coroutine_test
+    async def test_start_bug_before_yield(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.ERROR):
             crawler = get_crawler(BrokenStartSpider)
-            yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
+            await crawler.crawl_async(fail_before_yield=1, mockserver=mockserver)
 
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert record.exc_info is not None
         assert record.exc_info[0] is ZeroDivisionError
 
-    @inline_callbacks_test
-    def test_start_bug_yielding(self):
-        with LogCapture("scrapy", level=logging.ERROR) as log:
+    @coroutine_test
+    async def test_start_bug_yielding(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.ERROR):
             crawler = get_crawler(BrokenStartSpider)
-            yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
+            await crawler.crawl_async(fail_yielding=1, mockserver=mockserver)
 
-        assert len(log.records) == 1
-        record = log.records[0]
+        assert len(caplog.records) == 1
+        record = caplog.records[0]
         assert record.exc_info is not None
         assert record.exc_info[0] is ZeroDivisionError
 
-    @inline_callbacks_test
-    def test_start_items(self):
+    @coroutine_test
+    async def test_start_items(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         items = []
 
         def _on_item_scraped(item):
             items.append(item)
 
-        with LogCapture("scrapy", level=logging.ERROR) as log:
+        with caplog.at_level(logging.ERROR):
             crawler = get_crawler(StartItemSpider)
             crawler.signals.connect(_on_item_scraped, signals.item_scraped)
-            yield crawler.crawl(mockserver=self.mockserver)
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert len(log.records) == 0
+        assert len(caplog.records) == 0
         assert items == [{"name": "test item"}]
 
-    @inline_callbacks_test
-    def test_start_unsupported_output(self):
+    @coroutine_test
+    async def test_start_unsupported_output(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         """Anything that is not a request is assumed to be an item, avoiding a
         potentially expensive call to itemadapter.is_item(), and letting
         instead things fail when ItemAdapter is actually used on the
@@ -226,35 +233,39 @@ class TestCrawl:
         def _on_item_scraped(item):
             items.append(item)
 
-        with LogCapture("scrapy", level=logging.ERROR) as log:
+        with caplog.at_level(logging.ERROR):
             crawler = get_crawler(StartGoodAndBadOutput)
             crawler.signals.connect(_on_item_scraped, signals.item_scraped)
-            yield crawler.crawl(mockserver=self.mockserver)
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert len(log.records) == 0
+        assert len(caplog.records) == 0
         assert len(items) == 3
         assert not any(isinstance(item, Request) for item in items)
 
-    @inline_callbacks_test
-    def test_start_dupes(self):
+    @coroutine_test
+    async def test_start_dupes(self, mockserver: MockServer) -> None:
         settings = {"CONCURRENT_REQUESTS": 1}
         crawler = get_crawler(DuplicateStartSpider, settings)
-        yield crawler.crawl(
-            dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver
+        await crawler.crawl_async(
+            dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=mockserver
         )
+        assert isinstance(crawler.spider, DuplicateStartSpider)
         assert crawler.spider.visited == 6
 
         crawler = get_crawler(DuplicateStartSpider, settings)
-        yield crawler.crawl(
+        await crawler.crawl_async(
             dont_filter=False,
             distinct_urls=3,
             dupe_factor=4,
-            mockserver=self.mockserver,
+            mockserver=mockserver,
         )
+        assert isinstance(crawler.spider, DuplicateStartSpider)
         assert crawler.spider.visited == 3
 
-    @inline_callbacks_test
-    def test_unbounded_response(self):
+    @coroutine_test
+    async def test_unbounded_response(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         # Completeness of responses without Content-Length or Transfer-Encoding
         # can not be determined, we treat them as valid but flagged as "partial"
         query = urlencode(
@@ -279,41 +290,45 @@ with multiples lines
             }
         )
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                mockserver.url(f"/raw?{query}"), mockserver=mockserver
             )
-        assert str(log).count("Got response 200") == 1
+        assert caplog.text.count("Got response 200") == 1
 
-    @inline_callbacks_test
-    def test_retry_conn_lost(self):
+    @coroutine_test
+    async def test_retry_conn_lost(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         # connection lost after receiving data
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                mockserver.url("/drop?abort=0"), mockserver=mockserver
             )
-        self._assert_retried(log)
+        self._assert_retried(caplog.text)
 
-    @inline_callbacks_test
-    def test_retry_conn_aborted(self):
+    @coroutine_test
+    async def test_retry_conn_aborted(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         # connection lost before receiving data
         crawler = get_crawler(SimpleSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                mockserver.url("/drop?abort=1"), mockserver=mockserver
             )
-        self._assert_retried(log)
+        self._assert_retried(caplog.text)
 
     @staticmethod
-    def _assert_retried(log: LogCapture | str) -> None:
+    def _assert_retried(log: str) -> None:
         assert str(log).count("Retrying") == 2
         assert str(log).count("Gave up retrying") == 1
 
-    @inline_callbacks_test
-    def test_referer_header(self):
+    @coroutine_test
+    async def test_referer_header(self, mockserver: MockServer) -> None:
         """Referer header is set by RefererMiddleware unless it is already set"""
-        req0 = Request(self.mockserver.url("/echo?headers=1&body=0"), dont_filter=1)
+        req0 = Request(mockserver.url("/echo?headers=1&body=0"), dont_filter=True)
         req1 = req0.replace()
         req2 = req0.replace(headers={"Referer": None})
         req3 = req0.replace(headers={"Referer": "http://example.com"})
@@ -321,7 +336,8 @@ with multiples lines
         req1.meta["next"] = req2
         req2.meta["next"] = req3
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(seed=req0, mockserver=self.mockserver)
+        await crawler.crawl_async(seed=req0, mockserver=mockserver)
+        assert isinstance(crawler.spider, SingleRequestSpider)
         # basic asserts in case of weird communication errors
         assert "responses" in crawler.spider.meta
         assert "failures" not in crawler.spider.meta
@@ -338,33 +354,35 @@ with multiples lines
         echo3 = json.loads(to_unicode(crawler.spider.meta["responses"][3].body))
         assert echo3["headers"].get("Referer") == ["http://example.com"]
 
-    @inline_callbacks_test
-    def test_engine_status(self):
+    @coroutine_test
+    async def test_engine_status(self, mockserver: MockServer) -> None:
         est = []
 
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(
-            seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver
+        await crawler.crawl_async(
+            seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver
         )
+        assert isinstance(crawler.spider, SingleRequestSpider)
         assert len(est) == 1, est
         s = dict(est[0])
         assert s["engine.spider.name"] == crawler.spider.name
         assert s["len(engine.scraper.slot.active)"] == 1
 
-    @inline_callbacks_test
-    def test_format_engine_status(self):
+    @coroutine_test
+    async def test_format_engine_status(self, mockserver: MockServer) -> None:
         est = []
 
         def cb(response):
             est.append(format_engine_status(crawler.engine))
 
         crawler = get_crawler(SingleRequestSpider)
-        yield crawler.crawl(
-            seed=self.mockserver.url("/"), callback_func=cb, mockserver=self.mockserver
+        await crawler.crawl_async(
+            seed=mockserver.url("/"), callback_func=cb, mockserver=mockserver
         )
+        assert isinstance(crawler.spider, SingleRequestSpider)
         assert len(est) == 1, est
         est = est[0].split("\n")[2:-2]  # remove header & footer
         # convert to dict
@@ -377,8 +395,10 @@ with multiples lines
         assert s["engine.spider.name"] == crawler.spider.name
         assert s["len(engine.scraper.slot.active)"] == "1"
 
-    @inline_callbacks_test
-    def test_open_spider_error_on_faulty_pipeline(self):
+    @coroutine_test
+    async def test_open_spider_error_on_faulty_pipeline(
+        self, mockserver: MockServer
+    ) -> None:
         settings = {
             "ITEM_PIPELINES": {
                 "tests.pipelines.ZeroDivisionErrorPipeline": 300,
@@ -386,25 +406,48 @@ with multiples lines
         }
         crawler = get_crawler(SimpleSpider, settings)
         with pytest.raises(ZeroDivisionError):
-            yield crawler.crawl(
-                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200"), mockserver=mockserver
             )
         assert not crawler.crawling
 
-    @inline_callbacks_test
-    def test_crawlerrunner_accepts_crawler(self):
-        crawler = get_crawler(SimpleSpider)
-        runner = CrawlerRunner()
-        with LogCapture() as log:
-            yield runner.crawl(
-                crawler,
-                self.mockserver.url("/status?n=200"),
-                mockserver=self.mockserver,
+    @coroutine_test
+    async def test_open_spider_error_on_faulty_pipeline_crawl(
+        self, mockserver: MockServer
+    ) -> None:
+        # cover the except block in Crawler.crawl()
+        settings = {
+            "ITEM_PIPELINES": {
+                "tests.pipelines.ZeroDivisionErrorPipeline": 300,
+            }
+        }
+        crawler = get_crawler(SimpleSpider, settings)
+        with pytest.raises(ZeroDivisionError):
+            await maybe_deferred_to_future(
+                crawler.crawl(mockserver.url("/status?n=200"), mockserver=mockserver)
             )
-        assert "Got response 200" in str(log)
+        assert not crawler.crawling
 
     @coroutine_test
-    async def test_crawl_multiple(self, caplog: pytest.LogCaptureFixture) -> None:
+    async def test_crawlerrunner_accepts_crawler(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        crawler = get_crawler(SimpleSpider)
+        runner = CrawlerRunner()
+        with caplog.at_level(logging.DEBUG):
+            await maybe_deferred_to_future(
+                runner.crawl(
+                    crawler,
+                    mockserver.url("/status?n=200"),
+                    mockserver=mockserver,
+                )
+            )
+        assert "Got response 200" in caplog.text
+
+    @coroutine_test
+    async def test_crawl_multiple(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         settings_dict = get_reactor_settings()
         runner_cls = (
             CrawlerRunner
@@ -414,13 +457,13 @@ with multiples lines
         runner = runner_cls(settings_dict)
         runner.crawl(
             SimpleSpider,
-            self.mockserver.url("/status?n=200"),
-            mockserver=self.mockserver,
+            mockserver.url("/status?n=200"),
+            mockserver=mockserver,
         )
         runner.crawl(
             SimpleSpider,
-            self.mockserver.url("/status?n=503"),
-            mockserver=self.mockserver,
+            mockserver.url("/status?n=503"),
+            mockserver=mockserver,
         )
 
         with caplog.at_level(logging.DEBUG):
@@ -432,25 +475,15 @@ with multiples lines
     @coroutine_test
     async def test_unknown_url_scheme(self, caplog: pytest.LogCaptureFixture) -> None:
         crawler = get_crawler(SimpleSpider)
-        await maybe_deferred_to_future(crawler.crawl("foo://bar"))
+        await crawler.crawl_async("foo://bar")
         assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text
 
 
 class TestCrawlSpider:
-    mockserver: MockServer
-
-    @classmethod
-    def setup_class(cls):
-        cls.mockserver = MockServer()
-        cls.mockserver.__enter__()
-
-    @classmethod
-    def teardown_class(cls):
-        cls.mockserver.__exit__(None, None, None)
-
+    @staticmethod
     async def _run_spider(
-        self, spider_cls: type[Spider]
-    ) -> tuple[LogCapture, list[Any], StatsCollector]:
+        spider_cls: type[Spider], mockserver: MockServer
+    ) -> tuple[list[Any], StatsCollector]:
         items = []
 
         def _on_item_scraped(item):
@@ -458,103 +491,119 @@ class TestCrawlSpider:
 
         crawler = get_crawler(spider_cls)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
-        with LogCapture() as log:
-            await maybe_deferred_to_future(
-                crawler.crawl(
-                    self.mockserver.url("/status?n=200"), mockserver=self.mockserver
-                )
-            )
+        await crawler.crawl_async(
+            mockserver.url("/status?n=200"), mockserver=mockserver
+        )
         assert crawler.stats
-        return log, items, crawler.stats
+        return items, crawler.stats
 
-    @inline_callbacks_test
-    def test_crawlspider_with_parse(self):
+    @coroutine_test
+    async def test_crawlspider_with_parse(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(CrawlSpiderWithParseMethod)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert "[parse] status 200 (foo: None)" in str(log)
-        assert "[parse] status 201 (foo: None)" in str(log)
-        assert "[parse] status 202 (foo: bar)" in str(log)
+        assert "[parse] status 200 (foo: None)" in caplog.text
+        assert "[parse] status 201 (foo: None)" in caplog.text
+        assert "[parse] status 202 (foo: bar)" in caplog.text
 
-    @inline_callbacks_test
-    def test_crawlspider_with_async_callback(self):
+    @coroutine_test
+    async def test_crawlspider_with_async_callback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(CrawlSpiderWithAsyncCallback)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert "[parse_async] status 200 (foo: None)" in str(log)
-        assert "[parse_async] status 201 (foo: None)" in str(log)
-        assert "[parse_async] status 202 (foo: bar)" in str(log)
+        assert "[parse_async] status 200 (foo: None)" in caplog.text
+        assert "[parse_async] status 201 (foo: None)" in caplog.text
+        assert "[parse_async] status 202 (foo: bar)" in caplog.text
 
-    @inline_callbacks_test
-    def test_crawlspider_with_async_generator_callback(self):
+    @coroutine_test
+    async def test_crawlspider_with_async_generator_callback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert "[parse_async_gen] status 200 (foo: None)" in str(log)
-        assert "[parse_async_gen] status 201 (foo: None)" in str(log)
-        assert "[parse_async_gen] status 202 (foo: bar)" in str(log)
+        assert "[parse_async_gen] status 200 (foo: None)" in caplog.text
+        assert "[parse_async_gen] status 201 (foo: None)" in caplog.text
+        assert "[parse_async_gen] status 202 (foo: bar)" in caplog.text
 
-    @inline_callbacks_test
-    def test_crawlspider_with_errback(self):
+    @coroutine_test
+    async def test_crawlspider_with_errback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(CrawlSpiderWithErrback)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert "[parse] status 200 (foo: None)" in str(log)
-        assert "[parse] status 201 (foo: None)" in str(log)
-        assert "[parse] status 202 (foo: bar)" in str(log)
-        assert "[errback] status 404" in str(log)
-        assert "[errback] status 500" in str(log)
-        assert "[errback] status 501" in str(log)
+        assert "[parse] status 200 (foo: None)" in caplog.text
+        assert "[parse] status 201 (foo: None)" in caplog.text
+        assert "[parse] status 202 (foo: bar)" in caplog.text
+        assert "[errback] status 404" in caplog.text
+        assert "[errback] status 500" in caplog.text
+        assert "[errback] status 501" in caplog.text
 
-    @inline_callbacks_test
-    def test_crawlspider_process_request_cb_kwargs(self):
+    @coroutine_test
+    async def test_crawlspider_process_request_cb_kwargs(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(CrawlSpiderWithProcessRequestCallbackKeywordArguments)
-        with LogCapture() as log:
-            yield crawler.crawl(mockserver=self.mockserver)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(mockserver=mockserver)
 
-        assert "[parse] status 200 (foo: process_request)" in str(log)
-        assert "[parse] status 201 (foo: process_request)" in str(log)
-        assert "[parse] status 202 (foo: bar)" in str(log)
+        assert "[parse] status 200 (foo: process_request)" in caplog.text
+        assert "[parse] status 201 (foo: process_request)" in caplog.text
+        assert "[parse] status 202 (foo: bar)" in caplog.text
 
-    @inline_callbacks_test
-    def test_async_def_parse(self):
+    @coroutine_test
+    async def test_async_def_parse(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(AsyncDefSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200"), mockserver=mockserver
             )
-        assert "Got response 200" in str(log)
+        assert "Got response 200" in caplog.text
 
     @pytest.mark.only_asyncio
-    @inline_callbacks_test
-    def test_async_def_asyncio_parse(self):
+    @coroutine_test
+    async def test_async_def_asyncio_parse(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(
             AsyncDefAsyncioSpider,
             {
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
             },
         )
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200"), mockserver=mockserver
             )
-        assert "Got response 200" in str(log)
+        assert "Got response 200" in caplog.text
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncio_parse_items_list(self):
-        log, items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider)
-        assert "Got response 200" in str(log)
+    async def test_async_def_asyncio_parse_items_list(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.INFO):
+            items, _ = await self._run_spider(AsyncDefAsyncioReturnSpider, mockserver)
+        assert "Got response 200" in caplog.text
         assert {"id": 1} in items
         assert {"id": 2} in items
 
     @pytest.mark.only_asyncio
-    @inline_callbacks_test
-    def test_async_def_asyncio_parse_items_single_element(self):
+    @coroutine_test
+    async def test_async_def_asyncio_parse_items_single_element(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         items = []
 
         def _on_item_scraped(item):
@@ -562,26 +611,34 @@ class TestCrawlSpider:
 
         crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                self.mockserver.url("/status?n=200"), mockserver=self.mockserver
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                mockserver.url("/status?n=200"), mockserver=mockserver
             )
-        assert "Got response 200" in str(log)
+        assert "Got response 200" in caplog.text
         assert {"foo": 42} in items
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncgen_parse(self):
-        log, _, stats = await self._run_spider(AsyncDefAsyncioGenSpider)
-        assert "Got response 200" in str(log)
+    async def test_async_def_asyncgen_parse(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.INFO):
+            _, stats = await self._run_spider(AsyncDefAsyncioGenSpider, mockserver)
+        assert "Got response 200" in caplog.text
         itemcount = stats.get_value("item_scraped_count")
         assert itemcount == 1
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncgen_parse_loop(self):
-        log, items, stats = await self._run_spider(AsyncDefAsyncioGenLoopSpider)
-        assert "Got response 200" in str(log)
+    async def test_async_def_asyncgen_parse_loop(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.INFO):
+            items, stats = await self._run_spider(
+                AsyncDefAsyncioGenLoopSpider, mockserver
+            )
+        assert "Got response 200" in caplog.text
         itemcount = stats.get_value("item_scraped_count")
         assert itemcount == 10
         for i in range(10):
@@ -589,11 +646,15 @@ class TestCrawlSpider:
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncgen_parse_exc(self):
-        log, items, stats = await self._run_spider(AsyncDefAsyncioGenExcSpider)
-        log = str(log)
-        assert "Spider error processing" in log
-        assert "ValueError" in log
+    async def test_async_def_asyncgen_parse_exc(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.INFO):
+            items, stats = await self._run_spider(
+                AsyncDefAsyncioGenExcSpider, mockserver
+            )
+        assert "Spider error processing" in caplog.text
+        assert "ValueError" in caplog.text
         itemcount = stats.get_value("item_scraped_count")
         assert itemcount == 7
         for i in range(7):
@@ -601,8 +662,12 @@ class TestCrawlSpider:
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncgen_parse_complex(self):
-        _, items, stats = await self._run_spider(AsyncDefAsyncioGenComplexSpider)
+    async def test_async_def_asyncgen_parse_complex(
+        self, mockserver: MockServer
+    ) -> None:
+        items, stats = await self._run_spider(
+            AsyncDefAsyncioGenComplexSpider, mockserver
+        )
         itemcount = stats.get_value("item_scraped_count")
         assert itemcount == 156
         # some random items
@@ -613,33 +678,41 @@ class TestCrawlSpider:
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_asyncio_parse_reqs_list(self):
-        log, *_ = await self._run_spider(AsyncDefAsyncioReqsReturnSpider)
+    async def test_async_def_asyncio_parse_reqs_list(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
+        with caplog.at_level(logging.INFO):
+            await self._run_spider(AsyncDefAsyncioReqsReturnSpider, mockserver)
         for req_id in range(3):
-            assert f"Got response 200, req_id {req_id}" in str(log)
+            assert f"Got response 200, req_id {req_id}" in caplog.text
 
     @pytest.mark.only_not_asyncio
     @coroutine_test
-    async def test_async_def_deferred_direct(self):
-        _, items, _ = await self._run_spider(AsyncDefDeferredDirectSpider)
+    async def test_async_def_deferred_direct(self, mockserver: MockServer) -> None:
+        items, _ = await self._run_spider(AsyncDefDeferredDirectSpider, mockserver)
         assert items == [{"code": 200}]
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_async_def_deferred_wrapped(self):
-        _, items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider)
+    async def test_async_def_deferred_wrapped(self, mockserver: MockServer) -> None:
+        items, _ = await self._run_spider(AsyncDefDeferredWrappedSpider, mockserver)
         assert items == [{"code": 200}]
 
     @coroutine_test
-    async def test_async_def_deferred_maybe_wrapped(self):
-        _, items, _ = await self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
+    async def test_async_def_deferred_maybe_wrapped(
+        self, mockserver: MockServer
+    ) -> None:
+        items, _ = await self._run_spider(
+            AsyncDefDeferredMaybeWrappedSpider, mockserver
+        )
         assert items == [{"code": 200}]
 
-    @inline_callbacks_test
-    def test_response_ssl_certificate_none(self):
+    @coroutine_test
+    async def test_response_ssl_certificate_none(self, mockserver: MockServer) -> None:
         crawler = get_crawler(SingleRequestSpider)
-        url = self.mockserver.url("/echo?body=test", is_secure=False)
-        yield crawler.crawl(seed=url, mockserver=self.mockserver)
+        url = mockserver.url("/echo?body=test", is_secure=False)
+        await crawler.crawl_async(seed=url, mockserver=mockserver)
+        assert isinstance(crawler.spider, SingleRequestSpider)
         assert crawler.spider.meta["responses"][0].certificate is None
 
     @pytest.mark.parametrize(
@@ -699,10 +772,13 @@ class TestCrawlSpider:
         assert isinstance(ip_address, IPv4Address)
         assert str(ip_address) == gethostbyname(expected_netloc)
 
-    @inline_callbacks_test
-    def test_bytes_received_stop_download_callback(self):
+    @coroutine_test
+    async def test_bytes_received_stop_download_callback(
+        self, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(BytesReceivedCallbackSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, BytesReceivedCallbackSpider)
         assert crawler.spider.meta.get("failure") is None
         assert isinstance(crawler.spider.meta["response"], Response)
         assert crawler.spider.meta["response"].body == crawler.spider.meta.get(
@@ -713,10 +789,13 @@ class TestCrawlSpider:
             < crawler.spider.full_response_length
         )
 
-    @inline_callbacks_test
-    def test_bytes_received_stop_download_errback(self):
+    @coroutine_test
+    async def test_bytes_received_stop_download_errback(
+        self, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(BytesReceivedErrbackSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, BytesReceivedErrbackSpider)
         assert crawler.spider.meta.get("response") is None
         assert isinstance(crawler.spider.meta["failure"], Failure)
         assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
@@ -729,20 +808,26 @@ class TestCrawlSpider:
             < crawler.spider.full_response_length
         )
 
-    @inline_callbacks_test
-    def test_headers_received_stop_download_callback(self):
+    @coroutine_test
+    async def test_headers_received_stop_download_callback(
+        self, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(HeadersReceivedCallbackSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, HeadersReceivedCallbackSpider)
         assert crawler.spider.meta.get("failure") is None
         assert isinstance(crawler.spider.meta["response"], Response)
         assert crawler.spider.meta["response"].headers == crawler.spider.meta.get(
             "headers_received"
         )
 
-    @inline_callbacks_test
-    def test_headers_received_stop_download_errback(self):
+    @coroutine_test
+    async def test_headers_received_stop_download_errback(
+        self, mockserver: MockServer
+    ) -> None:
         crawler = get_crawler(HeadersReceivedErrbackSpider)
-        yield crawler.crawl(mockserver=self.mockserver)
+        await crawler.crawl_async(mockserver=mockserver)
+        assert isinstance(crawler.spider, HeadersReceivedErrbackSpider)
         assert crawler.spider.meta.get("response") is None
         assert isinstance(crawler.spider.meta["failure"], Failure)
         assert isinstance(crawler.spider.meta["failure"].value, StopDownload)
@@ -751,8 +836,10 @@ class TestCrawlSpider:
             "failure"
         ].value.response.headers == crawler.spider.meta.get("headers_received")
 
-    @inline_callbacks_test
-    def test_spider_callback_deferred_deprecated(self):
+    @coroutine_test
+    async def test_spider_callback_deferred_deprecated(
+        self, mockserver: MockServer
+    ) -> None:
         def cb(response: Response) -> Any:
             return succeed(None)
 
@@ -761,10 +848,12 @@ class TestCrawlSpider:
             ScrapyDeprecationWarning,
             match="Returning Deferreds from spider callbacks is deprecated",
         ):
-            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
+            await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb)
 
-    @inline_callbacks_test
-    def test_spider_errback(self):
+    @coroutine_test
+    async def test_spider_errback(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         failures = []
 
         def eb(failure: Failure) -> Failure:
@@ -772,72 +861,82 @@ class TestCrawlSpider:
             return failure
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
         assert len(failures) == 1
-        assert "HTTP status code is not handled or not allowed" in str(log)
-        assert "Spider error processing" not in str(log)
+        assert "HTTP status code is not handled or not allowed" in caplog.text
+        assert "Spider error processing" not in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_silence(self):
+    @coroutine_test
+    async def test_spider_errback_silence(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         failures = []
 
         def eb(failure: Failure) -> None:
             failures.append(failure)
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
         assert len(failures) == 1
-        assert "HTTP status code is not handled or not allowed" not in str(log)
-        assert "Spider error processing" not in str(log)
+        assert "HTTP status code is not handled or not allowed" not in caplog.text
+        assert "Spider error processing" not in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_exception(self):
+    @coroutine_test
+    async def test_spider_errback_exception(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> None:
             raise ValueError("foo")
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
-        assert "Spider error processing" in str(log)
+        assert "Spider error processing" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_item(self):
+    @coroutine_test
+    async def test_spider_errback_item(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> Any:
             return {"foo": "bar"}
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
-        assert "HTTP status code is not handled or not allowed" not in str(log)
-        assert "Spider error processing" not in str(log)
-        assert "'item_scraped_count': 1" in str(log)
+        assert "HTTP status code is not handled or not allowed" not in caplog.text
+        assert "Spider error processing" not in caplog.text
+        assert "'item_scraped_count': 1" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_request(self):
+    @coroutine_test
+    async def test_spider_errback_request(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> Request:
-            return Request(self.mockserver.url("/"))
+            return Request(mockserver.url("/"))
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
-        assert "HTTP status code is not handled or not allowed" not in str(log)
-        assert "Spider error processing" not in str(log)
-        assert "Crawled (200)" in str(log)
+        assert "HTTP status code is not handled or not allowed" not in caplog.text
+        assert "Spider error processing" not in caplog.text
+        assert "Crawled (200)" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_downloader_error(self):
+    @coroutine_test
+    async def test_spider_errback_downloader_error(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         failures = []
 
         def eb(failure: Failure) -> Failure:
@@ -845,57 +944,65 @@ class TestCrawlSpider:
             return failure
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/drop?abort=1"), errback_func=eb
             )
         assert len(failures) == 1
-        assert "Error downloading" in str(log)
-        assert "Spider error processing" not in str(log)
+        assert "Error downloading" in caplog.text
+        assert "Spider error processing" not in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_downloader_error_exception(self):
+    @coroutine_test
+    async def test_spider_errback_downloader_error_exception(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> None:
             raise ValueError("foo")
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/drop?abort=1"), errback_func=eb
             )
-        assert "Error downloading" in str(log)
-        assert "Spider error processing" in str(log)
+        assert "Error downloading" in caplog.text
+        assert "Spider error processing" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_downloader_error_item(self):
+    @coroutine_test
+    async def test_spider_errback_downloader_error_item(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> Any:
             return {"foo": "bar"}
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(
+                seed=mockserver.url("/drop?abort=1"), errback_func=eb
             )
-        assert "HTTP status code is not handled or not allowed" not in str(log)
-        assert "Spider error processing" not in str(log)
-        assert "'item_scraped_count': 1" in str(log)
+        assert "HTTP status code is not handled or not allowed" not in caplog.text
+        assert "Spider error processing" not in caplog.text
+        assert "'item_scraped_count': 1" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_downloader_error_request(self):
+    @coroutine_test
+    async def test_spider_errback_downloader_error_request(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> Request:
-            return Request(self.mockserver.url("/"))
+            return Request(mockserver.url("/"))
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(
-                seed=self.mockserver.url("/drop?abort=1"), errback_func=eb
+        with caplog.at_level(logging.DEBUG):
+            await crawler.crawl_async(
+                seed=mockserver.url("/drop?abort=1"), errback_func=eb
             )
-        assert "HTTP status code is not handled or not allowed" not in str(log)
-        assert "Spider error processing" not in str(log)
-        assert "Crawled (200)" in str(log)
+        assert "HTTP status code is not handled or not allowed" not in caplog.text
+        assert "Spider error processing" not in caplog.text
+        assert "Crawled (200)" in caplog.text
 
-    @inline_callbacks_test
-    def test_spider_errback_deferred_deprecated(self):
+    @coroutine_test
+    async def test_spider_errback_deferred_deprecated(
+        self, mockserver: MockServer
+    ) -> None:
         def eb(failure: Failure) -> Any:
             return succeed(None)
 
@@ -904,28 +1011,32 @@ class TestCrawlSpider:
             ScrapyDeprecationWarning,
             match="Returning Deferreds from spider errbacks is deprecated",
         ):
-            yield crawler.crawl(
-                seed=self.mockserver.url("/status?n=400"), errback_func=eb
+            await crawler.crawl_async(
+                seed=mockserver.url("/status?n=400"), errback_func=eb
             )
 
-    @inline_callbacks_test
-    def test_raise_closespider(self):
+    @coroutine_test
+    async def test_raise_closespider(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def cb(response):
             raise CloseSpider
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
-        assert "Closing spider (cancelled)" in str(log)
-        assert "Spider error processing" not in str(log)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb)
+        assert "Closing spider (cancelled)" in caplog.text
+        assert "Spider error processing" not in caplog.text
 
-    @inline_callbacks_test
-    def test_raise_closespider_reason(self):
+    @coroutine_test
+    async def test_raise_closespider_reason(
+        self, caplog: pytest.LogCaptureFixture, mockserver: MockServer
+    ) -> None:
         def cb(response):
             raise CloseSpider("my_reason")
 
         crawler = get_crawler(SingleRequestSpider)
-        with LogCapture() as log:
-            yield crawler.crawl(seed=self.mockserver.url("/"), callback_func=cb)
-        assert "Closing spider (my_reason)" in str(log)
-        assert "Spider error processing" not in str(log)
+        with caplog.at_level(logging.INFO):
+            await crawler.crawl_async(seed=mockserver.url("/"), callback_func=cb)
+        assert "Closing spider (my_reason)" in caplog.text
+        assert "Spider error processing" not in caplog.text
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index cbcb7e274..853d6cfaa 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,13 +1,13 @@
+from __future__ import annotations
+
 import asyncio
 import logging
 import re
 import warnings
-from collections.abc import Generator
 from pathlib import Path
-from typing import Any, cast
+from typing import Any, ClassVar
 
 import pytest
-from twisted.internet.defer import Deferred
 from zope.interface.exceptions import MultipleInvalid
 
 import scrapy
@@ -22,8 +22,8 @@ from scrapy.crawler import (
 )
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.throttle import AutoThrottle
-from scrapy.settings import Settings, default_settings
-from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
+from scrapy.settings import Settings, _SettingsKey, default_settings
+from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
 from scrapy.utils.log import (
     _uninstall_scrapy_root_handler,
     configure_logging,
@@ -31,12 +31,14 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
-from tests.utils.decorators import coroutine_test, inline_callbacks_test
+from tests.utils.decorators import coroutine_test
 
 BASE_SETTINGS: dict[str, Any] = {}
 
 
-def get_raw_crawler(spidercls=None, settings_dict=None):
+def get_raw_crawler(
+    spidercls: type[Spider] | None = None, settings_dict: dict[str, Any] | None = None
+) -> Crawler:
     """get_crawler alternative that only calls the __init__ method of the
     crawler."""
     settings = Settings()
@@ -46,14 +48,18 @@ def get_raw_crawler(spidercls=None, settings_dict=None):
 
 
 class TestBaseCrawler:
-    def assertOptionIsDefault(self, settings: Settings, key: str) -> None:
+    @staticmethod
+    def assertOptionIsDefault(settings: Settings, key: str) -> None:
         assert isinstance(settings, Settings)
         assert settings[key] == getattr(default_settings, key)
 
 
 class TestCrawler(TestBaseCrawler):
-    def test_populate_spidercls_settings(self):
-        spider_settings = {"TEST1": "spider", "TEST2": "spider"}
+    def test_populate_spidercls_settings(self) -> None:
+        spider_settings: dict[_SettingsKey, Any] = {
+            "TEST1": "spider",
+            "TEST2": "spider",
+        }
         project_settings = {
             **BASE_SETTINGS,
             "TEST1": "project",
@@ -76,47 +82,47 @@ class TestCrawler(TestBaseCrawler):
         assert not settings.frozen
         assert crawler.settings.frozen
 
-    def test_crawler_accepts_dict(self):
+    def test_crawler_accepts_dict(self) -> None:
         crawler = get_crawler(DefaultSpider, {"foo": "bar"})
         assert crawler.settings["foo"] == "bar"
         self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
-    def test_crawler_accepts_None(self):
+    def test_crawler_accepts_None(self) -> None:
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", ScrapyDeprecationWarning)
             crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, "RETRY_ENABLED")
 
-    def test_crawler_rejects_spider_objects(self):
+    def test_crawler_rejects_spider_objects(self) -> None:
         with pytest.raises(ValueError, match="spidercls argument must be a class"):
-            Crawler(DefaultSpider())
-
-    @inline_callbacks_test
-    def test_crawler_crawl_twice_seq_unsupported(self):
-        crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
-        yield crawler.crawl()
-        with pytest.raises(RuntimeError, match="more than once on the same instance"):
-            yield crawler.crawl()
+            Crawler(DefaultSpider())  # type: ignore[arg-type]
 
     @coroutine_test
-    async def test_crawler_crawl_async_twice_seq_unsupported(self):
+    async def test_crawler_crawl_twice_seq_unsupported(self) -> None:
+        crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
+        await maybe_deferred_to_future(crawler.crawl())
+        with pytest.raises(RuntimeError, match="more than once on the same instance"):
+            await maybe_deferred_to_future(crawler.crawl())
+
+    @coroutine_test
+    async def test_crawler_crawl_async_twice_seq_unsupported(self) -> None:
         crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         await crawler.crawl_async()
         with pytest.raises(RuntimeError, match="more than once on the same instance"):
             await crawler.crawl_async()
 
-    @inline_callbacks_test
-    def test_crawler_crawl_twice_parallel_unsupported(self):
+    @coroutine_test
+    async def test_crawler_crawl_twice_parallel_unsupported(self) -> None:
         crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         d1 = crawler.crawl()
         d2 = crawler.crawl()
-        yield d1
+        await maybe_deferred_to_future(d1)
         with pytest.raises(RuntimeError, match="Crawling already taking place"):
-            yield d2
+            await maybe_deferred_to_future(d2)
 
     @pytest.mark.only_asyncio
     @coroutine_test
-    async def test_crawler_crawl_async_twice_parallel_unsupported(self):
+    async def test_crawler_crawl_async_twice_parallel_unsupported(self) -> None:
         crawler = get_raw_crawler(NoRequestsSpider, BASE_SETTINGS)
         t1 = asyncio.create_task(crawler.crawl_async())
         t2 = asyncio.create_task(crawler.crawl_async())
@@ -124,12 +130,12 @@ class TestCrawler(TestBaseCrawler):
         with pytest.raises(RuntimeError, match="Crawling already taking place"):
             await t2
 
-    def test_get_addon(self):
+    def test_get_addon(self) -> None:
         class ParentAddon:
             pass
 
         class TrackingAddon(ParentAddon):
-            instances = []
+            instances: ClassVar[list[TrackingAddon]] = []
 
             def __init__(self):
                 TrackingAddon.instances.append(self)
@@ -150,7 +156,7 @@ class TestCrawler(TestBaseCrawler):
         addon = crawler.get_addon(TrackingAddon)
         assert addon == expected
 
-        addon = crawler.get_addon(DefaultSpider)
+        addon = crawler.get_addon(DefaultSpider)  # type: ignore[assignment]
         assert addon is None
 
         addon = crawler.get_addon(ParentAddon)
@@ -162,19 +168,21 @@ class TestCrawler(TestBaseCrawler):
         addon = crawler.get_addon(ChildAddon)
         assert addon is None
 
-    @inline_callbacks_test
-    def test_get_downloader_middleware(self):
+    @coroutine_test
+    async def test_get_downloader_middleware(self) -> None:
         class ParentDownloaderMiddleware:
             pass
 
         class TrackingDownloaderMiddleware(ParentDownloaderMiddleware):
-            instances = []
+            instances: ClassVar[list[TrackingDownloaderMiddleware]] = []
 
             def __init__(self):
                 TrackingDownloaderMiddleware.instances.append(self)
 
         class MySpider(Spider):
             name = "myspider"
+            cls: ClassVar[type[Any]]
+            result: ClassVar[Any]
 
             @classmethod
             def from_crawler(cls, crawler):
@@ -198,18 +206,18 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = TrackingDownloaderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert len(TrackingDownloaderMiddleware.instances) == 1
         assert MySpider.result == TrackingDownloaderMiddleware.instances[-1]
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = DefaultSpider
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ParentDownloaderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result == TrackingDownloaderMiddleware.instances[-1]
 
         class ChildDownloaderMiddleware(TrackingDownloaderMiddleware):
@@ -217,16 +225,16 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ChildDownloaderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
-    def test_get_downloader_middleware_not_crawling(self):
+    def test_get_downloader_middleware_not_crawling(self) -> None:
         crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
         with pytest.raises(RuntimeError):
             crawler.get_downloader_middleware(DefaultSpider)
 
-    @inline_callbacks_test
-    def test_get_downloader_middleware_no_engine(self):
+    @coroutine_test
+    async def test_get_downloader_middleware_no_engine(self) -> None:
         class MySpider(Spider):
             name = "myspider"
 
@@ -240,21 +248,23 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
-            yield crawler.crawl()
+            await crawler.crawl_async()
 
-    @inline_callbacks_test
-    def test_get_extension(self):
+    @coroutine_test
+    async def test_get_extension(self) -> None:
         class ParentExtension:
             pass
 
         class TrackingExtension(ParentExtension):
-            instances = []
+            instances: ClassVar[list[TrackingExtension]] = []
 
             def __init__(self):
                 TrackingExtension.instances.append(self)
 
         class MySpider(Spider):
             name = "myspider"
+            cls: ClassVar[type[Any]]
+            result: ClassVar[Any]
 
             @classmethod
             def from_crawler(cls, crawler):
@@ -278,18 +288,18 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = TrackingExtension
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert len(TrackingExtension.instances) == 1
         assert MySpider.result == TrackingExtension.instances[-1]
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = DefaultSpider
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ParentExtension
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result == TrackingExtension.instances[-1]
 
         class ChildExtension(TrackingExtension):
@@ -297,16 +307,16 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ChildExtension
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
-    def test_get_extension_not_crawling(self):
+    def test_get_extension_not_crawling(self) -> None:
         crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
         with pytest.raises(RuntimeError):
             crawler.get_extension(DefaultSpider)
 
-    @inline_callbacks_test
-    def test_get_extension_no_engine(self):
+    @coroutine_test
+    async def test_get_extension_no_engine(self) -> None:
         class MySpider(Spider):
             name = "myspider"
 
@@ -320,21 +330,23 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
-            yield crawler.crawl()
+            await crawler.crawl_async()
 
-    @inline_callbacks_test
-    def test_get_item_pipeline(self):
+    @coroutine_test
+    async def test_get_item_pipeline(self) -> None:
         class ParentItemPipeline:
             pass
 
         class TrackingItemPipeline(ParentItemPipeline):
-            instances = []
+            instances: ClassVar[list[TrackingItemPipeline]] = []
 
             def __init__(self):
                 TrackingItemPipeline.instances.append(self)
 
         class MySpider(Spider):
             name = "myspider"
+            cls: ClassVar[type[Any]]
+            result: ClassVar[Any]
 
             @classmethod
             def from_crawler(cls, crawler):
@@ -358,18 +370,18 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = TrackingItemPipeline
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert len(TrackingItemPipeline.instances) == 1
         assert MySpider.result == TrackingItemPipeline.instances[-1]
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = DefaultSpider
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ParentItemPipeline
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result == TrackingItemPipeline.instances[-1]
 
         class ChildItemPipeline(TrackingItemPipeline):
@@ -377,16 +389,16 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ChildItemPipeline
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
-    def test_get_item_pipeline_not_crawling(self):
+    def test_get_item_pipeline_not_crawling(self) -> None:
         crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
         with pytest.raises(RuntimeError):
             crawler.get_item_pipeline(DefaultSpider)
 
-    @inline_callbacks_test
-    def test_get_item_pipeline_no_engine(self):
+    @coroutine_test
+    async def test_get_item_pipeline_no_engine(self) -> None:
         class MySpider(Spider):
             name = "myspider"
 
@@ -400,21 +412,23 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
-            yield crawler.crawl()
+            await crawler.crawl_async()
 
-    @inline_callbacks_test
-    def test_get_spider_middleware(self):
+    @coroutine_test
+    async def test_get_spider_middleware(self) -> None:
         class ParentSpiderMiddleware:
             pass
 
         class TrackingSpiderMiddleware(ParentSpiderMiddleware):
-            instances = []
+            instances: ClassVar[list[TrackingSpiderMiddleware]] = []
 
             def __init__(self):
                 TrackingSpiderMiddleware.instances.append(self)
 
         class MySpider(Spider):
             name = "myspider"
+            cls: ClassVar[type[Any]]
+            result: ClassVar[Any]
 
             @classmethod
             def from_crawler(cls, crawler):
@@ -438,18 +452,18 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = TrackingSpiderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert len(TrackingSpiderMiddleware.instances) == 1
         assert MySpider.result == TrackingSpiderMiddleware.instances[-1]
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = DefaultSpider
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ParentSpiderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result == TrackingSpiderMiddleware.instances[-1]
 
         class ChildSpiderMiddleware(TrackingSpiderMiddleware):
@@ -457,16 +471,16 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, settings)
         MySpider.cls = ChildSpiderMiddleware
-        yield crawler.crawl()
+        await crawler.crawl_async()
         assert MySpider.result is None
 
-    def test_get_spider_middleware_not_crawling(self):
+    def test_get_spider_middleware_not_crawling(self) -> None:
         crawler = get_raw_crawler(settings_dict=BASE_SETTINGS)
         with pytest.raises(RuntimeError):
             crawler.get_spider_middleware(DefaultSpider)
 
-    @inline_callbacks_test
-    def test_get_spider_middleware_no_engine(self):
+    @coroutine_test
+    async def test_get_spider_middleware_no_engine(self) -> None:
         class MySpider(Spider):
             name = "myspider"
 
@@ -480,22 +494,23 @@ class TestCrawler(TestBaseCrawler):
 
         crawler = get_raw_crawler(MySpider, BASE_SETTINGS)
         with pytest.raises(RuntimeError):
-            yield crawler.crawl()
+            await crawler.crawl_async()
 
 
 class TestSpiderSettings:
-    def test_spider_custom_settings(self):
+    def test_spider_custom_settings(self) -> None:
         class MySpider(scrapy.Spider):
             name = "spider"
             custom_settings = {"AUTOTHROTTLE_ENABLED": True}
 
         crawler = get_crawler(MySpider)
+        assert crawler.extensions
         enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
         assert AutoThrottle in enabled_exts
 
 
 class TestCrawlerLogging:
-    def test_no_root_handler_installed(self):
+    def test_no_root_handler_installed(self) -> None:
         handler = get_scrapy_root_handler()
         if handler is not None:
             logging.root.removeHandler(handler)
@@ -507,7 +522,7 @@ class TestCrawlerLogging:
         assert get_scrapy_root_handler() is None
 
     @coroutine_test
-    async def test_spider_custom_settings_log_level(self, tmp_path):
+    async def test_spider_custom_settings_log_level(self, tmp_path: Path) -> None:
         log_file = Path(tmp_path, "log.txt")
         log_file.write_text("previous message\n", encoding="utf-8")
 
@@ -535,9 +550,13 @@ class TestCrawlerLogging:
 
         try:
             configure_logging()
-            assert get_scrapy_root_handler().level == logging.DEBUG
+            handler = get_scrapy_root_handler()
+            assert handler is not None
+            assert handler.level == logging.DEBUG
             crawler = get_crawler(MySpider)
-            assert get_scrapy_root_handler().level == logging.INFO
+            handler = get_scrapy_root_handler()
+            assert handler is not None
+            assert handler.level == logging.INFO
             await crawler.crawl_async()
         finally:
             _uninstall_scrapy_root_handler()
@@ -549,12 +568,13 @@ class TestCrawlerLogging:
         assert "info message" in logged
         assert "warning message" in logged
         assert "error message" in logged
+        assert crawler.stats
         assert crawler.stats.get_value("log_count/ERROR") == 1
         assert crawler.stats.get_value("log_count/WARNING") == 1
         assert info_count == 1
         assert crawler.stats.get_value("log_count/DEBUG", 0) == 0
 
-    def test_spider_custom_settings_log_append(self, tmp_path):
+    def test_spider_custom_settings_log_append(self, tmp_path: Path) -> None:
         log_file = Path(tmp_path, "log.txt")
         log_file.write_text("previous message\n", encoding="utf-8")
 
@@ -579,12 +599,12 @@ class TestCrawlerLogging:
 
 
 class SpiderLoaderWithWrongInterface:
-    def unneeded_method(self):
+    def unneeded_method(self) -> None:
         pass
 
 
 class TestCrawlerRunner(TestBaseCrawler):
-    def test_spider_manager_verify_interface(self):
+    def test_spider_manager_verify_interface(self) -> None:
         settings = Settings(
             {
                 "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
@@ -593,18 +613,18 @@ class TestCrawlerRunner(TestBaseCrawler):
         with pytest.raises(MultipleInvalid):
             CrawlerRunner(settings)
 
-    def test_crawler_runner_accepts_dict(self):
+    def test_crawler_runner_accepts_dict(self) -> None:
         runner = CrawlerRunner({"foo": "bar"})
         assert runner.settings["foo"] == "bar"
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
-    def test_crawler_runner_accepts_None(self):
+    def test_crawler_runner_accepts_None(self) -> None:
         runner = CrawlerRunner()
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
 
 class TestAsyncCrawlerRunner(TestBaseCrawler):
-    def test_spider_manager_verify_interface(self):
+    def test_spider_manager_verify_interface(self) -> None:
         settings = Settings(
             {
                 "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
@@ -613,23 +633,23 @@ class TestAsyncCrawlerRunner(TestBaseCrawler):
         with pytest.raises(MultipleInvalid):
             AsyncCrawlerRunner(settings)
 
-    def test_crawler_runner_accepts_dict(self):
+    def test_crawler_runner_accepts_dict(self) -> None:
         runner = AsyncCrawlerRunner({"foo": "bar"})
         assert runner.settings["foo"] == "bar"
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
-    def test_crawler_runner_accepts_None(self):
+    def test_crawler_runner_accepts_None(self) -> None:
         runner = AsyncCrawlerRunner()
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
 
 class TestCrawlerProcess(TestBaseCrawler):
-    def test_crawler_process_accepts_dict(self):
+    def test_crawler_process_accepts_dict(self) -> None:
         runner = CrawlerProcess({"foo": "bar"}, install_root_handler=False)
         assert runner.settings["foo"] == "bar"
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
-    def test_crawler_process_accepts_None(self):
+    def test_crawler_process_accepts_None(self) -> None:
         runner = CrawlerProcess(install_root_handler=False)
         self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED")
 
@@ -668,33 +688,35 @@ class NoRequestsSpider(scrapy.Spider):
 
 @pytest.mark.requires_reactor  # CrawlerRunner requires a reactor
 class TestCrawlerRunnerHasSpider:
-    @staticmethod
-    def _runner() -> CrawlerRunnerBase:
+    @pytest.fixture
+    def runner(self) -> CrawlerRunnerBase:
         return CrawlerRunner(get_reactor_settings())
 
     @staticmethod
-    def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
-        return cast("Deferred[None]", runner.crawl(spider))
+    async def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> None:
+        await ensure_awaitable(runner.crawl(spider))
 
-    @inline_callbacks_test
-    def test_crawler_runner_bootstrap_successful(self):
-        runner = self._runner()
-        yield self._crawl(runner, NoRequestsSpider)
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_successful(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        await self._crawl(runner, NoRequestsSpider)
         assert not runner.bootstrap_failed
 
-    @inline_callbacks_test
-    def test_crawler_runner_bootstrap_successful_for_several(self):
-        runner = self._runner()
-        yield self._crawl(runner, NoRequestsSpider)
-        yield self._crawl(runner, NoRequestsSpider)
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_successful_for_several(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
+        await self._crawl(runner, NoRequestsSpider)
+        await self._crawl(runner, NoRequestsSpider)
         assert not runner.bootstrap_failed
 
-    @inline_callbacks_test
-    def test_crawler_runner_bootstrap_failed(self):
-        runner = self._runner()
-
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_failed(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
         try:
-            yield self._crawl(runner, ExceptionSpider)
+            await self._crawl(runner, ExceptionSpider)
         except ValueError:
             pass
         else:
@@ -702,25 +724,25 @@ class TestCrawlerRunnerHasSpider:
 
         assert runner.bootstrap_failed
 
-    @inline_callbacks_test
-    def test_crawler_runner_bootstrap_failed_for_several(self):
-        runner = self._runner()
-
+    @coroutine_test
+    async def test_crawler_runner_bootstrap_failed_for_several(
+        self, runner: CrawlerRunnerBase
+    ) -> None:
         try:
-            yield self._crawl(runner, ExceptionSpider)
+            await self._crawl(runner, ExceptionSpider)
         except ValueError:
             pass
         else:
             pytest.fail("Exception should be raised from spider")
 
-        yield self._crawl(runner, NoRequestsSpider)
+        await self._crawl(runner, NoRequestsSpider)
 
         assert runner.bootstrap_failed
 
-    @inline_callbacks_test
-    def test_crawler_runner_asyncio_enabled_true(
+    @coroutine_test
+    async def test_crawler_runner_asyncio_enabled_true(
         self, reactor_pytest: str
-    ) -> Generator[Deferred[Any], Any, None]:
+    ) -> None:
         if reactor_pytest != "asyncio":
             runner = CrawlerRunner(
                 settings={
@@ -731,7 +753,7 @@ class TestCrawlerRunnerHasSpider:
                 Exception,
                 match=r"The installed reactor \(.*?\) does not match the requested one \(.*?\)",
             ):
-                yield self._crawl(runner, NoRequestsSpider)
+                await self._crawl(runner, NoRequestsSpider)
         else:
             CrawlerRunner(
                 settings={
@@ -746,11 +768,7 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
     def _runner() -> CrawlerRunnerBase:
         return AsyncCrawlerRunner(get_reactor_settings())
 
-    @staticmethod
-    def _crawl(runner: CrawlerRunnerBase, spider: type[Spider]) -> Deferred[None]:
-        return deferred_from_coro(runner.crawl(spider))
-
-    def test_crawler_runner_asyncio_enabled_true(self):
+    def test_crawler_runner_asyncio_enabled_true(self) -> None:  # type: ignore[override]
         pytest.skip("This test is only for CrawlerRunner")
 
 
@@ -762,7 +780,9 @@ class TestAsyncCrawlerRunnerHasSpider(TestCrawlerRunnerHasSpider):
         ({"LOG_VERSIONS": []}, None),
     ],
 )
-def test_log_scrapy_info(settings, items, caplog):
+def test_log_scrapy_info(
+    settings: dict[str, Any], items: list[str] | None, caplog: pytest.LogCaptureFixture
+) -> None:
     with caplog.at_level("INFO"):
         CrawlerProcess(settings, install_root_handler=False)
     assert (
diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.py
index beae4f277..146d94ecd 100644
--- a/tests/test_crawler_subprocess.py
+++ b/tests/test_crawler_subprocess.py
@@ -52,7 +52,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
     with the same file names and expectations.
     """
 
-    def test_simple(self):
+    def test_simple(self) -> None:
         log = self.run_script("simple.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -61,7 +61,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         )
         assert "is_reactorless(): False" in log
 
-    def test_multi(self):
+    def test_multi(self) -> None:
         log = self.run_script("multi.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -70,7 +70,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         )
         assert "ReactorAlreadyInstalledError" not in log
 
-    def test_reactor_default(self):
+    def test_reactor_default(self) -> None:
         log = self.run_script("reactor_default.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -78,7 +78,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
             "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
         ) in log
 
-    def test_asyncio_enabled_no_reactor(self):
+    def test_asyncio_enabled_no_reactor(self) -> None:
         log = self.run_script("asyncio_enabled_no_reactor.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -87,7 +87,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         )
         assert "RuntimeError" not in log
 
-    def test_asyncio_enabled_reactor(self):
+    def test_asyncio_enabled_reactor(self) -> None:
         log = self.run_script("asyncio_enabled_reactor.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -100,7 +100,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         parse_version(w3lib_version) >= parse_version("2.0.0"),
         reason="w3lib 2.0.0 and later do not allow invalid domains.",
     )
-    def test_ipv6_default_name_resolver(self):
+    def test_ipv6_default_name_resolver(self) -> None:
         log = self.run_script("default_name_resolver.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -112,7 +112,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
             in log
         )
 
-    def test_caching_hostname_resolver_ipv6(self):
+    def test_caching_hostname_resolver_ipv6(self) -> None:
         log = self.run_script("caching_hostname_resolver_ipv6.py")
         assert "Spider closed (finished)" in log
         assert "scrapy.exceptions.CannotResolveHostError" not in log
@@ -126,7 +126,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "TimeoutError" not in log
         assert "scrapy.exceptions.CannotResolveHostError" not in log
 
-    def test_twisted_reactor_asyncio(self):
+    def test_twisted_reactor_asyncio(self) -> None:
         log = self.run_script("twisted_reactor_asyncio.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -134,7 +134,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
             in log
         )
 
-    def test_twisted_reactor_asyncio_custom_settings(self):
+    def test_twisted_reactor_asyncio_custom_settings(self) -> None:
         log = self.run_script("twisted_reactor_custom_settings.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -142,7 +142,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
             in log
         )
 
-    def test_twisted_reactor_asyncio_custom_settings_same(self):
+    def test_twisted_reactor_asyncio_custom_settings_same(self) -> None:
         log = self.run_script("twisted_reactor_custom_settings_same.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -151,7 +151,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         )
 
     @pytest.mark.requires_uvloop
-    def test_custom_loop_asyncio(self):
+    def test_custom_loop_asyncio(self) -> None:
         log = self.run_script("asyncio_custom_loop.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -161,7 +161,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_custom_loop_asyncio_deferred_signal(self):
+    def test_custom_loop_asyncio_deferred_signal(self) -> None:
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         assert "Spider closed (finished)" in log
         assert (
@@ -172,7 +172,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "async pipeline opened!" in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_same_loop(self):
+    def test_asyncio_enabled_reactor_same_loop(self) -> None:
         log = self.run_script("asyncio_enabled_reactor_same_loop.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -182,7 +182,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_different_loop(self):
+    def test_asyncio_enabled_reactor_different_loop(self) -> None:
         log = self.run_script("asyncio_enabled_reactor_different_loop.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -190,7 +190,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
             "setting (uvloop.Loop)"
         ) in log
 
-    def test_default_loop_asyncio_deferred_signal(self):
+    def test_default_loop_asyncio_deferred_signal(self) -> None:
         log = self.run_script("asyncio_deferred_signal.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -200,7 +200,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" not in log
         assert "async pipeline opened!" in log
 
-    def test_args_change_settings(self):
+    def test_args_change_settings(self) -> None:
         log = self.run_script("args_settings.py")
         assert "Spider closed (finished)" in log
         assert "The value of FOO is 42" in log
@@ -243,7 +243,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
     def script_dir(self) -> Path:
         return self.get_script_dir("CrawlerProcess")
 
-    def test_reactor_default_twisted_reactor_select(self):
+    def test_reactor_default_twisted_reactor_select(self) -> None:
         log = self.run_script("reactor_default_twisted_reactor_select.py")
         if platform.system() in ["Windows", "Darwin"]:
             # The goal of this test function is to test that, when a reactor is
@@ -264,7 +264,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
                 "(twisted.internet.selectreactor.SelectReactor)"
             ) in log
 
-    def test_reactor_select(self):
+    def test_reactor_select(self) -> None:
         log = self.run_script("reactor_select.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -272,12 +272,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             "(twisted.internet.asyncioreactor.AsyncioSelectorReactor)"
         ) in log
 
-    def test_reactor_select_twisted_reactor_select(self):
+    def test_reactor_select_twisted_reactor_select(self) -> None:
         log = self.run_script("reactor_select_twisted_reactor_select.py")
         assert "Spider closed (finished)" in log
         assert "ReactorAlreadyInstalledError" not in log
 
-    def test_reactor_select_subclass_twisted_reactor_select(self):
+    def test_reactor_select_subclass_twisted_reactor_select(self) -> None:
         log = self.run_script("reactor_select_subclass_twisted_reactor_select.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -285,7 +285,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             "(twisted.internet.selectreactor.SelectReactor)"
         ) in log
 
-    def test_twisted_reactor_select(self):
+    def test_twisted_reactor_select(self) -> None:
         log = self.run_script("twisted_reactor_select.py")
         assert "Spider closed (finished)" in log
         assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
@@ -293,12 +293,12 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
     @pytest.mark.skipif(
         platform.system() == "Windows", reason="PollReactor is not supported on Windows"
     )
-    def test_twisted_reactor_poll(self):
+    def test_twisted_reactor_poll(self) -> None:
         log = self.run_script("twisted_reactor_poll.py")
         assert "Spider closed (finished)" in log
         assert "Using reactor: twisted.internet.pollreactor.PollReactor" in log
 
-    def test_twisted_reactor_asyncio_custom_settings_conflict(self):
+    def test_twisted_reactor_asyncio_custom_settings_conflict(self) -> None:
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
         assert "Using reactor: twisted.internet.selectreactor.SelectReactor" in log
         assert (
@@ -306,7 +306,7 @@ class TestCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             in log
         )
 
-    def test_reactorless(self):
+    def test_reactorless(self) -> None:
         log = self.run_script("reactorless.py")
         assert (
             "RuntimeError: CrawlerProcess doesn't support TWISTED_REACTOR_ENABLED=False"
@@ -319,7 +319,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
     def script_dir(self) -> Path:
         return self.get_script_dir("AsyncCrawlerProcess")
 
-    def test_twisted_reactor_custom_settings_select(self):
+    def test_twisted_reactor_custom_settings_select(self) -> None:
         log = self.run_script("twisted_reactor_custom_settings_select.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -329,7 +329,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         ) in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_same_loop(self):
+    def test_asyncio_enabled_reactor_same_loop(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_same.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -339,7 +339,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_asyncio_enabled_reactor_different_loop(self):
+    def test_asyncio_enabled_reactor_different_loop(self) -> None:
         log = self.run_script("asyncio_custom_loop_custom_settings_different.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -347,7 +347,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             "setting (uvloop.Loop)"
         ) in log
 
-    def test_reactorless_simple(self):
+    def test_reactorless_simple(self) -> None:
         log = self.run_script("reactorless_simple.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
@@ -356,7 +356,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
         assert log.count("WARNING: ") == 2
 
-    def test_reactorless_custom_settings(self):
+    def test_reactorless_custom_settings(self) -> None:
         """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not
         currently supported, AsyncCrawlerProcess will install a reactor in this
         case.
@@ -368,7 +368,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
             in log
         )
 
-    def test_reactorless_datauri(self):
+    def test_reactorless_datauri(self) -> None:
         log = self.run_script("reactorless_datauri.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
@@ -378,13 +378,13 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
         assert log.count("WARNING: ") == 2
 
-    def test_reactorless_import_hook(self):
+    def test_reactorless_import_hook(self) -> None:
         log = self.run_script("reactorless_import_hook.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
         assert "ImportError: Import of twisted.internet.reactor is forbidden" in log
 
-    def test_reactorless_telnetconsole_default(self):
+    def test_reactorless_telnetconsole_default(self) -> None:
         """By default TWISTED_REACTOR_ENABLED=False silently sets TELNETCONSOLE_ENABLED=False."""
         log = self.run_script("reactorless_simple.py")  # no need for a separate script
         assert "Not using a Twisted reactor" in log
@@ -392,7 +392,7 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "The TelnetConsole extension requires a Twisted reactor" not in log
         assert "scrapy.extensions.telnet.TelnetConsole" not in log
 
-    def test_reactorless_telnetconsole_disabled(self):
+    def test_reactorless_telnetconsole_disabled(self) -> None:
         """Explicit TELNETCONSOLE_ENABLED=False, there are no warnings."""
         log = self.run_script("reactorless_telnetconsole_disabled.py")
         assert "Not using a Twisted reactor" in log
@@ -400,14 +400,14 @@ class TestAsyncCrawlerProcessSubprocess(TestCrawlerProcessSubprocessBase):
         assert "The TelnetConsole extension requires a Twisted reactor" not in log
         assert "scrapy.extensions.telnet.TelnetConsole" not in log
 
-    def test_reactorless_telnetconsole_enabled(self):
+    def test_reactorless_telnetconsole_enabled(self) -> None:
         """Explicit TELNETCONSOLE_ENABLED=True, the user gets a warning."""
         log = self.run_script("reactorless_telnetconsole_enabled.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
         assert "The TelnetConsole extension requires a Twisted reactor" in log
 
-    def test_reactorless_reactor(self):
+    def test_reactorless_reactor(self) -> None:
         log = self.run_script("reactorless_reactor.py")
         assert (
             "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed"
@@ -427,7 +427,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
     with the same file names and expectations.
     """
 
-    def test_simple(self):
+    def test_simple(self) -> None:
         log = self.run_script("simple.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -436,7 +436,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
         )
         assert "is_reactorless(): False" in log
 
-    def test_multi_parallel(self):
+    def test_multi_parallel(self) -> None:
         log = self.run_script("multi_parallel.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -449,7 +449,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
             re.DOTALL,
         )
 
-    def test_multi_seq(self):
+    def test_multi_seq(self) -> None:
         log = self.run_script("multi_seq.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -463,7 +463,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
         )
 
     @pytest.mark.requires_uvloop
-    def test_custom_loop_same(self):
+    def test_custom_loop_same(self) -> None:
         log = self.run_script("custom_loop_same.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -473,7 +473,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
         assert "Using asyncio event loop: uvloop.Loop" in log
 
     @pytest.mark.requires_uvloop
-    def test_custom_loop_different(self):
+    def test_custom_loop_different(self) -> None:
         log = self.run_script("custom_loop_different.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -481,7 +481,7 @@ class TestCrawlerRunnerSubprocessBase(ScriptRunnerMixin):
             "setting (uvloop.Loop)"
         ) in log
 
-    def test_no_reactor(self):
+    def test_no_reactor(self) -> None:
         log = self.run_script("no_reactor.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -495,7 +495,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
     def script_dir(self) -> Path:
         return self.get_script_dir("CrawlerRunner")
 
-    def test_explicit_default_reactor(self):
+    def test_explicit_default_reactor(self) -> None:
         log = self.run_script("explicit_default_reactor.py")
         assert "Spider closed (finished)" in log
         assert (
@@ -503,14 +503,14 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
             not in log
         )
 
-    def test_response_ip_address(self):
+    def test_response_ip_address(self) -> None:
         log = self.run_script("ip_address.py")
         assert "INFO: Spider closed (finished)" in log
         assert "INFO: Host: not.a.real.domain" in log
         assert "INFO: Type: <class 'ipaddress.IPv4Address'>" in log
         assert "INFO: IP address: 127.0.0.1" in log
 
-    def test_change_default_reactor(self):
+    def test_change_default_reactor(self) -> None:
         log = self.run_script("change_reactor.py")
         assert (
             "DEBUG: Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
@@ -518,7 +518,7 @@ class TestCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
         )
         assert "DEBUG: Using asyncio event loop" in log
 
-    def test_reactorless(self):
+    def test_reactorless(self) -> None:
         log = self.run_script("reactorless.py")
         assert (
             "RuntimeError: CrawlerRunner doesn't support TWISTED_REACTOR_ENABLED=False"
@@ -531,7 +531,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
     def script_dir(self) -> Path:
         return self.get_script_dir("AsyncCrawlerRunner")
 
-    def test_simple_default_reactor(self):
+    def test_simple_default_reactor(self) -> None:
         log = self.run_script("simple_default_reactor.py")
         assert "Spider closed (finished)" not in log
         assert (
@@ -539,7 +539,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
             "AsyncCrawlerRunner requires that the installed Twisted reactor"
         ) in log
 
-    def test_reactorless_simple(self):
+    def test_reactorless_simple(self) -> None:
         log = self.run_script("reactorless_simple.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
@@ -548,7 +548,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
         assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
         assert log.count("WARNING: ") == 2
 
-    def test_reactorless_custom_settings(self):
+    def test_reactorless_custom_settings(self) -> None:
         """Setting TWISTED_REACTOR_ENABLED=False in spider settings is not
         currently supported, AsyncCrawlerRunner will expect a reactor installed
         by the user.
@@ -557,7 +557,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
         assert "Spider closed (finished)" not in log
         assert "We expected a Twisted reactor to be installed but it isn't." in log
 
-    def test_reactorless_datauri(self):
+    def test_reactorless_datauri(self) -> None:
         log = self.run_script("reactorless_datauri.py")
         assert "Not using a Twisted reactor" in log
         assert "Spider closed (finished)" in log
@@ -567,7 +567,7 @@ class TestAsyncCrawlerRunnerSubprocess(TestCrawlerRunnerSubprocessBase):
         assert log.count("WARNING: HttpxDownloadHandler is experimental") == 2
         assert log.count("WARNING: ") == 2
 
-    def test_reactorless_reactor(self):
+    def test_reactorless_reactor(self) -> None:
         log = self.run_script("reactorless_reactor.py")
         assert (
             "RuntimeError: TWISTED_REACTOR_ENABLED is False but a Twisted reactor is installed"

From df2f3d708e1ca313380a9e28e26f752b49c2693b Mon Sep 17 00:00:00 2001
From: Ethan Kuo <ethankuocodes@gmail.com>
Date: Thu, 4 Jun 2026 07:24:50 -0700
Subject: [PATCH 35/36] fix open_in_browser() logic issue plus add new tests
 (#7506)

---
 scrapy/utils/response.py     |   3 +-
 tests/test_utils_response.py | 116 ++++++++++++++++++++++++++++++++---
 2 files changed, 108 insertions(+), 11 deletions(-)

diff --git a/scrapy/utils/response.py b/scrapy/utils/response.py
index abb5f6a70..c068d9b1e 100644
--- a/scrapy/utils/response.py
+++ b/scrapy/utils/response.py
@@ -97,8 +97,7 @@ def open_in_browser(
     # XXX: this implementation is a bit dirty and could be improved
     body = response.body
     if isinstance(response, HtmlResponse):
-        if b"<base" not in body:
-            _remove_html_comments(body)
+        if b"<base" not in _remove_html_comments(body):
             repl = rf'\g<0><base href="{response.url}">'
             body = re.sub(rb"<head(?:[^<>]*?>)", to_bytes(repl), body, count=1)
         ext = ".html"
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 381a9c3ff..e02bdfb69 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -4,7 +4,7 @@ from urllib.parse import urlparse
 
 import pytest
 
-from scrapy.http import HtmlResponse, Response
+from scrapy.http import HtmlResponse, Response, TextResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (
     _remove_html_comments,
@@ -15,6 +15,13 @@ from scrapy.utils.response import (
 )
 
 
+def _read_browser_output(burl: str):
+    path = urlparse(burl).path
+    if not path or not Path(path).exists():
+        path = burl.replace("file://", "")
+    return Path(path).read_bytes()
+
+
 def test_open_in_browser():
     url = "http:///www.example.com/some/page.html"
     body = (
@@ -22,10 +29,7 @@ def test_open_in_browser():
     )
 
     def browser_open(burl: str) -> bool:
-        path = urlparse(burl).path
-        if not path or not Path(path).exists():
-            path = burl.replace("file://", "")
-        bbody = Path(path).read_bytes()
+        bbody = _read_browser_output(burl)
         assert b'<base href="' + to_bytes(url) + b'">' in bbody
         return True
 
@@ -169,10 +173,7 @@ def test_inject_base_url(body: bytes) -> None:
     url = "http://www.example.com"
 
     def check_base_url(burl):
-        path = urlparse(burl).path
-        if not path or not Path(path).exists():
-            path = burl.replace("file://", "")
-        bbody = Path(path).read_bytes()
+        bbody = _read_browser_output(burl)
         assert bbody.count(b'><base href="' + to_bytes(url) + b'">') == 1
         assert b"<head" in bbody
         return True
@@ -219,7 +220,104 @@ def test_open_in_browser_redos_head():
         (b"a<!--b-->c<!--d", b"ac"),
         (b"a<!--b-->c<!---->d", b"acd"),
         (b"a<!--b--><!--c-->d", b"ad"),
+        (b"a<!-- <!-- inner --> -->b", b"a -->b"),
+        (b"<!-- <head>fake</head> --><head>real</head>", b"<head>real</head>"),
     ],
 )
 def test_remove_html_comments(input_body, output_body):
     assert _remove_html_comments(input_body) == output_body
+
+
+def test_open_in_browser_preserves_html_comments():
+    url = "http://www.example.com"
+    body = (
+        b"<html>"
+        b"<!-- preserved comment -->"
+        b"<head><title>Real</title></head>"
+        b"<body>content</body>"
+        b"</html>"
+    )
+
+    def check(burl):
+        bbody = _read_browser_output(burl)
+        assert b"<!-- preserved comment -->" in bbody
+        return True
+
+    response = HtmlResponse(url, body=body)
+    assert open_in_browser(response, _openfunc=check)
+
+
+def test_open_in_browser_does_not_inject_base_when_present():
+    url = "http://www.example.com"
+    body = (
+        b"<html>"
+        b'<head><base href="http://real.com"><title>T</title></head>'
+        b"<body>hi</body>"
+        b"</html>"
+    )
+
+    def check(burl):
+        bbody = _read_browser_output(burl)
+        assert b'<base href="' + to_bytes(url) + b'">' not in bbody
+        assert b'<base href="http://real.com">' in bbody
+        return True
+
+    response = HtmlResponse(url, body=body)
+    assert open_in_browser(response, _openfunc=check)
+
+
+def test_open_in_browser_injects_base_when_only_in_comment():
+    url = "http://www.example.com"
+    body = (
+        b"<html>"
+        b"<!-- <base href='http://other.com'> -->"
+        b"<head><title>Real</title></head>"
+        b"<body>content</body>"
+        b"</html>"
+    )
+
+    def check(burl):
+        bbody = _read_browser_output(burl)
+        assert b'<base href="' + to_bytes(url) + b'">' in bbody
+        return True
+
+    response = HtmlResponse(url, body=body)
+    assert open_in_browser(response, _openfunc=check)
+
+
+def test_open_in_browser_injects_base_at_real_head_not_commented_head():
+    url = "http://www.example.com"
+    body = (
+        b"<html>"
+        b"<!--<head>comment head</head>-->"
+        b"<head><title>Actual</title></head>"
+        b"<body>hello</body>"
+        b"</html>"
+    )
+
+    def check(burl):
+        bbody = _read_browser_output(burl)
+        assert bbody.count(b'<base href="' + to_bytes(url) + b'">') == 1
+        base_pos = bbody.find(b'<base href="' + to_bytes(url) + b'">')
+        title_pos = bbody.find(b"<title>Actual</title>")
+        assert base_pos < title_pos
+        return True
+
+    response = HtmlResponse(url, body=body)
+    assert open_in_browser(response, _openfunc=check)
+
+
+def test_open_in_browser_text_response_uses_txt_extension():
+    response = TextResponse("http://www.example.com", body=b"plain text content")
+
+    def check(burl):
+        assert burl.endswith(".txt")
+        return True
+
+    assert open_in_browser(response, _openfunc=check)
+
+
+def test_open_in_browser_raises_for_unsupported_response_type():
+    response = Response("http://www.example.com", body=b"binary")
+    with pytest.raises(TypeError):
+        open_in_browser(response, _openfunc=lambda _: True)

From 13c1c1faf81100394cffc6cf088c18c315f09ede Mon Sep 17 00:00:00 2001
From: Ayush Singh <135635937+Ayush442842q@users.noreply.github.com>
Date: Thu, 4 Jun 2026 20:41:50 +0530
Subject: [PATCH 36/36] Close GCS feed temp files after upload (#7546)

---
 scrapy/extensions/feedexport.py   | 13 ++++++++-----
 tests/test_feedexport_storages.py | 28 ++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+), 5 deletions(-)

diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index d0430d0db..8029f85c9 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -306,12 +306,15 @@ class GCSFeedStorage(BlockingFeedStorage):
 
     def _store_in_thread(self, file: IO[bytes]) -> None:
         file.seek(0)
-        from google.cloud.storage import Client  # noqa: PLC0415
+        try:
+            from google.cloud.storage import Client  # noqa: PLC0415
 
-        client = Client(project=self.project_id)
-        bucket = client.get_bucket(self.bucket_name)
-        blob = bucket.blob(self.blob_name)
-        blob.upload_from_file(file, predefined_acl=self.acl)
+            client = Client(project=self.project_id)
+            bucket = client.get_bucket(self.bucket_name)
+            blob = bucket.blob(self.blob_name)
+            blob.upload_from_file(file, predefined_acl=self.acl)
+        finally:
+            file.close()
 
 
 class FTPFeedStorage(BlockingFeedStorage):
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index 5102f8e37..11e79775f 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -513,6 +513,34 @@ class TestGCSFeedStorage:
             client_mock.get_bucket.assert_called_once_with("mybucket")
             bucket_mock.blob.assert_called_once_with("export.csv")
             blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+            f.close.assert_called_once_with()
+
+    @coroutine_test
+    async def test_store_closes_file_on_upload_error(self):
+        try:
+            from google.cloud.storage import Client  # noqa: F401,PLC0415
+        except ImportError:
+            pytest.skip("GCSFeedStorage requires google-cloud-storage")
+
+        uri = "gs://mybucket/export.csv"
+        project_id = "myproject-123"
+        acl = "publicRead"
+        (client_mock, bucket_mock, blob_mock) = mock_google_cloud_storage()
+        blob_mock.upload_from_file.side_effect = OSError("Upload failed")
+        with mock.patch("google.cloud.storage.Client") as m:
+            m.return_value = client_mock
+
+            f = mock.Mock()
+            storage = GCSFeedStorage(uri, project_id, acl)
+            with pytest.raises(OSError, match="Upload failed"):
+                await maybe_deferred_to_future(storage.store(f))
+
+            f.seek.assert_called_once_with(0)
+            m.assert_called_once_with(project=project_id)
+            client_mock.get_bucket.assert_called_once_with("mybucket")
+            bucket_mock.blob.assert_called_once_with("export.csv")
+            blob_mock.upload_from_file.assert_called_once_with(f, predefined_acl=acl)
+            f.close.assert_called_once_with()
 
     def test_overwrite_default(self):
         with LogCapture() as log: