Merge branch 'master' into issue-6844-improve-command-docs

2025-11-25 19:47:24 +05:00 · 2025-11-25 19:47:24 +05:00 · 0c8ad6b0c3
parent 37ec53e748 813fd9f1ac
commit 0c8ad6b0c3
301 changed files with 9805 additions and 8035 deletions
--- a/.git-blame-ignore-revs
+++ b/.git-blame-ignore-revs
@ -4,4 +4,4 @@ e211ec0aa26ecae0da8ae55d064ea60e1efe4d0d
 # reapplying black to the code with default line length
 303f0a70fcf8067adf0a909c2096a5009162383a
 # reapplying black again and removing line length on pre-commit black config
-c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
+c5cdd0d30ceb68ccba04af0e71d1b8e6678e2962
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@ -20,10 +20,10 @@ jobs:
        - python-version: "3.13"
          env:
            TOXENV: pylint
-        - python-version: "3.9"
+        - python-version: "3.10"
          env:
            TOXENV: typing
-        - python-version: "3.9"
+        - python-version: "3.10"
          env:
            TOXENV: typing-tests
        - python-version: "3.13"  # Keep in sync with .readthedocs.yml
@ -34,10 +34,10 @@ jobs:
            TOXENV: twinecheck

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

@ -50,5 +50,5 @@ jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5
    - uses: pre-commit/action@v3.0.1
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@ -18,8 +18,8 @@ jobs:
    permissions:
      id-token: write
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
+      - uses: actions/checkout@v5
+      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"
      - run: |
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@ -16,13 +16,13 @@ jobs:
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@ -17,9 +17,6 @@ jobs:
      fail-fast: false
      matrix:
        include:
-        - python-version: "3.9"
-          env:
-            TOXENV: py
        - python-version: "3.10"
          env:
            TOXENV: py
@ -35,27 +32,24 @@ jobs:
        - python-version: "3.13"
          env:
            TOXENV: default-reactor
-        - python-version: pypy3.10
-          env:
-            TOXENV: pypy3
        - python-version: pypy3.11
          env:
            TOXENV: pypy3

        # pinned deps
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
          env:
            TOXENV: pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
          env:
            TOXENV: default-reactor-pinned
-        - python-version: pypy3.10
+        - python-version: pypy3.11
          env:
            TOXENV: pypy3-pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
          env:
            TOXENV: extra-deps-pinned
-        - python-version: "3.9.21"
+        - python-version: "3.10.19"
          env:
            TOXENV: botocore-pinned

@ -68,12 +62,15 @@ jobs:
        - python-version: "3.13"
          env:
            TOXENV: botocore
+        - python-version: "3.13"
+          env:
+            TOXENV: mitmproxy

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@ -17,9 +17,6 @@ jobs:
      fail-fast: false
      matrix:
        include:
-        - python-version: "3.9"
-          env:
-            TOXENV: py
        - python-version: "3.10"
          env:
            TOXENV: py
@ -37,10 +34,10 @@ jobs:
            TOXENV: default-reactor

        # pinned deps
-        - python-version: "3.9.13"
+        - python-version: "3.10.11"
          env:
            TOXENV: pinned
-        - python-version: "3.9.13"
+        - python-version: "3.10.11"
          env:
            TOXENV: extra-deps-pinned

@ -49,10 +46,10 @@ jobs:
            TOXENV: extra-deps

    steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v5

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
+      uses: actions/setup-python@v6
      with:
        python-version: ${{ matrix.python-version }}

--- a/.gitignore
+++ b/.gitignore
@ -5,14 +5,16 @@ _trial_temp*
 dropin.cache
 docs/build
 *egg-info
-.tox
-venv
-build
-dist
-.idea
+.tox/
+venv/
+.venv/
+build/
+dist/
+.idea/
+.vscode/
 htmlcov/
-.coverage
 .pytest_cache/
+.coverage
 .coverage.*
 coverage.*
 *.junit.xml
@ -26,4 +28,4 @@ test-output.*
 Thumbs.db

 # OSX miscellaneous
-.DS_Store
+.DS_Store
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@ -1,17 +1,28 @@
+exclude: |
+  (?x)(
+      ^docs/_static|
+      ^docs/_tests|
+      ^tests/sample_data
+  )
 repos:
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.9.3
+  rev: v0.14.2
  hooks:
-    - id: ruff
+    - id: ruff-check
      args: [ --fix ]
    - id: ruff-format
 - repo: https://github.com/adamchainz/blacken-docs
-  rev: 1.19.1
+  rev: 1.20.0
  hooks:
  - id: blacken-docs
    additional_dependencies:
-    - black==24.10.0
+    - black==25.9.0
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v5.0.0
+  rev: v6.0.0
  hooks:
+  - id: end-of-file-fixer
  - id: trailing-whitespace
+- repo: https://github.com/sphinx-contrib/sphinx-lint
+  rev: v1.0.0
+  hooks:
+    - id: sphinx-lint
--- a/README.rst
+++ b/README.rst
@ -40,7 +40,7 @@
   :alt: Ask DeepWiki

 Scrapy_ is a web scraping framework to extract structured data from websites.
-It is cross-platform, and requires Python 3.9+. It is maintained by Zyte_
+It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_
 (formerly Scrapinghub) and `many other contributors`_.

 .. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
--- a/conftest.py
+++ b/conftest.py
@ -1,10 +1,17 @@
+from __future__ import annotations
+
 from pathlib import Path
+from typing import TYPE_CHECKING

 import pytest
 from twisted.web.http import H2_ENABLED

-from scrapy.utils.reactor import install_reactor
+from scrapy.utils.reactor import set_asyncio_event_loop_policy
 from tests.keys import generate_keys
+from tests.mockserver.http import MockServer
+
+if TYPE_CHECKING:
+    from collections.abc import Generator


 def _py_files(folder):
@ -48,36 +55,30 @@ if not H2_ENABLED:
    )


-def pytest_addoption(parser):
-    parser.addoption(
-        "--reactor",
-        default="asyncio",
-        choices=["default", "asyncio"],
-    )
+@pytest.fixture(scope="session")
+def mockserver() -> Generator[MockServer]:
+    with MockServer() as mockserver:
+        yield mockserver


-@pytest.fixture(scope="class")
-def reactor_pytest(request):
-    if not request.cls:
-        # doctests
-        return None
-    request.cls.reactor_pytest = request.config.getoption("--reactor")
-    return request.cls.reactor_pytest
+@pytest.fixture(scope="session")
+def reactor_pytest(request) -> str:
+    return request.config.getoption("--reactor")


@pytest.fixture(autouse=True)
 def only_asyncio(request, reactor_pytest):
-    if request.node.get_closest_marker("only_asyncio") and reactor_pytest == "default":
-        pytest.skip("This test is only run without --reactor=default")
+    if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio":
+        pytest.skip("This test is only run with --reactor=asyncio")


@pytest.fixture(autouse=True)
 def only_not_asyncio(request, reactor_pytest):
    if (
        request.node.get_closest_marker("only_not_asyncio")
-        and reactor_pytest != "default"
+        and reactor_pytest == "asyncio"
    ):
-        pytest.skip("This test is only run with --reactor=default")
+        pytest.skip("This test is only run without --reactor=asyncio")


@pytest.fixture(autouse=True)
@ -85,7 +86,7 @@ def requires_uvloop(request):
    if not request.node.get_closest_marker("requires_uvloop"):
        return
    try:
-        import uvloop
+        import uvloop  # noqa: PLC0415

        del uvloop
    except ImportError:
@ -97,7 +98,7 @@ def requires_botocore(request):
    if not request.node.get_closest_marker("requires_botocore"):
        return
    try:
-        import botocore
+        import botocore  # noqa: PLC0415

        del botocore
    except ImportError:
@ -109,19 +110,28 @@ def requires_boto3(request):
    if not request.node.get_closest_marker("requires_boto3"):
        return
    try:
-        import boto3
+        import boto3  # noqa: PLC0415

        del boto3
    except ImportError:
        pytest.skip("boto3 is not installed")


+@pytest.fixture(autouse=True)
+def requires_mitmproxy(request):
+    if not request.node.get_closest_marker("requires_mitmproxy"):
+        return
+    try:
+        import mitmproxy  # noqa: F401, PLC0415
+    except ImportError:
+        pytest.skip("mitmproxy is not installed")
+
+
 def pytest_configure(config):
-    if config.getoption("--reactor") != "default":
-        install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
-    else:
-        # install the reactor explicitly
-        from twisted.internet import reactor  # noqa: F401
+    if config.getoption("--reactor") == "asyncio":
+        # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
+        # If we decide to run tests with both, we will need to add a new option and check it here.
+        set_asyncio_event_loop_policy()


 # Generate localhost certificate files, needed by some tests
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@ -29,14 +29,14 @@ def is_setting_index(node: Node) -> bool:
    if node.tagname == "index" and node["entries"]:  # type: ignore[index,attr-defined]
        # index entries for setting directives look like:
        # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
-        entry_type, info, refid = node["entries"][0][:3]  # type: ignore[index]
+        entry_type, info, _ = node["entries"][0][:3]  # type: ignore[index]
        return entry_type == "pair" and info.endswith("; setting")
    return False


 def get_setting_name_and_refid(node: Node) -> tuple[str, str]:
    """Extract setting name from directive index node"""
-    entry_type, info, refid = node["entries"][0][:3]  # type: ignore[index]
+    _, info, refid = node["entries"][0][:3]  # type: ignore[index]
    return info.replace("; setting", ""), refid


--- a/docs/conf.py
+++ b/docs/conf.py
@ -26,7 +26,6 @@ author = "Scrapy developers"
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration

 extensions = [
-    "hoverxref.extension",
    "notfound.extension",
    "scrapydocs",
    "sphinx.ext.autodoc",
@ -69,6 +68,14 @@ html_css_files = [
    "custom.css",
 ]

+html_context = {
+    "display_github": True,
+    "github_user": "scrapy",
+    "github_repo": "scrapy",
+    "github_version": "master",
+    "conf_py_path": "/docs/",
+}
+
 # Set canonical URL from the Read the Docs Domain
 html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "")

@ -119,7 +126,7 @@ coverage_ignore_pyobjects = [
    # The interface methods of duplicate request filtering classes are already
    # covered in the interface documentation part of the DUPEFILTER_CLASS
    # setting documentation.
-    r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_settings|request_seen|open|close|log)$",
+    r"^scrapy\.dupefilters\.[A-Z]\w*?\.(from_crawler|request_seen|open|close|log)$",
    # Private exception used by the command-line interface implementation.
    r"^scrapy\.exceptions\.UsageError",
    # Methods of BaseItemExporter subclasses are only documented in
@ -157,22 +164,5 @@ intersphinx_mapping = {
 }
 intersphinx_disabled_reftypes: Sequence[str] = []

-
-# -- Options for sphinx-hoverxref extension ----------------------------------
-# https://sphinx-hoverxref.readthedocs.io/en/latest/configuration.html
-
-hoverxref_auto_ref = True
-hoverxref_role_types = {
-    "class": "tooltip",
-    "command": "tooltip",
-    "confval": "tooltip",
-    "hoverxref": "tooltip",
-    "mod": "tooltip",
-    "ref": "tooltip",
-    "reqmeta": "tooltip",
-    "setting": "tooltip",
-    "signal": "tooltip",
-}
-hoverxref_roles = ["command", "reqmeta", "setting", "signal"]
-
+# -- Other options ------------------------------------------------------------
 default_dark_mode = False
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@ -251,10 +251,10 @@ Coding style
 Please follow these coding conventions when writing code for inclusion in
 Scrapy:

-* We use `black <https://black.readthedocs.io/en/stable/>`_ for code formatting.
+* We use `Ruff <https://docs.astral.sh/ruff/>`_ for code formatting.
  There is a hook in the pre-commit config
  that will automatically format your code before every commit. You can also
-  run black manually with ``tox -e pre-commit``.
+  run Ruff manually with ``tox -e pre-commit``.

 * Don't put your name in the code you contribute; git provides enough
  metadata to identify author of the code.
--- a/docs/faq.rst
+++ b/docs/faq.rst
@ -349,7 +349,7 @@ method for this purpose. For example:


    class MultiplyItemsMiddleware:
-        def process_spider_output(self, response, result, spider):
+        def process_spider_output(self, response, result):
            for item_or_request in result:
                if isinstance(item_or_request, Request):
                    continue
--- a/docs/index.rst
+++ b/docs/index.rst
@ -132,7 +132,7 @@ Built-in services
   topics/telnetconsole

 :doc:`topics/logging`
-    Learn how to use Python's builtin logging on Scrapy.
+    Learn how to use Python's built-in logging on Scrapy.

 :doc:`topics/stats`
    Collect statistics about your scraping crawler.
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@ -9,7 +9,7 @@ Installation guide
 Supported Python versions
 =========================

-Scrapy requires Python 3.9+, either the CPython implementation (default) or
+Scrapy requires Python 3.10+, either the CPython implementation (default) or
 the PyPy implementation (see :ref:`python:implementations`).

 .. _intro-install-scrapy:
--- a/docs/news.rst
+++ b/docs/news.rst
@ -15,6 +15,156 @@ Backward-incompatible changes
    ``True`` when running Scrapy via :ref:`its command-line tool
    <topics-commands-crawlerprocess>` to avoid a reactor mismatch exception.

+-   The ``log_count/*`` stats no longer count some of the early messages that
+    they counted before. While the earliest log messages, emitted before the
+    counter is initialized, were never counted, the counter initialization now
+    happens later than in previous Scrapy versions. You may need to adjust
+    expected values if you retrieve and compare values of these stats in your
+    code.
+    (:issue:`7046`)
+
+-   The classes listed below are now :term:`abstract base classes <abstract
+    base class>`. They cannot be instantiated directly and their subclasses
+    need to override the abstract methods listed below to be able to be
+    instantiated. If you previously instantiated these classes directly, you
+    will now need to subclass them and provide trivial (e.g. empty)
+    implementations for the abstract methods.
+
+    - :class:`scrapy.commands.ScrapyCommand`
+
+        - :meth:`~scrapy.commands.ScrapyCommand.run`
+
+        - :meth:`~scrapy.commands.ScrapyCommand.short_desc`
+
+    - :class:`scrapy.exporters.BaseItemExporter`
+
+        - :meth:`~scrapy.exporters.BaseItemExporter.export_item`
+
+    - :class:`scrapy.extensions.feedexport.BlockingFeedStorage`
+
+        - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread`
+
+    - :class:`scrapy.middleware.MiddlewareManager`
+
+        - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings`
+
+    - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`
+
+        - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
+
+-   :class:`scrapy.middleware.MiddlewareManager` no longer includes code for
+    handling ``open_spider()`` and ``close_spider()`` component methods. As
+    this code was only used for pipelines it was moved into
+    :class:`scrapy.pipelines.ItemPipelineManager`. This change should only
+    affect custom subclasses of :class:`~scrapy.middleware.MiddlewareManager`.
+    The following code was moved:
+
+    - ``scrapy.middleware.MiddlewareManager.open_spider()``
+
+    - ``scrapy.middleware.MiddlewareManager.close_spider()``
+
+    - Code in ``scrapy.middleware.MiddlewareManager._add_middleware()`` that
+      processes ``open_spider()`` and ``close_spider()`` component methods.
+
+-   :meth:`scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware.process_request`
+    now returns a coroutine, previously it returned a
+    :class:`~twisted.internet.defer.Deferred` object or ``None``. The
+    ``robot_parser()`` method was also changed to return a coroutine. This
+    change only impacts code that subclasses
+    :class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` or
+    calls its methods directly.
+
+.. _release-2.13.4:
+
+Scrapy 2.13.4 (2025-11-17)
+--------------------------
+
+Security bug fixes
+~~~~~~~~~~~~~~~~~~
+
+-   Improved protection against decompression bombs in
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    for responses compressed using the ``br`` and ``deflate`` methods: if a
+    single compressed chunk would be larger than the response size limit (see
+    :setting:`DOWNLOAD_MAXSIZE`) when decompressed, decompression is no longer
+    carried out. This is especially important for the ``br`` (Brotli) method
+    that can provide a very high compression ratio. Please, see the
+    `CVE-2025-6176`_ and `GHSA-2qfp-q593-8484`_ security advisories for more
+    information.
+    (:issue:`7134`)
+
+    .. _CVE-2025-6176: https://nvd.nist.gov/vuln/detail/CVE-2025-6176
+    .. _GHSA-2qfp-q593-8484: https://github.com/advisories/GHSA-2qfp-q593-8484
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+-   The minimum supported version of the optional ``brotli`` package is now
+    ``1.2.0``.
+    (:issue:`7134`)
+
+-   The ``brotlicffi`` and ``brotlipy`` packages can no longer be used to
+    decompress Brotli-compressed responses. Please install the ``brotli``
+    package instead.
+    (:issue:`7134`)
+
+Other changes
+~~~~~~~~~~~~~
+
+-   Restricted the maximum supported Twisted version to ``25.5.0``, as Scrapy
+    currently uses some private APIs changed in later Twisted versions.
+    (:issue:`7142`)
+
+-   Stopped setting the ``COVERAGE_CORE`` environment variable in tests, it
+    didn't have an effect but caused the ``coverage`` module to produce a
+    warning or an error.
+    (:issue:`7137`)
+
+-   Removed the documentation build dependency on the deprecated
+    ``sphinx-hoverxref`` module.
+    (:issue:`6786`, :issue:`6922`)
+
+.. _release-2.13.3:
+
+Scrapy 2.13.3 (2025-07-02)
+--------------------------
+
+-   Changed the values for :setting:`DOWNLOAD_DELAY` (from ``0`` to ``1``) and
+    :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` (from ``8`` to ``1``) in the
+    default project template.
+    (:issue:`6597`, :issue:`6918`, :issue:`6923`)
+
+-   Improved :class:`scrapy.core.engine.ExecutionEngine` logic related to
+    initialization and exception handling, fixing several cases where the
+    spider would crash, hang or log an unhandled exception.
+    (:issue:`6783`, :issue:`6784`, :issue:`6900`, :issue:`6908`, :issue:`6910`,
+    :issue:`6911`)
+
+-   Fixed a Windows issue with :ref:`feed exports <topics-feed-exports>` using
+    :class:`scrapy.extensions.feedexport.FileFeedStorage` that caused the file
+    to be created on the wrong drive.
+    (:issue:`6894`, :issue:`6897`)
+
+-   Allowed running tests with Twisted 25.5.0+ again. Pytest 8.4.1+ is now
+    required for running tests in non-pinned envs as support for the new
+    Twisted version was added in that version.
+    (:issue:`6893`)
+
+-   Fixed running tests with lxml 6.0.0+.
+    (:issue:`6919`)
+
+-   Added a deprecation notice for
+    ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` to :ref:`the Scrapy
+    2.11.2 release notes <release-2.11.2>`.
+    (:issue:`6926`)
+
+-   Updated :ref:`contribution docs <topics-contributing>` to refer to ruff_
+    instead of black_.
+    (:issue:`6903`)
+
+-   Added ``.venv/`` and ``.vscode/`` to ``.gitignore``.
+    (:issue:`6901`, :issue:`6907`)
+

 .. _release-2.13.2:

@ -340,7 +490,7 @@ Deprecations
    (:issue:`6708`, :issue:`6714`)

 -   ``scrapy.utils.versions.scrapy_components_versions()`` is deprecated, use
-    :func:`scrapy.utils.versions.get_versions()` instead.
+    :func:`scrapy.utils.versions.get_versions` instead.
    (:issue:`6582`)

 -   ``BaseDupeFilter.log()`` is deprecated. It does nothing and shouldn't be
@ -1233,6 +1383,17 @@ Security bug fixes

    .. _defusedxml: https://github.com/tiran/defusedxml

+Deprecations
+~~~~~~~~~~~~
+
+-   ``scrapy.spidermiddlewares.offsite.OffsiteMiddleware`` (a spider
+    middleware) is now deprecated and not enabled by default. The new
+    downloader middleware with the same functionality,
+    :class:`scrapy.downloadermiddlewares.offsite.OffsiteMiddleware`, is enabled
+    instead.
+    (:issue:`2241`, :issue:`6358`)
+
+
 Bug fixes
 ~~~~~~~~~

@ -1786,7 +1947,7 @@ Bug fixes
    (:issue:`5914`, :issue:`5917`)

 -   Fixed an error breaking user handling of send failures in
-    :meth:`scrapy.mail.MailSender.send()`. (:issue:`1611`, :issue:`5880`)
+    :meth:`scrapy.mail.MailSender.send`. (:issue:`1611`, :issue:`5880`)

 Documentation
 ~~~~~~~~~~~~~
@ -5336,7 +5497,7 @@ Docs
 - Added missing bullet point for the ``AUTOTHROTTLE_TARGET_CONCURRENCY``
  setting. (:issue:`2756`)
 - Update Contributing docs, document new support channels
-  (:issue:`2762`, issue:`3038`)
+  (:issue:`2762`, :issue:`3038`)
 - Include references to Scrapy subreddit in the docs
 - Fix broken links; use ``https://`` for external links
  (:issue:`2978`, :issue:`2982`, :issue:`2958`)
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@ -1,5 +1,6 @@
+pydantic==2.12.3
+scrapy-spider-metadata==0.2.0
 sphinx==8.1.3
-sphinx-hoverxref==1.4.2
 sphinx-notfound-page==1.0.4
 sphinx-rtd-theme==3.0.2
 sphinx-rtd-dark-mode==1.3.0
--- a/docs/topics/addons.rst
+++ b/docs/topics/addons.rst
@ -88,7 +88,7 @@ recommend that such custom components should be written in the following way:

 1. The custom component (e.g. ``MyDownloadHandler``) shouldn't inherit from the
   default Scrapy one (e.g.
-   ``scrapy.core.downloader.handlers.http.HTTPDownloadHandler``), but instead
+   ``scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler``), but instead
   be able to load the class of the fallback component from a special setting
   (e.g. ``MY_FALLBACK_DOWNLOAD_HANDLER``), create an instance of it and use
   it.
@ -166,7 +166,6 @@ Use a fallback component:

 .. code-block:: python

-    from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
    from scrapy.utils.misc import build_from_crawler


--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@ -280,13 +280,13 @@ class (which they all inherit from).
    The following methods are not part of the stats collection api but instead
    used when implementing custom stats collectors:

-    .. method:: open_spider(spider)
+    .. method:: open_spider()

-        Open the given spider for stats collection.
+        Open the spider for stats collection.

-    .. method:: close_spider(spider)
+    .. method:: close_spider()

-        Close the given spider. After this is called, no more specific stats
+        Close the spider. After this is called, no more specific stats
        can be accessed or collected.

 Engine API
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@ -49,6 +49,7 @@ You can usually fix the issue by moving those offending module-level Twisted
 imports to the method or function definitions where they are used. For example,
 if you have something like:

+.. skip: next
 .. code-block:: python

    from twisted.internet import reactor
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@ -37,8 +37,7 @@ processed in parallel.

 Instead of adjusting the delays one can just set a small fixed
 download delay and impose hard limits on concurrency using
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
-:setting:`CONCURRENT_REQUESTS_PER_IP` options. It will provide a similar
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. It will provide a similar
 effect, but there are some important differences:

 * because the download delay is small there will be occasional bursts
@ -71,7 +70,6 @@ AutoThrottle algorithm adjusts download delays based on the following rules:
 .. note:: The AutoThrottle extension honours the standard Scrapy settings for
   concurrency and delay. This means that it will respect
   :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` and
-   :setting:`CONCURRENT_REQUESTS_PER_IP` options and
   never set a download delay lower than :setting:`DOWNLOAD_DELAY`.

 .. _download-latency:
@ -123,7 +121,6 @@ The settings used to control the AutoThrottle extension are:
 * :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY`
 * :setting:`AUTOTHROTTLE_DEBUG`
 * :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-* :setting:`CONCURRENT_REQUESTS_PER_IP`
 * :setting:`DOWNLOAD_DELAY`

 For more information see :ref:`autothrottle-algorithm`.
@ -171,12 +168,10 @@ a higher value (e.g. ``2.0``) to increase the throughput and the load on remote
 servers. A lower ``AUTOTHROTTLE_TARGET_CONCURRENCY`` value
 (e.g. ``0.5``) makes the crawler more conservative and polite.

-Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`
-and :setting:`CONCURRENT_REQUESTS_PER_IP` options are still respected
+Note that :setting:`CONCURRENT_REQUESTS_PER_DOMAIN` is still respected
 when AutoThrottle extension is enabled. This means that if
 ``AUTOTHROTTLE_TARGET_CONCURRENCY`` is set to a value higher than
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` or
-:setting:`CONCURRENT_REQUESTS_PER_IP`, the crawler won't reach this number
+:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`, the crawler won't reach this number
 of concurrent requests.

 At every given time point Scrapy can be sending more or less concurrent
--- a/docs/topics/benchmarking.rst
+++ b/docs/topics/benchmarking.rst
@ -83,4 +83,4 @@ and how well it's written.

 Use scrapy-bench_ for more complex benchmarking.

-.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
+.. _scrapy-bench: https://github.com/scrapy/scrapy-bench
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@ -61,12 +61,7 @@ Increase concurrency

 Concurrency is the number of requests that are processed in parallel. There is
 a global limit (:setting:`CONCURRENT_REQUESTS`) and an additional limit that
-can be set either per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`) or per
-IP (:setting:`CONCURRENT_REQUESTS_PER_IP`).
-
-.. note:: The scheduler priority queue :ref:`recommended for broad crawls
-          <broad-crawls-scheduler-priority-queue>` does not support
-          :setting:`CONCURRENT_REQUESTS_PER_IP`.
+can be set per domain (:setting:`CONCURRENT_REQUESTS_PER_DOMAIN`).

 The default global concurrency limit in Scrapy is not suitable for crawling
 many different domains in parallel, so you will want to increase it. How much
@ -143,7 +138,7 @@ To disable cookies use:
 Disable retries
 ===============

-Retrying failed HTTP requests can slow down the crawls substantially, specially
+Retrying failed HTTP requests can slow down the crawls substantially, especially
 when sites causes are very slow (or fail) to respond, thus causing a timeout
 error which gets retried many times, unnecessarily, preventing crawler capacity
 to be reused for other domains.
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@ -191,7 +191,7 @@ shorter and cleaner:
            adapter["field"] = data
            return item

-        def process_item(self, item, spider):
+        def process_item(self, item):
            adapter = ItemAdapter(item)
            dfd = db.get_some_data(adapter["id"])
            dfd.addCallback(self._update_item, item)
@ -205,7 +205,7 @@ becomes:


    class DbPipeline:
-        async def process_item(self, item, spider):
+        async def process_item(self, item):
            adapter = ItemAdapter(item)
            adapter["field"] = await db.get_some_data(adapter["id"])
            return item
@ -266,7 +266,6 @@ within a spider callback:
 .. code-block:: python

    from scrapy import Spider, Request
-    from scrapy.utils.defer import maybe_deferred_to_future


    class SingleRequestSpider(Spider):
@ -275,8 +274,9 @@ within a spider callback:

        async def parse(self, response, **kwargs):
            additional_request = Request("https://example.org/price")
-            deferred = self.crawler.engine.download(additional_request)
-            additional_response = await maybe_deferred_to_future(deferred)
+            additional_response = await self.crawler.engine.download_async(
+                additional_request
+            )
            yield {
                "h1": response.css("h1").get(),
                "price": additional_response.css("#price").get(),
@ -286,9 +286,9 @@ You can also send multiple requests in parallel:

 .. code-block:: python

+    import asyncio
+
    from scrapy import Spider, Request
-    from scrapy.utils.defer import maybe_deferred_to_future
-    from twisted.internet.defer import DeferredList


    class MultipleRequestsSpider(Spider):
@ -300,11 +300,11 @@ You can also send multiple requests in parallel:
                Request("https://example.com/price"),
                Request("https://example.com/color"),
            ]
-            deferreds = []
+            tasks = []
            for r in additional_requests:
-                deferred = self.crawler.engine.download(r)
-                deferreds.append(deferred)
-            responses = await maybe_deferred_to_future(DeferredList(deferreds))
+                task = self.crawler.engine.download_async(r)
+                tasks.append(task)
+            responses = await asyncio.gather(*tasks)
            yield {
                "h1": response.css("h1::text").get(),
                "price": responses[0][1].css(".price::text").get(),
@ -421,12 +421,12 @@ For example:
 .. code-block:: python

    class UniversalSpiderMiddleware:
-        def process_spider_output(self, response, result, spider):
+        def process_spider_output(self, response, result):
            for r in result:
                # ... do something with r
                yield r

-        async def process_spider_output_async(self, response, result, spider):
+        async def process_spider_output_async(self, response, result):
            async for r in result:
                # ... do something with r
                yield r
--- a/docs/topics/developer-tools.rst
+++ b/docs/topics/developer-tools.rst
@ -280,7 +280,7 @@ In more complex websites, it could be difficult to easily reproduce the
 requests, as we could need to add ``headers`` or ``cookies`` to make it work.
 In those cases you can export the requests in `cURL <https://curl.se/>`_
 format, by right-clicking on each of them in the network tool and using the
-:meth:`~scrapy.Request.from_curl()` method to generate an equivalent
+:meth:`~scrapy.Request.from_curl` method to generate an equivalent
 request:

 .. code-block:: python
@ -317,4 +317,3 @@ to identifying the correct request and replicating it in your spider.
 .. _quotes.toscrape.com/scroll: https://quotes.toscrape.com/scroll
 .. _quotes.toscrape.com/api/quotes?page=10: https://quotes.toscrape.com/api/quotes?page=10
 .. _has-class-extension: https://parsel.readthedocs.io/en/latest/usage.html#other-xpath-extensions
-
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@ -70,7 +70,7 @@ defines one or more of these methods:

   .. note::  Any of the downloader middleware methods may also return a deferred.

-   .. method:: process_request(request, spider)
+   .. method:: process_request(request)

      This method is called for each request that goes through the download
      middleware.
@ -102,10 +102,7 @@ defines one or more of these methods:
      :param request: the request being processed
      :type request: :class:`~scrapy.Request` object

-      :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.Spider` object
-
-   .. method:: process_response(request, response, spider)
+   .. method:: process_response(request, response)

      :meth:`process_response` should either: return a :class:`~scrapy.http.Response`
      object, return a :class:`~scrapy.Request` object or
@ -129,10 +126,7 @@ defines one or more of these methods:
      :param response: the response being processed
      :type response: :class:`~scrapy.http.Response` object

-      :param spider: the spider for which this response is intended
-      :type spider: :class:`~scrapy.Spider` object
-
-   .. method:: process_exception(request, exception, spider)
+   .. method:: process_exception(request, exception)

      Scrapy calls :meth:`process_exception` when a download handler
      or a :meth:`process_request` (from a downloader middleware) raises an
@ -160,9 +154,6 @@ defines one or more of these methods:
      :param exception: the raised exception
      :type exception: an ``Exception`` object

-      :param spider: the spider for which this request is intended
-      :type spider: :class:`~scrapy.Spider` object
-
 .. _topics-downloader-middleware-ref:

 Built-in downloader middleware reference
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@ -86,7 +86,7 @@ method and URL. However, you may also need to reproduce the body, headers and
 form parameters (see :class:`~scrapy.FormRequest`) of that request.

 As all major browsers allow to export the requests in curl_ format, Scrapy
-incorporates the method :meth:`~scrapy.Request.from_curl()` to generate an equivalent
+incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent
 :class:`~scrapy.Request` from a cURL command. To get more information
 visit :ref:`request from curl <requests-from-curl>` inside the network
 tool section.
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@ -67,7 +67,7 @@ value of one of their fields:
                self.year_to_exporter[year] = (exporter, xml_file)
            return self.year_to_exporter[year][0]

-        def process_item(self, item, spider):
+        def process_item(self, item):
            exporter = self._exporter_for_item(item)
            exporter.export_item(item)
            return item
@ -116,10 +116,10 @@ Example:
 2. Overriding the serialize_field() method
 ------------------------------------------

-You can also override the :meth:`~BaseItemExporter.serialize_field()` method to
+You can also override the :meth:`~BaseItemExporter.serialize_field` method to
 customize how your field value will be exported.

-Make sure you call the base class :meth:`~BaseItemExporter.serialize_field()` method
+Make sure you call the base class :meth:`~BaseItemExporter.serialize_field` method
 after your custom code.

 Example:
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@ -138,6 +138,14 @@ enabled (see :ref:`topics-stats`).

 .. _topics-extensions-ref-telnetconsole:

+Log Count extension
+~~~~~~~~~~~~~~~~~~~
+
+.. module:: scrapy.extensions.logcount
+   :synopsis: Basic stats logging
+
+.. autoclass:: LogCount
+
 Telnet console extension
 ~~~~~~~~~~~~~~~~~~~~~~~~

@ -259,7 +267,7 @@ CLOSESPIDER_TIMEOUT
 Default: ``0``

 An integer which specifies a number of seconds. If the spider remains open for
-more than that number of second, it will be automatically closed with the
+more than that number of seconds, it will be automatically closed with the
 reason ``closespider_timeout``. If zero (or non set), spiders won't be closed by
 timeout.

--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@ -26,7 +26,7 @@ Writing your own item pipeline
 Each item pipeline is a :ref:`component <topics-components>` that must
 implement the following method:

-.. method:: process_item(self, item, spider)
+.. method:: process_item(self, item)

   This method is called for every item pipeline component.

@ -42,25 +42,16 @@ implement the following method:
   :param item: the scraped item
   :type item: :ref:`item object <item-types>`

-   :param spider: the spider which scraped the item
-   :type spider: :class:`~scrapy.Spider` object
-
 Additionally, they may also implement the following methods:

-.. method:: open_spider(self, spider)
+.. method:: open_spider(self)

   This method is called when the spider is opened.

-   :param spider: the spider which was opened
-   :type spider: :class:`~scrapy.Spider` object
-
-.. method:: close_spider(self, spider)
+.. method:: close_spider(self)

   This method is called when the spider is closed.

-   :param spider: the spider which was closed
-   :type spider: :class:`~scrapy.Spider` object
-

 Item pipeline example
 =====================
@ -82,7 +73,7 @@ contain a price:
    class PricePipeline:
        vat_factor = 1.15

-        def process_item(self, item, spider):
+        def process_item(self, item):
            adapter = ItemAdapter(item)
            if adapter.get("price"):
                if adapter.get("price_excludes_vat"):
@ -107,13 +98,13 @@ format:


   class JsonWriterPipeline:
-       def open_spider(self, spider):
+       def open_spider(self):
           self.file = open("items.jsonl", "w")

-       def close_spider(self, spider):
+       def close_spider(self):
           self.file.close()

-       def process_item(self, item, spider):
+       def process_item(self, item):
           line = json.dumps(ItemAdapter(item).asdict()) + "\n"
           self.file.write(line)
           return item
@ -153,14 +144,14 @@ The main point of this example is to show how to :ref:`get the crawler
                mongo_db=crawler.settings.get("MONGO_DATABASE", "items"),
            )

-        def open_spider(self, spider):
+        def open_spider(self):
            self.client = pymongo.MongoClient(self.mongo_uri)
            self.db = self.client[self.mongo_db]

-        def close_spider(self, spider):
+        def close_spider(self):
            self.client.close()

-        def process_item(self, item, spider):
+        def process_item(self, item):
            self.db[self.collection_name].insert_one(ItemAdapter(item).asdict())
            return item

@ -190,7 +181,6 @@ item.
    import scrapy
    from itemadapter import ItemAdapter
    from scrapy.http.request import NO_CALLBACK
-    from scrapy.utils.defer import maybe_deferred_to_future


    class ScreenshotPipeline:
@ -199,14 +189,19 @@ item.

        SPLASH_URL = "http://localhost:8050/render.png?url={}"

-        async def process_item(self, item, spider):
+        def __init__(crawler):
+            self.crawler = crawler
+
+        @classmethod
+        def from_crawler(cls, crawler):
+            return cls(crawler)
+
+        async def process_item(self, item):
            adapter = ItemAdapter(item)
            encoded_item_url = quote(adapter["url"])
            screenshot_url = self.SPLASH_URL.format(encoded_item_url)
            request = scrapy.Request(screenshot_url, callback=NO_CALLBACK)
-            response = await maybe_deferred_to_future(
-                spider.crawler.engine.download(request)
-            )
+            response = await self.crawler.engine.download_async(request)

            if response.status != 200:
                # Error happened, return item.
@ -241,7 +236,7 @@ returns multiples items with the same id:
        def __init__(self):
            self.ids_seen = set()

-        def process_item(self, item, spider):
+        def process_item(self, item):
            adapter = ItemAdapter(item)
            if adapter["id"] in self.ids_seen:
                raise DropItem(f"Item ID already seen: {adapter['id']}")
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@ -162,7 +162,7 @@ Too many spiders?
 -----------------

 If your project has too many spiders executed in parallel,
-the output of :func:`prefs()` can be difficult to read.
+the output of :func:`prefs` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
 ignore a particular class (and all its subclasses). For
 example, this won't show any live references to spiders:
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.

-The Images Pipeline requires Pillow_ 8.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 8.3.2 or greater. It is used for
 thumbnailing and normalizing images to JPEG/RGB format.

 .. _Pillow: https://github.com/python-pillow/Pillow
@ -238,7 +238,7 @@ Amazon S3 storage
 .. setting:: FILES_STORE_S3_ACL
 .. setting:: IMAGES_STORE_S3_ACL

-If botocore_ >= 1.4.87 is installed, :setting:`FILES_STORE` and
+If botocore_ >= 1.13.45 is installed, :setting:`FILES_STORE` and
 :setting:`IMAGES_STORE` can represent an Amazon S3 bucket. Scrapy will
 automatically upload the files to the bucket.

--- a/docs/topics/practices.rst
+++ b/docs/topics/practices.rst
@ -281,7 +281,7 @@ finishes before starting the next one:
 Distributed crawls
 ==================

-Scrapy doesn't provide any built-in facility for running crawls in a distribute
+Scrapy doesn't provide any built-in facility for running crawls in a distributed
 (multi-server) manner. However, there are some ways to distribute crawls, which
 vary depending on how you plan to distribute them.

@ -289,10 +289,10 @@ If you have many spiders, the obvious way to distribute the load is to setup
 many Scrapyd instances and distribute spider runs among those.

 If you instead want to run a single (big) spider through many machines, what
-you usually do is partition the urls to crawl and send them to each separate
+you usually do is partition the URLs to crawl and send them to each separate
 spider. Here is a concrete example:

-First, you prepare the list of urls to crawl and put them into separate
+First, you prepare the list of URLs to crawl and put them into separate
 files/urls::

    http://somedomain.com/urls-to-crawl/spider1/part1.list
@ -319,7 +319,7 @@ consider contacting `commercial support`_ if in doubt.

 Here are some tips to keep in mind when dealing with these kinds of sites:

-* rotate your user agent from a pool of well-known ones from browsers (google
+* rotate your user agent from a pool of well-known ones from browsers (Google
  around to get a list of them)
 * disable cookies (see :setting:`COOKIES_ENABLED`) as some sites may use
  cookies to spot bot behaviour
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@ -873,7 +873,7 @@ It is usual for web sites to provide pre-populated form fields through ``<input
 type="hidden">`` elements, such as session related data or authentication
 tokens (for login pages). When scraping, you'll want these fields to be
 automatically pre-populated and only override a couple of them, such as the
-user name and password. You can use the :meth:`.FormRequest.from_response()`
+user name and password. You can use the :meth:`.FormRequest.from_response`
 method for this job. Here's an example spider which uses it:

 .. code-block:: python
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@ -517,7 +517,7 @@ performed by the Scrapy downloader.
 CONCURRENT_REQUESTS_PER_DOMAIN
 ------------------------------

-Default: ``8``
+Default: ``1`` (:ref:`fallback <default-settings>`: ``8``)

 The maximum number of concurrent (i.e. simultaneous) requests that will be
 performed to any single domain.
@ -526,23 +526,6 @@ See also: :ref:`topics-autothrottle` and its
 :setting:`AUTOTHROTTLE_TARGET_CONCURRENCY` option.


-.. setting:: CONCURRENT_REQUESTS_PER_IP
-
-CONCURRENT_REQUESTS_PER_IP
--------------------------
-
-Default: ``0``
-
-The maximum number of concurrent (i.e. simultaneous) requests that will be
-performed to any single IP. If non-zero, the
-:setting:`CONCURRENT_REQUESTS_PER_DOMAIN` setting is ignored, and this one is
-used instead. In other words, concurrency limits will be applied per IP, not
-per domain.
-
-This setting also affects :setting:`DOWNLOAD_DELAY` and
-:ref:`topics-autothrottle`: if :setting:`CONCURRENT_REQUESTS_PER_IP`
-is non-zero, download delay is enforced per IP, not per domain.
-
 .. setting:: DEFAULT_DROPITEM_LOG_LEVEL

 DEFAULT_DROPITEM_LOG_LEVEL
@ -571,7 +554,7 @@ When writing an item pipeline, you can force a different log level by setting


   class MyPipeline:
-       def process_item(self, item, spider):
+       def process_item(self, item):
           if not item.get("price"):
               raise DropItem("Missing price data", log_level="INFO")
           return item
@ -728,7 +711,7 @@ connections (for ``HTTP10DownloadHandler``).
    so you can safely ignore this setting,
    unless you really want to use HTTP/1.0 and override
    :setting:`DOWNLOAD_HANDLERS` for ``http(s)`` scheme accordingly,
-    i.e. to ``'scrapy.core.downloader.handlers.http.HTTP10DownloadHandler'``.
+    i.e. to ``'scrapy.core.downloader.handlers.http10.HTTP10DownloadHandler'``.

 .. setting:: DOWNLOADER_CLIENTCONTEXTFACTORY

@ -869,7 +852,7 @@ Whether to enable downloader stats collection.
 DOWNLOAD_DELAY
 --------------

-Default: ``0``
+Default: ``1`` (:ref:`fallback <default-settings>`: ``0``)

 Minimum seconds to wait between 2 consecutive requests to the same domain.

@ -884,9 +867,6 @@ every 10 seconds::
 This setting is also affected by the :setting:`RANDOMIZE_DOWNLOAD_DELAY`
 setting, which is enabled by default.

-When :setting:`CONCURRENT_REQUESTS_PER_IP` is non-zero, delays are enforced
-per IP address instead of per domain.
-
 Note that :setting:`DOWNLOAD_DELAY` can lower the effective per-domain
 concurrency below :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`. If the response
 time of a domain is lower than :setting:`DOWNLOAD_DELAY`, the effective
@ -929,8 +909,8 @@ Default:
    {
        "data": "scrapy.core.downloader.handlers.datauri.DataURIDownloadHandler",
        "file": "scrapy.core.downloader.handlers.file.FileDownloadHandler",
-        "http": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
-        "https": "scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
+        "http": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
+        "https": "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler",
        "s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
        "ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
    }
@ -1151,9 +1131,9 @@ interface::
    class MyDupeFilter:

        @classmethod
-        def from_settings(cls, settings):
+        def from_crawler(cls, crawler):
            """Returns an instance of this duplicate request filtering class
-            based on the current crawl settings."""
+            based on the current Crawler instance."""
            return cls()

        def request_seen(self, request):
@ -1765,8 +1745,7 @@ Type of priority queue used by the scheduler. Another available type is
 ``scrapy.pqueues.DownloaderAwarePriorityQueue``.
 ``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
 ``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
-domains in parallel. But currently ``scrapy.pqueues.DownloaderAwarePriorityQueue``
-does not work together with :setting:`CONCURRENT_REQUESTS_PER_IP`.
+domains in parallel.


 .. setting:: SCHEDULER_START_DISK_QUEUE
@ -2024,6 +2003,7 @@ reactor is installed.

 In order to use the reactor installed by Scrapy:

+.. skip: next
 .. code-block:: python

    import scrapy
--- a/docs/topics/shell.rst
+++ b/docs/topics/shell.rst
@ -24,7 +24,7 @@ If you have `IPython`_ installed, the Scrapy shell will use it (instead of the
 standard Python console). The `IPython`_ console is much more powerful and
 provides smart auto-completion and colorized output, among other things.

-We highly recommend you install `IPython`_, specially if you're working on
+We highly recommend you install `IPython`_, especially if you're working on
 Unix systems (where `IPython`_ excels). See the `IPython installation guide`_
 for more info.

--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@ -94,7 +94,7 @@ one or more of these methods:
            def process_start_requests(self, start, spider):
                yield from start

-    .. method:: process_spider_input(response, spider)
+    .. method:: process_spider_input(response)

        This method is called for each response that goes through the spider
        middleware and into the spider, for processing.
@ -116,11 +116,7 @@ one or more of these methods:
        :param response: the response being processed
        :type response: :class:`~scrapy.http.Response` object

-        :param spider: the spider for which this response is intended
-        :type spider: :class:`~scrapy.Spider` object
-
-
-    .. method:: process_spider_output(response, result, spider)
+    .. method:: process_spider_output(response, result)

        This method is called with the results returned from the Spider, after
        it has processed the response.
@ -149,10 +145,7 @@ one or more of these methods:
        :type result: an iterable of :class:`~scrapy.Request` objects and
          :ref:`item objects <topics-items>`

-        :param spider: the spider whose result is being processed
-        :type spider: :class:`~scrapy.Spider` object
-
-    .. method:: process_spider_output_async(response, result, spider)
+    .. method:: process_spider_output_async(response, result)
        :async:

        .. versionadded:: 2.7
@ -161,7 +154,7 @@ one or more of these methods:
        which will be called instead of :meth:`process_spider_output` if
        ``result`` is an :term:`asynchronous iterable`.

-    .. method:: process_spider_exception(response, exception, spider)
+    .. method:: process_spider_exception(response, exception)

        This method is called when a spider or :meth:`process_spider_output`
        method (from a previous spider middleware) raises an exception.
@ -186,8 +179,6 @@ one or more of these methods:
        :param exception: the exception raised
        :type exception: :exc:`Exception` object

-        :param spider: the spider which raised the exception
-        :type spider: :class:`~scrapy.Spider` object

 Base class for custom spider middlewares
 ----------------------------------------
@ -354,7 +345,7 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
 Acceptable values for REFERRER_POLICY
 *************************************

- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
+- either a path to a :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`
  subclass — a custom policy or one of the built-in ones (see classes below),
 - or one or more comma-separated standard W3C-defined string values,
 - or the special ``"scrapy-default"``.
@ -373,6 +364,8 @@ String value                             Class name (as a string)
 `"unsafe-url"`_                          :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
 =======================================  ========================================================================

+.. autoclass:: ReferrerPolicy
+
 .. autoclass:: DefaultReferrerPolicy
 .. warning::
    Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@ -364,6 +364,52 @@ used by :class:`~scrapy.downloadermiddlewares.useragent.UserAgentMiddleware`::
 Spider arguments can also be passed through the Scrapyd ``schedule.json`` API.
 See `Scrapyd documentation`_.

+scrapy-spider-metadata parameters
+---------------------------------
+
+Another alternative to pass spider arguments is the library `scrapy-spider-metadata`_.
+
+This allows for Scrapy spiders to define, validate, document and pre-process
+their arguments as Pydantic models.
+
+The example shows how to define typed parameters where a string argument
+is automatically converted to an integer:
+
+.. code-block:: python
+
+    import scrapy
+    from pydantic import BaseModel
+    from scrapy_spider_metadata import Args
+
+
+    class MyParams(BaseModel):
+        pages: int
+
+
+    class BookSpider(Args[MyParams], scrapy.Spider):
+        name = "bookspider"
+        start_urls = ["http://books.toscrape.com/catalogue"]
+
+        async def start(self):
+            for start_url in self.start_urls:
+                for index in range(1, self.args.pages + 1):
+                    yield scrapy.Request(f"{start_url}/page-{index}.html")
+
+        def parse(self, response):
+            book_links = response.css("article.product_pod h3 a::attr(href)").getall()
+            for book_link in book_links:
+                yield response.follow(book_link, self.parse_book)
+
+        def parse_book(self, response):
+            yield {
+                "title": response.css("h1::text").get(),
+                "price": response.css("p.price_color::text").get(),
+            }
+
+This spider can be called from the command line::
+
+    scrapy crawl bookspider -a pages=2
+
 .. _start-requests:

 Start requests
@ -628,7 +674,7 @@ XMLFeedSpider
        This method is called for the nodes matching the provided tag name
        (``itertag``).  Receives the response and an
        :class:`~scrapy.Selector` for each node.  Overriding this
-        method is mandatory. Otherwise, you spider won't work.  This method
+        method is mandatory. Otherwise, your spider won't work.  This method
        must return an :ref:`item object <topics-items>`, a
        :class:`~scrapy.Request` object, or an iterable containing any of
        them.
@ -938,6 +984,7 @@ Combine SitemapSpider with other sources of urls:
        def parse_other(self, response):
            pass  # ... scrape other here ...

+.. _scrapy-spider-metadata: https://scrapy-spider-metadata.readthedocs.io/en/latest/params.html
 .. _Sitemaps: https://www.sitemaps.org/index.html
 .. _Sitemap index files: https://www.sitemaps.org/protocol.html#index
 .. _robots.txt: https://www.robotstxt.org/
--- a/docs/topics/stats.rst
+++ b/docs/topics/stats.rst
@ -121,4 +121,3 @@ DummyStatsCollector
    setting, to disable stats collect in order to improve performance. However,
    the performance penalty of stats collection is usually marginal compared to
    other Scrapy workload like parsing pages.
-
--- a/docs/versioning.rst
+++ b/docs/versioning.rst
@ -66,4 +66,3 @@ the :ref:`release notes <news>`.


 .. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases
-
--- a/pyproject.toml
+++ b/pyproject.toml
@ -7,13 +7,14 @@ name = "Scrapy"
 dynamic = ["version"]
 description = "A high-level Web Crawling and Web Scraping framework"
 dependencies = [
-    "Twisted>=21.7.0",
+    # Twisted pinned until Scrapy is updated for its internal TLS API changes
+    "Twisted>=21.7.0,<=25.5.0",
    "cryptography>=37.0.0",
    "cssselect>=0.9.1",
    "defusedxml>=0.7.1",
    "itemadapter>=0.1.0",
    "itemloaders>=1.0.1",
-    "lxml>=4.6.0",
+    "lxml>=4.6.4",
    "packaging",
    "parsel>=1.5.0",
    "protego>=0.1.15",
@ -35,7 +36,6 @@ classifiers = [
    "Operating System :: OS Independent",
    "Programming Language :: Python",
    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
@ -49,7 +49,7 @@ classifiers = [
 license = "BSD-3-Clause"
 license-files = ["LICENSE", "AUTHORS"]
 readme = "README.rst"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 authors = [{ name = "Scrapy developers", email = "pablo@pablohoffman.com" }]
 maintainers = [{ name = "Pablo Hoffman", email = "pablo@pablohoffman.com" }]

@ -106,16 +106,12 @@ follow_imports = "skip"
 module = "scrapy.settings.default_settings"
 ignore_errors = true

-[[tool.mypy.overrides]]
-module = "itemadapter"
-implicit_reexport = true
-
 [[tool.mypy.overrides]]
 module = "twisted"
 implicit_reexport = true

 [tool.bumpversion]
-current_version = "2.13.2"
+current_version = "2.13.4"
 commit = true
 tag = true
 tag_name = "{new_version}"
@ -139,6 +135,9 @@ branch = true
 include = ["scrapy/*"]
 omit = ["tests/*"]
 disable_warnings = ["include-ignored"]
+patch = [
+    "subprocess",
+]

 [tool.coverage.paths]
 source = [
@ -217,12 +216,14 @@ disable = [
    "keyword-arg-before-vararg",
    "pointless-statement",
    "raise-missing-from",
-    "unbalanced-tuple-unpacking",
    "unnecessary-dunder-call",
    "used-before-assignment",
 ]

 [tool.pytest.ini_options]
+addopts = [
+    "--reactor=asyncio",
+]
 xfail_strict = true
 python_files = ["test_*.py", "test_*/__init__.py"]
 markers = [
@ -231,6 +232,7 @@ markers = [
    "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
    "requires_botocore: marks tests that need botocore (but not boto3)",
    "requires_boto3: marks tests that need botocore and boto3",
+    "requires_mitmproxy: marks tests that need mitmproxy",
 ]
 filterwarnings = [
    "ignore::DeprecationWarning:twisted.web.static"
@ -238,10 +240,16 @@ filterwarnings = [

 [tool.ruff.lint]
 extend-select = [
+    # flake8-builtins
+    "A",
+    # flake8-async
+    "ASYNC",
    # flake8-bugbear
    "B",
    # flake8-comprehensions
    "C4",
+    # flake8-commas
+    "COM",
    # pydocstyle
    "D",
    # flake8-future-annotations
@ -300,6 +308,8 @@ extend-select = [
 ignore = [
    # Ones we want to ignore

+    # Trailing comma missing
+    "COM812",
    # Missing docstring in public module
    "D100",
    # Missing docstring in public class
@ -392,8 +402,15 @@ ignore = [
 [tool.ruff.lint.flake8-tidy-imports]
 banned-module-level-imports = [
    "twisted.internet.reactor",
+    # indirectly imports twisted.conch.insults.helper which imports twisted.internet.reactor
+    "twisted.conch.manhole",
+    # directly imports twisted.internet.reactor
+    "twisted.protocols.ftp",
 ]

+[tool.ruff.lint.isort]
+split-on-trailing-comma = false
+
 [tool.ruff.lint.per-file-ignores]
 # Circular import workarounds
 "scrapy/linkextractors/__init__.py" = ["E402"]
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@ -1 +1 @@
-2.13.2
+2.13.4
--- a/scrapy/init.py
+++ b/scrapy/init.py
@ -29,23 +29,6 @@ __version__ = (pkgutil.get_data(__package__, "VERSION") or b"").decode("ascii").
 version_info = tuple(int(v) if v.isdigit() else v for v in __version__.split("."))


-def __getattr__(name: str):
-    if name == "twisted_version":
-        import warnings  # pylint: disable=reimported
-
-        from twisted import version as _txv
-
-        from scrapy.exceptions import ScrapyDeprecationWarning
-
-        warnings.warn(
-            "The scrapy.twisted_version attribute is deprecated, use twisted.version instead",
-            ScrapyDeprecationWarning,
-        )
-        return _txv.major, _txv.minor, _txv.micro
-
-    raise AttributeError
-
-
 # Ignore noisy twisted deprecation warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning, module="twisted")

--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@ -6,7 +6,7 @@ import inspect
 import os
 import sys
 from importlib.metadata import entry_points
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, ParamSpec

 import scrapy
 from scrapy.commands import BaseRunSpiderCommand, ScrapyCommand, ScrapyHelpFormatter
@ -20,12 +20,9 @@ from scrapy.utils.reactor import _asyncio_reactor_path
 if TYPE_CHECKING:
    from collections.abc import Callable, Iterable

-    # typing.ParamSpec requires Python 3.10
-    from typing_extensions import ParamSpec
-
    from scrapy.settings import BaseSettings, Settings

-    _P = ParamSpec("_P")
+_P = ParamSpec("_P")


 class ScrapyArgumentParser(argparse.ArgumentParser):
@ -67,11 +64,7 @@ def _get_commands_from_entry_points(
    inproject: bool, group: str = "scrapy.commands"
 ) -> dict[str, ScrapyCommand]:
    cmds: dict[str, ScrapyCommand] = {}
-    if sys.version_info >= (3, 10):
-        eps = entry_points(group=group)
-    else:
-        eps = entry_points().get(group, ())
-    for entry_point in eps:
+    for entry_point in entry_points(group=group):
        obj = entry_point.load()
        if inspect.isclass(obj):
            cmds[entry_point.name] = obj()
--- a/scrapy/commands/init.py
+++ b/scrapy/commands/init.py
@ -7,6 +7,7 @@ from __future__ import annotations
 import argparse
 import builtins
 import os
+from abc import ABC, abstractmethod
 from pathlib import Path
 from typing import TYPE_CHECKING, Any

@ -22,7 +23,7 @@ if TYPE_CHECKING:
    from scrapy.settings import Settings


-class ScrapyCommand:
+class ScrapyCommand(ABC):
    """Base class for all Scrapy commands."""

    requires_project: bool = False
@ -48,6 +49,7 @@ class ScrapyCommand:
        """
        return ""

+    @abstractmethod
    def short_desc(self) -> str:
        """
        A short description of the command
@ -130,6 +132,7 @@ class ScrapyCommand:
        if opts.pdb:
            failure.startDebugMode()

+    @abstractmethod
    def run(self, args: list[str], opts: argparse.Namespace) -> None:
        """
        Entry point for running commands
--- a/scrapy/commands/bench.py
+++ b/scrapy/commands/bench.py
@ -10,6 +10,7 @@ import scrapy
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Response, TextResponse
 from scrapy.linkextractors import LinkExtractor
+from scrapy.utils.test import get_testenv

 if TYPE_CHECKING:
    import argparse
@ -35,8 +36,6 @@ class Command(ScrapyCommand):

 class _BenchServer:
    def __enter__(self) -> None:
-        from scrapy.utils.test import get_testenv
-
        pargs = [sys.executable, "-u", "-m", "scrapy.utils.benchserver"]
        self.proc = subprocess.Popen(  # noqa: S603
            pargs, stdout=subprocess.PIPE, env=get_testenv()
--- a/scrapy/commands/genspider.py
+++ b/scrapy/commands/genspider.py
@ -206,7 +206,7 @@ class Command(ScrapyCommand):

        # a file with the same name exists in the target directory
        spiders_module = import_module(self.settings["NEWSPIDER_MODULE"])
-        spiders_dir = Path(cast(str, spiders_module.__file__)).parent
+        spiders_dir = Path(cast("str", spiders_module.__file__)).parent
        spiders_dir_abs = spiders_dir.resolve()
        path = spiders_dir_abs / (name + ".py")
        if path.exists():
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@ -15,7 +15,7 @@ from scrapy.exceptions import UsageError
 from scrapy.http import Request, Response
 from scrapy.utils import display
 from scrapy.utils.asyncgen import collect_asyncgen
-from scrapy.utils.defer import aiter_errback, deferred_from_coro
+from scrapy.utils.defer import _schedule_coro, aiter_errback, deferred_from_coro
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.spider import spidercls_for_request
@ -284,8 +284,12 @@ class Command(BaseRunSpiderCommand):
        if opts.pipelines:
            assert self.pcrawler.engine
            itemproc = self.pcrawler.engine.scraper.itemproc
-            for item in items:
-                itemproc.process_item(item, spider)
+            if hasattr(itemproc, "process_item_async"):
+                for item in items:
+                    _schedule_coro(itemproc.process_item_async(item))
+            else:
+                for item in items:
+                    itemproc.process_item(item, spider)
        self.add_items(depth, items)
        self.add_requests(depth, requests)

--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
 from scrapy.commands import ScrapyCommand
 from scrapy.http import Request
 from scrapy.shell import Shell
+from scrapy.utils.defer import _schedule_coro
 from scrapy.utils.spider import DefaultSpider, spidercls_for_request
 from scrapy.utils.url import guess_scheme

@ -56,7 +57,7 @@ class Command(ScrapyCommand):
            help="do not handle HTTP 3xx status codes and print response as-is",
        )

-    def update_vars(self, vars: dict[str, Any]) -> None:
+    def update_vars(self, vars: dict[str, Any]) -> None:  # noqa: A002
        """You can use this function to update the Scrapy objects that will be
        available in the shell
        """
@ -84,7 +85,7 @@ class Command(ScrapyCommand):
        crawler._apply_settings()
        # The Shell class needs a persistent engine in the crawler
        crawler.engine = crawler._create_engine()
-        crawler.engine.start(_start_request_processing=False)
+        _schedule_coro(crawler.engine.start_async(_start_request_processing=False))

        self._start_crawler_thread()

--- a/scrapy/contracts/init.py
+++ b/scrapy/contracts/init.py
@ -52,7 +52,7 @@ class Contract:
                cb_result = cb(response, **cb_kwargs)
                if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                    raise TypeError("Contracts don't support async callbacks")
-                return list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+                return list(cast("Iterable[Any]", iterate_spider_output(cb_result)))

            request.callback = wrapper

@ -68,7 +68,7 @@ class Contract:
                cb_result = cb(response, **cb_kwargs)
                if isinstance(cb_result, (AsyncGenerator, CoroutineType)):
                    raise TypeError("Contracts don't support async callbacks")
-                output = list(cast(Iterable[Any], iterate_spider_output(cb_result)))
+                output = list(cast("Iterable[Any]", iterate_spider_output(cb_result)))
                try:
                    results.startTest(self.testcase_post)
                    self.post_process(output)
@ -181,7 +181,7 @@ class ContractsManager:
        def cb_wrapper(response: Response, **cb_kwargs: Any) -> None:
            try:
                output = cb(response, **cb_kwargs)
-                output = list(cast(Iterable[Any], iterate_spider_output(output)))
+                output = list(cast("Iterable[Any]", iterate_spider_output(output)))
            except Exception:
                case = _create_testcase(method, "callback")
                results.addError(case, sys.exc_info())
--- a/scrapy/contracts/default.py
+++ b/scrapy/contracts/default.py
@ -1,7 +1,7 @@
 from __future__ import annotations

 import json
-from typing import Any, Callable
+from typing import TYPE_CHECKING, Any

 from itemadapter import ItemAdapter, is_item

@ -9,6 +9,9 @@ from scrapy.contracts import Contract
 from scrapy.exceptions import ContractFail
 from scrapy.http import Request

+if TYPE_CHECKING:
+    from collections.abc import Callable
+

 # contracts
 class UrlContract(Contract):
--- a/scrapy/core/downloader/init.py
+++ b/scrapy/core/downloader/init.py
@ -8,6 +8,7 @@ from time import time
 from typing import TYPE_CHECKING, Any, cast

 from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.python.failure import Failure

 from scrapy import Request, Spider, signals
 from scrapy.core.downloader.handlers import DownloadHandlers
@ -20,10 +21,11 @@ from scrapy.utils.asyncio import (
    call_later,
    create_looping_call,
 )
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.defer import (
-    deferred_from_coro,
+    _defer_sleep_async,
+    _schedule_coro,
    maybe_deferred_to_future,
-    mustbe_deferred,
 )
 from scrapy.utils.httpobj import urlparse_cached

@ -96,6 +98,13 @@ def _get_concurrency_delay(
        delay = spider.download_delay

    if hasattr(spider, "max_concurrent_requests"):
+        warnings.warn(
+            "The 'max_concurrent_requests' spider attribute is deprecated. "
+            "Use Spider.custom_settings or Spider.update_settings() instead. "
+            "The corresponding setting name is 'CONCURRENT_REQUESTS'.",
+            category=ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
        concurrency = spider.max_concurrent_requests

    return concurrency, delay
@ -105,6 +114,7 @@ class Downloader:
    DOWNLOAD_SLOT = "download_slot"

    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
        self.settings: BaseSettings = crawler.settings
        self.signals: SignalManager = crawler.signals
        self.slots: dict[str, Slot] = {}
@ -128,28 +138,30 @@ class Downloader:
        )

    @inlineCallbacks
+    @_warn_spider_arg
    def fetch(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Generator[Deferred[Any], Any, Response | Request]:
        self.active.add(request)
        try:
-            return (
-                yield self.middleware.download(self._enqueue_request, request, spider)
-            )
+            return (yield self.middleware.download(self._enqueue_request, request))
        finally:
            self.active.remove(request)

    def needs_backout(self) -> bool:
        return len(self.active) >= self.total_concurrency

-    def _get_slot(self, request: Request, spider: Spider) -> tuple[str, Slot]:
+    def _get_slot(self, request: Request) -> tuple[str, Slot]:
        key = self.get_slot_key(request)
        if key not in self.slots:
+            assert self.crawler.spider
            slot_settings = self.per_slot_settings.get(key, {})
            conc = (
                self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
            )
-            conc, delay = _get_concurrency_delay(conc, spider, self.settings)
+            conc, delay = _get_concurrency_delay(
+                conc, self.crawler.spider, self.settings
+            )
            conc, delay = (
                slot_settings.get("concurrency", conc),
                slot_settings.get("delay", delay),
@ -162,7 +174,7 @@ class Downloader:

    def get_slot_key(self, request: Request) -> str:
        if self.DOWNLOAD_SLOT in request.meta:
-            return cast(str, request.meta[self.DOWNLOAD_SLOT])
+            return cast("str", request.meta[self.DOWNLOAD_SLOT])

        key = urlparse_cached(request).hostname or ""
        if self.ip_concurrency:
@ -170,33 +182,28 @@ class Downloader:

        return key

-    def _get_slot_key(self, request: Request, spider: Spider | None) -> str:
-        warnings.warn(
-            "Use of this protected method is deprecated. Consider using its corresponding public method get_slot_key() instead.",
-            ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return self.get_slot_key(request)
-
+    # passed as download_func into self.middleware.download() in self.fetch()
    @inlineCallbacks
    def _enqueue_request(
-        self, request: Request, spider: Spider
+        self, request: Request
    ) -> Generator[Deferred[Any], Any, Response]:
-        key, slot = self._get_slot(request, spider)
+        key, slot = self._get_slot(request)
        request.meta[self.DOWNLOAD_SLOT] = key
        slot.active.add(request)
        self.signals.send_catch_log(
-            signal=signals.request_reached_downloader, request=request, spider=spider
+            signal=signals.request_reached_downloader,
+            request=request,
+            spider=self.crawler.spider,
        )
        d: Deferred[Response] = Deferred()
        slot.queue.append((request, d))
-        self._process_queue(spider, slot)
+        self._process_queue(slot)
        try:
-            return (yield d)
+            return (yield d)  # fired in _wait_for_download()
        finally:
            slot.active.remove(request)

-    def _process_queue(self, spider: Spider, slot: Slot) -> None:
+    def _process_queue(self, slot: Slot) -> None:
        if slot.latercall:
            # block processing until slot.latercall is called
            return
@ -207,31 +214,30 @@ class Downloader:
        if delay:
            penalty = delay - now + slot.lastseen
            if penalty > 0:
-                slot.latercall = call_later(penalty, self._latercall, spider, slot)
+                slot.latercall = call_later(penalty, self._latercall, slot)
                return

        # Process enqueued requests if there are free slots to transfer for this slot
        while slot.queue and slot.free_transfer_slots() > 0:
            slot.lastseen = now
-            request, deferred = slot.queue.popleft()
-            dfd = deferred_from_coro(self._download(slot, request, spider))
-            dfd.chainDeferred(deferred)
+            request, queue_dfd = slot.queue.popleft()
+            _schedule_coro(self._wait_for_download(slot, request, queue_dfd))
            # prevent burst if inter-request delays were configured
            if delay:
-                self._process_queue(spider, slot)
+                self._process_queue(slot)
                break

-    def _latercall(self, spider: Spider, slot: Slot) -> None:
+    def _latercall(self, slot: Slot) -> None:
        slot.latercall = None
-        self._process_queue(spider, slot)
+        self._process_queue(slot)

-    async def _download(self, slot: Slot, request: Request, spider: Spider) -> Response:
+    async def _download(self, slot: Slot, request: Request) -> Response:
        # The order is very important for the following logic. Do not change!
        slot.transferring.add(request)
        try:
            # 1. Download the response
            response: Response = await maybe_deferred_to_future(
-                mustbe_deferred(self.handlers.download_request, request, spider)
+                self.handlers.download_request(request)
            )
            # 2. Notify response_downloaded listeners about the recent download
            # before querying queue for next request
@ -239,20 +245,35 @@ class Downloader:
                signal=signals.response_downloaded,
                response=response,
                request=request,
-                spider=spider,
+                spider=self.crawler.spider,
            )
            return response
+        except Exception:
+            await _defer_sleep_async()
+            raise
        finally:
            # 3. After response arrives, remove the request from transferring
            # state to free up the transferring slot so it can be used by the
            # following requests (perhaps those which came from the downloader
            # middleware itself)
            slot.transferring.remove(request)
-            self._process_queue(spider, slot)
+            self._process_queue(slot)
            self.signals.send_catch_log(
-                signal=signals.request_left_downloader, request=request, spider=spider
+                signal=signals.request_left_downloader,
+                request=request,
+                spider=self.crawler.spider,
            )

+    async def _wait_for_download(
+        self, slot: Slot, request: Request, queue_dfd: Deferred[Response]
+    ) -> None:
+        try:
+            response = await self._download(slot, request)
+        except Exception:
+            queue_dfd.errback(Failure())
+        else:
+            queue_dfd.callback(response)  # awaited in _enqueue_request()
+
    def close(self) -> None:
        self._slot_gc_loop.stop()
        for slot in self.slots.values():
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@ -71,21 +71,6 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
                stacklevel=2,
            )

-    @classmethod
-    def from_settings(
-        cls,
-        settings: BaseSettings,
-        method: int = SSL.SSLv23_METHOD,
-        *args: Any,
-        **kwargs: Any,
-    ) -> Self:
-        warnings.warn(
-            f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return cls._from_settings(settings, method, *args, **kwargs)
-
    @classmethod
    def from_crawler(
        cls,
@ -94,20 +79,10 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
        *args: Any,
        **kwargs: Any,
    ) -> Self:
-        return cls._from_settings(crawler.settings, method, *args, **kwargs)
-
-    @classmethod
-    def _from_settings(
-        cls,
-        settings: BaseSettings,
-        method: int = SSL.SSLv23_METHOD,
-        *args: Any,
-        **kwargs: Any,
-    ) -> Self:
-        tls_verbose_logging: bool = settings.getbool(
+        tls_verbose_logging: bool = crawler.settings.getbool(
            "DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING"
        )
-        tls_ciphers: str | None = settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
+        tls_ciphers: str | None = crawler.settings["DOWNLOADER_CLIENT_TLS_CIPHERS"]
        return cls(  # type: ignore[misc]
            method=method,
            tls_verbose_logging=tls_verbose_logging,
--- a/scrapy/core/downloader/handlers/init.py
+++ b/scrapy/core/downloader/handlers/init.py
@ -9,6 +9,7 @@ from twisted.internet import defer

 from scrapy import Request, Spider, signals
 from scrapy.exceptions import NotConfigured, NotSupported
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values
@ -93,14 +94,18 @@ class DownloadHandlers:
        self._handlers[scheme] = dh
        return dh

-    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
+    @_warn_spider_arg
+    def download_request(
+        self, request: Request, spider: Spider | None = None
+    ) -> Deferred[Response]:
        scheme = urlparse_cached(request).scheme
        handler = self._get_handler(scheme)
        if not handler:
            raise NotSupported(
                f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
            )
-        return handler.download_request(request, spider)
+        assert self._crawler.spider
+        return handler.download_request(request, self._crawler.spider)

    @defer.inlineCallbacks
    def _close(self, *_a: Any, **_kw: Any) -> Generator[Deferred[Any], Any, None]:
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@ -37,7 +37,6 @@ from typing import TYPE_CHECKING, Any, BinaryIO
 from urllib.parse import unquote

 from twisted.internet.protocol import ClientCreator, Protocol
-from twisted.protocols.ftp import CommandFailed, FTPClient

 from scrapy.http import Response
 from scrapy.responsetypes import responsetypes
@ -46,6 +45,7 @@ from scrapy.utils.python import to_bytes

 if TYPE_CHECKING:
    from twisted.internet.defer import Deferred
+    from twisted.protocols.ftp import FTPClient
    from twisted.python.failure import Failure

    # typing.Self requires Python 3.11
@ -101,6 +101,7 @@ class FTPDownloadHandler:

    def download_request(self, request: Request, spider: Spider) -> Deferred[Response]:
        from twisted.internet import reactor
+        from twisted.protocols.ftp import FTPClient

        parsed_url = urlparse_cached(request)
        user = request.meta.get("ftp_user", self.default_user)
@ -138,6 +139,8 @@ class FTPDownloadHandler:
        return respcls(url=request.url, status=200, body=body, headers=headers)  # type: ignore[arg-type]

    def _failed(self, result: Failure, request: Request) -> Response:
+        from twisted.protocols.ftp import CommandFailed
+
        message = result.getErrorMessage()
        if result.type == CommandFailed:
            m = _CODE_RE.search(message)
--- a/scrapy/core/downloader/handlers/http.py
+++ b/scrapy/core/downloader/handlers/http.py
@ -1,7 +1,18 @@
+import warnings
+
 from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import (
    HTTP11DownloadHandler as HTTPDownloadHandler,
 )
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+warnings.warn(
+    "The scrapy.core.downloader.handlers.http module is deprecated,"
+    " please import scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler"
+    " instead of its deprecated alias scrapy.core.downloader.handlers.http.HTTPDownloadHandler",
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)

 __all__ = [
    "HTTP10DownloadHandler",
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@ -14,7 +14,7 @@ from urllib.parse import urldefrag, urlparse
 from twisted.internet import ssl
 from twisted.internet.defer import CancelledError, Deferred, succeed
 from twisted.internet.endpoints import TCP4ClientEndpoint
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.internet.protocol import Factory, Protocol, connectionDone
 from twisted.python.failure import Failure
 from twisted.web.client import (
@ -386,6 +386,7 @@ class ScrapyAgent:
            if not proxy_port:
                proxy_port = 443 if proxy_parsed.scheme == "https" else 80
            if urlparse_cached(request).scheme == "https":
+                assert proxy_host is not None
                proxyAuth = request.headers.get(b"Proxy-Authorization", None)
                proxyConf = (proxy_host, proxy_port, proxyAuth)
                return self._TunnelingAgent(
@ -430,7 +431,7 @@ class ScrapyAgent:
            method,
            to_bytes(url, encoding="ascii"),
            headers,
-            cast(IBodyProducer, bodyproducer),
+            cast("IBodyProducer", bodyproducer),
        )
        # set download latency
        d.addCallback(self._cb_latency, request, start_time)
@ -451,7 +452,7 @@ class ScrapyAgent:
        if self._txresponse:
            self._txresponse._transport.stopProducing()

-        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
+        raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")

    def _cb_latency(self, result: _T, request: Request, start_time: float) -> _T:
        request.meta["download_latency"] = time() - start_time
--- a/scrapy/core/downloader/handlers/http2.py
+++ b/scrapy/core/downloader/handlers/http2.py
@ -4,7 +4,7 @@ from time import time
 from typing import TYPE_CHECKING
 from urllib.parse import urldefrag

-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.web.client import URI

 from scrapy.core.downloader.contextfactory import load_context_factory_from_settings
@ -127,4 +127,4 @@ class ScrapyH2Agent:
            return response

        url = urldefrag(request.url)[0]
-        raise TimeoutError(f"Getting {url} took longer than {timeout} seconds.")
+        raise TxTimeoutError(f"Getting {url} took longer than {timeout} seconds.")
--- a/scrapy/core/downloader/handlers/s3.py
+++ b/scrapy/core/downloader/handlers/s3.py
@ -2,7 +2,7 @@ from __future__ import annotations

 from typing import TYPE_CHECKING, Any

-from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
+from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.exceptions import NotConfigured
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.httpobj import urlparse_cached
@ -29,7 +29,7 @@ class S3DownloadHandler:
        aws_access_key_id: str | None = None,
        aws_secret_access_key: str | None = None,
        aws_session_token: str | None = None,
-        httpdownloadhandler: type[HTTPDownloadHandler] = HTTPDownloadHandler,
+        httpdownloadhandler: type[HTTP11DownloadHandler] = HTTP11DownloadHandler,
        **kw: Any,
    ):
        if not is_botocore_available():
@ -51,8 +51,8 @@ class S3DownloadHandler:
        self.anon = kw.get("anon")

        self._signer = None
-        import botocore.auth
-        import botocore.credentials
+        import botocore.auth  # noqa: PLC0415
+        import botocore.credentials  # noqa: PLC0415

        kw.pop("anon", None)
        if kw:
@ -87,7 +87,7 @@ class S3DownloadHandler:
        if self.anon:
            request = request.replace(url=url)
        else:
-            import botocore.awsrequest
+            import botocore.awsrequest  # noqa: PLC0415

            awsrequest = botocore.awsrequest.AWSRequest(
                method=request.method,
--- a/scrapy/core/downloader/middleware.py
+++ b/scrapy/core/downloader/middleware.py
@ -6,19 +6,20 @@ See documentation in docs/topics/downloader-middleware.rst

 from __future__ import annotations

-from collections.abc import Callable
+import warnings
 from typing import TYPE_CHECKING, Any, cast

 from twisted.internet.defer import Deferred, inlineCallbacks

-from scrapy.exceptions import _InvalidOutput
+from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import deferred_from_coro, mustbe_deferred
+from scrapy.utils.defer import _defer_sleep, deferred_from_coro
+from scrapy.utils.deprecate import argument_is_required

 if TYPE_CHECKING:
-    from collections.abc import Generator
+    from collections.abc import Callable, Generator

    from scrapy import Spider
    from scrapy.settings import BaseSettings
@ -34,27 +35,44 @@ class DownloaderMiddlewareManager(MiddlewareManager):
    def _add_middleware(self, mw: Any) -> None:
        if hasattr(mw, "process_request"):
            self.methods["process_request"].append(mw.process_request)
+            self._check_mw_method_spider_arg(mw.process_request)
        if hasattr(mw, "process_response"):
            self.methods["process_response"].appendleft(mw.process_response)
+            self._check_mw_method_spider_arg(mw.process_response)
        if hasattr(mw, "process_exception"):
            self.methods["process_exception"].appendleft(mw.process_exception)
+            self._check_mw_method_spider_arg(mw.process_exception)

    @inlineCallbacks
    def download(
        self,
-        download_func: Callable[[Request, Spider], Deferred[Response]],
+        download_func: Callable[[Request], Deferred[Response]],
        request: Request,
-        spider: Spider,
+        spider: Spider | None = None,
    ) -> Generator[Deferred[Any], Any, Response | Request]:
+        if argument_is_required(download_func, "spider"):
+            warnings.warn(
+                "The spider argument of download_func is deprecated"
+                " and will not be passed in future Scrapy versions.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            need_spider_arg = True
+        else:
+            need_spider_arg = False
+
        @inlineCallbacks
        def process_request(
            request: Request,
        ) -> Generator[Deferred[Any], Any, Response | Request]:
            for method in self.methods["process_request"]:
-                method = cast(Callable, method)
-                response = yield deferred_from_coro(
-                    method(request=request, spider=spider)
-                )
+                method = cast("Callable", method)
+                if method in self._mw_methods_requiring_spider:
+                    response = yield deferred_from_coro(
+                        method(request=request, spider=self._spider)
+                    )
+                else:
+                    response = yield deferred_from_coro(method(request=request))
                if response is not None and not isinstance(
                    response, (Response, Request)
                ):
@ -64,7 +82,9 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                    )
                if response:
                    return response
-            return (yield download_func(request, spider))
+            if need_spider_arg:
+                return (yield download_func(request, self._spider))  # type: ignore[call-arg]
+            return (yield download_func(request))

        @inlineCallbacks
        def process_response(
@ -76,10 +96,15 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                return response

            for method in self.methods["process_response"]:
-                method = cast(Callable, method)
-                response = yield deferred_from_coro(
-                    method(request=request, response=response, spider=spider)
-                )
+                method = cast("Callable", method)
+                if method in self._mw_methods_requiring_spider:
+                    response = yield deferred_from_coro(
+                        method(request=request, response=response, spider=self._spider)
+                    )
+                else:
+                    response = yield deferred_from_coro(
+                        method(request=request, response=response)
+                    )
                if not isinstance(response, (Response, Request)):
                    raise _InvalidOutput(
                        f"Middleware {method.__qualname__} must return Response or Request, "
@ -94,10 +119,17 @@ class DownloaderMiddlewareManager(MiddlewareManager):
            exception: Exception,
        ) -> Generator[Deferred[Any], Any, Response | Request]:
            for method in self.methods["process_exception"]:
-                method = cast(Callable, method)
-                response = yield deferred_from_coro(
-                    method(request=request, exception=exception, spider=spider)
-                )
+                method = cast("Callable", method)
+                if method in self._mw_methods_requiring_spider:
+                    response = yield deferred_from_coro(
+                        method(
+                            request=request, exception=exception, spider=self._spider
+                        )
+                    )
+                else:
+                    response = yield deferred_from_coro(
+                        method(request=request, exception=exception)
+                    )
                if response is not None and not isinstance(
                    response, (Response, Request)
                ):
@ -109,9 +141,13 @@ class DownloaderMiddlewareManager(MiddlewareManager):
                    return response
            raise exception

+        if spider:
+            self._warn_spider_arg("download")
+            self._set_compat_spider(spider)
        try:
-            result: Response | Request = yield mustbe_deferred(process_request, request)
+            result: Response | Request = yield process_request(request)
        except Exception as ex:
+            yield _defer_sleep()
            # either returns a request or response (which we pass to process_response())
            # or reraises the exception
            result = yield process_exception(ex)
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@ -7,38 +7,49 @@ For more information see docs/topics/architecture.rst

 from __future__ import annotations

+import asyncio
 import logging
+import warnings
 from time import time
 from traceback import format_exc
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any

-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import CancelledError, Deferred, inlineCallbacks
 from twisted.python.failure import Failure

 from scrapy import signals
+from scrapy.core.scheduler import BaseScheduler
 from scrapy.core.scraper import Scraper
-from scrapy.exceptions import CloseSpider, DontCloseSpider, IgnoreRequest
+from scrapy.exceptions import (
+    CloseSpider,
+    DontCloseSpider,
+    IgnoreRequest,
+    ScrapyDeprecationWarning,
+)
 from scrapy.http import Request, Response
 from scrapy.utils.asyncio import (
    AsyncioLoopingCall,
    create_looping_call,
+    is_asyncio_available,
 )
 from scrapy.utils.defer import (
-    deferred_f_from_coro_f,
+    _schedule_coro,
    deferred_from_coro,
+    ensure_awaitable,
    maybe_deferred_to_future,
 )
+from scrapy.utils.deprecate import argument_is_required
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import build_from_crawler, load_object
+from scrapy.utils.python import global_object_name
 from scrapy.utils.reactor import CallLaterOnce

 if TYPE_CHECKING:
-    from collections.abc import AsyncIterator, Callable, Generator
+    from collections.abc import AsyncIterator, Callable, Coroutine, Generator

    from twisted.internet.task import LoopingCall

    from scrapy.core.downloader import Downloader
-    from scrapy.core.scheduler import BaseScheduler
    from scrapy.crawler import Crawler
    from scrapy.logformatter import LogFormatter
    from scrapy.settings import BaseSettings, Settings
@ -72,10 +83,10 @@ class _Slot:
        self.inprogress.remove(request)
        self._maybe_fire_closing()

-    def close(self) -> Deferred[None]:
+    async def close(self) -> None:
        self.closing = Deferred()
        self._maybe_fire_closing()
-        return self.closing
+        await maybe_deferred_to_future(self.closing)

    def _maybe_fire_closing(self) -> None:
        if self.closing is not None and not self.inprogress:
@ -92,7 +103,9 @@ class ExecutionEngine:
    def __init__(
        self,
        crawler: Crawler,
-        spider_closed_callback: Callable[[Spider], Deferred[None] | None],
+        spider_closed_callback: Callable[
+            [Spider], Coroutine[Any, Any, None] | Deferred[None] | None
+        ],
    ) -> None:
        self.crawler: Crawler = crawler
        self.settings: Settings = crawler.settings
@ -103,25 +116,39 @@ class ExecutionEngine:
        self.spider: Spider | None = None
        self.running: bool = False
        self.paused: bool = False
-        self._spider_closed_callback: Callable[[Spider], Deferred[None] | None] = (
-            spider_closed_callback
-        )
+        self._spider_closed_callback: Callable[
+            [Spider], Coroutine[Any, Any, None] | Deferred[None] | None
+        ] = spider_closed_callback
        self.start_time: float | None = None
        self._start: AsyncIterator[Any] | None = None
+        self._closewait: Deferred[None] | None = None
+        self._start_request_processing_awaitable: (
+            asyncio.Future[None] | Deferred[None] | None
+        ) = None
        downloader_cls: type[Downloader] = load_object(self.settings["DOWNLOADER"])
        try:
            self.scheduler_cls: type[BaseScheduler] = self._get_scheduler_class(
                crawler.settings
            )
            self.downloader: Downloader = downloader_cls(crawler)
+            self._downloader_fetch_needs_spider: bool = argument_is_required(
+                self.downloader.fetch, "spider"
+            )
+            if self._downloader_fetch_needs_spider:
+                warnings.warn(
+                    f"The fetch() method of {global_object_name(downloader_cls)} requires a spider argument,"
+                    f" this is deprecated and the argument will not be passed in future Scrapy versions.",
+                    ScrapyDeprecationWarning,
+                    stacklevel=2,
+                )
+
            self.scraper: Scraper = Scraper(crawler)
        except Exception:
-            self.close()
+            if hasattr(self, "downloader"):
+                self.downloader.close()
            raise

    def _get_scheduler_class(self, settings: BaseSettings) -> type[BaseScheduler]:
-        from scrapy.core.scheduler import BaseScheduler
-
        scheduler_cls: type[BaseScheduler] = load_object(settings["SCHEDULER"])
        if not issubclass(scheduler_cls, BaseScheduler):
            raise TypeError(
@ -131,52 +158,87 @@ class ExecutionEngine:
        return scheduler_cls

    def start(self, _start_request_processing=True) -> Deferred[None]:
-        return deferred_from_coro(self.start_async(_start_request_processing))
+        warnings.warn(
+            "ExecutionEngine.start() is deprecated, use start_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(
+            self.start_async(_start_request_processing=_start_request_processing)
+        )

-    async def start_async(self, _start_request_processing=True) -> None:
+    async def start_async(self, *, _start_request_processing: bool = True) -> None:
+        """Start the execution engine.
+
+        .. versionadded:: VERSION
+        """
        if self.running:
            raise RuntimeError("Engine already running")
        self.start_time = time()
        await self.signals.send_catch_log_async(signal=signals.engine_started)
+        if _start_request_processing and self.spider is None:
+            # require an opened spider when not run in scrapy shell
+            return
        self.running = True
-        self._closewait: Deferred[None] = Deferred()
+        self._closewait = Deferred()
        if _start_request_processing:
-            self._start_request_processing()
+            coro = self._start_request_processing()
+            if is_asyncio_available():
+                # not wrapping in a Deferred here to avoid https://github.com/twisted/twisted/issues/12470
+                # (can happen when this is cancelled, e.g. in test_close_during_start_iteration())
+                self._start_request_processing_awaitable = asyncio.ensure_future(coro)
+            else:
+                self._start_request_processing_awaitable = Deferred.fromCoroutine(coro)
        await maybe_deferred_to_future(self._closewait)

    def stop(self) -> Deferred[None]:
-        """Gracefully stop the execution engine"""
+        warnings.warn(
+            "ExecutionEngine.stop() is deprecated, use stop_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.stop_async())

-        @deferred_f_from_coro_f
-        async def _finish_stopping_engine(_: Any) -> None:
-            await self.signals.send_catch_log_async(signal=signals.engine_stopped)
-            self._closewait.callback(None)
+    async def stop_async(self) -> None:
+        """Gracefully stop the execution engine.
+
+        .. versionadded:: VERSION
+        """

        if not self.running:
            raise RuntimeError("Engine not running")

        self.running = False
-        dfd = (
-            self.close_spider(self.spider, reason="shutdown")
-            if self.spider is not None
-            else succeed(None)
-        )
-        return dfd.addBoth(_finish_stopping_engine)
+        if self._start_request_processing_awaitable is not None:
+            self._start_request_processing_awaitable.cancel()
+            self._start_request_processing_awaitable = None
+        if self.spider is not None:
+            await self.close_spider_async(reason="shutdown")
+        await self.signals.send_catch_log_async(signal=signals.engine_stopped)
+        if self._closewait:
+            self._closewait.callback(None)

    def close(self) -> Deferred[None]:
+        warnings.warn(
+            "ExecutionEngine.close() is deprecated, use close_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.close_async())
+
+    async def close_async(self) -> None:
        """
        Gracefully close the execution engine.
        If it has already been started, stop it. In all cases, close the spider and the downloader.
        """
        if self.running:
-            return self.stop()  # will also close spider and downloader
-        if self.spider is not None:
-            return self.close_spider(
-                self.spider, reason="shutdown"
+            await self.stop_async()  # will also close spider and downloader
+        elif self.spider is not None:
+            await self.close_spider_async(
+                reason="shutdown"
            )  # will also close downloader
-        if hasattr(self, "downloader"):
+        elif hasattr(self, "downloader"):
            self.downloader.close()
-        return succeed(None)

    def pause(self) -> None:
        self.paused = True
@ -207,27 +269,41 @@ class ExecutionEngine:
            if isinstance(item_or_request, Request):
                self.crawl(item_or_request)
            else:
-                self.scraper.start_itemproc(item_or_request, response=None)
+                _schedule_coro(
+                    self.scraper.start_itemproc_async(item_or_request, response=None)
+                )
                self._slot.nextcall.schedule()

-    @deferred_f_from_coro_f
    async def _start_request_processing(self) -> None:
        """Starts consuming Spider.start() output and sending scheduled
        requests."""
        # Starts the processing of scheduled requests, as well as a periodic
        # call to that processing method for scenarios where the scheduler
        # reports having pending requests but returns none.
-        assert self._slot is not None  # typing
-        self._slot.nextcall.schedule()
-        self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL)
+        try:
+            assert self._slot is not None  # typing
+            self._slot.nextcall.schedule()
+            self._slot.heartbeat.start(self._SLOT_HEARTBEAT_INTERVAL)

-        while self._start and self.spider:
-            await self._process_start_next()
-            if not self.needs_backout():
-                # Give room for the outcome of self._process_start_next() to be
-                # processed before continuing with the next iteration.
-                self._slot.nextcall.schedule()
-                await self._slot.nextcall.wait()
+            while self._start and self.spider:
+                await self._process_start_next()
+                if not self.needs_backout():
+                    # Give room for the outcome of self._process_start_next() to be
+                    # processed before continuing with the next iteration.
+                    self._slot.nextcall.schedule()
+                    await self._slot.nextcall.wait()
+        except (asyncio.exceptions.CancelledError, CancelledError):
+            # self.stop() has cancelled us, nothing to do
+            return
+        except Exception:
+            # an error happened, log it and stop the engine
+            self._start_request_processing_awaitable = None
+            logger.error(
+                "Error while processing requests from start()",
+                exc_info=True,
+                extra={"spider": self.spider},
+            )
+            await self.stop_async()

    def _start_scheduled_requests(self) -> None:
        if self._slot is None or self._slot.closing is not None or self.paused:
@ -246,10 +322,10 @@ class ExecutionEngine:

        See :ref:`start-requests-lazy` for an example.
        """
-        assert self._slot is not None  # typing
        assert self.scraper.slot is not None  # typing
        return (
            not self.running
+            or not self._slot
            or bool(self._slot.closing)
            or self.downloader.needs_backout()
            or self.scraper.slot.needs_backout()
@ -354,18 +430,33 @@ class ExecutionEngine:
                signals.request_dropped, request=request, spider=self.spider
            )

-    @inlineCallbacks
-    def download(self, request: Request) -> Generator[Deferred[Any], Any, Response]:
+    def download(self, request: Request) -> Deferred[Response]:
        """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
+        warnings.warn(
+            "ExecutionEngine.download() is deprecated, use download_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.download_async(request))
+
+    async def download_async(self, request: Request) -> Response:
+        """Return a coroutine which fires with a Response as result.
+
+         Only downloader middlewares are applied.
+
+        .. versionadded:: VERSION
+        """
        if self.spider is None:
            raise RuntimeError(f"No open spider to crawl: {request}")
        try:
-            response_or_request = yield self._download(request)
+            response_or_request = await maybe_deferred_to_future(
+                self._download(request)
+            )
        finally:
            assert self._slot is not None
            self._slot.remove_request(request)
        if isinstance(response_or_request, Request):
-            return (yield self.download(response_or_request))
+            return await self.download_async(response_or_request)
        return response_or_request

    @inlineCallbacks
@ -377,9 +468,11 @@ class ExecutionEngine:

        self._slot.add_request(request)
        try:
-            result: Response | Request = yield self.downloader.fetch(
-                request, self.spider
-            )
+            result: Response | Request
+            if self._downloader_fetch_needs_spider:
+                result = yield self.downloader.fetch(request, self.spider)
+            else:
+                result = yield self.downloader.fetch(request)
            if not isinstance(result, (Response, Request)):
                raise TypeError(
                    f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}"
@ -387,7 +480,6 @@ class ExecutionEngine:
            if isinstance(result, Response):
                if result.request is None:
                    result.request = request
-                assert self.spider is not None
                logkws = self.logformatter.crawled(result.request, result, self.spider)
                if logkws is not None:
                    logger.log(
@ -404,30 +496,33 @@ class ExecutionEngine:
            self._slot.nextcall.schedule()

    def open_spider(self, spider: Spider, close_if_idle: bool = True) -> Deferred[None]:
-        return deferred_from_coro(
-            self.open_spider_async(spider, close_if_idle=close_if_idle)
+        warnings.warn(
+            "ExecutionEngine.open_spider() is deprecated, use open_spider_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
        )
+        return deferred_from_coro(self.open_spider_async(close_if_idle=close_if_idle))

-    async def open_spider_async(
-        self,
-        spider: Spider,
-        *,
-        close_if_idle: bool = True,
-    ) -> None:
+    async def open_spider_async(self, *, close_if_idle: bool = True) -> None:
+        assert self.crawler.spider
        if self._slot is not None:
-            raise RuntimeError(f"No free spider slot when opening {spider.name!r}")
-        logger.info("Spider opened", extra={"spider": spider})
-        self.spider = spider
+            raise RuntimeError(
+                f"No free spider slot when opening {self.crawler.spider.name!r}"
+            )
+        logger.info("Spider opened", extra={"spider": self.crawler.spider})
+        self.spider = self.crawler.spider
        nextcall = CallLaterOnce(self._start_scheduled_requests)
        scheduler = build_from_crawler(self.scheduler_cls, self.crawler)
        self._slot = _Slot(close_if_idle, nextcall, scheduler)
-        self._start = await self.scraper.spidermw.process_start(spider)
-        if hasattr(scheduler, "open") and (d := scheduler.open(spider)):
+        self._start = await self.scraper.spidermw.process_start()
+        if hasattr(scheduler, "open") and (d := scheduler.open(self.crawler.spider)):
            await maybe_deferred_to_future(d)
-        await maybe_deferred_to_future(self.scraper.open_spider(spider))
+        await self.scraper.open_spider_async()
        assert self.crawler.stats
-        self.crawler.stats.open_spider(spider)
-        await self.signals.send_catch_log_async(signals.spider_opened, spider=spider)
+        self.crawler.stats.open_spider()
+        await self.signals.send_catch_log_async(
+            signals.spider_opened, spider=self.crawler.spider
+        )

    def _spider_idle(self) -> None:
        """
@ -452,70 +547,87 @@ class ExecutionEngine:
        if self.spider_is_idle():
            ex = detected_ex.get(CloseSpider, CloseSpider(reason="finished"))
            assert isinstance(ex, CloseSpider)  # typing
-            self.close_spider(self.spider, reason=ex.reason)
+            _schedule_coro(self.close_spider_async(reason=ex.reason))

    def close_spider(self, spider: Spider, reason: str = "cancelled") -> Deferred[None]:
-        """Close (cancel) spider and clear all its outstanding requests"""
+        warnings.warn(
+            "ExecutionEngine.close_spider() is deprecated, use close_spider_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.close_spider_async(reason=reason))
+
+    async def close_spider_async(self, *, reason: str = "cancelled") -> None:
+        """Close (cancel) spider and clear all its outstanding requests.
+
+        .. versionadded:: VERSION
+        """
+        if self.spider is None:
+            raise RuntimeError("Spider not opened")
+
        if self._slot is None:
            raise RuntimeError("Engine slot not assigned")

        if self._slot.closing is not None:
-            return self._slot.closing
+            await maybe_deferred_to_future(self._slot.closing)
+            return
+
+        spider = self.spider

        logger.info(
            "Closing spider (%(reason)s)", {"reason": reason}, extra={"spider": spider}
        )

-        dfd = self._slot.close()
+        def log_failure(msg: str) -> None:
+            logger.error(msg, exc_info=True, extra={"spider": spider})  # noqa: LOG014

-        def log_failure(msg: str) -> Callable[[Failure], None]:
-            def errback(failure: Failure) -> None:
-                logger.error(
-                    msg, exc_info=failure_to_exc_info(failure), extra={"spider": spider}
-                )
+        try:
+            await self._slot.close()
+        except Exception:
+            log_failure("Slot close failure")

-            return errback
+        try:
+            self.downloader.close()
+        except Exception:
+            log_failure("Downloader close failure")

-        dfd.addBoth(lambda _: self.downloader.close())
-        dfd.addErrback(log_failure("Downloader close failure"))
-
-        dfd.addBoth(lambda _: self.scraper.close_spider())
-        dfd.addErrback(log_failure("Scraper close failure"))
+        try:
+            await self.scraper.close_spider_async()
+        except Exception:
+            log_failure("Scraper close failure")

        if hasattr(self._slot.scheduler, "close"):
-            dfd.addBoth(lambda _: cast(_Slot, self._slot).scheduler.close(reason))
-            dfd.addErrback(log_failure("Scheduler close failure"))
+            try:
+                if (d := self._slot.scheduler.close(reason)) is not None:
+                    await maybe_deferred_to_future(d)
+            except Exception:
+                log_failure("Scheduler close failure")

-        dfd.addBoth(
-            lambda _: self.signals.send_catch_log_deferred(
+        try:
+            await self.signals.send_catch_log_async(
                signal=signals.spider_closed,
                spider=spider,
                reason=reason,
            )
-        )
-        dfd.addErrback(log_failure("Error while sending spider_close signal"))
+        except Exception:
+            log_failure("Error while sending spider_close signal")

-        def close_stats(_: Any) -> None:
-            assert self.crawler.stats
-            self.crawler.stats.close_spider(spider, reason=reason)
+        assert self.crawler.stats
+        try:
+            self.crawler.stats.close_spider(reason=reason)
+        except Exception:
+            log_failure("Stats close failure")

-        dfd.addBoth(close_stats)
-        dfd.addErrback(log_failure("Stats close failure"))
-
-        dfd.addBoth(
-            lambda _: logger.info(
-                "Spider closed (%(reason)s)",
-                {"reason": reason},
-                extra={"spider": spider},
-            )
+        logger.info(
+            "Spider closed (%(reason)s)",
+            {"reason": reason},
+            extra={"spider": spider},
        )

-        dfd.addBoth(lambda _: setattr(self, "slot", None))
-        dfd.addErrback(log_failure("Error while unassigning slot"))
+        self._slot = None
+        self.spider = None

-        dfd.addBoth(lambda _: setattr(self, "spider", None))
-        dfd.addErrback(log_failure("Error while unassigning spider"))
-
-        dfd.addBoth(lambda _: self._spider_closed_callback(spider))
-
-        return dfd
+        try:
+            await ensure_awaitable(self._spider_closed_callback(spider))
+        except Exception:
+            log_failure("Error running spider_closed_callback")
--- a/scrapy/core/http2/protocol.py
+++ b/scrapy/core/http2/protocol.py
@ -21,7 +21,7 @@ from h2.events import (
    WindowUpdated,
 )
 from h2.exceptions import FrameTooLargeError, H2Error
-from twisted.internet.error import TimeoutError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.internet.interfaces import (
    IAddress,
    IHandshakeListener,
@ -322,7 +322,7 @@ class H2ClientProtocol(Protocol, TimeoutMixin):
        self._write_to_transport()

        self._lose_connection_with_error(
-            [TimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
+            [TxTimeoutError(f"Connection was IDLE for more than {self.IDLE_TIMEOUT}s")]
        )

    def connectionLost(self, reason: Failure = connectionDone) -> None:
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@ -348,11 +348,11 @@ class Scheduler(BaseScheduler):
        dqok = self._dqpush(request)
        assert self.stats is not None
        if dqok:
-            self.stats.inc_value("scheduler/enqueued/disk", spider=self.spider)
+            self.stats.inc_value("scheduler/enqueued/disk")
        else:
            self._mqpush(request)
-            self.stats.inc_value("scheduler/enqueued/memory", spider=self.spider)
-        self.stats.inc_value("scheduler/enqueued", spider=self.spider)
+            self.stats.inc_value("scheduler/enqueued/memory")
+        self.stats.inc_value("scheduler/enqueued")
        return True

    def next_request(self) -> Request | None:
@ -367,13 +367,13 @@ class Scheduler(BaseScheduler):
        request: Request | None = self.mqs.pop()
        assert self.stats is not None
        if request is not None:
-            self.stats.inc_value("scheduler/dequeued/memory", spider=self.spider)
+            self.stats.inc_value("scheduler/dequeued/memory")
        else:
            request = self._dqpop()
            if request is not None:
-                self.stats.inc_value("scheduler/dequeued/disk", spider=self.spider)
+                self.stats.inc_value("scheduler/dequeued/disk")
        if request is not None:
-            self.stats.inc_value("scheduler/dequeued", spider=self.spider)
+            self.stats.inc_value("scheduler/dequeued")
        return request

    def __len__(self) -> int:
@ -402,7 +402,7 @@ class Scheduler(BaseScheduler):
                )
                self.logunser = False
            assert self.stats is not None
-            self.stats.inc_value("scheduler/unserializable", spider=self.spider)
+            self.stats.inc_value("scheduler/unserializable")
            return False
        return True

@ -491,7 +491,7 @@ class Scheduler(BaseScheduler):
        if not path.exists():
            return []
        with path.open(encoding="utf-8") as f:
-            return cast(list[int], json.load(f))
+            return cast("list[int]", json.load(f))

    def _write_dqs_state(self, dqdir: str, state: list[int]) -> None:
        with Path(dqdir, "active.json").open("w", encoding="utf-8") as f:
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@ -7,9 +7,9 @@ import logging
 import warnings
 from collections import deque
 from collections.abc import AsyncIterator
-from typing import TYPE_CHECKING, Any, TypeVar, Union
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar

-from twisted.internet.defer import Deferred, inlineCallbacks, maybeDeferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure

 from scrapy import Spider, signals
@ -21,19 +21,24 @@ from scrapy.exceptions import (
    ScrapyDeprecationWarning,
 )
 from scrapy.http import Request, Response
+from scrapy.pipelines import ItemPipelineManager
 from scrapy.utils.asyncio import _parallel_asyncio, is_asyncio_available
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.defer import (
-    _defer_sleep,
+    _defer_sleep_async,
+    _schedule_coro,
    aiter_errback,
-    deferred_f_from_coro_f,
    deferred_from_coro,
+    ensure_awaitable,
    iter_errback,
    maybe_deferred_to_future,
    parallel,
    parallel_async,
 )
+from scrapy.utils.deprecate import method_is_overridden
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
+from scrapy.utils.python import global_object_name
 from scrapy.utils.spider import iterate_spider_output

 if TYPE_CHECKING:
@ -41,7 +46,6 @@ if TYPE_CHECKING:

    from scrapy.crawler import Crawler
    from scrapy.logformatter import LogFormatter
-    from scrapy.pipelines import ItemPipelineManager
    from scrapy.signalmanager import SignalManager


@ -49,7 +53,7 @@ logger = logging.getLogger(__name__)


 _T = TypeVar("_T")
-QueueTuple = tuple[Union[Response, Failure], Request, Deferred[None]]
+QueueTuple: TypeAlias = tuple[Response | Failure, Request, Deferred[None]]


 class Slot:
@ -106,33 +110,97 @@ class Scraper:
            crawler.settings["ITEM_PROCESSOR"]
        )
        self.itemproc: ItemPipelineManager = itemproc_cls.from_crawler(crawler)
+        self._itemproc_has_async: dict[str, bool] = {}
+        for method in [
+            "open_spider",
+            "close_spider",
+            "process_item",
+        ]:
+            self._check_deprecated_itemproc_method(method)
+
        self.concurrent_items: int = crawler.settings.getint("CONCURRENT_ITEMS")
        self.crawler: Crawler = crawler
        self.signals: SignalManager = crawler.signals
        assert crawler.logformatter
        self.logformatter: LogFormatter = crawler.logformatter

-    @deferred_f_from_coro_f
-    async def open_spider(self, spider: Spider) -> None:
-        """Open the given spider for scraping and allocate resources for it"""
-        self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
-        await maybe_deferred_to_future(self.itemproc.open_spider(spider))
-
-    def close_spider(self, spider: Spider | None = None) -> Deferred[Spider]:
-        """Close a spider being scraped and release its resources"""
-        if spider is not None:
+    def _check_deprecated_itemproc_method(self, method: str) -> None:
+        itemproc_cls = type(self.itemproc)
+        if not hasattr(self.itemproc, "process_item_async"):
            warnings.warn(
-                "Passing a 'spider' argument to Scraper.close_spider() is deprecated.",
-                category=ScrapyDeprecationWarning,
+                f"{global_object_name(itemproc_cls)} doesn't define a {method}_async() method,"
+                f" this is deprecated and the method will be required in future Scrapy versions.",
+                ScrapyDeprecationWarning,
                stacklevel=2,
            )
+            self._itemproc_has_async[method] = False
+        elif (
+            issubclass(itemproc_cls, ItemPipelineManager)
+            and method_is_overridden(itemproc_cls, ItemPipelineManager, method)
+            and not method_is_overridden(
+                itemproc_cls, ItemPipelineManager, f"{method}_async"
+            )
+        ):
+            warnings.warn(
+                f"{global_object_name(itemproc_cls)} overrides {method}() but doesn't override {method}_async()."
+                f" This is deprecated. {method}() will be used, but in future Scrapy versions {method}_async() will be used instead.",
+                ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
+            self._itemproc_has_async[method] = False
+        else:
+            self._itemproc_has_async[method] = True

+    def open_spider(self, spider: Spider | None = None) -> Deferred[None]:
+        warnings.warn(
+            "Scraper.open_spider() is deprecated, use open_spider_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.open_spider_async())
+
+    async def open_spider_async(self) -> None:
+        """Open the spider for scraping and allocate resources for it.
+
+        .. versionadded:: VERSION
+        """
+        self.slot = Slot(self.crawler.settings.getint("SCRAPER_SLOT_MAX_ACTIVE_SIZE"))
+        if not self.crawler.spider:
+            raise RuntimeError(
+                "Scraper.open_spider() called before Crawler.spider is set."
+            )
+        if self._itemproc_has_async["open_spider"]:
+            await self.itemproc.open_spider_async()
+        else:
+            await maybe_deferred_to_future(
+                self.itemproc.open_spider(self.crawler.spider)
+            )
+
+    def close_spider(self, spider: Spider | None = None) -> Deferred[None]:
+        warnings.warn(
+            "Scraper.close_spider() is deprecated, use close_spider_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.close_spider_async())
+
+    async def close_spider_async(self) -> None:
+        """Close the spider being scraped and release its resources.
+
+        .. versionadded:: VERSION
+        """
        if self.slot is None:
            raise RuntimeError("Scraper slot not assigned")
        self.slot.closing = Deferred()
-        self.slot.closing.addCallback(self.itemproc.close_spider)
        self._check_if_closing()
-        return self.slot.closing
+        await maybe_deferred_to_future(self.slot.closing)
+        if self._itemproc_has_async["close_spider"]:
+            await self.itemproc.close_spider_async()
+        else:
+            assert self.crawler.spider
+            await maybe_deferred_to_future(
+                self.itemproc.close_spider(self.crawler.spider)
+            )

    def is_idle(self) -> bool:
        """Return True if there isn't any more spiders to process"""
@ -140,28 +208,21 @@ class Scraper:

    def _check_if_closing(self) -> None:
        assert self.slot is not None  # typing
-        assert self.crawler.spider
        if self.slot.closing and self.slot.is_idle():
            assert self.crawler.spider
            self.slot.closing.callback(self.crawler.spider)

    @inlineCallbacks
+    @_warn_spider_arg
    def enqueue_scrape(
        self, result: Response | Failure, request: Request, spider: Spider | None = None
    ) -> Generator[Deferred[Any], Any, None]:
-        if spider is not None:
-            warnings.warn(
-                "Passing a 'spider' argument to Scraper.enqueue_scrape() is deprecated.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
        if self.slot is None:
            raise RuntimeError("Scraper slot not assigned")
        dfd = self.slot.add_response_request(result, request)
        self._scrape_next()
        try:
-            yield dfd
+            yield dfd  # fired in _wait_for_processing()
        except Exception:
            logger.error(
                "Scraper bug processing %(request)s",
@ -177,10 +238,9 @@ class Scraper:
    def _scrape_next(self) -> None:
        assert self.slot is not None  # typing
        while self.slot.queue:
-            result, request, deferred = self.slot.next_response_request_deferred()
-            self._scrape(result, request).chainDeferred(deferred)
+            result, request, queue_dfd = self.slot.next_response_request_deferred()
+            _schedule_coro(self._wait_for_processing(result, request, queue_dfd))

-    @deferred_f_from_coro_f
    async def _scrape(self, result: Response | Failure, request: Request) -> None:
        """Handle the downloaded response or failure through the spider callback/errback."""
        if not isinstance(result, (Response, Failure)):
@ -188,13 +248,12 @@ class Scraper:
                f"Incorrect type: expected Response or Failure, got {type(result)}: {result!r}"
            )

-        assert self.crawler.spider
        output: Iterable[Any] | AsyncIterator[Any]
        if isinstance(result, Response):
            try:
                # call the spider middlewares and the request callback with the response
                output = await self.spidermw.scrape_response_async(
-                    self.call_spider, result, request, self.crawler.spider
+                    self.call_spider_async, result, request
                )
            except Exception:
                self.handle_spider_error(Failure(), request, result)
@ -207,6 +266,7 @@ class Scraper:
            output = await self.call_spider_async(result, request)
        except Exception as spider_exc:
            # the errback didn't silence the exception
+            assert self.crawler.spider
            if not result.check(IgnoreRequest):
                logkws = self.logformatter.download_error(
                    result, request, self.crawler.spider
@ -222,22 +282,34 @@ class Scraper:
        else:
            await self.handle_spider_output_async(output, request, result)

+    async def _wait_for_processing(
+        self, result: Response | Failure, request: Request, queue_dfd: Deferred[None]
+    ) -> None:
+        try:
+            await self._scrape(result, request)
+        except Exception:
+            queue_dfd.errback(Failure())
+        else:
+            queue_dfd.callback(None)  # awaited in enqueue_scrape()
+
    def call_spider(
        self, result: Response | Failure, request: Request, spider: Spider | None = None
    ) -> Deferred[Iterable[Any] | AsyncIterator[Any]]:
-        if spider is not None:
-            warnings.warn(
-                "Passing a 'spider' argument to Scraper.call_spider() is deprecated.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "Scraper.call_spider() is deprecated, use call_spider_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
        return deferred_from_coro(self.call_spider_async(result, request))

    async def call_spider_async(
        self, result: Response | Failure, request: Request
    ) -> Iterable[Any] | AsyncIterator[Any]:
-        """Call the request callback or errback with the response or failure."""
-        await maybe_deferred_to_future(_defer_sleep())
+        """Call the request callback or errback with the response or failure.
+
+        .. versionadded:: 2.13
+        """
+        await _defer_sleep_async()
        assert self.crawler.spider
        if isinstance(result, Response):
            if getattr(result, "request", None) is None:
@ -257,10 +329,9 @@ class Scraper:
                output.raiseException()
            # else the errback returned actual output (like a callback),
            # which needs to be passed to iterate_spider_output()
-        return await maybe_deferred_to_future(
-            maybeDeferred(iterate_spider_output, output)
-        )
+        return await ensure_awaitable(iterate_spider_output(output))

+    @_warn_spider_arg
    def handle_spider_error(
        self,
        _failure: Failure,
@ -269,19 +340,12 @@ class Scraper:
        spider: Spider | None = None,
    ) -> None:
        """Handle an exception raised by a spider callback or errback."""
-        if spider is not None:
-            warnings.warn(
-                "Passing a 'spider' argument to Scraper.handle_spider_error() is deprecated.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
-
        assert self.crawler.spider
        exc = _failure.value
        if isinstance(exc, CloseSpider):
            assert self.crawler.engine is not None  # typing
-            self.crawler.engine.close_spider(
-                self.crawler.spider, exc.reason or "cancelled"
+            _schedule_coro(
+                self.crawler.engine.close_spider_async(reason=exc.reason or "cancelled")
            )
            return
        logkws = self.logformatter.spider_error(
@ -299,12 +363,9 @@ class Scraper:
            spider=self.crawler.spider,
        )
        assert self.crawler.stats
+        self.crawler.stats.inc_value("spider_exceptions/count")
        self.crawler.stats.inc_value(
-            "spider_exceptions/count", spider=self.crawler.spider
-        )
-        self.crawler.stats.inc_value(
-            f"spider_exceptions/{_failure.value.__class__.__name__}",
-            spider=self.crawler.spider,
+            f"spider_exceptions/{_failure.value.__class__.__name__}"
        )

    def handle_spider_output(
@ -315,12 +376,11 @@ class Scraper:
        spider: Spider | None = None,
    ) -> Deferred[None]:
        """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
-        if spider is not None:
-            warnings.warn(
-                "Passing a 'spider' argument to Scraper.handle_spider_output() is deprecated.",
-                category=ScrapyDeprecationWarning,
-                stacklevel=2,
-            )
+        warnings.warn(
+            "Scraper.handle_spider_output() is deprecated, use handle_spider_output_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
        return deferred_from_coro(
            self.handle_spider_output_async(result, request, response)
        )
@ -331,7 +391,10 @@ class Scraper:
        request: Request,
        response: Response | Failure,
    ) -> None:
-        """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel."""
+        """Pass items/requests produced by a callback to ``_process_spidermw_output()`` in parallel.
+
+        .. versionadded:: 2.13
+        """
        it: Iterable[_T] | AsyncIterator[_T]
        if is_asyncio_available():
            if isinstance(result, AsyncIterator):
@ -396,6 +459,11 @@ class Scraper:
        *response* is the source of the item data. If the item does not come
        from response data, e.g. it was hard-coded, set it to ``None``.
        """
+        warnings.warn(
+            "Scraper.start_itemproc() is deprecated, use start_itemproc_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
        return deferred_from_coro(self.start_itemproc_async(item, response=response))

    async def start_itemproc_async(
@ -405,14 +473,19 @@ class Scraper:

        *response* is the source of the item data. If the item does not come
        from response data, e.g. it was hard-coded, set it to ``None``.
+
+        .. versionadded:: VERSION
        """
        assert self.slot is not None  # typing
        assert self.crawler.spider is not None  # typing
        self.slot.itemproc_size += 1
        try:
-            output = await maybe_deferred_to_future(
-                self.itemproc.process_item(item, self.crawler.spider)
-            )
+            if self._itemproc_has_async["process_item"]:
+                output = await self.itemproc.process_item_async(item)
+            else:
+                output = await maybe_deferred_to_future(
+                    self.itemproc.process_item(item, self.crawler.spider)
+                )
        except DropItem as ex:
            logkws = self.logformatter.dropped(item, ex, response, self.crawler.spider)
            if logkws is not None:
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@ -7,10 +7,11 @@ See documentation in docs/topics/spider-middleware.rst
 from __future__ import annotations

 import logging
-from collections.abc import AsyncIterator, Callable, Iterable
+from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
+from functools import wraps
 from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
-from typing import TYPE_CHECKING, Any, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, TypeVar, cast
 from warnings import warn

 from twisted.internet.defer import Deferred, inlineCallbacks
@ -23,15 +24,16 @@ from scrapy.middleware import MiddlewareManager
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import (
+    _defer_sleep_async,
    deferred_from_coro,
    maybe_deferred_to_future,
-    mustbe_deferred,
 )
 from scrapy.utils.python import MutableAsyncChain, MutableChain, global_object_name

 if TYPE_CHECKING:
    from collections.abc import Generator

+    from scrapy.crawler import Crawler
    from scrapy.settings import BaseSettings


@ -39,9 +41,9 @@ logger = logging.getLogger(__name__)


 _T = TypeVar("_T")
-ScrapeFunc = Callable[
-    [Union[Response, Failure], Request],
-    Deferred[Union[Iterable[_T], AsyncIterator[_T]]],
+ScrapeFunc: TypeAlias = Callable[
+    [Response | Failure, Request],
+    Coroutine[Any, Any, Iterable[_T] | AsyncIterator[_T]],
 ]


@ -56,12 +58,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
    def _get_mwlist_from_settings(cls, settings: BaseSettings) -> list[Any]:
        return build_component_list(settings.getwithbase("SPIDER_MIDDLEWARES"))

-    def __init__(self, *middlewares: Any) -> None:
+    def __init__(self, *middlewares: Any, crawler: Crawler | None = None) -> None:
        self._check_deprecated_process_start_requests_use(middlewares)
-        super().__init__(*middlewares)
+        super().__init__(*middlewares, crawler=crawler)

    def _check_deprecated_process_start_requests_use(
-        self, middlewares: tuple[Any]
+        self, middlewares: tuple[Any, ...]
    ) -> None:
        deprecated_middlewares = [
            middleware
@ -115,9 +117,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
            )

    def _add_middleware(self, mw: Any) -> None:
-        super()._add_middleware(mw)
        if hasattr(mw, "process_spider_input"):
            self.methods["process_spider_input"].append(mw.process_spider_input)
+            self._check_mw_method_spider_arg(mw.process_spider_input)
+
        if self._use_start_requests:
            if hasattr(mw, "process_start_requests"):
                self.methods["process_start_requests"].appendleft(
@ -125,22 +128,33 @@ class SpiderMiddlewareManager(MiddlewareManager):
                )
        elif hasattr(mw, "process_start"):
            self.methods["process_start"].appendleft(mw.process_start)
+
        process_spider_output = self._get_async_method_pair(mw, "process_spider_output")
        self.methods["process_spider_output"].appendleft(process_spider_output)
+        if callable(process_spider_output):
+            self._check_mw_method_spider_arg(process_spider_output)
+        elif isinstance(process_spider_output, tuple):
+            for m in process_spider_output:
+                self._check_mw_method_spider_arg(m)
+
        process_spider_exception = getattr(mw, "process_spider_exception", None)
        self.methods["process_spider_exception"].appendleft(process_spider_exception)
+        if process_spider_exception is not None:
+            self._check_mw_method_spider_arg(process_spider_exception)

-    def _process_spider_input(
+    async def _process_spider_input(
        self,
        scrape_func: ScrapeFunc[_T],
        response: Response,
        request: Request,
-        spider: Spider,
-    ) -> Deferred[Iterable[_T] | AsyncIterator[_T]]:
+    ) -> Iterable[_T] | AsyncIterator[_T]:
        for method in self.methods["process_spider_input"]:
-            method = cast(Callable, method)
+            method = cast("Callable", method)
            try:
-                result = method(response=response, spider=spider)
+                if method in self._mw_methods_requiring_spider:
+                    result = method(response=response, spider=self._spider)
+                else:
+                    result = method(response=response)
                if result is not None:
                    msg = (
                        f"{global_object_name(method)} must return None "
@ -150,13 +164,12 @@ class SpiderMiddlewareManager(MiddlewareManager):
            except _InvalidOutput:
                raise
            except Exception:
-                return scrape_func(Failure(), request)
-        return scrape_func(response, request)
+                return await scrape_func(Failure(), request)
+        return await scrape_func(response, request)

    def _evaluate_iterable(
        self,
        response: Response,
-        spider: Spider,
        iterable: Iterable[_T] | AsyncIterator[_T],
        exception_processor_index: int,
        recover_to: MutableChain[_T] | MutableAsyncChain[_T],
@ -166,9 +179,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                yield from iterable
            except Exception as ex:
                exception_result = cast(
-                    Union[Failure, MutableChain[_T]],
+                    "Failure | MutableChain[_T]",
                    self._process_spider_exception(
-                        response, spider, ex, exception_processor_index
+                        response, ex, exception_processor_index
                    ),
                )
                if isinstance(exception_result, Failure):
@ -182,9 +195,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                    yield r
            except Exception as ex:
                exception_result = cast(
-                    Union[Failure, MutableAsyncChain[_T]],
+                    "Failure | MutableAsyncChain[_T]",
                    self._process_spider_exception(
-                        response, spider, ex, exception_processor_index
+                        response, ex, exception_processor_index
                    ),
                )
                if isinstance(exception_result, Failure):
@ -199,7 +212,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
    def _process_spider_exception(
        self,
        response: Response,
-        spider: Spider,
        exception: Exception,
        start_index: int = 0,
    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
@ -212,23 +224,24 @@ class SpiderMiddlewareManager(MiddlewareManager):
        for method_index, method in enumerate(method_list, start=start_index):
            if method is None:
                continue
-            method = cast(Callable, method)
-            result = method(response=response, exception=exception, spider=spider)
+            method = cast("Callable", method)
+            if method in self._mw_methods_requiring_spider:
+                result = method(
+                    response=response, exception=exception, spider=self._spider
+                )
+            else:
+                result = method(response=response, exception=exception)
            if _isiterable(result):
                # stop exception handling by handing control over to the
                # process_spider_output chain if an iterable has been returned
                dfd: Deferred[MutableChain[_T] | MutableAsyncChain[_T]] = (
-                    self._process_spider_output(
-                        response, spider, result, method_index + 1
-                    )
+                    self._process_spider_output(response, result, method_index + 1)
                )
                # _process_spider_output() returns a Deferred only because of downgrading so this can be
                # simplified when downgrading is removed.
                if dfd.called:
                    # the result is available immediately if _process_spider_output didn't do downgrading
-                    return cast(
-                        Union[MutableChain[_T], MutableAsyncChain[_T]], dfd.result
-                    )
+                    return cast("MutableChain[_T] | MutableAsyncChain[_T]", dfd.result)
                # we forbid waiting here because otherwise we would need to return a deferred from
                # _process_spider_exception too, which complicates the architecture
                msg = f"Async iterable returned from {global_object_name(method)} cannot be downgraded"
@ -249,7 +262,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
    def _process_spider_output(
        self,
        response: Response,
-        spider: Spider,
        result: Iterable[_T] | AsyncIterator[_T],
        start_index: int = 0,
    ) -> Generator[Deferred[Any], Any, MutableChain[_T] | MutableAsyncChain[_T]]:
@ -302,19 +314,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
                        )
                        recovered = MutableChain(recovered_collected)
                # might fail directly if the output value is not a generator
-                result = method(response=response, result=result, spider=spider)
+                if method in self._mw_methods_requiring_spider:
+                    result = method(
+                        response=response, result=result, spider=self._spider
+                    )
+                else:
+                    result = method(response=response, result=result)
            except Exception as ex:
                exception_result: Failure | MutableChain[_T] | MutableAsyncChain[_T] = (
-                    self._process_spider_exception(
-                        response, spider, ex, method_index + 1
-                    )
+                    self._process_spider_exception(response, ex, method_index + 1)
                )
                if isinstance(exception_result, Failure):
                    raise
                return exception_result
            if _isiterable(result):
                result = self._evaluate_iterable(
-                    response, spider, result, method_index + 1, recovered
+                    response, result, method_index + 1, recovered
                )
            else:
                if iscoroutine(result):
@ -338,7 +353,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
    async def _process_callback_output(
        self,
        response: Response,
-        spider: Spider,
        result: Iterable[_T] | AsyncIterator[_T],
    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
        recovered: MutableChain[_T] | MutableAsyncChain[_T]
@ -346,11 +360,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
            recovered = MutableAsyncChain()
        else:
            recovered = MutableChain()
-        result = self._evaluate_iterable(response, spider, result, 0, recovered)
+        result = self._evaluate_iterable(response, result, 0, recovered)
        result = await maybe_deferred_to_future(
            cast(
                "Deferred[Iterable[_T] | AsyncIterator[_T]]",
-                self._process_spider_output(response, spider, result),
+                self._process_spider_output(response, result),
            )
        )
        if isinstance(result, AsyncIterator):
@ -362,13 +376,29 @@ class SpiderMiddlewareManager(MiddlewareManager):

    def scrape_response(
        self,
-        scrape_func: ScrapeFunc[_T],
+        scrape_func: Callable[
+            [Response | Failure, Request],
+            Deferred[Iterable[_T] | AsyncIterator[_T]],
+        ],
        response: Response,
        request: Request,
        spider: Spider,
    ) -> Deferred[MutableChain[_T] | MutableAsyncChain[_T]]:
+        warn(
+            "SpiderMiddlewareManager.scrape_response() is deprecated, use scrape_response_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+
+        @wraps(scrape_func)
+        async def scrape_func_wrapped(
+            response: Response | Failure, request: Request
+        ) -> Iterable[_T] | AsyncIterator[_T]:
+            return await maybe_deferred_to_future(scrape_func(response, request))
+
+        self._set_compat_spider(spider)
        return deferred_from_coro(
-            self.scrape_response_async(scrape_func, response, request, spider)
+            self.scrape_response_async(scrape_func_wrapped, response, request)
        )

    async def scrape_response_async(
@ -376,47 +406,54 @@ class SpiderMiddlewareManager(MiddlewareManager):
        scrape_func: ScrapeFunc[_T],
        response: Response,
        request: Request,
-        spider: Spider,
    ) -> MutableChain[_T] | MutableAsyncChain[_T]:
+        if not self.crawler:
+            raise RuntimeError(
+                "scrape_response_async() called on a SpiderMiddlewareManager"
+                " instance created without a crawler."
+            )
+
        async def process_callback_output(
            result: Iterable[_T] | AsyncIterator[_T],
        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return await self._process_callback_output(response, spider, result)
+            return await self._process_callback_output(response, result)

        def process_spider_exception(
            exception: Exception,
        ) -> MutableChain[_T] | MutableAsyncChain[_T]:
-            return self._process_spider_exception(response, spider, exception)
+            return self._process_spider_exception(response, exception)

        try:
-            it: Iterable[_T] | AsyncIterator[_T] = await maybe_deferred_to_future(
-                mustbe_deferred(
-                    self._process_spider_input, scrape_func, response, request, spider
-                )
+            it: Iterable[_T] | AsyncIterator[_T] = await self._process_spider_input(
+                scrape_func, response, request
            )
            return await process_callback_output(it)
        except Exception as ex:
+            await _defer_sleep_async()
            return process_spider_exception(ex)

-    async def process_start(self, spider: Spider) -> AsyncIterator[Any] | None:
-        self._check_deprecated_start_requests_use(spider)
+    async def process_start(
+        self, spider: Spider | None = None
+    ) -> AsyncIterator[Any] | None:
+        if spider:
+            self._warn_spider_arg("process_start")
+            self._set_compat_spider(spider)
+        self._check_deprecated_start_requests_use()
        if self._use_start_requests:
-            sync_start = iter(spider.start_requests())
-            sync_start = await maybe_deferred_to_future(
-                self._process_chain("process_start_requests", sync_start, spider)
+            sync_start = iter(self._spider.start_requests())
+            sync_start = await self._process_chain(
+                "process_start_requests", sync_start, always_add_spider=True
            )
            start: AsyncIterator[Any] = as_async_generator(sync_start)
        else:
-            start = spider.start()
-            start = await maybe_deferred_to_future(
-                self._process_chain("process_start", start)
-            )
+            start = self._spider.start()
+            start = await self._process_chain("process_start", start)
        return start

-    def _check_deprecated_start_requests_use(self, spider: Spider):
+    def _check_deprecated_start_requests_use(self):
        start_requests_cls = None
        start_cls = None
-        spidercls = spider.__class__
+        spidercls = self._spider.__class__
        mro = spidercls.__mro__

        for cls in mro:
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@ -5,26 +5,23 @@ import contextlib
 import logging
 import pprint
 import signal
+import warnings
 from abc import ABC, abstractmethod
 from typing import TYPE_CHECKING, Any, TypeVar

-from twisted.internet.defer import (
-    Deferred,
-    DeferredList,
-    inlineCallbacks,
-)
+from twisted.internet.defer import Deferred, DeferredList, inlineCallbacks

-from scrapy import Spider, signals
+from scrapy import Spider
 from scrapy.addons import AddonManager
 from scrapy.core.engine import ExecutionEngine
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extension import ExtensionManager
 from scrapy.settings import Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
 from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
 from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.defer import deferred_from_coro, deferred_to_future
+from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.log import (
-    LogCounterHandler,
    configure_logging,
    get_scrapy_root_handler,
    install_scrapy_root_handler,
@ -99,13 +96,6 @@ class Crawler:
        self.addons.load_settings(self.settings)
        self.stats = load_object(self.settings["STATS_CLASS"])(self)

-        handler = LogCounterHandler(self, level=self.settings.get("LOG_LEVEL"))
-        logging.root.addHandler(handler)
-        # lambda is assigned to Crawler attribute because this way it is not
-        # garbage collected after leaving the scope
-        self.__remove_handler = lambda: logging.root.removeHandler(handler)
-        self.signals.connect(self.__remove_handler, signals.engine_stopped)
-
        lf_cls: type[LogFormatter] = load_object(self.settings["LOG_FORMATTER"])
        self.logformatter = lf_cls.from_crawler(self)

@ -163,12 +153,12 @@ class Crawler:
            self._apply_settings()
            self._update_root_log_handler()
            self.engine = self._create_engine()
-            yield self.engine.open_spider(self.spider)
-            yield self.engine.start()
+            yield deferred_from_coro(self.engine.open_spider_async())
+            yield deferred_from_coro(self.engine.start_async())
        except Exception:
            self.crawling = False
            if self.engine is not None:
-                yield self.engine.close()
+                yield deferred_from_coro(self.engine.close_async())
            raise

    async def crawl_async(self, *args: Any, **kwargs: Any) -> None:
@ -199,41 +189,40 @@ class Crawler:
            self._apply_settings()
            self._update_root_log_handler()
            self.engine = self._create_engine()
-            await self.engine.open_spider_async(self.spider)
+            await self.engine.open_spider_async()
            await self.engine.start_async()
        except Exception:
            self.crawling = False
            if self.engine is not None:
-                await deferred_to_future(self.engine.close())
+                await self.engine.close_async()
            raise

    def _create_spider(self, *args: Any, **kwargs: Any) -> Spider:
        return self.spidercls.from_crawler(self, *args, **kwargs)

    def _create_engine(self) -> ExecutionEngine:
-        return ExecutionEngine(self, lambda _: self.stop())
+        return ExecutionEngine(self, lambda _: self.stop_async())

-    @inlineCallbacks
-    def stop(self) -> Generator[Deferred[Any], Any, None]:
+    def stop(self) -> Deferred[None]:
        """Start a graceful stop of the crawler and return a deferred that is
        fired when the crawler is stopped."""
-        if self.crawling:
-            self.crawling = False
-            assert self.engine
-            yield self.engine.stop()
+        warnings.warn(
+            "Crawler.stop() is deprecated, use stop_async() instead",
+            ScrapyDeprecationWarning,
+            stacklevel=2,
+        )
+        return deferred_from_coro(self.stop_async())

    async def stop_async(self) -> None:
        """Start a graceful stop of the crawler and complete when the crawler is stopped.

        .. versionadded:: VERSION
-
-        This function requires
-        :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor` to be
-        installed.
        """
-        if not is_asyncio_available():
-            raise RuntimeError("Crawler.stop_async() requires AsyncioSelectorReactor.")
-        await deferred_to_future(self.stop())
+        if self.crawling:
+            self.crawling = False
+            assert self.engine
+            if self.engine.running:
+                await self.engine.stop_async()

    @staticmethod
    def _get_component(
@ -450,7 +439,7 @@ class CrawlerRunner(CrawlerRunnerBase):

        Returns a deferred that is fired when they all have ended.
        """
-        return DeferredList(c.stop() for c in self.crawlers)
+        return DeferredList(deferred_from_coro(c.stop_async()) for c in self.crawlers)

    @inlineCallbacks
    def join(self) -> Generator[Deferred[Any], Any, None]:
@ -666,6 +655,7 @@ class CrawlerProcess(CrawlerProcessBase, CrawlerRunner):
    ):
        super().__init__(settings, install_root_handler)
        self._initialized_reactor: bool = False
+        logger.debug("Using CrawlerProcess")

    def _create_crawler(self, spidercls: type[Spider] | str) -> Crawler:
        if isinstance(spidercls, str):
@ -740,6 +730,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
        install_root_handler: bool = True,
    ):
        super().__init__(settings, install_root_handler)
+        logger.debug("Using AsyncCrawlerProcess")
        # We want the asyncio event loop to be installed early, so that it's
        # always the correct one. And as we do that, we can also install the
        # reactor here.
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@ -9,6 +9,7 @@ from tldextract import TLDExtract
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_unicode

@ -39,6 +40,8 @@ def _is_public_domain(domain: str) -> bool:
 class CookiesMiddleware:
    """This middleware enables working with sites that need cookies"""

+    crawler: Crawler
+
    def __init__(self, debug: bool = False):
        self.jars: defaultdict[Any, CookieJar] = defaultdict(CookieJar)
        self.debug: bool = debug
@ -47,7 +50,9 @@ class CookiesMiddleware:
    def from_crawler(cls, crawler: Crawler) -> Self:
        if not crawler.settings.getbool("COOKIES_ENABLED"):
            raise NotConfigured
-        return cls(crawler.settings.getbool("COOKIES_DEBUG"))
+        o = cls(crawler.settings.getbool("COOKIES_DEBUG"))
+        o.crawler = crawler
+        return o

    def _process_cookies(
        self, cookies: Iterable[Cookie], *, jar: CookieJar, request: Request
@ -67,8 +72,9 @@ class CookiesMiddleware:

            jar.set_cookie_if_ok(cookie, request)

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        if request.meta.get("dont_merge_cookies", False):
            return None
@ -81,11 +87,12 @@ class CookiesMiddleware:
        # set Cookie header
        request.headers.pop("Cookie", None)
        jar.add_cookie_header(request)
-        self._debug_cookie(request, spider)
+        self._debug_cookie(request)
        return None

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if request.meta.get("dont_merge_cookies", False):
            return response
@ -96,11 +103,11 @@ class CookiesMiddleware:
        cookies = jar.make_cookies(response, request)
        self._process_cookies(cookies, jar=jar, request=request)

-        self._debug_set_cookie(response, spider)
+        self._debug_set_cookie(response)

        return response

-    def _debug_cookie(self, request: Request, spider: Spider) -> None:
+    def _debug_cookie(self, request: Request) -> None:
        if self.debug:
            cl = [
                to_unicode(c, errors="replace")
@ -109,9 +116,9 @@ class CookiesMiddleware:
            if cl:
                cookies = "\n".join(f"Cookie: {c}\n" for c in cl)
                msg = f"Sending cookies to: {request}\n{cookies}"
-                logger.debug(msg, extra={"spider": spider})
+                logger.debug(msg, extra={"spider": self.crawler.spider})

-    def _debug_set_cookie(self, response: Response, spider: Spider) -> None:
+    def _debug_set_cookie(self, response: Response) -> None:
        if self.debug:
            cl = [
                to_unicode(c, errors="replace")
@ -120,7 +127,7 @@ class CookiesMiddleware:
            if cl:
                cookies = "\n".join(f"Set-Cookie: {c}\n" for c in cl)
                msg = f"Received cookies from: {response}\n{cookies}"
-                logger.debug(msg, extra={"spider": spider})
+                logger.debug(msg, extra={"spider": self.crawler.spider})

    def _format_cookie(self, cookie: VerboseCookie, request: Request) -> str | None:
        """
--- a/scrapy/downloadermiddlewares/defaultheaders.py
+++ b/scrapy/downloadermiddlewares/defaultheaders.py
@ -8,6 +8,7 @@ from __future__ import annotations

 from typing import TYPE_CHECKING

+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.python import without_none_values

 if TYPE_CHECKING:
@ -30,8 +31,9 @@ class DefaultHeadersMiddleware:
        headers = without_none_values(crawler.settings["DEFAULT_REQUEST_HEADERS"])
        return cls(headers.items())

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        for k, v in self._headers:
            request.headers.setdefault(k, v)
--- a/scrapy/downloadermiddlewares/downloadtimeout.py
+++ b/scrapy/downloadermiddlewares/downloadtimeout.py
@ -9,6 +9,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING

 from scrapy import Request, Spider, signals
+from scrapy.utils.decorators import _warn_spider_arg

 if TYPE_CHECKING:
    # typing.Self requires Python 3.11
@ -31,8 +32,9 @@ class DownloadTimeoutMiddleware:
    def spider_opened(self, spider: Spider) -> None:
        self._timeout = getattr(spider, "download_timeout", self._timeout)

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        if self._timeout:
            request.meta.setdefault("download_timeout", self._timeout)
--- a/scrapy/downloadermiddlewares/httpauth.py
+++ b/scrapy/downloadermiddlewares/httpauth.py
@ -11,6 +11,7 @@ from typing import TYPE_CHECKING
 from w3lib.http import basic_auth_header

 from scrapy import Request, Spider, signals
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.url import url_is_from_any_domain

 if TYPE_CHECKING:
@ -38,8 +39,9 @@ class HttpAuthMiddleware:
            self.auth = basic_auth_header(usr, pwd)
            self.domain = spider.http_auth_domain  # type: ignore[attr-defined]

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        auth = getattr(self, "auth", None)
        if (
--- a/scrapy/downloadermiddlewares/httpcache.py
+++ b/scrapy/downloadermiddlewares/httpcache.py
@ -8,15 +8,16 @@ from twisted.internet.error import (
    ConnectError,
    ConnectionDone,
    ConnectionLost,
-    ConnectionRefusedError,
    DNSLookupError,
    TCPTimedOutError,
-    TimeoutError,
 )
+from twisted.internet.error import ConnectionRefusedError as TxConnectionRefusedError
+from twisted.internet.error import TimeoutError as TxTimeoutError
 from twisted.web.client import ResponseFailed

 from scrapy import signals
 from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.misc import load_object

 if TYPE_CHECKING:
@ -34,9 +35,9 @@ if TYPE_CHECKING:
 class HttpCacheMiddleware:
    DOWNLOAD_EXCEPTIONS = (
        defer.TimeoutError,
-        TimeoutError,
+        TxTimeoutError,
        DNSLookupError,
-        ConnectionRefusedError,
+        TxConnectionRefusedError,
        ConnectionDone,
        ConnectError,
        ConnectionLost,
@ -45,6 +46,8 @@ class HttpCacheMiddleware:
        OSError,
    )

+    crawler: Crawler
+
    def __init__(self, settings: Settings, stats: StatsCollector) -> None:
        if not settings.getbool("HTTPCACHE_ENABLED"):
            raise NotConfigured
@ -59,6 +62,7 @@ class HttpCacheMiddleware:
        o = cls(crawler.settings, crawler.stats)
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
+        o.crawler = crawler
        return o

    def spider_opened(self, spider: Spider) -> None:
@ -67,8 +71,9 @@ class HttpCacheMiddleware:
    def spider_closed(self, spider: Spider) -> None:
        self.storage.close_spider(spider)

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        if request.meta.get("dont_cache", False):
            return None
@ -80,19 +85,19 @@ class HttpCacheMiddleware:

        # Look for cached response and check if expired
        cachedresponse: Response | None = self.storage.retrieve_response(
-            spider, request
+            self.crawler.spider, request
        )
        if cachedresponse is None:
-            self.stats.inc_value("httpcache/miss", spider=spider)
+            self.stats.inc_value("httpcache/miss")
            if self.ignore_missing:
-                self.stats.inc_value("httpcache/ignore", spider=spider)
+                self.stats.inc_value("httpcache/ignore")
                raise IgnoreRequest(f"Ignored request not in cache: {request}")
            return None  # first time request

        # Return cached response only if not expired
        cachedresponse.flags.append("cached")
        if self.policy.is_cached_response_fresh(cachedresponse, request):
-            self.stats.inc_value("httpcache/hit", spider=spider)
+            self.stats.inc_value("httpcache/hit")
            return cachedresponse

        # Keep a reference to cached response to avoid a second cache lookup on
@ -101,8 +106,9 @@ class HttpCacheMiddleware:

        return None

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if request.meta.get("dont_cache", False):
            return response
@ -120,38 +126,33 @@ class HttpCacheMiddleware:
        # Do not validate first-hand responses
        cachedresponse: Response | None = request.meta.pop("cached_response", None)
        if cachedresponse is None:
-            self.stats.inc_value("httpcache/firsthand", spider=spider)
-            self._cache_response(spider, response, request, cachedresponse)
+            self.stats.inc_value("httpcache/firsthand")
+            self._cache_response(response, request)
            return response

        if self.policy.is_cached_response_valid(cachedresponse, response, request):
-            self.stats.inc_value("httpcache/revalidate", spider=spider)
+            self.stats.inc_value("httpcache/revalidate")
            return cachedresponse

-        self.stats.inc_value("httpcache/invalidate", spider=spider)
-        self._cache_response(spider, response, request, cachedresponse)
+        self.stats.inc_value("httpcache/invalidate")
+        self._cache_response(response, request)
        return response

+    @_warn_spider_arg
    def process_exception(
-        self, request: Request, exception: Exception, spider: Spider
+        self, request: Request, exception: Exception, spider: Spider | None = None
    ) -> Request | Response | None:
        cachedresponse: Response | None = request.meta.pop("cached_response", None)
        if cachedresponse is not None and isinstance(
            exception, self.DOWNLOAD_EXCEPTIONS
        ):
-            self.stats.inc_value("httpcache/errorrecovery", spider=spider)
+            self.stats.inc_value("httpcache/errorrecovery")
            return cachedresponse
        return None

-    def _cache_response(
-        self,
-        spider: Spider,
-        response: Response,
-        request: Request,
-        cachedresponse: Response | None,
-    ) -> None:
+    def _cache_response(self, response: Response, request: Request) -> None:
        if self.policy.should_cache_response(response, request):
-            self.stats.inc_value("httpcache/store", spider=spider)
-            self.storage.store_response(spider, request, response)
+            self.stats.inc_value("httpcache/store")
+            self.storage.store_response(self.crawler.spider, request, response)
        else:
-            self.stats.inc_value("httpcache/uncacheable", spider=spider)
+            self.stats.inc_value("httpcache/uncacheable")
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@ -1,11 +1,12 @@
 from __future__ import annotations

+import warnings
 from itertools import chain
 from logging import getLogger
 from typing import TYPE_CHECKING, Any

 from scrapy import Request, Spider, signals
-from scrapy.exceptions import IgnoreRequest, NotConfigured
+from scrapy.exceptions import IgnoreRequest, NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.utils._compression import (
@ -14,6 +15,7 @@ from scrapy.utils._compression import (
    _unbrotli,
    _unzstd,
 )
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.gz import gunzip

 if TYPE_CHECKING:
@ -30,13 +32,22 @@ ACCEPTED_ENCODINGS: list[bytes] = [b"gzip", b"deflate"]

 try:
    try:
-        import brotli  # noqa: F401
+        import brotli
    except ImportError:
-        import brotlicffi  # noqa: F401
+        import brotlicffi as brotli
 except ImportError:
    pass
 else:
-    ACCEPTED_ENCODINGS.append(b"br")
+    try:
+        brotli.Decompressor.can_accept_more_data
+    except AttributeError:  # pragma: no cover
+        warnings.warn(
+            "You have brotli installed. But 'br' encoding support now requires "
+            "brotli's or brotlicffi's version >= 1.2.0. Please upgrade "
+            "brotli/brotlicffi to make Scrapy decode 'br' encoded responses.",
+        )
+    else:
+        ACCEPTED_ENCODINGS.append(b"br")

 try:
    import zstandard  # noqa: F401
@ -74,18 +85,34 @@ class HttpCompressionMiddleware:

    def open_spider(self, spider: Spider) -> None:
        if hasattr(spider, "download_maxsize"):
+            warnings.warn(
+                "The 'download_maxsize' spider attribute is deprecated. "
+                "Use Spider.custom_settings or Spider.update_settings() instead. "
+                "The corresponding setting name is 'DOWNLOAD_MAXSIZE'.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
            self._max_size = spider.download_maxsize
        if hasattr(spider, "download_warnsize"):
+            warnings.warn(
+                "The 'download_warnsize' spider attribute is deprecated. "
+                "Use Spider.custom_settings or Spider.update_settings() instead. "
+                "The corresponding setting name is 'DOWNLOAD_WARNSIZE'.",
+                category=ScrapyDeprecationWarning,
+                stacklevel=2,
+            )
            self._warn_size = spider.download_warnsize

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
        return None

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if request.method == "HEAD":
            return response
@ -98,13 +125,13 @@ class HttpCompressionMiddleware:
                    decoded_body, content_encoding = self._handle_encoding(
                        response.body, content_encoding, max_size
                    )
-                except _DecompressionMaxSizeExceeded:
+                except _DecompressionMaxSizeExceeded as e:
                    raise IgnoreRequest(
                        f"Ignored response {response} because its body "
-                        f"({len(response.body)} B compressed) exceeded "
-                        f"DOWNLOAD_MAXSIZE ({max_size} B) during "
-                        f"decompression."
-                    )
+                        f"({len(response.body)} B compressed, "
+                        f"{e.decompressed_size} B decompressed so far) exceeded "
+                        f"DOWNLOAD_MAXSIZE ({max_size} B) during decompression."
+                    ) from e
                if len(response.body) < warn_size <= len(decoded_body):
                    logger.warning(
                        f"{response} body size after decompression "
@ -118,11 +145,8 @@ class HttpCompressionMiddleware:
                    self.stats.inc_value(
                        "httpcompression/response_bytes",
                        len(decoded_body),
-                        spider=spider,
-                    )
-                    self.stats.inc_value(
-                        "httpcompression/response_count", spider=spider
                    )
+                    self.stats.inc_value("httpcompression/response_count")
                respcls = responsetypes.from_args(
                    headers=response.headers, url=response.url, body=decoded_body
                )
@ -187,7 +211,7 @@ class HttpCompressionMiddleware:
            f"from unsupported encoding(s) '{encodings_str}'."
        )
        if b"br" in encodings:
-            msg += " You need to install brotli or brotlicffi to decode 'br'."
+            msg += " You need to install brotli or brotlicffi >= 1.2.0 to decode 'br'."
        if b"zstd" in encodings:
            msg += " You need to install zstandard to decode 'zstd'."
        logger.warning(msg)
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@ -10,6 +10,7 @@ from urllib.request import (  # type: ignore[attr-defined]
 )

 from scrapy.exceptions import NotConfigured
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.python import to_bytes

@ -55,8 +56,9 @@ class HttpProxyMiddleware:

        return creds, proxy_url

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        creds, proxy_url, scheme = None, None, None
        if "proxy" in request.meta:
--- a/scrapy/downloadermiddlewares/offsite.py
+++ b/scrapy/downloadermiddlewares/offsite.py
@ -7,6 +7,7 @@ from typing import TYPE_CHECKING

 from scrapy import Request, Spider, signals
 from scrapy.exceptions import IgnoreRequest
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached

 if TYPE_CHECKING:
@ -21,29 +22,34 @@ logger = logging.getLogger(__name__)


 class OffsiteMiddleware:
+    crawler: Crawler
+
+    def __init__(self, stats: StatsCollector):
+        self.stats = stats
+        self.domains_seen: set[str] = set()
+
    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
        assert crawler.stats
        o = cls(crawler.stats)
        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
        crawler.signals.connect(o.request_scheduled, signal=signals.request_scheduled)
+        o.crawler = crawler
        return o

-    def __init__(self, stats: StatsCollector):
-        self.stats = stats
-        self.domains_seen: set[str] = set()
-
    def spider_opened(self, spider: Spider) -> None:
        self.host_regex: re.Pattern[str] = self.get_host_regex(spider)

    def request_scheduled(self, request: Request, spider: Spider) -> None:
-        self.process_request(request, spider)
+        self.process_request(request)

-    def process_request(self, request: Request, spider: Spider) -> None:
+    @_warn_spider_arg
+    def process_request(self, request: Request, spider: Spider | None = None) -> None:
+        assert self.crawler.spider
        if (
            request.dont_filter
            or request.meta.get("allow_offsite")
-            or self.should_follow(request, spider)
+            or self.should_follow(request, self.crawler.spider)
        ):
            return
        domain = urlparse_cached(request).hostname
@ -52,10 +58,10 @@ class OffsiteMiddleware:
            logger.debug(
                "Filtered offsite request to %(domain)r: %(request)s",
                {"domain": domain, "request": request},
-                extra={"spider": spider},
+                extra={"spider": self.crawler.spider},
            )
-            self.stats.inc_value("offsite/domains", spider=spider)
-        self.stats.inc_value("offsite/filtered", spider=spider)
+            self.stats.inc_value("offsite/domains")
+        self.stats.inc_value("offsite/filtered")
        raise IgnoreRequest

    def should_follow(self, request: Request, spider: Spider) -> bool:
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@ -8,6 +8,7 @@ from w3lib.url import safe_url_string

 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import HtmlResponse, Response
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.response import get_meta_refresh

@ -79,6 +80,7 @@ def _build_redirect_request(


 class BaseRedirectMiddleware:
+    crawler: Crawler
    enabled_setting: str = "REDIRECT_ENABLED"

    def __init__(self, settings: BaseSettings):
@ -90,11 +92,11 @@ class BaseRedirectMiddleware:

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler.settings)
+        o = cls(crawler.settings)
+        o.crawler = crawler
+        return o

-    def _redirect(
-        self, redirected: Request, request: Request, spider: Spider, reason: Any
-    ) -> Request:
+    def _redirect(self, redirected: Request, request: Request, reason: Any) -> Request:
        ttl = request.meta.setdefault("redirect_ttl", self.max_redirect_times)
        redirects = request.meta.get("redirect_times", 0) + 1

@ -114,13 +116,13 @@ class BaseRedirectMiddleware:
            logger.debug(
                "Redirecting (%(reason)s) to %(redirected)s from %(request)s",
                {"reason": reason, "redirected": redirected, "request": request},
-                extra={"spider": spider},
+                extra={"spider": self.crawler.spider},
            )
            return redirected
        logger.debug(
            "Discarding %(request)s: max redirections reached",
            {"request": request},
-            extra={"spider": spider},
+            extra={"spider": self.crawler.spider},
        )
        raise IgnoreRequest("max redirections reached")

@ -144,12 +146,14 @@ class RedirectMiddleware(BaseRedirectMiddleware):
    and meta-refresh html tag.
    """

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if (
            request.meta.get("dont_redirect", False)
-            or response.status in getattr(spider, "handle_httpstatus_list", [])
+            or response.status
+            in getattr(self.crawler.spider, "handle_httpstatus_list", [])
            or response.status in request.meta.get("handle_httpstatus_list", [])
            or request.meta.get("handle_httpstatus_all", False)
        ):
@ -171,10 +175,10 @@ class RedirectMiddleware(BaseRedirectMiddleware):
            return response

        if response.status in (301, 307, 308) or request.method == "HEAD":
-            return self._redirect(redirected, request, spider, response.status)
+            return self._redirect(redirected, request, response.status)

        redirected = self._redirect_request_using_get(request, redirected_url)
-        return self._redirect(redirected, request, spider, response.status)
+        return self._redirect(redirected, request, response.status)


 class MetaRefreshMiddleware(BaseRedirectMiddleware):
@ -185,8 +189,9 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
        self._ignore_tags: list[str] = settings.getlist("METAREFRESH_IGNORE_TAGS")
        self._maxdelay: int = settings.getint("METAREFRESH_MAXDELAY")

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if (
            request.meta.get("dont_redirect", False)
@ -202,6 +207,6 @@ class MetaRefreshMiddleware(BaseRedirectMiddleware):
        redirected = self._redirect_request_using_get(request, url)
        if urlparse_cached(redirected).scheme not in {"http", "https"}:
            return response
-        if cast(float, interval) < self._maxdelay:
-            return self._redirect(redirected, request, spider, "meta refresh")
+        if cast("float", interval) < self._maxdelay:
+            return self._redirect(redirected, request, "meta refresh")
        return response
--- a/scrapy/downloadermiddlewares/retry.py
+++ b/scrapy/downloadermiddlewares/retry.py
@ -16,6 +16,7 @@ from logging import Logger, getLogger
 from typing import TYPE_CHECKING

 from scrapy.exceptions import NotConfigured
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import global_object_name
 from scrapy.utils.response import response_status_message
@ -123,6 +124,8 @@ def get_retry_request(


 class RetryMiddleware:
+    crawler: Crawler
+
    def __init__(self, settings: BaseSettings):
        if not settings.getbool("RETRY_ENABLED"):
            raise NotConfigured
@ -136,39 +139,41 @@ class RetryMiddleware:

    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
-        return cls(crawler.settings)
+        o = cls(crawler.settings)
+        o.crawler = crawler
+        return o

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
        if request.meta.get("dont_retry", False):
            return response
        if response.status in self.retry_http_codes:
            reason = response_status_message(response.status)
-            return self._retry(request, reason, spider) or response
+            return self._retry(request, reason) or response
        return response

+    @_warn_spider_arg
    def process_exception(
-        self, request: Request, exception: Exception, spider: Spider
+        self, request: Request, exception: Exception, spider: Spider | None = None
    ) -> Request | Response | None:
        if isinstance(exception, self.exceptions_to_retry) and not request.meta.get(
            "dont_retry", False
        ):
-            return self._retry(request, exception, spider)
+            return self._retry(request, exception)
        return None

    def _retry(
-        self,
-        request: Request,
-        reason: str | Exception | type[Exception],
-        spider: Spider,
+        self, request: Request, reason: str | Exception | type[Exception]
    ) -> Request | None:
        max_retry_times = request.meta.get("max_retry_times", self.max_retry_times)
        priority_adjust = request.meta.get("priority_adjust", self.priority_adjust)
+        assert self.crawler.spider
        return get_retry_request(
            request,
            reason=reason,
-            spider=spider,
+            spider=self.crawler.spider,
            max_retry_times=max_retry_times,
            priority_adjust=priority_adjust,
        )
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@ -9,18 +9,17 @@ from __future__ import annotations
 import logging
 from typing import TYPE_CHECKING

-from twisted.internet.defer import Deferred, maybeDeferred
+from twisted.internet.defer import Deferred

 from scrapy.exceptions import IgnoreRequest, NotConfigured
 from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
+from scrapy.utils.decorators import _warn_spider_arg
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.misc import load_object

 if TYPE_CHECKING:
-    from twisted.python.failure import Failure
-
    # typing.Self requires Python 3.11
    from typing_extensions import Self

@ -53,24 +52,18 @@ class RobotsTxtMiddleware:
    def from_crawler(cls, crawler: Crawler) -> Self:
        return cls(crawler)

-    def process_request(
-        self, request: Request, spider: Spider
-    ) -> Deferred[None] | None:
-        if request.meta.get("dont_obey_robotstxt"):
-            return None
-        if request.url.startswith("data:") or request.url.startswith("file:"):
-            return None
-        d: Deferred[RobotParser | None] = maybeDeferred(
-            self.robot_parser,
-            request,
-            spider,  # type: ignore[call-overload]
-        )
-        d2: Deferred[None] = d.addCallback(self.process_request_2, request, spider)
-        return d2
-
-    def process_request_2(
-        self, rp: RobotParser | None, request: Request, spider: Spider
+    @_warn_spider_arg
+    async def process_request(
+        self, request: Request, spider: Spider | None = None
    ) -> None:
+        if request.meta.get("dont_obey_robotstxt"):
+            return
+        if request.url.startswith("data:") or request.url.startswith("file:"):
+            return
+        rp = await self.robot_parser(request)
+        self.process_request_2(rp, request)
+
+    def process_request_2(self, rp: RobotParser | None, request: Request) -> None:
        if rp is None:
            return

@ -82,15 +75,13 @@ class RobotsTxtMiddleware:
            logger.debug(
                "Forbidden by robots.txt: %(request)s",
                {"request": request},
-                extra={"spider": spider},
+                extra={"spider": self.crawler.spider},
            )
            assert self.crawler.stats
            self.crawler.stats.inc_value("robotstxt/forbidden")
            raise IgnoreRequest("Forbidden by robots.txt")

-    def robot_parser(
-        self, request: Request, spider: Spider
-    ) -> RobotParser | Deferred[RobotParser | None] | None:
+    async def robot_parser(self, request: Request) -> RobotParser | None:
        url = urlparse_cached(request)
        netloc = url.netloc

@ -105,35 +96,26 @@ class RobotsTxtMiddleware:
            )
            assert self.crawler.engine
            assert self.crawler.stats
-            dfd = self.crawler.engine.download(robotsreq)
-            dfd.addCallback(self._parse_robots, netloc, spider)
-            dfd.addErrback(self._logerror, robotsreq, spider)
-            dfd.addErrback(self._robots_error, netloc)
+            try:
+                resp = await self.crawler.engine.download_async(robotsreq)
+                self._parse_robots(resp, netloc)
+            except Exception as e:
+                if not isinstance(e, IgnoreRequest):
+                    logger.error(
+                        "Error downloading %(request)s: %(f_exception)s",
+                        {"request": request, "f_exception": e},
+                        exc_info=True,
+                        extra={"spider": self.crawler.spider},
+                    )
+                self._robots_error(e, netloc)
            self.crawler.stats.inc_value("robotstxt/request_count")

        parser = self._parsers[netloc]
        if isinstance(parser, Deferred):
-            d: Deferred[RobotParser | None] = Deferred()
-
-            def cb(result: RobotParser | None) -> RobotParser | None:
-                d.callback(result)
-                return result
-
-            parser.addCallback(cb)
-            return d
+            return await maybe_deferred_to_future(parser)
        return parser

-    def _logerror(self, failure: Failure, request: Request, spider: Spider) -> Failure:
-        if failure.type is not IgnoreRequest:
-            logger.error(
-                "Error downloading %(request)s: %(f_exception)s",
-                {"request": request, "f_exception": failure.value},
-                exc_info=failure_to_exc_info(failure),
-                extra={"spider": spider},
-            )
-        return failure
-
-    def _parse_robots(self, response: Response, netloc: str, spider: Spider) -> None:
+    def _parse_robots(self, response: Response, netloc: str) -> None:
        assert self.crawler.stats
        self.crawler.stats.inc_value("robotstxt/response_count")
        self.crawler.stats.inc_value(
@ -145,9 +127,9 @@ class RobotsTxtMiddleware:
        self._parsers[netloc] = rp
        rp_dfd.callback(rp)

-    def _robots_error(self, failure: Failure, netloc: str) -> None:
-        if failure.type is not IgnoreRequest:
-            key = f"robotstxt/exception_count/{failure.type}"
+    def _robots_error(self, exc: Exception, netloc: str) -> None:
+        if not isinstance(exc, IgnoreRequest):
+            key = f"robotstxt/exception_count/{type(exc)}"
            assert self.crawler.stats
            self.crawler.stats.inc_value(key)
        rp_dfd = self._parsers[netloc]
--- a/scrapy/downloadermiddlewares/stats.py
+++ b/scrapy/downloadermiddlewares/stats.py
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING
 from twisted.web import http

 from scrapy.exceptions import NotConfigured
+from scrapy.utils.decorators import _warn_spider_arg
 from scrapy.utils.python import global_object_name, to_bytes
 from scrapy.utils.request import request_httprepr

@ -45,24 +46,22 @@ class DownloaderStats:
        assert crawler.stats
        return cls(crawler.stats)

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
-        self.stats.inc_value("downloader/request_count", spider=spider)
-        self.stats.inc_value(
-            f"downloader/request_method_count/{request.method}", spider=spider
-        )
+        self.stats.inc_value("downloader/request_count")
+        self.stats.inc_value(f"downloader/request_method_count/{request.method}")
        reqlen = len(request_httprepr(request))
-        self.stats.inc_value("downloader/request_bytes", reqlen, spider=spider)
+        self.stats.inc_value("downloader/request_bytes", reqlen)
        return None

+    @_warn_spider_arg
    def process_response(
-        self, request: Request, response: Response, spider: Spider
+        self, request: Request, response: Response, spider: Spider | None = None
    ) -> Request | Response:
-        self.stats.inc_value("downloader/response_count", spider=spider)
-        self.stats.inc_value(
-            f"downloader/response_status_count/{response.status}", spider=spider
-        )
+        self.stats.inc_value("downloader/response_count")
+        self.stats.inc_value(f"downloader/response_status_count/{response.status}")
        reslen = (
            len(response.body)
            + get_header_size(response.headers)
@ -70,15 +69,14 @@ class DownloaderStats:
            + 4
        )
        # response.body + b"\r\n"+ response.header + b"\r\n" + response.status
-        self.stats.inc_value("downloader/response_bytes", reslen, spider=spider)
+        self.stats.inc_value("downloader/response_bytes", reslen)
        return response

+    @_warn_spider_arg
    def process_exception(
-        self, request: Request, exception: Exception, spider: Spider
+        self, request: Request, exception: Exception, spider: Spider | None = None
    ) -> Request | Response | None:
        ex_class = global_object_name(exception.__class__)
-        self.stats.inc_value("downloader/exception_count", spider=spider)
-        self.stats.inc_value(
-            f"downloader/exception_type_count/{ex_class}", spider=spider
-        )
+        self.stats.inc_value("downloader/exception_count")
+        self.stats.inc_value(f"downloader/exception_type_count/{ex_class}")
        return None
--- a/scrapy/downloadermiddlewares/useragent.py
+++ b/scrapy/downloadermiddlewares/useragent.py
@ -5,6 +5,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING

 from scrapy import Request, Spider, signals
+from scrapy.utils.decorators import _warn_spider_arg

 if TYPE_CHECKING:
    # typing.Self requires Python 3.11
@ -29,8 +30,9 @@ class UserAgentMiddleware:
    def spider_opened(self, spider: Spider) -> None:
        self.user_agent = getattr(spider, "user_agent", self.user_agent)

+    @_warn_spider_arg
    def process_request(
-        self, request: Request, spider: Spider
+        self, request: Request, spider: Spider | None = None
    ) -> Request | Response | None:
        if self.user_agent:
            request.headers.setdefault(b"User-Agent", self.user_agent)
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@ -1,7 +1,6 @@
 from __future__ import annotations

 import logging
-import warnings
 from pathlib import Path
 from typing import TYPE_CHECKING
 from warnings import warn
@ -22,7 +21,6 @@ if TYPE_CHECKING:

    from scrapy.crawler import Crawler
    from scrapy.http.request import Request
-    from scrapy.settings import BaseSettings
    from scrapy.spiders import Spider


@ -30,15 +28,6 @@ class BaseDupeFilter:
    """Dummy duplicate request filtering class (:setting:`DUPEFILTER_CLASS`)
    that does not filter out any request."""

-    @classmethod
-    def from_settings(cls, settings: BaseSettings) -> Self:
-        warnings.warn(
-            f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return cls()
-
    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
        return cls()
@ -84,42 +73,24 @@ class RFPDupeFilter(BaseDupeFilter):
        self.debug = debug
        self.logger = logging.getLogger(__name__)
        if path:
-            self.file = Path(path, "requests.seen").open("a+", encoding="utf-8")
+            # line-by-line writing, see: https://github.com/scrapy/scrapy/issues/6019
+            self.file = Path(path, "requests.seen").open(
+                "a+", buffering=1, encoding="utf-8"
+            )
+            self.file.reconfigure(write_through=True)
            self.file.seek(0)
            self.fingerprints.update(x.rstrip() for x in self.file)

-    @classmethod
-    def from_settings(
-        cls,
-        settings: BaseSettings,
-        *,
-        fingerprinter: RequestFingerprinterProtocol | None = None,
-    ) -> Self:
-        warnings.warn(
-            f"{cls.__name__}.from_settings() is deprecated, use from_crawler() instead.",
-            category=ScrapyDeprecationWarning,
-            stacklevel=2,
-        )
-        return cls._from_settings(settings, fingerprinter=fingerprinter)
-
    @classmethod
    def from_crawler(cls, crawler: Crawler) -> Self:
        assert crawler.request_fingerprinter
-        return cls._from_settings(
-            crawler.settings,
+        debug = crawler.settings.getbool("DUPEFILTER_DEBUG")
+        return cls(
+            job_dir(crawler.settings),
+            debug,
            fingerprinter=crawler.request_fingerprinter,
        )

-    @classmethod
-    def _from_settings(
-        cls,
-        settings: BaseSettings,
-        *,
-        fingerprinter: RequestFingerprinterProtocol | None = None,
-    ) -> Self:
-        debug = settings.getbool("DUPEFILTER_DEBUG")
-        return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
-
    def request_seen(self, request: Request) -> bool:
        fp = self.request_fingerprint(request)
        if fp in self.fingerprints:
@ -152,4 +123,4 @@ class RFPDupeFilter(BaseDupeFilter):
            self.logdupes = False

        assert spider.crawler.stats
-        spider.crawler.stats.inc_value("dupefilter/filtered", spider=spider)
+        spider.crawler.stats.inc_value("dupefilter/filtered")
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@ -8,6 +8,7 @@ import csv
 import marshal
 import pickle
 import pprint
+from abc import ABC, abstractmethod
 from collections.abc import Callable, Iterable, Mapping
 from io import BytesIO, TextIOWrapper
 from typing import TYPE_CHECKING, Any
@ -35,7 +36,7 @@ __all__ = [
 ]


-class BaseItemExporter:
+class BaseItemExporter(ABC):
    def __init__(self, *, dont_fail: bool = False, **kwargs: Any):
        self._kwargs: dict[str, Any] = kwargs
        self._configure(kwargs, dont_fail=dont_fail)
@ -54,6 +55,7 @@ class BaseItemExporter:
        if not dont_fail and options:
            raise TypeError(f"Unexpected options: {', '.join(options.keys())}")

+    @abstractmethod
    def export_item(self, item: Any) -> None:
        raise NotImplementedError

@ -63,10 +65,10 @@ class BaseItemExporter:
        serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
        return serializer(value)

-    def start_exporting(self) -> None:
+    def start_exporting(self) -> None:  # noqa: B027
        pass

-    def finish_exporting(self) -> None:
+    def finish_exporting(self) -> None:  # noqa: B027
        pass

    def _get_serialized_fields(
--- a/scrapy/extensions/closespider.py
+++ b/scrapy/extensions/closespider.py
@ -18,6 +18,7 @@ from scrapy.utils.asyncio import (
    call_later,
    create_looping_call,
 )
+from scrapy.utils.defer import _schedule_coro

 if TYPE_CHECKING:
    from twisted.internet.task import LoopingCall
@ -86,38 +87,31 @@ class CloseSpider:
    def error_count(self, failure: Failure, response: Response, spider: Spider) -> None:
        self.counter["errorcount"] += 1
        if self.counter["errorcount"] == self.close_on["errorcount"]:
-            assert self.crawler.engine
-            self.crawler.engine.close_spider(spider, "closespider_errorcount")
+            self._close_spider("closespider_errorcount")

    def page_count(self, response: Response, request: Request, spider: Spider) -> None:
        self.counter["pagecount"] += 1
        self.counter["pagecount_since_last_item"] += 1
        if self.counter["pagecount"] == self.close_on["pagecount"]:
-            assert self.crawler.engine
-            self.crawler.engine.close_spider(spider, "closespider_pagecount")
+            self._close_spider("closespider_pagecount")
            return
        if self.close_on["pagecount_no_item"] and (
            self.counter["pagecount_since_last_item"]
            >= self.close_on["pagecount_no_item"]
        ):
-            assert self.crawler.engine
-            self.crawler.engine.close_spider(spider, "closespider_pagecount_no_item")
+            self._close_spider("closespider_pagecount_no_item")

    def spider_opened(self, spider: Spider) -> None:
        assert self.crawler.engine
        self.task = call_later(
-            self.close_on["timeout"],
-            self.crawler.engine.close_spider,
-            spider,
-            "closespider_timeout",
+            self.close_on["timeout"], self._close_spider, "closespider_timeout"
        )

    def item_scraped(self, item: Any, spider: Spider) -> None:
        self.counter["itemcount"] += 1
        self.counter["pagecount_since_last_item"] = 0
        if self.counter["itemcount"] == self.close_on["itemcount"]:
-            assert self.crawler.engine
-            self.crawler.engine.close_spider(spider, "closespider_itemcount")
+            self._close_spider("closespider_itemcount")

    def spider_closed(self, spider: Spider) -> None:
        if self.task:
@ -130,7 +124,7 @@ class CloseSpider:
            self.task_no_item = None

    def spider_opened_no_item(self, spider: Spider) -> None:
-        self.task_no_item = create_looping_call(self._count_items_produced, spider)
+        self.task_no_item = create_looping_call(self._count_items_produced)
        self.task_no_item.start(self.timeout_no_item, now=False)

        logger.info(
@ -141,7 +135,7 @@ class CloseSpider:
    def item_scraped_no_item(self, item: Any, spider: Spider) -> None:
        self.items_in_period += 1

-    def _count_items_produced(self, spider: Spider) -> None:
+    def _count_items_produced(self) -> None:
        if self.items_in_period >= 1:
            self.items_in_period = 0
        else:
@ -149,5 +143,8 @@ class CloseSpider:
                f"Closing spider since no items were produced in the last "
                f"{self.timeout_no_item} seconds."
            )
-            assert self.crawler.engine
-            self.crawler.engine.close_spider(spider, "closespider_timeout_no_item")
+            self._close_spider("closespider_timeout_no_item")
+
+    def _close_spider(self, reason: str) -> None:
+        assert self.crawler.engine
+        _schedule_coro(self.crawler.engine.close_spider_async(reason=reason))
--- a/scrapy/extensions/corestats.py
+++ b/scrapy/extensions/corestats.py
@ -35,26 +35,24 @@ class CoreStats:

    def spider_opened(self, spider: Spider) -> None:
        self.start_time = datetime.now(tz=timezone.utc)
-        self.stats.set_value("start_time", self.start_time, spider=spider)
+        self.stats.set_value("start_time", self.start_time)

    def spider_closed(self, spider: Spider, reason: str) -> None:
        assert self.start_time is not None
        finish_time = datetime.now(tz=timezone.utc)
        elapsed_time = finish_time - self.start_time
        elapsed_time_seconds = elapsed_time.total_seconds()
-        self.stats.set_value(
-            "elapsed_time_seconds", elapsed_time_seconds, spider=spider
-        )
-        self.stats.set_value("finish_time", finish_time, spider=spider)
-        self.stats.set_value("finish_reason", reason, spider=spider)
+        self.stats.set_value("elapsed_time_seconds", elapsed_time_seconds)
+        self.stats.set_value("finish_time", finish_time)
+        self.stats.set_value("finish_reason", reason)

    def item_scraped(self, item: Any, spider: Spider) -> None:
-        self.stats.inc_value("item_scraped_count", spider=spider)
+        self.stats.inc_value("item_scraped_count")

    def response_received(self, spider: Spider) -> None:
-        self.stats.inc_value("response_received_count", spider=spider)
+        self.stats.inc_value("response_received_count")

    def item_dropped(self, item: Any, spider: Spider, exception: BaseException) -> None:
        reason = exception.__class__.__name__
-        self.stats.inc_value("item_dropped_count", spider=spider)
-        self.stats.inc_value(f"item_dropped_reasons_count/{reason}", spider=spider)
+        self.stats.inc_value("item_dropped_count")
+        self.stats.inc_value(f"item_dropped_reasons_count/{reason}")
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@ -11,11 +11,12 @@ import logging
 import re
 import sys
 import warnings
+from abc import ABC, abstractmethod
 from collections.abc import Callable
 from datetime import datetime, timezone
 from pathlib import Path, PureWindowsPath
 from tempfile import NamedTemporaryFile
-from typing import IO, TYPE_CHECKING, Any, Optional, Protocol, TypeVar, cast
+from typing import IO, TYPE_CHECKING, Any, Protocol, TypeAlias, cast
 from urllib.parse import unquote, urlparse

 from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
@ -34,8 +35,6 @@ from scrapy.utils.misc import build_from_crawler, load_object
 from scrapy.utils.python import without_none_values

 if TYPE_CHECKING:
-    from collections.abc import Iterable
-
    from _typeshed import OpenBinaryMode
    from twisted.python.failure import Failure

@ -49,26 +48,9 @@ if TYPE_CHECKING:

 logger = logging.getLogger(__name__)

-UriParamsCallableT = Callable[[dict[str, Any], Spider], Optional[dict[str, Any]]]
-
-_StorageT = TypeVar("_StorageT", bound="FeedStorageProtocol")
-
-
-def build_storage(
-    builder: Callable[..., _StorageT],
-    uri: str,
-    *args: Any,
-    feed_options: dict[str, Any] | None = None,
-    preargs: Iterable[Any] = (),
-    **kwargs: Any,
-) -> _StorageT:
-    warnings.warn(
-        "scrapy.extensions.feedexport.build_storage() is deprecated, call the builder directly.",
-        category=ScrapyDeprecationWarning,
-        stacklevel=2,
-    )
-    kwargs["feed_options"] = feed_options
-    return builder(*preargs, uri, *args, **kwargs)
+UriParamsCallableT: TypeAlias = Callable[
+    [dict[str, Any], Spider], dict[str, Any] | None
+]


 class ItemFilter:
@ -140,7 +122,7 @@ class FeedStorageProtocol(Protocol):


@implementer(IFeedStorage)
-class BlockingFeedStorage:
+class BlockingFeedStorage(ABC):
    def open(self, spider: Spider) -> IO[bytes]:
        path = spider.crawler.settings["FEED_TEMPDIR"]
        if path and not Path(path).is_dir():
@ -151,6 +133,7 @@ class BlockingFeedStorage:
    def store(self, file: IO[bytes]) -> Deferred[None] | None:
        return deferToThread(self._store_in_thread, file)

+    @abstractmethod
    def _store_in_thread(self, file: IO[bytes]) -> None:
        raise NotImplementedError

@ -185,7 +168,7 @@ class StdoutFeedStorage:
@implementer(IFeedStorage)
 class FileFeedStorage:
    def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
-        self.path: str = file_uri_to_path(uri)
+        self.path: str = file_uri_to_path(uri) if uri.startswith("file://") else uri
        feed_options = feed_options or {}
        self.write_mode: OpenBinaryMode = (
            "wb" if feed_options.get("overwrite", False) else "ab"
@ -216,7 +199,7 @@ class S3FeedStorage(BlockingFeedStorage):
        region_name: str | None = None,
    ):
        try:
-            import boto3.session
+            import boto3.session  # noqa: PLC0415
        except ImportError:
            raise NotConfigured("missing boto3 library")
        u = urlparse(uri)
@ -315,7 +298,7 @@ class GCSFeedStorage(BlockingFeedStorage):

    def _store_in_thread(self, file: IO[bytes]) -> None:
        file.seek(0)
-        from google.cloud.storage import Client
+        from google.cloud.storage import Client  # noqa: PLC0415

        client = Client(project=self.project_id)
        bucket = client.get_bucket(self.bucket_name)
@ -374,11 +357,11 @@ class FeedSlot:
        self,
        storage: FeedStorageProtocol,
        uri: str,
-        format: str,
+        format: str,  # noqa: A002
        store_empty: bool,
        batch_id: int,
        uri_template: str,
-        filter: ItemFilter,
+        filter: ItemFilter,  # noqa: A002
        feed_options: dict[str, Any],
        spider: Spider,
        exporters: dict[str, type[BaseItemExporter]],
@ -411,7 +394,7 @@ class FeedSlot:
            self.file = self.storage.open(self.spider)
            if "postprocessing" in self.feed_options:
                self.file = cast(
-                    IO[bytes],
+                    "IO[bytes]",
                    PostProcessingManager(
                        self.feed_options["postprocessing"],
                        self.file,
@ -420,7 +403,7 @@ class FeedSlot:
                )
            self.exporter = self._get_exporter(
                file=self.file,
-                format=self.feed_options["format"],
+                format_=self.feed_options["format"],
                fields_to_export=self.feed_options["fields"],
                encoding=self.feed_options["encoding"],
                indent=self.feed_options["indent"],
@ -434,10 +417,10 @@ class FeedSlot:
            self._exporting = True

    def _get_exporter(
-        self, file: IO[bytes], format: str, *args: Any, **kwargs: Any
+        self, file: IO[bytes], format_: str, *args: Any, **kwargs: Any
    ) -> BaseItemExporter:
        return build_from_crawler(
-            self.exporters[format], self.crawler, file, *args, **kwargs
+            self.exporters[format_], self.crawler, file, *args, **kwargs
        )

    def finish_exporting(self) -> None:
@ -660,7 +643,7 @@ class FeedExporter:

    def _load_components(self, setting_prefix: str) -> dict[str, Any]:
        conf = without_none_values(
-            cast(dict[str, str], self.settings.getwithbase(setting_prefix))
+            cast("dict[str, str]", self.settings.getwithbase(setting_prefix))
        )
        d = {}
        for k, v in conf.items():
@ -668,10 +651,10 @@ class FeedExporter:
                d[k] = load_object(v)
        return d

-    def _exporter_supported(self, format: str) -> bool:
-        if format in self.exporters:
+    def _exporter_supported(self, format_: str) -> bool:
+        if format_ in self.exporters:
            return True
-        logger.error("Unknown feed format: %(format)s", {"format": format})
+        logger.error("Unknown feed format: %(format)s", {"format": format_})
        return False

    def _settings_are_valid(self) -> bool:
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@ -7,7 +7,7 @@ from email.utils import mktime_tz, parsedate_tz
 from importlib import import_module
 from pathlib import Path
 from time import time
-from typing import IO, TYPE_CHECKING, Any, cast
+from typing import IO, TYPE_CHECKING, Any, Concatenate, cast
 from weakref import WeakKeyDictionary

 from w3lib.http import headers_dict_to_raw, headers_raw_to_dict
@ -23,9 +23,6 @@ if TYPE_CHECKING:
    from collections.abc import Callable
    from types import ModuleType

-    # typing.Concatenate requires Python 3.10
-    from typing_extensions import Concatenate
-
    from scrapy.http.request import Request
    from scrapy.settings import BaseSettings
    from scrapy.spiders import Spider
@ -307,7 +304,7 @@ class DbmCacheStorage:
        if 0 < self.expiration_secs < time() - float(ts):
            return None  # expired

-        return cast(dict[str, Any], pickle.loads(db[f"{key}_data"]))  # noqa: S301
+        return cast("dict[str, Any]", pickle.loads(db[f"{key}_data"]))  # noqa: S301


 class FilesystemCacheStorage:
@ -389,7 +386,7 @@ class FilesystemCacheStorage:
        if 0 < self.expiration_secs < time() - mtime:
            return None  # expired
        with self._open(metapath, "rb") as f:
-            return cast(dict[str, Any], pickle.load(f))  # noqa: S301
+            return cast("dict[str, Any]", pickle.load(f))  # noqa: S301


 def parse_cachecontrol(header: bytes) -> dict[bytes, bytes | None]:
--- a/scrapy/extensions/logcount.py
+++ b/scrapy/extensions/logcount.py
@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+from scrapy import Spider, signals
+from scrapy.utils.log import LogCounterHandler
+
+if TYPE_CHECKING:
+    # typing.Self requires Python 3.11
+    from typing_extensions import Self
+
+    from scrapy.crawler import Crawler
+
+
+logger = logging.getLogger(__name__)
+
+
+class LogCount:
+    """Install a log handler that counts log messages by level.
+
+    The handler installed is :class:`scrapy.utils.log.LogCounterHandler`.
+    The counts are stored in stats as ``log_count/<level>``.
+
+    .. versionadded:: VERSION
+    """
+
+    def __init__(self, crawler: Crawler):
+        self.crawler: Crawler = crawler
+        self.handler: LogCounterHandler | None = None
+
+    @classmethod
+    def from_crawler(cls, crawler: Crawler) -> Self:
+        o = cls(crawler)
+        crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
+        crawler.signals.connect(o.spider_closed, signal=signals.spider_closed)
+        return o
+
+    def spider_opened(self, spider: Spider) -> None:
+        self.handler = LogCounterHandler(
+            self.crawler, level=self.crawler.settings.get("LOG_LEVEL")
+        )
+        logging.root.addHandler(self.handler)
+
+    def spider_closed(self, spider: Spider, reason: str) -> None:
+        if self.handler:
+            logging.root.removeHandler(self.handler)
+            self.handler = None
--- a/scrapy/extensions/logstats.py
+++ b/scrapy/extensions/logstats.py
@ -5,10 +5,7 @@ from typing import TYPE_CHECKING

 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
-    AsyncioLoopingCall,
-    create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call

 if TYPE_CHECKING:
    from twisted.internet.task import LoopingCall
--- a/scrapy/extensions/memdebug.py
+++ b/scrapy/extensions/memdebug.py
@ -36,12 +36,8 @@ class MemoryDebugger:

    def spider_closed(self, spider: Spider, reason: str) -> None:
        gc.collect()
-        self.stats.set_value(
-            "memdebug/gc_garbage_count", len(gc.garbage), spider=spider
-        )
+        self.stats.set_value("memdebug/gc_garbage_count", len(gc.garbage))
        for cls, wdict in live_refs.items():
            if not wdict:
                continue
-            self.stats.set_value(
-                f"memdebug/live_refs/{cls.__name__}", len(wdict), spider=spider
-            )
+            self.stats.set_value(f"memdebug/live_refs/{cls.__name__}", len(wdict))
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@ -16,10 +16,8 @@ from typing import TYPE_CHECKING
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.mail import MailSender
-from scrapy.utils.asyncio import (
-    AsyncioLoopingCall,
-    create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
+from scrapy.utils.defer import _schedule_coro
 from scrapy.utils.engine import get_engine_status

 if TYPE_CHECKING:
@ -113,11 +111,11 @@ class MemoryUsage:
                self.crawler.stats.set_value("memusage/limit_notified", 1)

            if self.crawler.engine.spider is not None:
-                self.crawler.engine.close_spider(
-                    self.crawler.engine.spider, "memusage_exceeded"
+                _schedule_coro(
+                    self.crawler.engine.close_spider_async(reason="memusage_exceeded")
                )
            else:
-                self.crawler.stop()
+                _schedule_coro(self.crawler.stop_async())
        else:
            logger.info(
                "Peak memory usage is %(virtualsize)dMiB",
--- a/scrapy/extensions/periodic_log.py
+++ b/scrapy/extensions/periodic_log.py
@ -6,10 +6,7 @@ from typing import TYPE_CHECKING, Any

 from scrapy import Spider, signals
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncio import (
-    AsyncioLoopingCall,
-    create_looping_call,
-)
+from scrapy.utils.asyncio import AsyncioLoopingCall, create_looping_call
 from scrapy.utils.serialize import ScrapyJSONEncoder

 if TYPE_CHECKING:
--- a/scrapy/extensions/postprocessing.py
+++ b/scrapy/extensions/postprocessing.py
@ -92,14 +92,14 @@ class LZMAPlugin:
        self.file = file
        self.feed_options = feed_options

-        format = self.feed_options.get("lzma_format")
+        format_ = self.feed_options.get("lzma_format")
        check = self.feed_options.get("lzma_check", -1)
        preset = self.feed_options.get("lzma_preset")
        filters = self.feed_options.get("lzma_filters")
        self.lzmafile = LZMAFile(
            filename=self.file,
            mode="wb",
-            format=format,
+            format=format_,
            check=check,
            preset=preset,
            filters=filters,
@ -142,7 +142,7 @@ class PostProcessingManager(IOBase):
        :return: returns number of bytes written
        :rtype: int
        """
-        return cast(int, self.head_plugin.write(data))
+        return cast("int", self.head_plugin.write(data))

    def tell(self) -> int:
        return self.file.tell()
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@ -40,7 +40,7 @@ class StatsMailer:
        return o

    def spider_closed(self, spider: Spider) -> Deferred[None] | None:
-        spider_stats = self.stats.get_stats(spider)
+        spider_stats = self.stats.get_stats()
        body = "Global stats\n\n"
        body += "\n".join(f"{k:<50} : {v}" for k, v in self.stats.get_stats().items())
        body += f"\n\n{spider.name} stats\n\n"
--- a/scrapy/extensions/telnet.py
+++ b/scrapy/extensions/telnet.py
@ -12,6 +12,8 @@ import os
 import pprint
 from typing import TYPE_CHECKING, Any

+from twisted.conch import telnet
+from twisted.conch.insults import insults
 from twisted.internet import protocol

 from scrapy import signals
@ -22,7 +24,6 @@ from scrapy.utils.reactor import listen_tcp
 from scrapy.utils.trackref import print_live_refs

 if TYPE_CHECKING:
-    from twisted.conch import telnet
    from twisted.internet.tcp import Port

    # typing.Self requires Python 3.11
@ -76,10 +77,6 @@ class TelnetConsole(protocol.ServerFactory):
        self.port.stopListening()

    def protocol(self) -> telnet.TelnetTransport:
-        # these import twisted.internet.reactor
-        from twisted.conch import manhole, telnet
-        from twisted.conch.insults import insults
-
        class Portal:
            """An implementation of IPortal"""

@ -91,6 +88,8 @@ class TelnetConsole(protocol.ServerFactory):
                ):
                    raise ValueError("Invalid credentials")

+                from twisted.conch import manhole
+
                protocol = telnet.TelnetBootstrapProtocol(
                    insults.ServerProtocol, manhole.Manhole, self._get_telnet_vars()
                )
--- a/scrapy/http/cookies.py
+++ b/scrapy/http/cookies.py
@ -161,7 +161,7 @@ class WrappedRequest:
        HTML document, and the user had no option to approve the automatic
        fetching of the image, this should be true.
        """
-        return cast(bool, self.request.meta.get("is_unverifiable", False))
+        return cast("bool", self.request.meta.get("is_unverifiable", False))

    @property
    def full_url(self) -> str:
@ -181,7 +181,7 @@ class WrappedRequest:

    @property
    def origin_req_host(self) -> str:
-        return cast(str, urlparse_cached(self.request).hostname)
+        return cast("str", urlparse_cached(self.request).hostname)

    def has_header(self, name: str) -> bool:
        return name in self.request.headers
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@ -1,7 +1,7 @@
 from __future__ import annotations

 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, AnyStr, Union, cast
+from typing import TYPE_CHECKING, Any, AnyStr, TypeAlias, cast

 from w3lib.http import headers_dict_to_raw

@ -15,7 +15,7 @@ if TYPE_CHECKING:
    from typing_extensions import Self


-_RawValueT = Union[bytes, str, int]
+_RawValue: TypeAlias = bytes | str | int


 # isn't fully compatible typing-wise with either dict or CaselessDict,
@ -44,9 +44,9 @@ class Headers(CaselessDict):
        """Normalize key to bytes"""
        return self._tobytes(key.title())

-    def normvalue(self, value: _RawValueT | Iterable[_RawValueT]) -> list[bytes]:
+    def normvalue(self, value: _RawValue | Iterable[_RawValue]) -> list[bytes]:
        """Normalize values to bytes"""
-        _value: Iterable[_RawValueT]
+        _value: Iterable[_RawValue]
        if value is None:
            _value = []
        elif isinstance(value, (str, bytes)):
@ -58,7 +58,7 @@ class Headers(CaselessDict):

        return [self._tobytes(x) for x in _value]

-    def _tobytes(self, x: _RawValueT) -> bytes:
+    def _tobytes(self, x: _RawValue) -> bytes:
        if isinstance(x, bytes):
            return x
        if isinstance(x, str):
@ -69,33 +69,33 @@ class Headers(CaselessDict):

    def __getitem__(self, key: AnyStr) -> bytes | None:
        try:
-            return cast(list[bytes], super().__getitem__(key))[-1]
+            return cast("list[bytes]", super().__getitem__(key))[-1]
        except IndexError:
            return None

    def get(self, key: AnyStr, def_val: Any = None) -> bytes | None:
        try:
-            return cast(list[bytes], super().get(key, def_val))[-1]
+            return cast("list[bytes]", super().get(key, def_val))[-1]
        except IndexError:
            return None

    def getlist(self, key: AnyStr, def_val: Any = None) -> list[bytes]:
        try:
-            return cast(list[bytes], super().__getitem__(key))
+            return cast("list[bytes]", super().__getitem__(key))
        except KeyError:
            if def_val is not None:
                return self.normvalue(def_val)
            return []

-    def setlist(self, key: AnyStr, list_: Iterable[_RawValueT]) -> None:
+    def setlist(self, key: AnyStr, list_: Iterable[_RawValue]) -> None:
        self[key] = list_

    def setlistdefault(
-        self, key: AnyStr, default_list: Iterable[_RawValueT] = ()
+        self, key: AnyStr, default_list: Iterable[_RawValue] = ()
    ) -> Any:
        return self.setdefault(key, default_list)

-    def appendlist(self, key: AnyStr, value: Iterable[_RawValueT]) -> None:
+    def appendlist(self, key: AnyStr, value: Iterable[_RawValue]) -> None:
        lst = self.getlist(key)
        lst.extend(self.normvalue(value))
        self[key] = lst
--- a/scrapy/http/request/init.py
+++ b/scrapy/http/request/init.py
@ -12,10 +12,11 @@ from typing import (
    TYPE_CHECKING,
    Any,
    AnyStr,
+    Concatenate,
    NoReturn,
+    TypeAlias,
    TypedDict,
    TypeVar,
-    Union,
    overload,
 )

@ -33,13 +34,13 @@ if TYPE_CHECKING:

    from twisted.python.failure import Failure

-    # typing.Concatenate requires Python 3.10
    # typing.NotRequired and typing.Self require Python 3.11
-    from typing_extensions import Concatenate, NotRequired, Self
+    from typing_extensions import NotRequired, Self

+    # circular import
    from scrapy.http import Response

-    CallbackT = Callable[Concatenate[Response, ...], Any]
+    CallbackT: TypeAlias = Callable[Concatenate[Response, ...], Any]


 class VerboseCookie(TypedDict):
@ -50,7 +51,7 @@ class VerboseCookie(TypedDict):
    secure: NotRequired[bool]


-CookiesT = Union[dict[str, str], list[VerboseCookie]]
+CookiesT: TypeAlias = dict[str, str] | list[VerboseCookie]


 RequestTypeVar = TypeVar("RequestTypeVar", bound="Request")
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@ -8,16 +8,10 @@ See documentation in docs/topics/request-response.rst
 from __future__ import annotations

 from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit

-from lxml.html import (
-    FormElement,
-    InputElement,
-    MultipleSelectOptions,
-    SelectElement,
-    TextareaElement,
-)
+from parsel.csstranslator import HTMLTranslator
 from w3lib.html import strip_html5_whitespace

 from scrapy.http.request import Request
@ -25,14 +19,21 @@ from scrapy.utils.python import is_listlike, to_bytes

 if TYPE_CHECKING:
    # typing.Self requires Python 3.11
+    from lxml.html import (
+        FormElement,
+        InputElement,
+        MultipleSelectOptions,
+        SelectElement,
+        TextareaElement,
+    )
    from typing_extensions import Self

    from scrapy.http.response.text import TextResponse


-FormdataVType = Union[str, Iterable[str]]
-FormdataKVType = tuple[str, FormdataVType]
-FormdataType = Optional[Union[dict[str, FormdataVType], list[FormdataKVType]]]
+FormdataVType: TypeAlias = str | Iterable[str]
+FormdataKVType: TypeAlias = tuple[str, FormdataVType]
+FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None


 class FormRequest(Request):
@ -76,8 +77,6 @@ class FormRequest(Request):
        kwargs.setdefault("encoding", response.encoding)

        if formcss is not None:
-            from parsel.csstranslator import HTMLTranslator
-
            formxpath = HTMLTranslator().css_to_xpath(formcss)

        form = _get_form(response, formname, formid, formnumber, formxpath)
@ -107,7 +106,7 @@ def _urlencode(seq: Iterable[FormdataKVType], enc: str) -> str:
    values = [
        (to_bytes(k, enc), to_bytes(v, enc))
        for k, vs in seq
-        for v in (cast(Iterable[str], vs) if is_listlike(vs) else [cast(str, vs)])
+        for v in (cast("Iterable[str]", vs) if is_listlike(vs) else [cast("str", vs)])
    ]
    return urlencode(values, doseq=True)

@ -128,12 +127,12 @@ def _get_form(
    if formname is not None:
        f = root.xpath(f'//form[@name="{formname}"]')
        if f:
-            return cast(FormElement, f[0])
+            return cast("FormElement", f[0])

    if formid is not None:
        f = root.xpath(f'//form[@id="{formid}"]')
        if f:
-            return cast(FormElement, f[0])
+            return cast("FormElement", f[0])

    # Get form element from xpath, if not found, go up
    if formxpath is not None:
@ -142,7 +141,7 @@ def _get_form(
            el = nodes[0]
            while True:
                if el.tag == "form":
-                    return cast(FormElement, el)
+                    return cast("FormElement", el)
                el = el.getparent()
                if el is None:
                    break
@ -153,7 +152,7 @@ def _get_form(
        form = forms[formnumber]
    except IndexError:
        raise IndexError(f"Form number {formnumber} not found in {response}")
-    return cast(FormElement, form)
+    return cast("FormElement", form)


 def _get_inputs(
@ -201,7 +200,7 @@ def _value(
    n = ele.name
    v = ele.value
    if ele.tag == "select":
-        return _select_value(cast(SelectElement, ele), n, v)
+        return _select_value(cast("SelectElement", ele), n, v)
    return n, v


@ -251,7 +250,7 @@ def _get_clickable(
        except IndexError:
            pass
        else:
-            return (el.get("name"), el.get("value") or "")
+            return (cast("str", el.get("name")), el.get("value") or "")

    # We didn't find it, so now we build an XPath expression out of the other
    # arguments, because they can be used as such
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@ -104,13 +104,14 @@ class TextResponse(Response):

    @memoizemethod_noargs
    def _headers_encoding(self) -> str | None:
-        content_type = cast(bytes, self.headers.get(b"Content-Type", b""))
+        content_type = cast("bytes", self.headers.get(b"Content-Type", b""))
        return http_content_type_encoding(to_unicode(content_type, encoding="latin-1"))

    def _body_inferred_encoding(self) -> str:
        if self._cached_benc is None:
            content_type = to_unicode(
-                cast(bytes, self.headers.get(b"Content-Type", b"")), encoding="latin-1"
+                cast("bytes", self.headers.get(b"Content-Type", b"")),
+                encoding="latin-1",
            )
            benc, ubody = html_to_unicode(
                content_type,
@ -141,31 +142,25 @@ class TextResponse(Response):

    @property
    def selector(self) -> Selector:
-        from scrapy.selector import Selector
+        # circular import
+        from scrapy.selector import Selector  # noqa: PLC0415

        if self._cached_selector is None:
            self._cached_selector = Selector(self)
        return self._cached_selector

    def jmespath(self, query: str, **kwargs: Any) -> SelectorList:
-        from scrapy.selector import SelectorList
-
        if not hasattr(self.selector, "jmespath"):
            raise AttributeError(
                "Please install parsel >= 1.8.1 to get jmespath support"
            )
-
-        return cast(SelectorList, self.selector.jmespath(query, **kwargs))
+        return cast("SelectorList", self.selector.jmespath(query, **kwargs))

    def xpath(self, query: str, **kwargs: Any) -> SelectorList:
-        from scrapy.selector import SelectorList
-
-        return cast(SelectorList, self.selector.xpath(query, **kwargs))
+        return cast("SelectorList", self.selector.xpath(query, **kwargs))

    def css(self, query: str) -> SelectorList:
-        from scrapy.selector import SelectorList
-
-        return cast(SelectorList, self.selector.css(query))
+        return cast("SelectorList", self.selector.css(query))

    def follow(
        self,
--- a/scrapy/linkextractors/lxmlhtml.py
+++ b/scrapy/linkextractors/lxmlhtml.py
@ -9,7 +9,7 @@ import operator
 import re
 from collections.abc import Callable, Iterable
 from functools import partial
-from typing import TYPE_CHECKING, Any, Union, cast
+from typing import TYPE_CHECKING, Any, TypeAlias, cast
 from urllib.parse import urljoin, urlparse

 from lxml import etree
@ -71,12 +71,12 @@ class LxmlParserLinkExtractor:
        self.scan_tag: Callable[[str], bool] = (
            tag
            if callable(tag)
-            else cast(Callable[[str], bool], partial(operator.eq, tag))
+            else cast("Callable[[str], bool]", partial(operator.eq, tag))
        )
        self.scan_attr: Callable[[str], bool] = (
            attr
            if callable(attr)
-            else cast(Callable[[str], bool], partial(operator.eq, attr))
+            else cast("Callable[[str], bool]", partial(operator.eq, attr))
        )
        self.process_attr: Callable[[Any], Any] = (
            process if callable(process) else _identity
@ -84,7 +84,7 @@ class LxmlParserLinkExtractor:
        self.unique: bool = unique
        self.strip: bool = strip
        self.link_key: Callable[[Link], str] = (
-            cast(Callable[[Link], str], operator.attrgetter("url"))
+            cast("Callable[[Link], str]", operator.attrgetter("url"))
            if canonicalized
            else _canonicalize_link_url
        )
@ -157,8 +157,8 @@ class LxmlParserLinkExtractor:
        return links


-_RegexT = Union[str, re.Pattern[str]]
-_RegexOrSeveralT = Union[_RegexT, Iterable[_RegexT]]
+_Regex: TypeAlias = str | re.Pattern[str]
+_RegexOrSeveral: TypeAlias = _Regex | Iterable[_Regex]


 class LxmlLinkExtractor:
@ -166,8 +166,8 @@ class LxmlLinkExtractor:

    def __init__(
        self,
-        allow: _RegexOrSeveralT = (),
-        deny: _RegexOrSeveralT = (),
+        allow: _RegexOrSeveral = (),
+        deny: _RegexOrSeveral = (),
        allow_domains: str | Iterable[str] = (),
        deny_domains: str | Iterable[str] = (),
        restrict_xpaths: str | Iterable[str] = (),
@ -179,7 +179,7 @@ class LxmlLinkExtractor:
        deny_extensions: str | Iterable[str] | None = None,
        restrict_css: str | Iterable[str] = (),
        strip: bool = True,
-        restrict_text: _RegexOrSeveralT | None = None,
+        restrict_text: _RegexOrSeveral | None = None,
    ):
        tags, attrs = set(arg_to_iter(tags)), set(arg_to_iter(attrs))
        self.link_extractor = LxmlParserLinkExtractor(
@ -208,7 +208,7 @@ class LxmlLinkExtractor:
        self.restrict_text: list[re.Pattern[str]] = self._compile_regexes(restrict_text)

    @staticmethod
-    def _compile_regexes(value: _RegexOrSeveralT | None) -> list[re.Pattern[str]]:
+    def _compile_regexes(value: _RegexOrSeveral | None) -> list[re.Pattern[str]]:
        return [
            x if isinstance(x, re.Pattern) else re.compile(x)
            for x in arg_to_iter(value)
--- a/Show More
+++ b/Show More
				@ -66,4 +66,3 @@ the :ref:`release notes <news>`.


				`.. _odd-numbered versions for development releases: https://en.wikipedia.org/wiki/Software_versioning#Odd-numbered_versions_for_development_releases`
 @ -1 +1 @@
 .13.2
 .13.4