diff --git a/.bumpversion.cfg b/.bumpversion.cfg
index 1d9b9c02f..f88071685 100644
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.6.1
+current_version = 2.7.0
 commit = True
 tag = True
 tag_name = {new_version}
diff --git a/.flake8 b/.flake8
index 1c503fb0b..0c64d009e 100644
--- a/.flake8
+++ b/.flake8
@@ -4,16 +4,19 @@ max-line-length = 119
 ignore = W503
 
 exclude =
+    docs/conf.py
+
+per-file-ignores =
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
 # F401: Module imported but unused
-    scrapy/__init__.py E402
-    scrapy/core/downloader/handlers/http.py F401
-    scrapy/http/__init__.py F401
-    scrapy/linkextractors/__init__.py E402 F401
-    scrapy/selector/__init__.py F401
-    scrapy/spiders/__init__.py E402 F401
+    scrapy/__init__.py:E402
+    scrapy/core/downloader/handlers/http.py:F401
+    scrapy/http/__init__.py:F401
+    scrapy/linkextractors/__init__.py:E402,F401
+    scrapy/selector/__init__.py:F401
+    scrapy/spiders/__init__.py:E402,F401
 
     # Issues pending a review:
-    scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
+    scrapy/utils/url.py:F403,F405
+    tests/test_loader.py:E741
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 98fa44c7f..439dfee51 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -3,7 +3,7 @@ on: [push, pull_request]
 
 jobs:
   checks:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
@@ -19,18 +19,21 @@ jobs:
         - python-version: 3.8
           env:
             TOXENV: pylint
-        - python-version: 3.6
+        - python-version: 3.7
           env:
             TOXENV: typing
         - python-version: "3.10"  # Keep in sync with .readthedocs.yml
           env:
             TOXENV: docs
+        - python-version: "3.10"
+          env:
+            TOXENV: twinecheck
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 44b682830..f6b098b80 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -3,14 +3,14 @@ on: [push]
 
 jobs:
   publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     if: startsWith(github.event.ref, 'refs/tags/')
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: "3.10"
 
diff --git a/.github/workflows/tests-macos.yml b/.github/workflows/tests-macos.yml
index 3aaf688c7..61f1857f8 100644
--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@@ -3,17 +3,17 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: macos-10.15
+    runs-on: macos-11
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 1fc8d914b..d2bfe4a5f 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -3,14 +3,11 @@ on: [push, pull_request]
 
 jobs:
   tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.7
-          env:
-            TOXENV: py
         - python-version: 3.8
           env:
             TOXENV: py
@@ -23,22 +20,26 @@ jobs:
         - python-version: "3.10"
           env:
             TOXENV: asyncio
-        - python-version: pypy3
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: py
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: asyncio
+        - python-version: pypy3.9
           env:
             TOXENV: pypy3
-            PYPY_VERSION: 3.6-v7.3.3
 
         # pinned deps
-        - python-version: 3.6.12
+        - python-version: 3.7.13
           env:
             TOXENV: pinned
-        - python-version: 3.6.12
+        - python-version: 3.7.13
           env:
             TOXENV: asyncio-pinned
-        - python-version: pypy3
+        - python-version: pypy3.7
           env:
             TOXENV: pypy3-pinned
-            PYPY_VERSION: 3.6-v7.2.0
 
         # extras
         # extra-deps includes reppy, which does not support Python 3.9
@@ -48,30 +49,22 @@ jobs:
             TOXENV: extra-deps
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
       run: |
         sudo apt-get update
-        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
-        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
+        sudo apt-get install libxml2-dev libxslt-dev
 
     - name: Run tests
       env: ${{ matrix.env }}
       run: |
-        if [[ ! -z "$PYPY_VERSION" ]]; then
-          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
-          tar -jxf ${PYPY_VERSION}.tar.bz2
-          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
-          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-        fi
         pip install -U tox
         tox
 
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index ab7385118..14683fd53 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -8,12 +8,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-        - python-version: 3.6
-          env:
-            TOXENV: windows-pinned
         - python-version: 3.7
           env:
-            TOXENV: py
+            TOXENV: windows-pinned
         - python-version: 3.8
           env:
             TOXENV: py
@@ -28,10 +25,10 @@ jobs:
             TOXENV: asyncio
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
 
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
 
diff --git a/.gitignore b/.gitignore
index d77d24624..6c5c50e08 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,3 +23,6 @@ test-output.*
 
 # Windows
 Thumbs.db
+
+# OSX miscellaneous
+.DS_Store
\ No newline at end of file
diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 06e812936..000000000
--- a/INSTALL
+++ /dev/null
@@ -1,4 +0,0 @@
-For information about installing Scrapy see:
-
-* docs/intro/install.rst (local file)
-* https://docs.scrapy.org/en/latest/intro/install.html (online version)
diff --git a/INSTALL.md b/INSTALL.md
new file mode 100644
index 000000000..495413f97
--- /dev/null
+++ b/INSTALL.md
@@ -0,0 +1,4 @@
+For information about installing Scrapy see:
+
+* [Local docs](docs/intro/install.rst)
+* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)
diff --git a/README.rst b/README.rst
index 6b563d638..970bf2c35 100644
--- a/README.rst
+++ b/README.rst
@@ -1,4 +1,5 @@
 .. image:: https://scrapy.org/img/scrapylogo.png
+   :target: https://scrapy.org/
    
 ======
 Scrapy
@@ -57,13 +58,15 @@ including a list of features.
 Requirements
 ============
 
-* Python 3.6+
+* Python 3.7+
 * Works on Linux, Windows, macOS, BSD
 
 Install
 =======
 
-The quick way::
+The quick way:
+
+.. code:: bash
 
     pip install scrapy
 
@@ -94,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
 Code of Conduct
 ---------------
 
-Please note that this project is released with a Contributor Code of Conduct
-(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
+Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.
 
 By participating in this project you agree to abide by its terms.
 Please report unacceptable behavior to opensource@zyte.com.
diff --git a/conftest.py b/conftest.py
index 117087790..d7fe80321 100644
--- a/conftest.py
+++ b/conftest.py
@@ -42,16 +42,6 @@ def chdir(tmpdir):
     tmpdir.chdir()
 
 
-def pytest_collection_modifyitems(session, config, items):
-    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
-    try:
-        from pytest_flake8 import Flake8Item
-        if config.getoption('--flake8'):
-            items[:] = [item for item in items if isinstance(item, Flake8Item)]
-    except ImportError:
-        pass
-
-
 def pytest_addoption(parser):
     parser.addoption(
         "--reactor",
diff --git a/docs/README.rst b/docs/README.rst
index 0b7afa548..36dd5aea4 100644
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
 Start over
 ----------
 
-To cleanup all generated documentation files and start from scratch run::
+To clean up all generated documentation files and start from scratch run::
 
     make clean
 
diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
index 640660943..f0f382da3 100644
--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@@ -15,7 +15,7 @@ class SettingsListDirective(Directive):
 
 
 def is_setting_index(node):
-    if node.tagname == 'index':
+    if node.tagname == 'index' and node['entries']:
         # index entries for setting directives look like:
         # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
         entry_type, info, refid = node['entries'][0][:3]
@@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):
 
 def setup(app):
     app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
+        directivename="setting",
+        rolename="setting",
+        indextemplate="pair: %s; setting",
     )
     app.add_crossref_type(
-        directivename = "signal",
-        rolename      = "signal",
-        indextemplate = "pair: %s; signal",
+        directivename="signal",
+        rolename="signal",
+        indextemplate="pair: %s; signal",
     )
     app.add_crossref_type(
-        directivename = "command",
-        rolename      = "command",
-        indextemplate = "pair: %s; command",
+        directivename="command",
+        rolename="command",
+        indextemplate="pair: %s; command",
     )
     app.add_crossref_type(
-        directivename = "reqmeta",
-        rolename      = "reqmeta",
-        indextemplate = "pair: %s; reqmeta",
+        directivename="reqmeta",
+        rolename="reqmeta",
+        indextemplate="pair: %s; reqmeta",
     )
     app.add_role('source', source_role)
     app.add_role('commit', commit_role)
diff --git a/docs/_static/selectors-sample1.html b/docs/_static/selectors-sample1.html
index 8a79a3381..915718832 100644
--- a/docs/_static/selectors-sample1.html
+++ b/docs/_static/selectors-sample1.html
@@ -1,16 +1,17 @@
-<html>
- <head>
-  <base href='http://example.com/' />
-  <title>Example website</title>
- </head>
- <body>
-  <div id='images'>
-   <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
-   <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
-   <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
-   <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
-   <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
-  </div>
- </body>
-</html>
+<!DOCTYPE html>
 
+<html>
+  <head>
+    <base href='http://example.com/' />
+    <title>Example website</title>
+  </head>
+  <body>
+    <div id='images'>
+      <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
+      <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
+      <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
+      <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
+      <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
+    </div>
+  </body>
+</html>
\ No newline at end of file
diff --git a/docs/conf.py b/docs/conf.py
index 55aa72d5a..3241295af 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -291,10 +291,12 @@ intersphinx_mapping = {
     'pytest': ('https://docs.pytest.org/en/latest', None),
     'python': ('https://docs.python.org/3', None),
     'sphinx': ('https://www.sphinx-doc.org/en/master', None),
-    'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current', None),
-    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+    'tox': ('https://tox.wiki/en/latest/', None),
+    'twisted': ('https://docs.twisted.org/en/stable/', None),
+    'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
+    'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
 }
+intersphinx_disabled_reftypes = []
 
 
 # Options for sphinx-hoverxref options
diff --git a/docs/contributing.rst b/docs/contributing.rst
index 4d2580a6c..9cfe10012 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -214,7 +214,7 @@ Tests
 =====
 
 Tests are implemented using the :doc:`Twisted unit-testing framework
-<twisted:core/development/policy/test-standard>`. Running tests requires
+<twisted:development/test-standard>`. Running tests requires
 :doc:`tox <tox:index>`.
 
 .. _running-tests:
@@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
 
 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.6 use::
+the tests with Python 3.7 use::
 
-    tox -e py36
+    tox -e py37
 
 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::
 
-    tox -e py36,py38 -p auto
+    tox -e py37,py38 -p auto
 
 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
     tox -- scrapy tests -x  # stop after first failure
 
 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::
 
-    tox -e py36 -- scrapy tests -n auto
+    tox -e py37 -- scrapy tests -n auto
 
 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
diff --git a/docs/index.rst b/docs/index.rst
index 75e08f537..5404969e0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -130,7 +130,6 @@ Built-in services
    topics/stats
    topics/email
    topics/telnetconsole
-   topics/webservice
 
 :doc:`topics/logging`
     Learn how to use Python's builtin logging on Scrapy.
@@ -144,9 +143,6 @@ Built-in services
 :doc:`topics/telnetconsole`
     Inspect a running crawler using a built-in Python console.
 
-:doc:`topics/webservice`
-    Monitor and control a crawler using a web service.
-
 
 Solving specific problems
 =========================
@@ -229,10 +225,11 @@ Extending Scrapy
    topics/downloader-middleware
    topics/spider-middleware
    topics/extensions
-   topics/api
    topics/signals
    topics/scheduler
    topics/exporters
+   topics/components
+   topics/api
 
 
 :doc:`topics/architecture`
@@ -247,9 +244,6 @@ Extending Scrapy
 :doc:`topics/extensions`
     Extend Scrapy with your custom functionality
 
-:doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
-
 :doc:`topics/signals`
     See all available signals and how to work with them.
 
@@ -259,6 +253,13 @@ Extending Scrapy
 :doc:`topics/exporters`
     Quickly export your scraped items to a file (XML, CSV, etc).
 
+:doc:`topics/components`
+    Learn the common API and some good practices when building custom Scrapy
+    components.
+
+:doc:`topics/api`
+    Use it on extensions and middlewares to extend Scrapy functionality.
+
 
 All the rest
 ============
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index b8d3a16bc..2c2079f68 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -9,8 +9,8 @@ Installation guide
 Supported Python versions
 =========================
 
-Scrapy requires Python 3.6+, either the CPython implementation (default) or
-the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.7+, either the CPython implementation (default) or
+the PyPy implementation (see :ref:`python:implementations`).
 
 .. _intro-install-scrapy:
 
@@ -52,17 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs
 
-The minimal versions which Scrapy is tested against are:
-
-* Twisted 14.0
-* lxml 3.4
-* pyOpenSSL 0.14
-
-Scrapy may work with older versions of these packages
-but it is not guaranteed it will continue working
-because it’s not being tested against them.
-
-Some of these packages themselves depends on non-Python packages
+Some of these packages themselves depend on non-Python packages
 that might require additional installation steps depending on your platform.
 Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.
 
@@ -197,7 +187,7 @@ solutions:
   * Install `homebrew`_ following the instructions in https://brew.sh/
 
   * Update your ``PATH`` variable to state that homebrew packages should be
-    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
+    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
     if you're using `zsh`_ as default shell)::
 
       echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
@@ -229,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
 PyPy
 ----
 
-We recommend using the latest PyPy version. The version tested is 5.9.0.
+We recommend using the latest PyPy version.
 For PyPy3, only Linux installation was tested.
 
 Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
diff --git a/docs/intro/overview.rst b/docs/intro/overview.rst
index f3d652621..cfa6bfa83 100644
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
 Put this in a text file, name it to something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::
 
-    scrapy runspider quotes_spider.py -o quotes.jl
+    scrapy runspider quotes_spider.py -o quotes.jsonl
 
-When this finishes you will have in the ``quotes.jl`` file a list of the
+When this finishes you will have in the ``quotes.jsonl`` file a list of the
 quotes in JSON Lines format, containing text and author, looking like this::
 
     {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index cde1b1ef4..092123d1d 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -379,7 +379,7 @@ like this:
 Let's open up scrapy shell and play a bit to find out how to extract the data
 we want::
 
-    $ scrapy shell 'https://quotes.toscrape.com'
+    scrapy shell 'https://quotes.toscrape.com'
 
 We get a list of selectors for the quote HTML elements with:
 
@@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
 makes the file contents invalid JSON. When appending to a file, consider
 using a different serialization format, such as `JSON Lines`_::
 
-    scrapy crawl quotes -o quotes.jl
+    scrapy crawl quotes -o quotes.jsonl
 
 The `JSON Lines`_ format is useful because it's stream-like, you can easily
 append new records to it. It doesn't have the same problem of JSON when you run
diff --git a/docs/news.rst b/docs/news.rst
index 5d92067b5..1ec183a1d 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,293 @@
 Release notes
 =============
 
+.. _release-2.7.0:
+
+Scrapy 2.7.0 (2022-10-17)
+-----------------------------
+
+Highlights:
+
+-   Added Python 3.11 support, dropped Python 3.6 support
+-   Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
+-   :ref:`Asyncio support <using-asyncio>` is enabled by default on new
+    projects
+-   Output names of item fields can now be arbitrary strings
+-   Centralized :ref:`request fingerprinting <request-fingerprints>`
+    configuration is now possible
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
+Support for the upcoming Python 3.11 has been added.
+
+The minimum required version of some dependencies has changed as well:
+
+-   lxml_: 3.5.0 → 4.3.0
+
+-   Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
+
+-   zope.interface_: 5.0.0 → 5.1.0
+
+(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
+:issue:`5670`, :issue:`5678`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
+    ``item`` parameter (:issue:`5504`, :issue:`5508`).
+
+-   The ``scrapy.downloadermiddlewares.decompression`` module is now
+    deprecated (:issue:`5546`, :issue:`5547`).
+
+
+New features
+~~~~~~~~~~~~
+
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>` can now be
+    defined as an :term:`asynchronous generator` (:issue:`4978`).
+
+-   The output of :class:`~scrapy.Request` callbacks defined as
+    :ref:`coroutines <topics-coroutines>` is now processed asynchronously
+    (:issue:`4978`).
+
+-   :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
+    callbacks <topics-coroutines>` (:issue:`5657`).
+
+-   New projects created with the :command:`startproject` command have
+    :ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
+    :issue:`5679`).
+
+-   The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
+    dictionary to customize the output name of item fields, lifting the
+    restriction that required output names to be valid Python identifiers, e.g.
+    preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
+    :issue:`3696`).
+
+-   You can now customize :ref:`request fingerprinting <request-fingerprints>`
+    through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
+    having to change it on every Scrapy component that relies on request
+    fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
+    :issue:`4524`).
+
+-   ``jsonl`` is now supported and encouraged as a file extension for `JSON
+    Lines`_ files (:issue:`4848`).
+
+    .. _JSON Lines: https://jsonlines.org/
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
+    source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
+
+
+Bug fixes
+~~~~~~~~~
+
+-   When using Google Cloud Storage with a :ref:`media pipeline
+    <topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
+    :setting:`FILES_STORE` does not point at the root of your Google Cloud
+    Storage bucket (:issue:`5317`, :issue:`5318`).
+
+-   The :command:`parse` command now supports :ref:`asynchronous callbacks
+    <topics-coroutines>` (:issue:`5424`, :issue:`5577`).
+
+-   When using the :command:`parse` command with a URL for which there is no
+    available spider, an exception is no longer raised (:issue:`3264`,
+    :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
+
+-   :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
+    order mark`_ when determining the text encoding of the response body,
+    following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
+
+    .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
+    .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+
+-   MIME sniffing takes the response body into account in FTP and HTTP/1.0
+    requests, as well as in cached requests (:issue:`4873`).
+
+-   MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
+    is missing (:issue:`4873`).
+
+-   An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
+    that does not match the asyncio event loop actually installed
+    (:issue:`5529`).
+
+-   Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
+    returning only the last header (:issue:`5515`, :issue:`5526`).
+
+-   Fixed :class:`LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
+    ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
+    :issue:`4066`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
+    (:issue:`5602`, :issue:`5608`).
+
+-   To enable
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    to do `brotli compression`_, installing brotli_ is now recommended instead
+    of installing brotlipy_, as the former provides a more recent version of
+    brotli.
+
+    .. _brotli: https://github.com/google/brotli
+    .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
+
+-   :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
+    support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
+    :issue:`5358`).
+
+-   :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
+    (:issue:`3582`, :issue:`5432`).
+
+    .. _Common Crawl: https://commoncrawl.org/
+    .. _Google cache: http://www.googleguide.com/cached_pages.html
+
+-   The new :ref:`topics-components` topic covers enforcing requirements on
+    Scrapy components, like :ref:`downloader middlewares
+    <topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
+    :ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
+    <topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
+    has also been added (:issue:`4978`).
+
+-   :ref:`topics-settings` now indicates that setting values must be
+    :ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
+
+-   Removed outdated documentation (:issue:`5446`, :issue:`5373`,
+    :issue:`5369`, :issue:`5370`, :issue:`5554`).
+
+-   Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
+    :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
+
+-   Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
+    :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Added a continuous integration job to run `twine check`_ (:issue:`5655`,
+    :issue:`5656`).
+
+    .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
+
+-   Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
+    :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
+    :issue:`5671`, :issue:`5675`).
+
+-   Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
+    :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
+
+-   Applied minor code improvements (:issue:`5661`).
+
+
+.. _release-2.6.3:
+
+Scrapy 2.6.3 (2022-09-27)
+-------------------------
+
+-   Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
+    (:issue:`5634`, :issue:`5635`, :issue:`5636`).
+
+-   Upgraded the minimum versions of the following dependencies:
+
+    -   cryptography_: 2.0 → 3.3
+
+    -   pyOpenSSL_: 16.2.0 → 21.0.0
+
+    -   service_identity_: 16.0.0 → 18.1.0
+
+    -   Twisted_: 17.9.0 → 18.9.0
+
+    -   zope.interface_: 4.1.3 → 5.0.0
+
+    (:issue:`5621`, :issue:`5632`)
+
+-   Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
+    :issue:`5631`).
+
+
+.. _release-2.6.2:
+
+Scrapy 2.6.2 (2022-07-25)
+-------------------------
+
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+Also fixes the following regressions introduced in 2.6.0:
+
+-   :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
+    spiders (:issue:`5435`, :issue:`5436`)
+
+-   Installing a Twisted reactor before Scrapy does (e.g. importing
+    :mod:`twisted.internet.reactor` somewhere at the module level) no longer
+    prevents Scrapy from starting, as long as a different reactor is not
+    specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`)
+
+-   Fixed an exception that was being logged after the spider finished under
+    certain conditions (:issue:`5437`, :issue:`5440`)
+
+-   The ``--output``/``-o`` command-line parameter supports again a value
+    starting with a hyphen (:issue:`5444`, :issue:`5445`)
+
+-   The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`,
+    :issue:`5482`)
+
+
 .. _release-2.6.1:
 
 Scrapy 2.6.1 (2022-03-01)
@@ -113,6 +400,9 @@ Backward-incompatible changes
     meet expectations, :exc:`TypeError` is now raised at startup time. Before,
     other exceptions would be raised at run time. (:issue:`3559`)
 
+-   The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects
+    is now named ``encoding``, in line with all other fields (:issue:`5130`)
+
 
 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~
@@ -1643,7 +1933,7 @@ New features
     :issue:`4370`)
 
 *   A new ``keep_fragments`` parameter of
-    :func:`scrapy.utils.request.request_fingerprint` allows to generate
+    ``scrapy.utils.request.request_fingerprint`` allows to generate
     different fingerprints for requests with different fragments in their URL
     (:issue:`4104`)
 
@@ -1897,6 +2187,59 @@ affect subclasses:
 (:issue:`3884`)
 
 
+.. _release-1.8.3:
+
+Scrapy 1.8.3 (2022-07-25)
+-------------------------
+
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+
 .. _release-1.8.2:
 
 Scrapy 1.8.2 (2022-03-01)
@@ -2985,7 +3328,7 @@ New Features
 ~~~~~~~~~~~~
 
 - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
-- Support `brotli`_-compressed content; requires optional `brotlipy`_
+- Support `brotli-compressed`_ content; requires optional `brotlipy`_
   (:issue:`2535`)
 - New :ref:`response.follow <response-follow-example>` shortcut
   for creating requests (:issue:`1940`)
@@ -3022,7 +3365,7 @@ New Features
 - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
   (:issue:`2740`)
 
-.. _brotli: https://github.com/google/brotli
+.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://github.com/python-hyper/brotlipy/
 
 Bug fixes
diff --git a/docs/requirements.txt b/docs/requirements.txt
index a0930ba1e..9f9aef711 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,4 +1,4 @@
-Sphinx>=3.0
-sphinx-hoverxref>=0.2b1
-sphinx-notfound-page>=0.4
-sphinx-rtd-theme>=0.5.2
\ No newline at end of file
+sphinx==5.0.2
+sphinx-hoverxref==1.1.1
+sphinx-notfound-page==0.8
+sphinx-rtd-theme==1.0.0
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 900b19c7a..60b5acd10 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
     :class:`scrapy.Spider` subclass and a
     :class:`scrapy.settings.Settings` object.
 
+    .. attribute:: request_fingerprinter
+
+        The request fingerprint builder of this crawler.
+
+        This is used from extensions and middlewares to build short, unique
+        identifiers for requests. See :ref:`request-fingerprints`.
+
     .. attribute:: settings
 
         The settings manager of this crawler.
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 3a6941a2c..dbee7146d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this:
          down to Scrapy 2.0 (earlier versions do not support
          :mod:`asyncio`), you can copy the implementation of these functions
          into your own code.
+
+
+.. _enforce-asyncio-requirement:
+
+Enforcing asyncio as a requirement
+==================================
+
+If you are writing a :ref:`component <topics-components>` that requires asyncio
+to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
+:ref:`enforce it as a requirement <enforce-component-requirements>`. For
+example::
+
+    from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    class MyComponent:
+
+        def __init__(self):
+            if not is_asyncio_reactor_installed():
+                raise ValueError(
+                    f"{MyComponent.__qualname__} requires the asyncio Twisted "
+                    f"reactor. Make sure you have it configured in the "
+                    f"TWISTED_REACTOR setting. See the asyncio documentation "
+                    f"of Scrapy for more information."
+                )
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
new file mode 100644
index 000000000..ca301b827
--- /dev/null
+++ b/docs/topics/components.rst
@@ -0,0 +1,84 @@
+.. _topics-components:
+
+==========
+Components
+==========
+
+A Scrapy component is any class whose objects are created using
+:func:`scrapy.utils.misc.create_instance`.
+
+That includes the classes that you may assign to the following settings:
+
+-   :setting:`DNS_RESOLVER`
+
+-   :setting:`DOWNLOAD_HANDLERS`
+
+-   :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
+
+-   :setting:`DOWNLOADER_MIDDLEWARES`
+
+-   :setting:`DUPEFILTER_CLASS`
+
+-   :setting:`EXTENSIONS`
+
+-   :setting:`FEED_EXPORTERS`
+
+-   :setting:`FEED_STORAGES`
+
+-   :setting:`ITEM_PIPELINES`
+
+-   :setting:`SCHEDULER`
+
+-   :setting:`SCHEDULER_DISK_QUEUE`
+
+-   :setting:`SCHEDULER_MEMORY_QUEUE`
+
+-   :setting:`SCHEDULER_PRIORITY_QUEUE`
+
+-   :setting:`SPIDER_MIDDLEWARES`
+
+Third-party Scrapy components may also let you define additional Scrapy
+components, usually configurable through :ref:`settings <topics-settings>`, to
+modify their behavior.
+
+.. _enforce-component-requirements:
+
+Enforcing component requirements
+================================
+
+Sometimes, your components may only be intended to work under certain
+conditions. For example, they may require a minimum version of Scrapy to work as
+intended, or they may require certain settings to have specific values.
+
+In addition to describing those conditions in the documentation of your
+component, it is a good practice to raise an exception from the ``__init__``
+method of your component if those conditions are not met at run time.
+
+In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
+:ref:`extensions <topics-extensions>`, :ref:`item pipelines
+<topics-item-pipeline>`, and :ref:`spider middlewares
+<topics-spider-middleware>`, you should raise
+:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
+parameter to the exception so that it is printed in the logs, for the user to
+see. For other components, feel free to raise whatever other exception feels
+right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
+version mismatch, while :exc:`ValueError` may be better if the issue is the
+value of a setting.
+
+If your requirement is a minimum Scrapy version, you may use
+:attr:`scrapy.__version__` to enforce your requirement. For example::
+
+    from pkg_resources import parse_version
+
+    import scrapy
+
+    class MyComponent:
+
+        def __init__(self):
+            if parse_version(scrapy.__version__) < parse_version('2.7'):
+                raise RuntimeError(
+                    f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
+                    f"later, which allow defining the process_spider_output "
+                    f"method of spider middlewares as an asynchronous "
+                    f"generator."
+                )
diff --git a/docs/topics/contracts.rst b/docs/topics/contracts.rst
index ef296dc9e..c29a3a410 100644
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@@ -102,7 +102,7 @@ override three methods:
     .. method:: Contract.post_process(output)
 
         This allows processing the output of the callback. Iterators are
-        converted listified before being passed to this hook.
+        converted to lists before being passed to this hook.
 
 Raise :class:`~scrapy.exceptions.ContractFail` from
 :class:`~scrapy.contracts.Contract.pre_process` or
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 2aef755c7..a1ba4ba5c 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -1,3 +1,5 @@
+.. _topics-coroutines:
+
 ==========
 Coroutines
 ==========
@@ -17,14 +19,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.Request` callbacks.
 
-    .. note:: The callback output is not processed until the whole callback
-        finishes.
+    If you are using any custom or third-party :ref:`spider middleware
+    <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
 
-        As a side effect, if the callback raises an exception, none of its
-        output is processed.
-
-        This is a known caveat of the current implementation that we aim to
-        address in a future version of Scrapy.
+    .. versionchanged:: 2.7
+       Output of async callbacks is now processed asynchronously instead of
+       collecting all of it first.
 
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
@@ -39,12 +39,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
-Usage
-=====
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>`.
 
-There are several use cases for coroutines in Scrapy. Code that would
-return Deferreds when written for previous Scrapy versions, such as downloader
-middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+    It must be defined as an :term:`asynchronous generator`. The input
+    ``result`` parameter is an :term:`asynchronous iterable`.
+
+    See also :ref:`sync-async-spider-middleware` and
+    :ref:`universal-spider-middleware`.
+
+    .. versionadded:: 2.7
+
+General usage
+=============
+
+There are several use cases for coroutines in Scrapy.
+
+Code that would return Deferreds when written for previous Scrapy versions,
+such as downloader middlewares and signal handlers, can be rewritten to be
+shorter and cleaner::
 
     from itemadapter import ItemAdapter
 
@@ -104,7 +118,100 @@ Common use cases for asynchronous code include:
 * storing data in databases (in pipelines and middlewares);
 * delaying the spider initialization until some external event (in the
   :signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
-  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
+  (see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
+
+
+.. _sync-async-spider-middleware:
+
+Mixing synchronous and asynchronous spider middlewares
+======================================================
+
+.. versionadded:: 2.7
+
+The output of a :class:`~scrapy.Request` callback is passed as the ``result``
+parameter to the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of the first :ref:`spider middleware <topics-spider-middleware>` from the
+:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
+Then the output of that ``process_spider_output`` method is passed to the
+``process_spider_output`` method of the next spider middleware, and so on for
+every active spider middleware.
+
+Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
+in this chain of calls.
+
+However, if any of the ``process_spider_output`` methods is defined as a
+synchronous method, and the previous ``Request`` callback or
+``process_spider_output`` method is a coroutine, there are some drawbacks to
+the asynchronous-to-synchronous conversion that Scrapy does so that the
+synchronous ``process_spider_output`` method gets a synchronous iterable as its
+``result`` parameter:
+
+-   The whole output of the previous ``Request`` callback or
+    ``process_spider_output`` method is awaited at this point.
+
+-   If an exception raises while awaiting the output of the previous
+    ``Request`` callback or ``process_spider_output`` method, none of that
+    output will be processed.
+
+    This contrasts with the regular behavior, where all items yielded before
+    an exception raises are processed.
+
+Asynchronous-to-synchronous conversions are supported for backward
+compatibility, but they are deprecated and will stop working in a future
+version of Scrapy.
+
+To avoid asynchronous-to-synchronous conversions, when defining ``Request``
+callbacks as coroutine methods or when using spider middlewares whose
+``process_spider_output`` method is an :term:`asynchronous generator`, all
+active spider middlewares must either have their ``process_spider_output``
+method defined as an asynchronous generator or :ref:`define a
+process_spider_output_async method <universal-spider-middleware>`.
+
+.. note:: When using third-party spider middlewares that only define a
+          synchronous ``process_spider_output`` method, consider
+          :ref:`making them universal <universal-spider-middleware>` through
+          :ref:`subclassing <tut-inheritance>`.
+
+
+.. _universal-spider-middleware:
+
+Universal spider middlewares
+============================
+
+.. versionadded:: 2.7
+
+To allow writing a spider middleware that supports asynchronous execution of
+its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
+:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
+while maintaining support for older Scrapy versions, you may define
+``process_spider_output`` as a synchronous method and define an
+:term:`asynchronous generator` version of that method with an alternative name:
+``process_spider_output_async``.
+
+For example::
+
+    class UniversalSpiderMiddleware:
+        def process_spider_output(self, response, result, spider):
+            for r in result:
+                # ... do something with r
+                yield r
+
+        async def process_spider_output_async(self, response, result, spider):
+            async for r in result:
+                # ... do something with r
+                yield r
+
+.. note:: This is an interim measure to allow, for a time, to write code that
+          works in Scrapy 2.7 and later without requiring
+          asynchronous-to-synchronous conversions, and works in earlier Scrapy
+          versions as well.
+
+          In some future version of Scrapy, however, this feature will be
+          deprecated and, eventually, in a later version of Scrapy, this
+          feature will be removed, and all spider middlewares will be expected
+          to define their ``process_spider_output`` method as an asynchronous
+          generator.
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 29e350651..986da0476 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -955,7 +955,7 @@ default because HTTP specs say so.
 .. setting:: RETRY_PRIORITY_ADJUST
 
 RETRY_PRIORITY_ADJUST
----------------------
+^^^^^^^^^^^^^^^^^^^^^
 
 Default: ``-1``
 
@@ -1119,7 +1119,7 @@ In order to use this parser:
 .. _support-for-new-robots-parser:
 
 Implementing support for a new parser
--------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
 You can implement support for a new robots.txt_ parser by subclassing
 the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 923336769..9360ecf37 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -117,7 +117,7 @@ after your custom code.
 
 Example::
 
-      from scrapy.exporter import XmlItemExporter
+      from scrapy.exporters import XmlItemExporter
 
       class ProductXmlExporter(XmlItemExporter):
 
@@ -195,17 +195,25 @@ BaseItemExporter
 
    .. attribute:: fields_to_export
 
-      A list with the name of the fields that will be exported, or ``None`` if
-      you want to export all fields. Defaults to ``None``.
+      Fields to export, their order [1]_ and their output names.
 
-      Some exporters (like :class:`CsvItemExporter`) respect the order of the
-      fields defined in this attribute.
+      Possible values are:
 
-      When using :ref:`item objects <item-types>` that do not expose all their
-      possible fields, exporters that do not support exporting a different
-      subset of fields per item will only export the fields found in the first
-      item exported. Use ``fields_to_export`` to define all the fields to be
-      exported.
+      -   ``None`` (all fields [2]_, default)
+
+      -   A list of fields::
+
+              ['field1', 'field2']
+
+      -   A dict where keys are fields and values are output names::
+
+              {'field1': 'Field 1', 'field2': 'Field 2'}
+
+      .. [1] Not all exporters respect the specified field order.
+      .. [2] When using :ref:`item objects <item-types>` that do not expose
+             all their possible fields, exporters that do not support exporting
+             a different subset of fields per item will only export the fields 
+             found in the first item exported.
 
    .. attribute:: export_empty_fields
 
@@ -297,8 +305,8 @@ CsvItemExporter
 
    Exports items in CSV format to the given file-like object. If the
    :attr:`fields_to_export` attribute is set, it will be used to define the
-   CSV columns and their order. The :attr:`export_empty_fields` attribute has
-   no effect on this exporter.
+   CSV columns, their order and their column names. The
+   :attr:`export_empty_fields` attribute has no effect on this exporter.
 
    :param file: the file-like object to use for exporting the data. Its ``write`` method should
                 accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/extensions.rst b/docs/topics/extensions.rst
index 297e1fdc5..130657b0b 100644
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@@ -17,7 +17,7 @@ settings, just like any other Scrapy code.
 
 It is customary for extensions to prefix their settings with their own name, to
 avoid collision with existing (and future) extensions. For example, a
-hypothetic extension to handle `Google Sitemaps`_ would use settings like
+hypothetical extension to handle `Google Sitemaps`_ would use settings like
 ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.
 
 .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9a13eb82f..a620e2c04 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -58,7 +58,7 @@ CSV
 
 -   Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
 
--   To specify columns to export and their order use
+-   To specify columns to export, their order and their column names, use
     :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
     option, but it is important for CSV because unlike many other export
     formats CSV uses a fixed header.
@@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS
 
 Default: ``None``
 
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
-defined in :ref:`item objects <topics-items>` yielded by your spider.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
 
 .. setting:: FEED_EXPORT_INDENT
 
@@ -638,6 +629,7 @@ Default::
     {
         'json': 'scrapy.exporters.JsonItemExporter',
         'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+        'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
         'jl': 'scrapy.exporters.JsonLinesItemExporter',
         'csv': 'scrapy.exporters.CsvItemExporter',
         'xml': 'scrapy.exporters.XmlItemExporter',
@@ -763,7 +755,7 @@ source spider in the feed URI:
 
 #.  Use ``%(spider_name)s`` in your feed URI::
 
-        scrapy crawl <spider_name> -o "%(spider_name)s.jl"
+        scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"
 
 
 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
diff --git a/docs/topics/item-pipeline.rst b/docs/topics/item-pipeline.rst
index 391751364..af294f52c 100644
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
    :param spider: the spider which was closed
    :type spider: :class:`~scrapy.Spider` object
 
-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)
 
-   If present, this classmethod is called to create a pipeline instance
+   If present, this class method is called to create a pipeline instance
    from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
    of the pipeline. Crawler object provides access to all Scrapy core
    components like settings and signals; it is a way for pipeline to
@@ -99,11 +99,11 @@ contain a price::
                 raise DropItem(f"Missing price in {item}")
 
 
-Write items to a JSON file
---------------------------
+Write items to a JSON lines file
+--------------------------------
 
 The following pipeline stores all scraped items (from all spiders) into a
-single ``items.jl`` file, containing one item per line serialized in JSON
+single ``items.jsonl`` file, containing one item per line serialized in JSON
 format::
 
    import json
@@ -113,7 +113,7 @@ format::
    class JsonWriterPipeline:
 
        def open_spider(self, spider):
-           self.file = open('items.jl', 'w')
+           self.file = open('items.jsonl', 'w')
 
        def close_spider(self, spider):
            self.file.close()
diff --git a/docs/topics/items.rst b/docs/topics/items.rst
index 7cd482d07..167014381 100644
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
 * define custom field metadata through :func:`dataclasses.field`, which can be used to
   :ref:`customize serialization <topics-exporters-field-serialization>`.
 
-They work natively in Python 3.7 or later, or using the `dataclasses
-backport`_ in Python 3.6.
-
-.. _dataclasses backport: https://pypi.org/project/dataclasses/
-
 Example::
 
     from dataclasses import dataclass
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index 477652704..33441838a 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -154,7 +154,7 @@ Too many spiders?
 If your project has too many spiders executed in parallel,
 the output of :func:`prefs()` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
-ignore a particular class (and all its subclases). For
+ignore a particular class (and all its subclasses). For
 example, this won't show any live references to spiders:
 
 >>> from scrapy.spiders import Spider
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rst
index 7dff78390..0925e6bb5 100644
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.
 
-The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
 thumbnailing and normalizing images to JPEG/RGB format.
 
 .. _Pillow: https://github.com/python-pillow/Pillow
@@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
       .. versionadded:: 2.4
          The *item* parameter.
 
+   .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
+
+      This method is called for every item of  :setting:`IMAGES_THUMBS` per downloaded item. It returns the
+      thumbnail download path of the image originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>`,
+      ``thumb_id``,
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
+      :class:`item <scrapy.Item>`.
+
+      You can override this method to customize the thumbnail download path of each image.
+      You can use the ``item`` to determine the file path based on some item
+      property.
+
+      By default the :meth:`thumb_path` method returns
+      ``thumbs/<size name>/<request URL hash>.<extension>``.
+
+
    .. method:: ImagesPipeline.get_media_requests(item, info)
 
       Works the same way as :meth:`FilesPipeline.get_media_requests` method,
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 92a471faf..a0d9fc03e 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -339,6 +339,7 @@ errors if needed::
                 request = failure.request
                 self.logger.error('TimeoutError on %s', request.url)
 
+
 .. _errback-cb_kwargs:
 
 Accessing additional data in errback functions
@@ -364,6 +365,278 @@ achieve this by using ``Failure.request.cb_kwargs``::
             main_url=failure.request.cb_kwargs['main_url'],
         )
 
+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.7
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.7
+
+Default: ``'2.6'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+-   ``'2.6'`` (default)
+
+    This implementation uses the same request fingerprinting algorithm as
+    Scrapy 2.6 and earlier versions.
+
+    Even though this is the default value for backward compatibility reasons,
+    it is a deprecated value.
+
+-   ``'2.7'``
+
+    This implementation was introduced in Scrapy 2.7 to fix an issue of the
+    previous implementation.
+
+    New projects should use this value. The :command:`startproject` command
+    sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'2.6'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the 2.6 request
+fingerprinting algorithm and does not log this warning (
+:ref:`2.6-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidate the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'2.6'``).
+
+
+.. _2.6-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. currentmodule:: None
+
+.. method:: fingerprint(self, request)
+
+   Return a :class:`bytes` object that uniquely identifies *request*.
+
+   See also :ref:`request-fingerprint-restrictions`.
+
+   :param request: request to fingerprint
+   :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+   :noindex:
+
+   If present, this class method is called to create a request fingerprinter
+   instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+   new instance of the request fingerprinter.
+
+   *crawler* provides access to all Scrapy core components like settings and
+   signals; it is a way for the request fingerprinter to access them and hook
+   its functionality into Scrapy.
+
+   :param crawler: crawler that uses this request fingerprinter
+   :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+   If present, and ``from_crawler`` is not defined, this class method is called
+   to create a request fingerprinter instance from a
+   :class:`~scrapy.settings.Settings` object. It must return a new instance of
+   the request fingerprinter.
+
+.. currentmodule:: scrapy.http
+
+The :meth:`fingerprint` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
+in your :meth:`fingerprint` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+    # my_project/settings.py
+    REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+    # my_project/utils.py
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+-   Caching saves CPU by ensuring that fingerprints are calculated only once
+    per request, and not once per Scrapy component that needs the fingerprint
+    of a request.
+
+-   Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+    request objects do not stay in memory forever just because you have
+    references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.url))
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
+when available, and then falls back to
+:func:`scrapy.utils.request.fingerprint`. For example::
+
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            if 'fingerprint' in request.meta:
+                return request.meta['fingerprint']
+            return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
+without using the deprecated ``'2.6'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+    from w3lib.url import canonicalize_url
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.method))
+                fp.update(to_bytes(canonicalize_url(request.url)))
+                fp.update(request.body or b'')
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter <custom-request-fingerprinter>` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+-   :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+    value of :setting:`HTTPCACHE_STORAGE`)
+
+    Request fingerprints must be at least 1 byte long.
+
+    Path and filename length limits of the file system of
+    :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+    the following directory structure is created:
+
+    -   :attr:`Spider.name <scrapy.spiders.Spider.name>`
+
+        -   first byte of a request fingerprint as hexadecimal
+
+            -   fingerprint as hexadecimal
+
+                -   filenames up to 16 characters long
+
+    For example, if a request fingerprint is made of 20 bytes (default),
+    :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+    and the name of your spider is ``'my_spider'`` your file system must
+    support a file path like::
+
+        /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+-   :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+    The underlying DBM implementation must support keys as long as twice
+    the number of bytes of a request fingerprint, plus 5. For example,
+    if a request fingerprint is made of 20 bytes (default),
+    45-character-long keys must be supported.
+
+
 .. _topics-request-meta:
 
 Request.meta special keys
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 4e105642d..40bcda288 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -98,6 +98,10 @@ class.
 The global defaults are located in the ``scrapy.settings.default_settings``
 module and documented in the :ref:`topics-settings-ref` section.
 
+Compatibility with pickle
+=========================
+
+Setting values must be :ref:`picklable <pickle-picklable>`.
 
 Import paths and classes
 ========================
@@ -560,7 +564,6 @@ This setting must be one of these string values:
   set this if you want the behavior of Scrapy<1.1
 - ``'TLSv1.1'``: forces TLS version 1.1
 - ``'TLSv1.2'``: forces TLS version 1.2
-- ``'SSLv3'``: forces SSL version 3 (**not recommended**)
 
 
 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@@ -825,12 +828,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``
 
 The class used to detect and filter duplicate requests.
 
-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
 
 You can disable filtering of duplicate requests by setting
 :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
@@ -1638,9 +1637,15 @@ which raises :exc:`Exception`, becomes::
 
 
 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will install the default reactor defined by Twisted for the
-current platform. This is to maintain backward compatibility and avoid possible
-problems caused by using a non-default reactor.
+means that Scrapy will use the existing reactor if one is already installed, or
+install the default reactor defined by Twisted for the current platform. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
+
+.. versionchanged:: 2.7
+   The :command:`startproject` command now sets this setting to
+   ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
+   ``settings.py`` file.
 
 For additional information, see :doc:`core/howto/choosing-reactor`.
 
@@ -1656,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength``
 
 The maximum URL length to allow for crawled URLs.
 
-This setting can act as a stopping condition in case of URLs of ever-increasing 
-length, which may be caused for example by a programming error either in the 
-target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and 
+This setting can act as a stopping condition in case of URLs of ever-increasing
+length, which may be caused for example by a programming error either in the
+target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
 :setting:`DEPTH_LIMIT`.
 
 Use ``0`` to allow URLs of any length.
 
-The default value is copied from the `Microsoft Internet Explorer maximum URL 
+The default value is copied from the `Microsoft Internet Explorer maximum URL
 length`_, even though this setting exists for different reasons.
 
 .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index 2fbd0b51c..17bd16156 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -51,12 +51,12 @@ Deferred signal handlers
 ========================
 
 Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, allowing you to run asynchronous code that
-does not block Scrapy. If a signal handler returns a
-:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
-:class:`~twisted.internet.defer.Deferred` to fire.
+or :term:`awaitable objects <awaitable>` from their handlers, allowing
+you to run asynchronous code that does not block Scrapy. If a signal
+handler returns one of these objects, Scrapy waits for that asynchronous
+operation to finish.
 
-Let's take an example::
+Let's take an example using :ref:`coroutines <topics-coroutines>`::
 
     class SignalSpider(scrapy.Spider):
         name = 'signals'
@@ -68,17 +68,15 @@ Let's take an example::
             crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
             return spider
 
-        def item_scraped(self, item):
+        async def item_scraped(self, item):
             # Send the scraped item to the server
-            d = treq.post(
+            response = await treq.post(
                 'http://example.com/post',
                 json.dumps(item).encode('ascii'),
                 headers={b'Content-Type': [b'application/json']}
             )
 
-            # The next item will be scraped only after
-            # deferred (d) is fired
-            return d
+            return response
 
         def parse(self, response):
             for quote in response.css('div.quote'):
@@ -89,7 +87,7 @@ Let's take an example::
                 }
 
 See the :ref:`topics-signals-ref` below to know which signals support
-:class:`~twisted.internet.defer.Deferred`.
+:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.
 
 .. _topics-signals-ref:
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f27bc79c0..303401a3c 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -102,27 +102,47 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         it has processed the response.
 
         :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.Request` objects and :ref:`item object
+        :class:`~scrapy.Request` objects and :ref:`item objects
         <topics-items>`.
 
+        .. versionchanged:: 2.7
+           This method may be defined as an :term:`asynchronous generator`, in
+           which case ``result`` is an :term:`asynchronous iterable`.
+
+        Consider defining this method as an :term:`asynchronous generator`,
+        which will be a requirement in a future version of Scrapy. However, if
+        you plan on sharing your spider middleware with other people, consider
+        either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
+        as a minimum requirement of your spider middleware, or :ref:`making
+        your spider middleware universal <universal-spider-middleware>` so that
+        it works with Scrapy versions earlier than Scrapy 2.7.
+
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
         :type result: an iterable of :class:`~scrapy.Request` objects and
-          :ref:`item object <topics-items>`
+          :ref:`item objects <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.Spider` object
 
+    .. method:: process_spider_output_async(response, result, spider)
+
+        .. versionadded:: 2.7
+
+        If defined, this method must be an :term:`asynchronous generator`,
+        which will be called instead of :meth:`process_spider_output` if
+        ``result`` is an :term:`asynchronous iterable`.
+
     .. method:: process_spider_exception(response, exception, spider)
 
         This method is called when a spider or :meth:`process_spider_output`
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>` 
+        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
         objects.
 
         If it returns ``None``, Scrapy will continue processing this exception,
diff --git a/docs/topics/spiders.rst b/docs/topics/spiders.rst
index ece02ae47..ffe41cf3e 100644
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@@ -181,9 +181,10 @@ scrapy.Spider
        scraped data and/or more URLs to follow. Other Requests callbacks have
        the same requirements as the :class:`Spider` class.
 
-       This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.Request` and/or :ref:`item objects
-       <topics-items>`.
+       This method, as well as any other Request callback, must return a
+       :class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
+       iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
+       <topics-items>`, or ``None``.
 
        :param response: the response to parse
        :type response: :class:`~scrapy.http.Response`
diff --git a/docs/topics/webservice.rst b/docs/topics/webservice.rst
deleted file mode 100644
index 2c4052c04..000000000
--- a/docs/topics/webservice.rst
+++ /dev/null
@@ -1,11 +0,0 @@
-.. _topics-webservice:
-
-===========
-Web Service
-===========
-
-webservice has been moved into a separate project.
-
-It is hosted at:
-
-    https://github.com/scrapy-plugins/scrapy-jsonrpc
diff --git a/pylintrc b/pylintrc
index 2cdd6321e..18819feba 100644
--- a/pylintrc
+++ b/pylintrc
@@ -9,11 +9,9 @@ disable=abstract-method,
         arguments-renamed,
         attribute-defined-outside-init,
         bad-classmethod-argument,
-        bad-continuation,
         bad-indentation,
         bad-mcs-classmethod-argument,
         bad-super-call,
-        bad-whitespace,
         bare-except,
         blacklisted-name,
         broad-except,
@@ -49,10 +47,10 @@ disable=abstract-method,
         keyword-arg-before-vararg,
         line-too-long,
         logging-format-interpolation,
+        logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
         method-hidden,
-        misplaced-comparison-constant,
         missing-docstring,
         missing-final-newline,
         multiple-imports,
@@ -60,12 +58,10 @@ disable=abstract-method,
         no-else-continue,
         no-else-raise,
         no-else-return,
-        no-init,
         no-member,
         no-method-argument,
         no-name-in-module,
         no-self-argument,
-        no-self-use,
         no-value-for-parameter,
         not-an-iterable,
         not-callable,
@@ -102,6 +98,7 @@ disable=abstract-method,
         ungrouped-imports,
         unidiomatic-typecheck,
         unnecessary-comprehension,
+        unnecessary-dunder-call,
         unnecessary-lambda,
         unnecessary-pass,
         unreachable,
diff --git a/pytest.ini b/pytest.ini
index ae2ed2029..af0f2fb6e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,3 +21,6 @@ addopts =
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
     only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+filterwarnings =
+    ignore:scrapy.downloadermiddlewares.decompression is deprecated
+    ignore:Module scrapy.utils.reqser is deprecated
diff --git a/scrapy/VERSION b/scrapy/VERSION
index 6a6a3d8e3..24ba9a38d 100644
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@@ -1 +1 @@
-2.6.1
+2.7.0
diff --git a/scrapy/__init__.py b/scrapy/__init__.py
index 396f98219..86e584396 100644
--- a/scrapy/__init__.py
+++ b/scrapy/__init__.py
@@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)
 
 
 # Check minimum required Python version
-if sys.version_info < (3, 6):
-    print(f"Scrapy {__version__} requires Python 3.6+")
+if sys.version_info < (3, 7):
+    print(f"Scrapy {__version__} requires Python 3.7+")
     sys.exit(1)
 
 
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index 491c4beab..5ee1f0f44 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -14,6 +14,15 @@ from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect
 
 
+class ScrapyArgumentParser(argparse.ArgumentParser):
+    def _parse_optional(self, arg_string):
+        # if starts with -: it means that is a parameter not a argument
+        if arg_string[:2] == '-:':
+            return None
+
+        return super()._parse_optional(arg_string)
+
+
 def _iter_command_classes(module_name):
     # TODO: add `name` attribute to commands and and merge this function with
     # scrapy.utils.spider.iter_spider_classes
@@ -131,10 +140,10 @@ def execute(argv=None, settings=None):
         sys.exit(2)
 
     cmd = cmds[cmdname]
-    parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
-                                     usage=f"scrapy {cmdname} {cmd.syntax()}",
-                                     conflict_handler='resolve',
-                                     description=cmd.long_desc())
+    parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
+                                  usage=f"scrapy {cmdname} {cmd.syntax()}",
+                                  conflict_handler='resolve',
+                                  description=cmd.long_desc())
     settings.setdict(cmd.default_settings, priority='command')
     cmd.settings = settings
     cmd.add_options(parser)
diff --git a/scrapy/commands/parse.py b/scrapy/commands/parse.py
index a3f6b96f4..d93ab2ac5 100644
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@@ -5,6 +5,8 @@ from typing import Dict
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url
 
+from twisted.internet.defer import maybeDeferred
+
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
@@ -51,7 +53,7 @@ class Command(BaseRunSpiderCommand):
         parser.add_argument("--cbkwargs", dest="cbkwargs",
                             help="inject extra callback kwargs into the Request, it must be a valid raw json string")
         parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: %default]")
+                            help="maximum depth for parsing requests [default: %(default)s]")
         parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                             help="print each depth level one by one")
 
@@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand):
             if not opts.nolinks:
                 self.print_requests(colour=colour)
 
-    def run_callback(self, response, callback, cb_kwargs=None):
-        cb_kwargs = cb_kwargs or {}
+    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
         items, requests = [], []
-
-        for x in iterate_spider_output(callback(response, **cb_kwargs)):
+        for x in spider_output:
             if is_item(x):
                 items.append(x)
             elif isinstance(x, Request):
                 requests.append(x)
-        return items, requests
+        return items, requests, opts, depth, spider, callback
+
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
+        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
+        return d
 
     def get_callback_from_rules(self, spider, response):
         if getattr(spider, 'rules', None):
@@ -146,7 +151,8 @@ class Command(BaseRunSpiderCommand):
 
         def _start_requests(spider):
             yield self.prepare_request(spider, Request(url), opts)
-        self.spidercls.start_requests = _start_requests
+        if self.spidercls:
+            self.spidercls.start_requests = _start_requests
 
     def start_parsing(self, url, opts):
         self.crawler_process.crawl(self.spidercls, **opts.spargs)
@@ -157,6 +163,25 @@ class Command(BaseRunSpiderCommand):
             logger.error('No response downloaded for: %(url)s',
                          {'url': url})
 
+    def scraped_data(self, args):
+        items, requests, opts, depth, spider, callback = args
+        if opts.pipelines:
+            itemproc = self.pcrawler.engine.scraper.itemproc
+            for item in items:
+                itemproc.process_item(item, spider)
+        self.add_items(depth, items)
+        self.add_requests(depth, requests)
+
+        scraped_data = items if opts.output else []
+        if depth < opts.depth:
+            for req in requests:
+                req.meta['_depth'] = depth + 1
+                req.meta['_callback'] = req.callback
+                req.callback = callback
+            scraped_data += requests
+
+        return scraped_data
+
     def prepare_request(self, spider, request, opts):
         def callback(response, **cb_kwargs):
             # memorize first request
@@ -190,23 +215,10 @@ class Command(BaseRunSpiderCommand):
             # parse items and requests
             depth = response.meta['_depth']
 
-            items, requests = self.run_callback(response, cb, cb_kwargs)
-            if opts.pipelines:
-                itemproc = self.pcrawler.engine.scraper.itemproc
-                for item in items:
-                    itemproc.process_item(item, spider)
-            self.add_items(depth, items)
-            self.add_requests(depth, requests)
-
-            scraped_data = items if opts.output else []
-            if depth < opts.depth:
-                for req in requests:
-                    req.meta['_depth'] = depth + 1
-                    req.meta['_callback'] = req.callback
-                    req.callback = callback
-                scraped_data += requests
-
-            return scraped_data
+            d = self.run_callback(response, cb, cb_kwargs)
+            d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
+            d.addCallback(self.scraped_data)
+            return d
 
         # update request meta if any extra meta was passed through the --meta/-m opts.
         if opts.meta:
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index b5318c7bb..4abde2238 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
     which allows TLS protocol negotiation
 
     'A TLS/SSL connection established with [this method] may
-     understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+     understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
     """
 
     def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.py
index 3ef129587..a495874bd 100644
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@@ -102,11 +102,11 @@ class FTPDownloadHandler:
 
     def _build_response(self, result, request, protocol):
         self.result = result
-        respcls = responsetypes.from_args(url=request.url)
         protocol.close()
-        body = protocol.filename or protocol.body.read()
         headers = {"local filename": protocol.filename or '', "size": protocol.size}
-        return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
+        body = to_bytes(protocol.filename or protocol.body.read())
+        respcls = responsetypes.from_args(url=request.url, body=body)
+        return respcls(url=request.url, status=200, body=body, headers=headers)
 
     def _failed(self, result, request):
         message = result.getErrorMessage()
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 38935667d..6b8a18f1a 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -384,8 +384,7 @@ class ScrapyAgent:
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": request, "handler": handler.__qualname__})
                 txresponse._transport.stopProducing()
-                with suppress(AttributeError):
-                    txresponse._transport._producer.loseConnection()
+                txresponse._transport.loseConnection()
                 return {
                     "txresponse": txresponse,
                     "body": b"",
@@ -417,7 +416,7 @@ class ScrapyAgent:
 
             logger.warning(warning_msg, warning_args)
 
-            txresponse._transport._producer.loseConnection()
+            txresponse._transport.loseConnection()
             raise defer.CancelledError(warning_msg % warning_args)
 
         if warnsize and expected_size > warnsize:
@@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol):
                 logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                              {"request": self._request, "handler": handler.__qualname__})
                 self.transport.stopProducing()
-                self.transport._producer.loseConnection()
+                self.transport.loseConnection()
                 failure = result if result.value.fail else None
                 self._finish_response(flags=["download_stopped"], failure=failure)
 
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
index 19a56d9b6..698a1c85c 100644
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 logger = logging.getLogger(__name__)
 
 
-METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
 METHOD_TLSv11 = 'TLSv1.1'
@@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2'
 
 openssl_methods = {
     METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
     METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 06cb96489..7d048c1e4 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
         request.meta['download_latency'] = self.headers_time - self.start_time
         status = int(self.status)
         headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self._url)
+        respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
         return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))
 
     def _set_connection_attributes(self, request):
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index f9de7ee23..1228e78da 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -136,7 +136,9 @@ class ExecutionEngine:
         self.paused = False
 
     def _next_request(self) -> None:
-        assert self.slot is not None  # typing
+        if self.slot is None:
+            return
+
         assert self.spider is not None  # typing
 
         if self.paused:
@@ -184,7 +186,8 @@ class ExecutionEngine:
         d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))
-        d.addBoth(lambda _: self.slot.nextcall.schedule())
+        slot = self.slot
+        d.addBoth(lambda _: slot.nextcall.schedule())
         d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                            exc_info=failure_to_exc_info(f),
                                            extra={'spider': self.spider}))
@@ -254,9 +257,7 @@ class ExecutionEngine:
 
     def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
         """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
-        if spider is None:
-            spider = self.spider
-        else:
+        if spider is not None:
             warnings.warn(
                 "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
                 category=ScrapyDeprecationWarning,
@@ -264,7 +265,7 @@ class ExecutionEngine:
             )
             if spider is not self.spider:
                 logger.warning("The spider '%s' does not match the open spider", spider.name)
-        if spider is None:
+        if self.spider is None:
             raise RuntimeError(f"No open spider to crawl: {request}")
         return self._download(request, spider).addBoth(self._downloaded, request, spider)
 
@@ -275,11 +276,14 @@ class ExecutionEngine:
         self.slot.remove_request(request)
         return self.download(result, spider) if isinstance(result, Request) else result
 
-    def _download(self, request: Request, spider: Spider) -> Deferred:
+    def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
         assert self.slot is not None  # typing
 
         self.slot.add_request(request)
 
+        if spider is None:
+            spider = self.spider
+
         def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
             if not isinstance(result, (Response, Request)):
                 raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index f40bccbb3..e1fdd8d13 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,9 +1,8 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-
 import logging
 from collections import deque
-from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
+from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union
 
 from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
@@ -13,7 +12,15 @@ from scrapy import signals, Spider
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import (
+    aiter_errback,
+    defer_fail,
+    defer_succeed,
+    iter_errback,
+    parallel,
+    parallel_async,
+)
+
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
@@ -185,12 +192,19 @@ class Scraper:
             spider=spider
         )
 
-    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
+    def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
+                             response: Response, spider: Spider) -> Deferred:
         if not result:
             return defer_succeed(None)
-        it = iter_errback(result, self.handle_spider_error, request, response, spider)
-        dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
-                       request, response, spider)
+        it: Union[Generator, AsyncGenerator]
+        if isinstance(result, AsyncIterable):
+            it = aiter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                 request, response, spider)
+        else:
+            it = iter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+                           request, response, spider)
         return dfd
 
     def _process_spidermw_output(self, output: Any, request: Request, response: Response,
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 7cdc28284..1aa02f29f 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,32 +3,42 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import logging
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, cast
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
 
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.python import MutableChain
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.python import MutableAsyncChain, MutableChain
+
+
+logger = logging.getLogger(__name__)
 
 
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
 def _isiterable(o) -> bool:
-    return isinstance(o, Iterable)
+    return isinstance(o, (Iterable, AsyncIterable))
 
 
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
 
+    def __init__(self, *middlewares):
+        super().__init__(*middlewares)
+        self.downgrade_warning_done = False
+
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -39,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        process_spider_output = getattr(mw, 'process_spider_output', None)
+        process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
         self.methods['process_spider_output'].appendleft(process_spider_output)
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
@@ -51,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {method.__qualname__} must return None "
+                    msg = (f"{method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -60,17 +70,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return scrape_func(Failure(), request, spider)
         return scrape_func(response, request, spider)
 
-    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
-                           exception_processor_index: int, recover_to: MutableChain) -> Generator:
-        try:
-            for r in iterable:
-                yield r
-        except Exception as ex:
-            exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                              exception_processor_index)
-            if isinstance(exception_result, Failure):
-                raise
-            recover_to.extend(exception_result)
+    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
+                           exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
+                           ) -> Union[Generator, AsyncGenerator]:
+
+        def process_sync(iterable: Iterable):
+            try:
+                for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        async def process_async(iterable: AsyncIterable):
+            try:
+                async for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        if isinstance(iterable, AsyncIterable):
+            return process_async(iterable)
+        return process_sync(iterable)
 
     def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
                                   start_index: int = 0) -> Union[Failure, MutableChain]:
@@ -82,30 +110,83 @@ class SpiderMiddlewareManager(MiddlewareManager):
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
+            method = cast(Callable, method)
             result = method(response=response, exception=exception, spider=spider)
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                return self._process_spider_output(response, spider, result, method_index + 1)
+                dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
+                # _process_spider_output() returns a Deferred only because of downgrading so this can be
+                # simplified when downgrading is removed.
+                if dfd.called:
+                    # the result is available immediately if _process_spider_output didn't do downgrading
+                    return dfd.result
+                else:
+                    # we forbid waiting here because otherwise we would need to return a deferred from
+                    # _process_spider_exception too, which complicates the architecture
+                    msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
+                    raise _InvalidOutput(msg)
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {method.__qualname__} must return None "
+                msg = (f"{method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
 
+    # This method cannot be made async def, as _process_spider_exception relies on the Deferred result
+    # being available immediately which doesn't work when it's a wrapped coroutine.
+    # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
+    @inlineCallbacks
     def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Iterable, start_index: int = 0) -> MutableChain:
+                               result: Union[Iterable, AsyncIterable], start_index: int = 0
+                               ) -> Deferred:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        recovered = MutableChain()
+        recovered: Union[MutableChain, MutableAsyncChain]
+        last_result_is_async = isinstance(result, AsyncIterable)
+        if last_result_is_async:
+            recovered = MutableAsyncChain()
+        else:
+            recovered = MutableChain()
+
+        # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
+        # 1. def foo. Sync iterables are passed as is, async ones are downgraded.
+        # 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
+        # 3. def foo + async def foo_async. Iterables are passed to the respective method.
+        # Storing methods and method tuples in the same list is weird but we should be able to roll this back
+        # when we drop this compatibility feature.
 
         method_list = islice(self.methods['process_spider_output'], start_index, None)
-        for method_index, method in enumerate(method_list, start=start_index):
-            if method is None:
+        for method_index, method_pair in enumerate(method_list, start=start_index):
+            if method_pair is None:
                 continue
+            need_upgrade = need_downgrade = False
+            if isinstance(method_pair, tuple):
+                # This tuple handling is only needed until _async compatibility methods are removed.
+                method_sync, method_async = method_pair
+                method = method_async if last_result_is_async else method_sync
+            else:
+                method = method_pair
+                if not last_result_is_async and isasyncgenfunction(method):
+                    need_upgrade = True
+                elif last_result_is_async and not isasyncgenfunction(method):
+                    need_downgrade = True
             try:
+                if need_upgrade:
+                    # Iterable -> AsyncIterable
+                    result = as_async_generator(result)
+                elif need_downgrade:
+                    if not self.downgrade_warning_done:
+                        logger.warning(f"Async iterable passed to {method.__qualname__} "
+                                       f"was downgraded to a non-async one")
+                        self.downgrade_warning_done = True
+                    assert isinstance(result, AsyncIterable)
+                    # AsyncIterable -> Iterable
+                    result = yield deferred_from_coro(collect_asyncgen(result))
+                    if isinstance(recovered, AsyncIterable):
+                        recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
+                        recovered = MutableChain(recovered_collected)
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
@@ -116,28 +197,77 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {method.__qualname__} must return an "
-                       f"iterable, got {type(result)}")
+                if iscoroutine(result):
+                    result.close()  # Silence warning about not awaiting
+                    msg = (
+                        f"{method.__qualname__} must be an asynchronous "
+                        f"generator (i.e. use yield)"
+                    )
+                else:
+                    msg = (
+                        f"{method.__qualname__} must return an iterable, got "
+                        f"{type(result)}"
+                    )
                 raise _InvalidOutput(msg)
+            last_result_is_async = isinstance(result, AsyncIterable)
 
-        return MutableChain(result, recovered)
+        if last_result_is_async:
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
-        recovered = MutableChain()
+    async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                       ) -> Union[MutableChain, MutableAsyncChain]:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        if isinstance(result, AsyncIterable):
+            recovered = MutableAsyncChain()
+        else:
+            recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        return MutableChain(self._process_spider_output(response, spider, result), recovered)
+        result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
+        if isinstance(result, AsyncIterable):
+            return MutableAsyncChain(result, recovered)
+        else:
+            if isinstance(recovered, AsyncIterable):
+                recovered_collected = await collect_asyncgen(recovered)
+                recovered = MutableChain(recovered_collected)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                         spider: Spider) -> Deferred:
-        def process_callback_output(result: Iterable) -> MutableChain:
-            return self._process_callback_output(response, spider, result)
+        async def process_callback_output(result: Union[Iterable, AsyncIterable]
+                                          ) -> Union[MutableChain, MutableAsyncChain]:
+            return await self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
             return self._process_spider_exception(response, spider, _failure)
 
         dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
-        dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
+        dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
         return dfd
 
     def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
         return self._process_chain('process_start_requests', start_requests, spider)
+
+    # This method is only needed until _async compatibility methods are removed.
+    @staticmethod
+    def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
+        normal_method = getattr(mw, methodname, None)
+        methodname_async = methodname + "_async"
+        async_method = getattr(mw, methodname_async, None)
+        if not async_method:
+            return normal_method
+        if not normal_method:
+            logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
+                         f"without {methodname}, skipping this method.")
+            return None
+        if not isasyncgenfunction(async_method):
+            logger.error(f"{async_method.__qualname__} is not "
+                         f"an async generator function, skipping this method.")
+            return normal_method
+        if isasyncgenfunction(normal_method):
+            logger.error(f"{normal_method.__qualname__} is an async "
+                         f"generator function while {methodname_async} exists, "
+                         f"skipping both methods.")
+            return None
+        return normal_method, async_method
diff --git a/scrapy/crawler.py b/scrapy/crawler.py
index a638254f1..65174d846 100644
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@@ -31,7 +31,12 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+from scrapy.utils.reactor import (
+    install_reactor,
+    is_asyncio_reactor_installed,
+    verify_installed_asyncio_event_loop,
+    verify_installed_reactor,
+)
 
 
 logger = logging.getLogger(__name__)
@@ -51,6 +56,7 @@ class Crawler:
         self.spidercls.update_settings(self.settings)
 
         self.signals = SignalManager(self)
+
         self.stats = load_object(self.settings['STATS_CLASS'])(self)
 
         handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@@ -71,18 +77,26 @@ class Crawler:
         lf_cls = load_object(self.settings['LOG_FORMATTER'])
         self.logformatter = lf_cls.from_crawler(self)
 
-        reactor_class = self.settings.get("TWISTED_REACTOR")
+        self.request_fingerprinter = create_instance(
+            load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+            settings=self.settings,
+            crawler=self,
+        )
+
+        reactor_class = self.settings["TWISTED_REACTOR"]
+        event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
         if init_reactor:
             # this needs to be done after the spider settings are merged,
             # but before something imports twisted.internet.reactor
             if reactor_class:
-                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
+                install_reactor(reactor_class, event_loop)
             else:
-                from twisted.internet import default
-                default.install()
+                from twisted.internet import reactor  # noqa: F401
             log_reactor_info()
         if reactor_class:
             verify_installed_reactor(reactor_class)
+            if is_asyncio_reactor_installed() and event_loop:
+                verify_installed_asyncio_event_loop(event_loop)
 
         self.extensions = ExtensionManager.from_crawler(self)
 
@@ -290,6 +304,7 @@ class CrawlerProcess(CrawlerRunner):
         super().__init__(settings)
         configure_logging(self.settings, install_root_handler)
         log_scrapy_info(self.settings)
+        self._initialized_reactor = False
 
     def _signal_shutdown(self, signum, _):
         from twisted.internet import reactor
@@ -310,7 +325,9 @@ class CrawlerProcess(CrawlerRunner):
     def _create_crawler(self, spidercls):
         if isinstance(spidercls, str):
             spidercls = self.spider_loader.load(spidercls)
-        return Crawler(spidercls, self.settings, init_reactor=True)
+        init_reactor = not self._initialized_reactor
+        self._initialized_reactor = True
+        return Crawler(spidercls, self.settings, init_reactor=init_reactor)
 
     def start(self, stop_after_crawl=True, install_signal_handlers=True):
         """
diff --git a/scrapy/downloadermiddlewares/cookies.py b/scrapy/downloadermiddlewares/cookies.py
index 3afa06077..c592acb57 100644
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@@ -104,8 +104,8 @@ class CookiesMiddleware:
         for key in ("name", "value", "path", "domain"):
             if cookie.get(key) is None:
                 if key in ("name", "value"):
-                    msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
-                    logger.warning(msg.format(request, cookie, key))
+                    msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+                    logger.warning(msg)
                     return
                 continue
             if isinstance(cookie[key], (bool, float, int, str)):
diff --git a/scrapy/downloadermiddlewares/decompression.py b/scrapy/downloadermiddlewares/decompression.py
index 0fcf8fb8c..e01e9cc76 100644
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@@ -9,10 +9,19 @@ import tarfile
 import zipfile
 from io import BytesIO
 from tempfile import mktemp
+from warnings import warn
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes
 
 
+warn(
+    'scrapy.downloadermiddlewares.decompression is deprecated',
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
+
+
 logger = logging.getLogger(__name__)
 
 
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index d2665b655..dd8a7e797 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -45,31 +45,40 @@ class HttpProxyMiddleware:
         return creds, proxy_url
 
     def process_request(self, request, spider):
-        # ignore if proxy is already set
+        creds, proxy_url = None, None
         if 'proxy' in request.meta:
-            if request.meta['proxy'] is None:
-                return
-            # extract credentials if present
-            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+            if request.meta['proxy'] is not None:
+                creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+        elif self.proxies:
+            parsed = urlparse_cached(request)
+            scheme = parsed.scheme
+            if (
+                (
+                    # 'no_proxy' is only supported by http schemes
+                    scheme not in ('http', 'https')
+                    or not proxy_bypass(parsed.hostname)
+                )
+                and scheme in self.proxies
+            ):
+                creds, proxy_url = self.proxies[scheme]
+
+        self._set_proxy_and_creds(request, proxy_url, creds)
+
+    def _set_proxy_and_creds(self, request, proxy_url, creds):
+        if proxy_url:
             request.meta['proxy'] = proxy_url
-            if creds and not request.headers.get('Proxy-Authorization'):
-                request.headers['Proxy-Authorization'] = b'Basic ' + creds
-            return
-        elif not self.proxies:
-            return
-
-        parsed = urlparse_cached(request)
-        scheme = parsed.scheme
-
-        # 'no_proxy' is only supported by http schemes
-        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
-            return
-
-        if scheme in self.proxies:
-            self._set_proxy(request, scheme)
-
-    def _set_proxy(self, request, scheme):
-        creds, proxy = self.proxies[scheme]
-        request.meta['proxy'] = proxy
+        elif request.meta.get('proxy') is not None:
+            request.meta['proxy'] = None
         if creds:
-            request.headers['Proxy-Authorization'] = b'Basic ' + creds
+            request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
+            request.meta['_auth_proxy'] = proxy_url
+        elif '_auth_proxy' in request.meta:
+            if proxy_url != request.meta['_auth_proxy']:
+                if b'Proxy-Authorization' in request.headers:
+                    del request.headers[b'Proxy-Authorization']
+                del request.meta['_auth_proxy']
+        elif b'Proxy-Authorization' in request.headers:
+            if proxy_url:
+                request.meta['_auth_proxy'] = proxy_url
+            else:
+                del request.headers[b'Proxy-Authorization']
diff --git a/scrapy/dupefilters.py b/scrapy/dupefilters.py
index 292c68099..d1b0559ef 100644
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@@ -1,14 +1,16 @@
 import logging
 import os
 from typing import Optional, Set, Type, TypeVar
+from warnings import warn
 
 from twisted.internet.defer import Deferred
 
 from scrapy.http.request import Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter
 
 
 BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
 class RFPDupeFilter(BaseDupeFilter):
     """Request Fingerprint duplicates filter"""
 
-    def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+    def __init__(
+        self,
+        path: Optional[str] = None,
+        debug: bool = False,
+        *,
+        fingerprinter=None,
+    ) -> None:
         self.file = None
+        self.fingerprinter = fingerprinter or RequestFingerprinter()
         self.fingerprints: Set[str] = set()
         self.logdupes = True
         self.debug = debug
@@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
             self.fingerprints.update(x.rstrip() for x in self.file)
 
     @classmethod
-    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
         debug = settings.getbool('DUPEFILTER_DEBUG')
-        return cls(job_dir(settings), debug)
+        try:
+            return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their '__init__' "
+                "method to support a 'fingerprinter' parameter or reimplement "
+                "the 'from_settings' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls(job_dir(settings), debug)
+            result.fingerprinter = fingerprinter
+            return result
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        try:
+            return cls.from_settings(
+                crawler.settings,
+                fingerprinter=crawler.request_fingerprinter,
+            )
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their overridden "
+                "'__init__' method and 'from_settings' class method to "
+                "support a 'fingerprinter' parameter, or reimplement the "
+                "'from_crawler' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls.from_settings(crawler.settings)
+            result.fingerprinter = crawler.request_fingerprinter
+            return result
 
     def request_seen(self, request: Request) -> bool:
         fp = self.request_fingerprint(request)
@@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
         return False
 
     def request_fingerprint(self, request: Request) -> str:
-        return request_fingerprint(request)
+        return self.fingerprinter.fingerprint(request).hex()
 
     def close(self, reason: str) -> None:
         if self.file:
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 1c26e81db..76cbe4d4b 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,6 +8,7 @@ import marshal
 import pickle
 import pprint
 import warnings
+from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator
 
 from itemadapter import is_item, ItemAdapter
@@ -68,6 +69,14 @@ class BaseItemExporter:
                 field_iter = item.field_names()
             else:
                 field_iter = item.keys()
+        elif isinstance(self.fields_to_export, Mapping):
+            if include_empty:
+                field_iter = self.fields_to_export.items()
+            else:
+                field_iter = (
+                    (x, y) for x, y in self.fields_to_export.items()
+                    if x in item
+                )
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -75,13 +84,17 @@ class BaseItemExporter:
                 field_iter = (x for x in self.fields_to_export if x in item)
 
         for field_name in field_iter:
-            if field_name in item:
-                field_meta = item.get_field_meta(field_name)
-                value = self.serialize_field(field_meta, field_name, item[field_name])
+            if isinstance(field_name, str):
+                item_field, output_field = field_name, field_name
+            else:
+                item_field, output_field = field_name
+            if item_field in item:
+                field_meta = item.get_field_meta(item_field)
+                value = self.serialize_field(field_meta, output_field, item[item_field])
             else:
                 value = default_value
 
-            yield field_name, value
+            yield output_field, value
 
 
 class JsonLinesItemExporter(BaseItemExporter):
@@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
             if not self.fields_to_export:
                 # use declared field names, or keys if the item is a dict
                 self.fields_to_export = ItemAdapter(item).field_names()
-            row = list(self._build_row(self.fields_to_export))
+            if isinstance(self.fields_to_export, Mapping):
+                fields = self.fields_to_export.values()
+            else:
+                fields = self.fields_to_export
+            row = list(self._build_row(fields))
             self.csv_writer.writerow(row)
 
 
diff --git a/scrapy/extensions/httpcache.py b/scrapy/extensions/httpcache.py
index d0ae29b90..843e14812 100644
--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint
 
 
 logger = logging.getLogger(__name__)
@@ -228,6 +227,8 @@ class DbmCacheStorage:
 
         logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})
 
+        self._fingerprinter = spider.crawler.request_fingerprinter
+
     def close_spider(self, spider):
         self.db.close()
 
@@ -239,12 +240,12 @@ class DbmCacheStorage:
         status = data['status']
         headers = Headers(data['headers'])
         body = data['body']
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
     def store_response(self, spider, request, response):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         data = {
             'status': response.status,
             'url': response.url,
@@ -255,7 +256,7 @@ class DbmCacheStorage:
         self.db[f'{key}_time'] = str(time())
 
     def _read_data(self, spider, request):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         db = self.db
         tkey = f'{key}_time'
         if tkey not in db:
@@ -267,9 +268,6 @@ class DbmCacheStorage:
 
         return pickle.loads(db[f'{key}_data'])
 
-    def _request_key(self, request):
-        return request_fingerprint(request)
-
 
 class FilesystemCacheStorage:
 
@@ -283,6 +281,8 @@ class FilesystemCacheStorage:
         logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
                      extra={'spider': spider})
 
+        self._fingerprinter = spider.crawler.request_fingerprinter
+
     def close_spider(self, spider):
         pass
 
@@ -299,7 +299,7 @@ class FilesystemCacheStorage:
         url = metadata.get('response_url')
         status = metadata['status']
         headers = Headers(headers_raw_to_dict(rawheaders))
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
         response = respcls(url=url, headers=headers, status=status, body=body)
         return response
 
@@ -329,7 +329,7 @@ class FilesystemCacheStorage:
             f.write(request.body)
 
     def _get_request_path(self, spider, request):
-        key = request_fingerprint(request)
+        key = self._fingerprinter.fingerprint(request).hex()
         return os.path.join(self.cachedir, spider.name, key[0:2], key)
 
     def _read_meta(self, spider, request):
diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
index 9de119a10..f5081a7d7 100644
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@@ -33,8 +33,8 @@ class MemoryUsage:
         self.crawler = crawler
         self.warned = False
         self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
-        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
-        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
+        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
+        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
         self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
         self.mail = MailSender.from_settings(crawler.settings)
         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@@ -77,7 +77,7 @@ class MemoryUsage:
     def _check_limit(self):
         if self.get_virtual_size() > self.limit:
             self.crawler.stats.set_value('memusage/limit_reached', 1)
-            mem = self.limit/1024/1024
+            mem = self.limit / 1024 / 1024
             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                          {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
@@ -94,11 +94,11 @@ class MemoryUsage:
                 self.crawler.stop()
 
     def _check_warning(self):
-        if self.warned: # warn only once
+        if self.warned:  # warn only once
             return
         if self.get_virtual_size() > self.warning:
             self.crawler.stats.set_value('memusage/warning_reached', 1)
-            mem = self.warning/1024/1024
+            mem = self.warning / 1024 / 1024
             logger.warning("Memory usage reached %(memusage)dM",
                            {'memusage': mem}, extra={'crawler': self.crawler})
             if self.notify_mails:
diff --git a/scrapy/extensions/statsmailer.py b/scrapy/extensions/statsmailer.py
index bcdbaff24..739e6b958 100644
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@@ -8,6 +8,7 @@ from scrapy import signals
 from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured
 
+
 class StatsMailer:
 
     def __init__(self, stats, recipients, mail):
diff --git a/scrapy/http/headers.py b/scrapy/http/headers.py
index 1a2b99b0a..9c03fe54f 100644
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@@ -1,3 +1,5 @@
+from collections.abc import Mapping
+
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@@ -10,6 +12,13 @@ class Headers(CaselessDict):
         self.encoding = encoding
         super().__init__(seq)
 
+    def update(self, seq):
+        seq = seq.items() if isinstance(seq, Mapping) else seq
+        iseq = {}
+        for k, v in seq:
+            iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+        super().update(iseq)
+
     def normkey(self, key):
         """Normalize key to bytes"""
         return self._tobytes(key.title())
@@ -86,4 +95,5 @@ class Headers(CaselessDict):
 
     def __copy__(self):
         return self.__class__(self)
+
     copy = __copy__
diff --git a/scrapy/http/response/text.py b/scrapy/http/response/text.py
index 89516b9b6..bfcde878d 100644
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@@ -11,8 +11,13 @@ from typing import Generator, Tuple
 from urllib.parse import urljoin
 
 import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
-                            http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+    html_body_declared_encoding,
+    html_to_unicode,
+    http_content_type_encoding,
+    resolve_encoding,
+    read_bom,
+)
 from w3lib.html import strip_html5_whitespace
 
 from scrapy.http import Request
@@ -60,6 +65,7 @@ class TextResponse(Response):
     def _declared_encoding(self):
         return (
             self._encoding
+            or self._bom_encoding()
             or self._headers_encoding()
             or self._body_declared_encoding()
         )
@@ -117,6 +123,10 @@ class TextResponse(Response):
     def _body_declared_encoding(self):
         return html_body_declared_encoding(self.body)
 
+    @memoizemethod_noargs
+    def _bom_encoding(self):
+        return read_bom(self.body)[0]
+
     @property
     def selector(self):
         from scrapy.selector import Selector
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 2eb1d8609..8d7e5a602 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Optional, cast, Iterable
+from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -21,8 +21,9 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        # Optional because process_spider_output and process_spider_exception can be None
-        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
+        # Only process_spider_output and process_spider_exception can be None.
+        # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
+        self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 5c52c6c28..906e7eb24 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -222,8 +222,8 @@ class GCSFilesStore:
                 return {'checksum': checksum, 'last_modified': last_modified}
             else:
                 return {}
-
-        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+        blob_path = self._get_blob_path(path)
+        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)
 
     def _get_content_type(self, headers):
         if headers and 'Content-Type' in headers:
@@ -231,8 +231,12 @@ class GCSFilesStore:
         else:
             return 'application/octet-stream'
 
+    def _get_blob_path(self, path):
+        return self.prefix + path
+
     def persist_file(self, path, buf, info, meta=None, headers=None):
-        blob = self.bucket.blob(self.prefix + path)
+        blob_path = self._get_blob_path(path)
+        blob = self.bucket.blob(blob_path)
         blob.cache_control = self.CACHE_CONTROL
         blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index c7a04a0cf..4d59436ab 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -146,7 +146,7 @@ class ImagesPipeline(FilesPipeline):
         yield path, image, buf
 
         for thumb_id, size in self.thumbs.items():
-            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf
 
@@ -165,7 +165,14 @@ class ImagesPipeline(FilesPipeline):
 
         if size:
             image = image.copy()
-            image.thumbnail(size, self._Image.ANTIALIAS)
+            try:
+                # Image.Resampling.LANCZOS was added in Pillow 9.1.0
+                # remove this try except block,
+                # when updating the minimum requirements for Pillow.
+                resampling_filter = self._Image.Resampling.LANCZOS
+            except AttributeError:
+                resampling_filter = self._Image.ANTIALIAS
+            image.thumbnail(size, resampling_filter)
 
         buf = BytesIO()
         image.save(buf, 'JPEG')
@@ -184,6 +191,6 @@ class ImagesPipeline(FilesPipeline):
         image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'full/{image_guid}.jpg'
 
-    def thumb_path(self, request, thumb_id, response=None, info=None):
+    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
         thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
         return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index d1bccf323..5308a9793 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -11,7 +11,6 @@ from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.log import failure_to_exc_info
 
@@ -77,6 +76,7 @@ class MediaPipeline:
         except AttributeError:
             pipe = cls()
         pipe.crawler = crawler
+        pipe._fingerprinter = crawler.request_fingerprinter
         return pipe
 
     def open_spider(self, spider):
@@ -90,7 +90,7 @@ class MediaPipeline:
         return dfd.addCallback(self.item_completed, item, info)
 
     def _process_request(self, request, info, item):
-        fp = request_fingerprint(request)
+        fp = self._fingerprinter.fingerprint(request)
         cb = request.callback or (lambda _: _)
         eb = request.errback
         request.callback = None
@@ -121,7 +121,7 @@ class MediaPipeline:
     def _make_compatible(self):
         """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
         methods = [
-            "file_path", "media_to_download", "media_downloaded",
+            "file_path", "thumb_path", "media_to_download", "media_downloaded",
             "file_downloaded", "image_downloaded", "get_images"
         ]
 
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 6ed9f8b8f..3efd4d2fd 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -95,12 +95,14 @@ class ResponseTypes:
         chunk = to_bytes(chunk)
         if not binary_is_text(chunk):
             return self.from_mimetype('application/octet-stream')
-        elif b"<html>" in chunk.lower():
+        lowercase_chunk = chunk.lower()
+        if b"<html>" in lowercase_chunk:
             return self.from_mimetype('text/html')
-        elif b"<?xml" in chunk.lower():
+        if b"<?xml" in lowercase_chunk:
             return self.from_mimetype('text/xml')
-        else:
-            return self.from_mimetype('text')
+        if b'<!doctype html>' in lowercase_chunk:
+            return self.from_mimetype('text/html')
+        return self.from_mimetype('text')
 
     def from_args(self, headers=None, url=None, filename=None, body=None):
         """Guess the most appropriate Response class based on
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 6b1ad0828..6cacc63e1 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -197,6 +197,38 @@ class BaseSettings(MutableMapping):
             value = json.loads(value)
         return dict(value)
 
+    def getdictorlist(self, name, default=None):
+        """Get a setting value as either a :class:`dict` or a :class:`list`.
+
+        If the setting is already a dict or a list, a copy of it will be
+        returned.
+
+        If it is a string it will be evaluated as JSON, or as a comma-separated
+        list of strings as a fallback.
+
+        For example, settings populated from the command line will return:
+
+        -   ``{'key1': 'value1', 'key2': 'value2'}`` if set to
+            ``'{"key1": "value1", "key2": "value2"}'``
+
+        -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
+        value = self.get(name, default)
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except ValueError:
+                return value.split(',')
+        return copy.deepcopy(value)
+
     def getwithbase(self, name):
         """Get a composition of a dictionary-like setting and its `_BASE`
         counterpart.
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 8389a70cb..29ff028be 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
     'json': 'scrapy.exporters.JsonItemExporter',
     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+    'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
     'jl': 'scrapy.exporters.JsonLinesItemExporter',
     'csv': 'scrapy.exporters.CsvItemExporter',
     'xml': 'scrapy.exporters.XmlItemExporter',
@@ -246,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
 REFERER_ENABLED = True
 REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
 
+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
+
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..4c923b1b3 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -28,31 +28,39 @@ class DepthMiddleware:
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request):
-                depth = response.meta['depth'] + 1
-                request.meta['depth'] = depth
-                if self.prio:
-                    request.priority -= depth * self.prio
-                if self.maxdepth and depth > self.maxdepth:
-                    logger.debug(
-                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                        {'maxdepth': self.maxdepth, 'requrl': request.url},
-                        extra={'spider': spider}
-                    )
-                    return False
-                else:
-                    if self.verbose_stats:
-                        self.stats.inc_value(f'request_depth_count/{depth}',
-                                             spider=spider)
-                    self.stats.max_value('request_depth_max', depth,
-                                         spider=spider)
-            return True
+        self._init_depth(response, spider)
+        return (r for r in result or () if self._filter(r, response, spider))
 
+    async def process_spider_output_async(self, response, result, spider):
+        self._init_depth(response, spider)
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _init_depth(self, response, spider):
         # base case (depth=0)
         if 'depth' not in response.meta:
             response.meta['depth'] = 0
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+    def _filter(self, request, response, spider):
+        if not isinstance(request, Request):
+            return True
+        depth = response.meta['depth'] + 1
+        request.meta['depth'] = depth
+        if self.prio:
+            request.priority -= depth * self.prio
+        if self.maxdepth and depth > self.maxdepth:
+            logger.debug(
+                "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                {'maxdepth': self.maxdepth, 'requrl': request.url},
+                extra={'spider': spider}
+            )
+            return False
+        if self.verbose_stats:
+            self.stats.inc_value(f'request_depth_count/{depth}',
+                                 spider=spider)
+        self.stats.max_value('request_depth_max', depth,
+                             spider=spider)
+        return True
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..448bc1367 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -26,21 +26,27 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
-                else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+        return (r for r in result or () if self._filter(r, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider) -> bool:
+        if not isinstance(request, Request):
+            return True
+        if request.dont_filter or self.should_follow(request, spider):
+            return True
+        domain = urlparse_cached(request).hostname
+        if domain and domain not in self.domains_seen:
+            self.domains_seen.add(domain)
+            logger.debug(
+                "Filtered offsite request to %(domain)r: %(request)s",
+                {'domain': domain, 'request': request}, extra={'spider': spider})
+            self.stats.inc_value('offsite/domains', spider=spider)
+        self.stats.inc_value('offsite/filtered', spider=spider)
+        return False
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 608c0eea5..8027beb92 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -333,13 +333,18 @@ class RefererMiddleware:
         return cls() if cls else self.default_policy()
 
     def process_spider_output(self, response, result, spider):
-        def _set_referer(r):
-            if isinstance(r, Request):
-                referrer = self.policy(response, r).referrer(response.url, r.url)
-                if referrer is not None:
-                    r.headers.setdefault('Referer', referrer)
-            return r
-        return (_set_referer(r) for r in result or ())
+        return (self._set_referer(r, response) for r in result or ())
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            yield self._set_referer(r, response)
+
+    def _set_referer(self, r, response):
+        if isinstance(r, Request):
+            referrer = self.policy(response, r).referrer(response.url, r.url)
+            if referrer is not None:
+                r.headers.setdefault('Referer', referrer)
+        return r
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 450d4ff40..7ad64d2af 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -25,16 +25,20 @@ class UrlLengthMiddleware:
         return cls(maxlength)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.info(
-                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
-                    {'maxlength': self.maxlength, 'url': request.url},
-                    extra={'spider': spider}
-                )
-                spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
-                return False
-            else:
-                return True
+        return (r for r in result or () if self._filter(r, spider))
 
-        return (r for r in result or () if _filter(r))
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider):
+        if isinstance(request, Request) and len(request.url) > self.maxlength:
+            logger.info(
+                "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                {'maxlength': self.maxlength, 'url': request.url},
+                extra={'spider': spider}
+            )
+            spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
+            return False
+        return True
diff --git a/scrapy/spiders/crawl.py b/scrapy/spiders/crawl.py
index 1dcf2e6ab..edac082d0 100644
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst
 """
 
 import copy
-from typing import Sequence
+from typing import AsyncIterable, Awaitable, Sequence
 
-from scrapy.http import Request, HtmlResponse
+from scrapy.http import Request, Response, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.spider import iterate_spider_output
 
 
@@ -78,7 +79,7 @@ class CrawlSpider(Spider):
     def parse_start_url(self, response, **kwargs):
         return []
 
-    def process_results(self, response, results):
+    def process_results(self, response: Response, results: list):
         return results
 
     def _build_request(self, rule_index, link):
@@ -109,9 +110,13 @@ class CrawlSpider(Spider):
         rule = self._rules[failure.request.meta['rule']]
         return self._handle_failure(failure, rule.errback)
 
-    def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
         if callback:
             cb_res = callback(response, **cb_kwargs) or ()
+            if isinstance(cb_res, AsyncIterable):
+                cb_res = await collect_asyncgen(cb_res)
+            elif isinstance(cb_res, Awaitable):
+                cb_res = await cb_res
             cb_res = self.process_results(response, cb_res)
             for request_or_item in iterate_spider_output(cb_res):
                 yield request_or_item
diff --git a/scrapy/templates/project/module/settings.py.tmpl b/scrapy/templates/project/module/settings.py.tmpl
index a414b5fde..bbf60982c 100644
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@@ -86,3 +86,7 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index c290e376c..c84b51e8c 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,8 +1,18 @@
-from collections.abc import AsyncIterable
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 
-async def collect_asyncgen(result: AsyncIterable):
+async def collect_asyncgen(result: AsyncIterable) -> list:
     results = []
     async for x in result:
         results.append(x)
     return results
+
+
+async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    """ Wraps an iterable (sync or async) into an async generator. """
+    if isinstance(it, AsyncIterable):
+        async for r in it:
+            yield r
+    else:
+        for r in it:
+            yield r
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 24a6187b9..00cc53725 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
     out = feed.copy()
     out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
     out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
-    out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
     out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
     out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
     out.setdefault("item_export_kwargs", {})
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 7ecb8ea3f..8fcf31cab 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -7,10 +7,15 @@ from asyncio import Future
 from functools import wraps
 from typing import (
     Any,
+    AsyncGenerator,
+    AsyncIterable,
     Callable,
     Coroutine,
     Generator,
     Iterable,
+    Iterator,
+    List,
+    Optional,
     Union
 )
 
@@ -87,6 +92,109 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
+class _AsyncCooperatorAdapter(Iterator):
+    """ A class that wraps an async iterable into a normal iterator suitable
+    for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
+    it calls the callable directly in the callback, instead of providing a more
+    generic interface.
+
+    On the outside, this class behaves as an iterator that yields Deferreds.
+    Each Deferred is fired with the result of the callable which was called on
+    the next result from aiterator. It raises StopIteration when aiterator is
+    exhausted, as expected.
+
+    Cooperator calls __next__() multiple times and waits on the Deferreds
+    returned from it. As async generators (since Python 3.8) don't support
+    awaiting on __anext__() several times in parallel, we need to serialize
+    this. It's done by storing the Deferreds returned from __next__() and
+    firing the oldest one when a result from __anext__() is available.
+
+    The workflow:
+    1. When __next__() is called for the first time, it creates a Deferred, stores it
+    in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
+    for self.aiterator.__anext__() and puts it into self.anext_deferred.
+    2. If __next__() is called again before self.anext_deferred fires, more Deferreds
+    are added to self.waiting_deferreds.
+    3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
+    clear self.anext_deferred.
+    3.1. _callback() calls the callable passing the result value that it takes, pops a
+    Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
+    chains those Deferreds so that the waiting Deferred will fire when the result
+    Deferred does, otherwise it fires it directly. This causes one awaiting task to
+    receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
+    called and self.anext_deferred is populated.
+    3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
+    self.aiterator is exhausted and so it sets self.finished and fires all
+    self.waiting_deferreds. Other exceptions are propagated.
+    4. If __next__() is called after __anext__() was handled, then if self.finished is
+    True, it raises StopIteration, otherwise it acts like in step 2, but if
+    self.anext_deferred is now empty is also populates it with a new __anext__().
+
+    Note that CooperativeTask ignores the value returned from the Deferred that it waits
+    for, so we fire them with None when needed.
+
+    It may be possible to write an async iterator-aware replacement for
+    Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
+    goal.
+    """
+    def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterable.__aiter__()
+        self.callable = callable
+        self.callable_args = callable_args
+        self.callable_kwargs = callable_kwargs
+        self.finished = False
+        self.waiting_deferreds: List[Deferred] = []
+        self.anext_deferred: Optional[Deferred] = None
+
+    def _callback(self, result: Any) -> None:
+        # This gets called when the result from aiterator.__anext__() is available.
+        # It calls the callable on it and sends the result to the oldest waiting Deferred
+        # (by chaining if the result is a Deferred too or by firing if not).
+        self.anext_deferred = None
+        result = self.callable(result, *self.callable_args, **self.callable_kwargs)
+        d = self.waiting_deferreds.pop(0)
+        if isinstance(result, Deferred):
+            result.chainDeferred(d)
+        else:
+            d.callback(None)
+        if self.waiting_deferreds:
+            self._call_anext()
+
+    def _errback(self, failure: Failure) -> None:
+        # This gets called on any exceptions in aiterator.__anext__().
+        # It handles StopAsyncIteration by stopping the iteration and reraises all others.
+        self.anext_deferred = None
+        failure.trap(StopAsyncIteration)
+        self.finished = True
+        for d in self.waiting_deferreds:
+            d.callback(None)
+
+    def _call_anext(self) -> None:
+        # This starts waiting for the next result from aiterator.
+        # If aiterator is exhausted, _errback will be called.
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred.addCallbacks(self._callback, self._errback)
+
+    def __next__(self) -> Deferred:
+        # This puts a new Deferred into self.waiting_deferreds and returns it.
+        # It also calls __anext__() if needed.
+        if self.finished:
+            raise StopIteration
+        d = Deferred()
+        self.waiting_deferreds.append(d)
+        if not self.anext_deferred:
+            self._call_anext()
+        return d
+
+
+def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:
+    """ Like parallel but for async iterators """
+    coop = Cooperator()
+    work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
+    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
+    return dl
+
+
 def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
     """Return a Deferred built by chaining the given callbacks"""
     d = Deferred()
@@ -136,6 +244,20 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
             errback(failure.Failure(), *a, **kw)
 
 
+async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:
+    """Wraps an async iterable calling an errback if an error is caught while
+    iterating it. Similar to scrapy.utils.defer.iter_errback()
+    """
+    it = aiterable.__aiter__()
+    while True:
+        try:
+            yield await it.__anext__()
+        except StopAsyncIteration:
+            break
+        except Exception:
+            errback(failure.Failure(), *a, **kw)
+
+
 def deferred_from_coro(o) -> Any:
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, Deferred):
diff --git a/scrapy/utils/display.py b/scrapy/utils/display.py
index f4d17224b..d28df40c7 100644
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support
 import ctypes
 import platform
 import sys
-from distutils.version import LooseVersion as parse_version
+from packaging.version import Version as parse_version
 from pprint import pformat as pformat_
 
 
diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.py
index 1221b39b2..4d4fb9600 100644
--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@@ -226,7 +226,14 @@ def is_generator_with_return_value(callable):
         return value is None or isinstance(value, ast.NameConstant) and value.value is None
 
     if inspect.isgeneratorfunction(callable):
-        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        src = inspect.getsource(callable)
+        pattern = re.compile(r"(^[\t ]+)")
+        code = pattern.sub("", src)
+
+        match = pattern.match(src)  # finds indentation
+        if match:
+            code = re.sub(f"\n{match.group(0)}", "\n", code)  # remove indentation
+
         tree = ast.parse(code)
         for node in walk_callable(tree):
             if isinstance(node, ast.Return) and not returns_none(node):
diff --git a/scrapy/utils/py36.py b/scrapy/utils/py36.py
deleted file mode 100644
index 653e2bbbb..000000000
--- a/scrapy/utils/py36.py
+++ /dev/null
@@ -1,11 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncgen import collect_asyncgen  # noqa: F401
-
-
-warnings.warn(
-    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
-    category=ScrapyDeprecationWarning,
-    stacklevel=2,
-)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index bcc12f24f..8ce030d9d 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,11 +8,12 @@ import re
 import sys
 import warnings
 import weakref
-from collections.abc import Iterable
 from functools import partial, wraps
 from itertools import chain
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.decorators import deprecated
 
 
@@ -179,23 +180,6 @@ def binary_is_text(data):
     return all(c not in _BINARYCHARS for c in data)
 
 
-def _getargspec_py23(func):
-    """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
-                                                        defaults)
-
-    Was identical to inspect.getargspec() in python2, but uses
-    inspect.getfullargspec() for python3 behind the scenes to avoid
-    DeprecationWarning.
-
-    >>> def f(a, b=2, *ar, **kw):
-    ...     pass
-
-    >>> _getargspec_py23(f)
-    ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
-    """
-    return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
-
-
 def get_func_args(func, stripself=False):
     """Return the argument name list of a callable"""
     if inspect.isfunction(func):
@@ -247,9 +231,9 @@ def get_spec(func):
     """
 
     if inspect.isfunction(func) or inspect.ismethod(func):
-        spec = _getargspec_py23(func)
+        spec = inspect.getfullargspec(func)
     elif hasattr(func, '__call__'):
-        spec = _getargspec_py23(func.__call__)
+        spec = inspect.getfullargspec(func.__call__)
     else:
         raise TypeError(f'{type(func)} is not callable')
 
@@ -344,7 +328,7 @@ class MutableChain(Iterable):
     def __init__(self, *args: Iterable):
         self.data = chain.from_iterable(args)
 
-    def extend(self, *iterables: Iterable):
+    def extend(self, *iterables: Iterable) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):
@@ -356,3 +340,27 @@ class MutableChain(Iterable):
     @deprecated("scrapy.utils.python.MutableChain.__next__")
     def next(self):
         return self.__next__()
+
+
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    for it in iterables:
+        async for o in as_async_generator(it):
+            yield o
+
+
+class MutableAsyncChain(AsyncIterable):
+    """
+    Similar to MutableChain but for async iterables
+    """
+
+    def __init__(self, *args: Union[Iterable, AsyncIterable]):
+        self.data = _async_chain(*args)
+
+    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+        self.data = _async_chain(self.data, _async_chain(*iterables))
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        return await self.data.__anext__()
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index 96395543c..652733ce8 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -83,13 +83,31 @@ def verify_installed_reactor(reactor_path):
     path."""
     from twisted.internet import reactor
     reactor_class = load_object(reactor_path)
-    if not isinstance(reactor, reactor_class):
+    if not reactor.__class__ == reactor_class:
         msg = ("The installed reactor "
                f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
                f"match the requested one ({reactor_path})")
         raise Exception(msg)
 
 
+def verify_installed_asyncio_event_loop(loop_path):
+    from twisted.internet import reactor
+    loop_class = load_object(loop_path)
+    if isinstance(reactor._asyncioEventloop, loop_class):
+        return
+    installed = (
+        f"{reactor._asyncioEventloop.__class__.__module__}"
+        f".{reactor._asyncioEventloop.__class__.__qualname__}"
+    )
+    specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
+    raise Exception(
+        "Scrapy found an asyncio Twisted reactor already "
+        f"installed, and its event loop class ({installed}) does "
+        "not match the one specified in the ASYNCIO_EVENT_LOOP "
+        f"setting ({specified})"
+    )
+
+
 def is_asyncio_reactor_installed():
     from twisted.internet import reactor
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 70ef3ba2b..fbddc41fb 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -4,7 +4,9 @@ scrapy.http.Request objects
 """
 
 import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary
 
@@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url
 
 from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 
 
-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+    for header in headers:
+        if header in request.headers:
+            yield header
+            for value in request.headers.getlist(header):
+                yield value
 
 
 def request_fingerprint(
@@ -26,6 +37,123 @@ def request_fingerprint(
     include_headers: Optional[Iterable[Union[bytes, str]]] = None,
     keep_fragments: bool = False,
 ) -> str:
+    """
+    Return the request fingerprint as an hexadecimal string.
+
+    The request fingerprint is a hash that uniquely identifies the resource the
+    request points to. For example, take the following two urls:
+
+    http://www.example.com/query?id=111&cat=222
+    http://www.example.com/query?cat=222&id=111
+
+    Even though those are two different URLs both point to the same resource
+    and are equivalent (i.e. they should return the same response).
+
+    Another example are cookies used to store session ids. Suppose the
+    following page is only accessible to authenticated users:
+
+    http://www.example.com/members/offers.html
+
+    Lots of sites use a cookie to store the session id, which adds a random
+    component to the HTTP Request and thus should be ignored when calculating
+    the fingerprint.
+
+    For this reason, request headers are ignored by default when calculating
+    the fingerprint. If you want to include specific headers use the
+    include_headers argument, which is a list of Request headers to include.
+
+    Also, servers usually ignore fragments in urls when handling requests,
+    so they are also ignored by default when calculating the fingerprint.
+    If you want to include them, set the keep_fragments argument to True
+    (for instance when handling requests with a headless browser).
+    """
+    if include_headers or keep_fragments:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component because you '
+            'need a non-default fingerprinting algorithm, and you are OK '
+            'with that non-default fingerprinting algorithm being used by '
+            'all Scrapy components and not just the one calling this '
+            'function, use crawler.request_fingerprinter.fingerprint() '
+            'instead in your Scrapy component (you can get the crawler '
+            'object from the \'from_crawler\' class method), and use the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+            'non-default fingerprinting algorithm.\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'If you switch to \'fingerprint()\', or assign the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+            '\'fingerprint()\', the generated fingerprints will not only be '
+            'bytes instead of a string, but they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    else:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component, and you '
+            'are OK with users of your component changing the fingerprinting '
+            'algorithm through settings, use '
+            'crawler.request_fingerprinter.fingerprint() instead in your '
+            'Scrapy component (you can get the crawler object from the '
+            '\'from_crawler\' class method).\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'Either way, the resulting fingerprints will be returned as '
+            'bytes, not as a string, and they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
+    if include_headers:
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
+    cache = _deprecated_fingerprint_cache.setdefault(request, {})
+    cache_key = (processed_include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = hashlib.sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+        fp.update(request.body or b'')
+        if processed_include_headers:
+            for part in _serialize_headers(processed_include_headers, request):
+                fp.update(part)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
+def _request_fingerprint_as_bytes(*args, **kwargs):
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+    request: Request,
+    *,
+    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+    keep_fragments: bool = False,
+) -> bytes:
     """
     Return the request fingerprint.
 
@@ -43,7 +171,7 @@ def request_fingerprint(
 
     http://www.example.com/members/offers.html
 
-    Lot of sites use a cookie to store the session id, which adds a random
+    Lots of sites use a cookie to store the session id, which adds a random
     component to the HTTP Request and thus should be ignored when calculating
     the fingerprint.
 
@@ -55,29 +183,96 @@ def request_fingerprint(
     so they are also ignored by default when calculating the fingerprint.
     If you want to include them, set the keep_fragments argument to True
     (for instance when handling requests with a headless browser).
-
     """
-    headers: Optional[Tuple[bytes, ...]] = None
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
     if include_headers:
-        headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
     cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (headers, keep_fragments)
+    cache_key = (processed_include_headers, keep_fragments)
     if cache_key not in cache:
-        fp = hashlib.sha1()
-        fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
-        fp.update(request.body or b'')
-        if headers:
-            for hdr in headers:
-                if hdr in request.headers:
-                    fp.update(hdr)
-                    for v in request.headers.getlist(hdr):
-                        fp.update(v)
-        cache[cache_key] = fp.hexdigest()
+        # To decode bytes reliably (JSON does not support bytes), regardless of
+        # character encoding, we use bytes.hex()
+        headers: Dict[str, List[str]] = {}
+        if processed_include_headers:
+            for header in processed_include_headers:
+                if header in request.headers:
+                    headers[header.hex()] = [
+                        header_value.hex()
+                        for header_value in request.headers.getlist(header)
+                    ]
+        fingerprint_data = {
+            'method': to_unicode(request.method),
+            'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+            'body': (request.body or b'').hex(),
+            'headers': headers,
+        }
+        fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
     return cache[cache_key]
 
 
-def request_authenticate(request: Request, username: str, password: str) -> None:
+class RequestFingerprinter:
+    """Default fingerprinter.
+
+    It takes into account a canonical version
+    (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+    <scrapy.http.Request.url>` and the values of :attr:`request.method
+    <scrapy.http.Request.method>` and :attr:`request.body
+    <scrapy.http.Request.body>`. It then generates an `SHA1
+    <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+
+    .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler=None):
+        if crawler:
+            implementation = crawler.settings.get(
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+            )
+        else:
+            implementation = '2.6'
+        if implementation == '2.6':
+            message = (
+                '\'2.6\' is a deprecated value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+                '\n'
+                'It is also the default value. In other words, it is normal '
+                'to get this warning if you have not defined a value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+                'for backward compatibility reasons, but it will change in a '
+                'future version of Scrapy.\n'
+                '\n'
+                'See the documentation of the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+                'information on how to handle this deprecation.'
+            )
+            warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+            self._fingerprint = _request_fingerprint_as_bytes
+        elif implementation == '2.7':
+            self._fingerprint = fingerprint
+        else:
+            raise ValueError(
+                f'Got an invalid value on setting '
+                f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+                f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
+                f'and \'2.7\'.'
+            )
+
+    def fingerprint(self, request):
+        return self._fingerprint(request)
+
+
+def request_authenticate(
+    request: Request,
+    username: str,
+    password: str,
+) -> None:
     """Authenticate the given request (in place) using the HTTP basic access
     authentication mechanism (RFC 2617) and the given username and password
     """
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 59fc9202f..d0fd1757d 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,7 +4,6 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.asyncgen import collect_asyncgen
 
 
 logger = logging.getLogger(__name__)
@@ -12,14 +11,13 @@ logger = logging.getLogger(__name__)
 
 def iterate_spider_output(result):
     if inspect.isasyncgen(result):
-        d = deferred_from_coro(collect_asyncgen(result))
-        d.addCallback(iterate_spider_output)
-        return d
+        return result
     elif inspect.iscoroutine(result):
         d = deferred_from_coro(result)
         d.addCallback(iterate_spider_output)
         return d
-    return arg_to_iter(result)
+    else:
+        return arg_to_iter(deferred_from_coro(result))
 
 
 def iter_spider_classes(module):
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..445cd2e3a 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
     return "".join(ftp_data)
 
 
-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
     """Return an unconfigured Crawler object. If settings_dict is given, it
     will be used to populate the crawler settings with a project level
     priority.
@@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
     from scrapy.crawler import CrawlerRunner
     from scrapy.spiders import Spider
 
-    runner = CrawlerRunner(settings_dict)
+    # Set by default settings that prevent deprecation warnings.
+    settings = {}
+    if prevent_warnings:
+        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7'
+    settings.update(settings_dict or {})
+    runner = CrawlerRunner(settings)
     return runner.create_crawler(spidercls or Spider)
 
 
@@ -110,3 +115,10 @@ def mock_google_cloud_storage():
     bucket_mock.blob.return_value = blob_mock
 
     return (client_mock, bucket_mock, blob_mock)
+
+
+def get_web_client_agent_req(url):
+    from twisted.internet import reactor
+    from twisted.web.client import Agent  # imports twisted.internet.reactor
+    agent = Agent(reactor)
+    return agent.request(b'GET', url.encode('utf-8'))
diff --git a/scrapy/utils/testsite.py b/scrapy/utils/testsite.py
index fce77be32..5d3710391 100644
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect):
     def render(self, request):
         content = util.Redirect.render(self, request)
         return content.replace(b'http-equiv=\"refresh\"',
-            b'http-no-equiv=\"do-not-refresh-me\"')
+                               b'http-no-equiv=\"do-not-refresh-me\"')
 
 
 def test_site():
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index a6a2a9e8b..21201ace5 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -5,7 +5,6 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
-import posixpath
 import re
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse
 
@@ -31,7 +30,9 @@ def url_is_from_spider(url, spider):
 
 
 def url_has_any_extension(url, extensions):
-    return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
+    """Return True if the url ends with one of the extensions provided"""
+    lowercase_path = parse_url(url).path.lower()
+    return any(lowercase_path.endswith(ext) for ext in extensions)
 
 
 def parse_url(url, encoding=None):
diff --git a/sep/sep-014.rst b/sep/sep-014.rst
index 0859e3f7c..2521aa0e5 100644
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@@ -590,11 +590,11 @@ Request Generator
 
        def generate_requests(self, response):
            """
-           Extract and process new requets from response
+           Extract and process new requests from response
            """
            requests = []
            for ext in self._request_extractors:
-               requets.extend(ext.extract_requests(response))
+               requests.extend(ext.extract_requests(response))
 
            for proc in self._request_processors:
                requests = proc(requests)
diff --git a/setup.py b/setup.py
index d86c0f285..a43cf08c8 100644
--- a/setup.py
+++ b/setup.py
@@ -19,35 +19,30 @@ def has_environment_marker_platform_impl_support():
 
 
 install_requires = [
-    'Twisted>=17.9.0',
-    'cryptography>=2.0',
+    'Twisted>=18.9.0',
+    'cryptography>=3.3',
     'cssselect>=0.9.1',
     'itemloaders>=1.0.1',
     'parsel>=1.5.0',
-    'pyOpenSSL>=16.2.0',
+    'pyOpenSSL>=21.0.0',
     'queuelib>=1.4.2',
-    'service_identity>=16.0.0',
+    'service_identity>=18.1.0',
     'w3lib>=1.17.0',
-    'zope.interface>=4.1.3',
+    'zope.interface>=5.1.0',
     'protego>=0.1.15',
     'itemadapter>=0.1.0',
     'setuptools',
+    'packaging',
     'tldextract',
+    'lxml>=4.3.0',
 ]
 extras_require = {}
 cpython_dependencies = [
-    'lxml>=3.5.0',
     'PyDispatcher>=2.0.5',
 ]
 if has_environment_marker_platform_impl_support():
     extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
     extras_require[':platform_python_implementation == "PyPy"'] = [
-        # Earlier lxml versions are affected by
-        # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
-        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
-        # generate the C headers of lxml release tarballs published since then, the
-        # first of which was:
-        'lxml>=4.0.0',
         'PyPyDispatcher>=2.1.0',
     ]
 else:
@@ -84,18 +79,18 @@ setup(
         'Operating System :: OS Independent',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
         'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
         'Programming Language :: Python :: Implementation :: CPython',
         'Programming Language :: Python :: Implementation :: PyPy',
         'Topic :: Internet :: WWW/HTTP',
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    python_requires='>=3.6',
+    python_requires='>=3.7',
     install_requires=install_requires,
     extras_require=extras_require,
 )
diff --git a/tests/CrawlerProcess/asyncio_deferred_signal.py b/tests/CrawlerProcess/asyncio_deferred_signal.py
index bdd3c1fef..b83f6a585 100644
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@@ -5,7 +5,6 @@ from typing import Optional
 from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.defer import deferred_from_coro
-from twisted.internet.defer import Deferred
 
 
 class UppercasePipeline:
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py
index f2a93074b..e561d63c7 100644
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())
 
-import scrapy
-from scrapy.crawler import CrawlerProcess
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
 
 
 class NoRequestsSpider(scrapy.Spider):
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
new file mode 100644
index 000000000..ea8242f67
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
@@ -0,0 +1,25 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
new file mode 100644
index 000000000..d24bf3031
--- /dev/null
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+from uvloop import Loop
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncio.set_event_loop(Loop())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/multi.py b/tests/CrawlerProcess/multi.py
new file mode 100644
index 000000000..aaa1af5c5
--- /dev/null
+++ b/tests/CrawlerProcess/multi.py
@@ -0,0 +1,16 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_default.py b/tests/CrawlerProcess/reactor_default.py
new file mode 100644
index 000000000..2c867df61
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default.py
@@ -0,0 +1,16 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor  # noqa: F401
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
new file mode 100644
index 000000000..c2b30b044
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_default_twisted_reactor_select.py
@@ -0,0 +1,18 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import reactor  # noqa: F401
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_select.py b/tests/CrawlerProcess/reactor_select.py
new file mode 100644
index 000000000..ca70c06a0
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select.py
@@ -0,0 +1,17 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
new file mode 100644
index 000000000..0035daf1e
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_subclass_twisted_reactor_select.py
@@ -0,0 +1,27 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet.main import installReactor
+from twisted.internet.selectreactor import SelectReactor
+
+
+class SelectReactorSubclass(SelectReactor):
+    pass
+
+
+reactor = SelectReactorSubclass()
+installReactor(reactor)
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
new file mode 100644
index 000000000..4f8394edb
--- /dev/null
+++ b/tests/CrawlerProcess/reactor_select_twisted_reactor_select.py
@@ -0,0 +1,19 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+from twisted.internet import selectreactor
+selectreactor.install()
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.selectreactor.SelectReactor",
+})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
index 1f5a44010..72bb986bc 100644
--- a/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
+++ b/tests/CrawlerProcess/twisted_reactor_custom_settings_same.py
@@ -8,6 +8,7 @@ class AsyncioReactorSpider1(scrapy.Spider):
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
     }
 
+
 class AsyncioReactorSpider2(scrapy.Spider):
     name = 'asyncio_reactor2'
     custom_settings = {
diff --git a/tests/__init__.py b/tests/__init__.py
index 12ce79fa9..bb62851dc 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -5,6 +5,7 @@ see https://docs.scrapy.org/en/latest/contributing.html#running-tests
 """
 
 import os
+import socket
 
 # ignore system-wide proxies for tests
 # which would send requests to a totally unsuspecting server
@@ -25,6 +26,15 @@ tests_datadir = os.path.join(os.path.abspath(os.path.dirname(__file__)),
                              'sample_data')
 
 
+# In some environments accessing a non-existing host doesn't raise an
+# error. In such cases we're going to skip tests which rely on it.
+try:
+    socket.getaddrinfo('non-existing-host', 80)
+    NON_EXISTING_RESOLVABLE = True
+except socket.gaierror:
+    NON_EXISTING_RESOLVABLE = False
+
+
 def get_testdata(*paths):
     """Return test data"""
     path = os.path.join(tests_datadir, *paths)
diff --git a/tests/requirements.txt b/tests/requirements.txt
index d2a8aae1b..d9373dfa8 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,14 +1,12 @@
 # Tests requirements
 attrs
-dataclasses; python_version == '3.6'
 pyftpdlib
 pytest
 pytest-cov==3.0.0
 pytest-xdist
 sybil >= 1.3.0  # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422
 testfixtures
-uvloop < 0.15.0; platform_system != "Windows" and python_version == '3.6'
-uvloop; platform_system != "Windows" and python_version > '3.6'
+uvloop; platform_system != "Windows"
 
 # optional for shell wrapper tests
 bpython
diff --git a/tests/sample_data/link_extractor/linkextractor.html b/tests/sample_data/link_extractor/linkextractor.html
index 2307ea865..e3a2a4145 100644
--- a/tests/sample_data/link_extractor/linkextractor.html
+++ b/tests/sample_data/link_extractor/linkextractor.html
@@ -1,20 +1,22 @@
+<!DOCTYPE html>
+
 <html>
-<head>
-<base href='http://example.com' />
-<title>Sample page with links for testing LinkExtractor</title>
-</head>
-<body>
-<div id='wrapper'>
-<div id='subwrapper'>
-<area href='sample1.html' />
-<a href='sample2.html'>sample 2<img src='sample2.jpg'/></a>
-</div>
-<a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
-<a href='sample3.html'>sample 3 repetition</a>
-<a href='sample3.html#foo'>sample 3 repetition with fragment</a>
-<a href='http://www.google.com/something'></a>
-<a href='http://example.com/innertag.html'><b>inner</b> tag</a>
-<a href=' page 4.html '>href with whitespaces</a>
-</div>
-</body>
-</html>
+  <head>
+    <base href='http://example.com' />
+    <title>Sample page with links for testing LinkExtractor</title>
+  </head>
+  <body>
+    <div id='wrapper'>
+      <div id='subwrapper'>
+        <area href='sample1.html' alt='sample1'/>
+        <a href='sample2.html'>sample 2<img src='sample2.jpg' alt='sample2'/></a>
+      </div>
+      <a href='http://example.com/sample3.html' title='sample 3'>sample 3 text</a>
+      <a href='sample3.html'>sample 3 repetition</a>
+      <a href='sample3.html#foo'>sample 3 repetition with fragment</a>
+      <a href='http://www.google.com/something'></a>
+      <a href='http://example.com/innertag.html'><strong>inner</strong> tag</a>
+      <a href='page 4.html'>href with whitespaces</a>
+    </div>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_latin1.html b/tests/sample_data/link_extractor/linkextractor_latin1.html
index e7eee18de..1e05bf0f0 100644
--- a/tests/sample_data/link_extractor/linkextractor_latin1.html
+++ b/tests/sample_data/link_extractor/linkextractor_latin1.html
@@ -1,3 +1,5 @@
+<!DOCTYPE html>
+
 <html>
   <head>
     <meta http-equiv="Content-Type" content="text/html; charset=latin-1">
@@ -7,11 +9,11 @@
 <body>
   <div id='wrapper'>
     <div id='subwrapper'>
-      <a href='sample_�.html'><img src='sample2.jpg'/></a>
+      <a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
     </div>
-    <a href='sample_�.html' title='sample �'>sample � text</a>
+    <a href='sample_á.html' title='sample á'>sample á text</a>
     <div id='subwrapper2'>
-      <a href='sample_�.html?price=�32&�=unit'><img src='sample3.jpg'/></a>
+      <a href='sample_ö.html?price=£32&µ=unit'><img src='sample3.jpg' alt='sample3'/></a>
     </div>
   </div>
 </body>
diff --git a/tests/sample_data/link_extractor/linkextractor_no_href.html b/tests/sample_data/link_extractor/linkextractor_no_href.html
index 0b01cede8..2d67ec6ff 100644
--- a/tests/sample_data/link_extractor/linkextractor_no_href.html
+++ b/tests/sample_data/link_extractor/linkextractor_no_href.html
@@ -1,3 +1,5 @@
+<!DOCTYPE html>
+
 <html>
   <head>
     <base href='http://example.com' />
@@ -21,5 +23,4 @@
     </div>
   </div>
 </body>
-
 </html>
\ No newline at end of file
diff --git a/tests/sample_data/link_extractor/linkextractor_noenc.html b/tests/sample_data/link_extractor/linkextractor_noenc.html
index f9166adbe..6fa137cd9 100644
--- a/tests/sample_data/link_extractor/linkextractor_noenc.html
+++ b/tests/sample_data/link_extractor/linkextractor_noenc.html
@@ -1,14 +1,17 @@
+<!DOCTYPE html>
+
 <html>
-<head>
-<base href='http://example.com' />
-<title>Sample page without encoding for testing LinkExtractor</title>
-</head>
+  <head>
+    <base href='http://example.com' />
+    <title>Sample page without encoding for testing LinkExtractor</title>
+  </head>
+
 <body>
-<div id='wrapper'>
-<div id='subwrapper'>
-<a href='sample_ñ.html'><img src='sample2.jpg'/></a>
-</div>
-<a href='sample_€.html' title='sample €'>sample € text</a>
-</div>
+  <div id='wrapper'>
+    <div id='subwrapper'>
+      <a href='sample_ñ.html'><img src='sample2.jpg' alt='sample2'/></a>
+    </div>
+    <a href='sample_€.html' title='sample €'>sample € text</a>
+  </div>
 </body>
 </html>
diff --git a/tests/sample_data/test_site/index.html b/tests/sample_data/test_site/index.html
index d268c846a..afe17d8e2 100644
--- a/tests/sample_data/test_site/index.html
+++ b/tests/sample_data/test_site/index.html
@@ -1,18 +1,15 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Scrapy test site</h1>
-
-<ul>
-<li><a href="item1.html">Item 1</li>
-<li><a href="item2.html">Item 2</li>
-<li><a href="item999.html">Item 999 (not found)</li>
-</ul>
-
-</body>
-</html>
+  <head>
+    <title>Scrapy test site</title>
+  </head>
+  <body>
+    <h1>Scrapy test site</h1>
+    <ul>
+      <li><a href="item1.html">Item 1</a></li>
+      <li><a href="item2.html">Item 2</a></li>
+      <li><a href="item999.html">Item 999 (not found)</a></li>
+    </ul>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/sample_data/test_site/item1.html b/tests/sample_data/test_site/item1.html
index ceeb6dc87..ee39f16f3 100644
--- a/tests/sample_data/test_site/item1.html
+++ b/tests/sample_data/test_site/item1.html
@@ -1,17 +1,14 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Item 1 - Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Item 1 name</h1>
-
-<ul>
-<li>Price: $100</li>
-<li>Stock: 12</li>
-</ul>
-
-</body>
+  <head>
+    <title>Item 1 - Scrapy test site</title>
+  </head>
+    <body>
+      <h1>Item 1 name</h1>
+      <ul>
+        <li>Price: $100</li>
+        <li>Stock: 12</li>
+      </ul>
+    </body>
 </html>
diff --git a/tests/sample_data/test_site/item2.html b/tests/sample_data/test_site/item2.html
index a64c92810..f40f70750 100644
--- a/tests/sample_data/test_site/item2.html
+++ b/tests/sample_data/test_site/item2.html
@@ -1,17 +1,14 @@
+<!DOCTYPE html>
+
 <html>
-
-<head>
-<title>Item 2 - Scrapy test site</title>
-</head>
-
-<body>
-
-<h1>Item 2 name</h1>
-
-<ul>
-<li>Price: $200</li>
-<li>Stock: 5</li>
-</ul>
-
-</body>
-</html>
+  <head>
+    <title>Item 2 - Scrapy test site</title>
+  </head>
+  <body>
+    <h1>Item 2 name</h1>
+    <ul>
+      <li>Price: $200</li>
+      <li>Stock: 5</li>
+    </ul>
+  </body>
+</html>
\ No newline at end of file
diff --git a/tests/spiders.py b/tests/spiders.py
index 67dbbbe0f..5ea8a4a21 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -14,7 +14,8 @@ from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.utils.test import get_from_asyncio_queue
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
+from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
 
 
 class MockServerSpider(Spider):
@@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         return reqs
 
 
+class AsyncDefAsyncioGenExcSpider(SimpleSpider):
+    name = 'asyncdef_asyncio_gen_exc'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+            if i > 5:
+                raise ValueError("Stopping the processing")
+
+
+class AsyncDefDeferredDirectSpider(SimpleSpider):
+    name = 'asyncdef_deferred_direct'
+
+    async def parse(self, response):
+        resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
 class AsyncDefAsyncioGenSpider(SimpleSpider):
 
     name = 'asyncdef_asyncio_gen'
@@ -333,6 +369,30 @@ class CrawlSpiderWithParseMethod(MockServerSpider, CrawlSpider):
         yield Request(self.mockserver.url("/status?n=202"), self.parse, cb_kwargs={"foo": "bar"})
 
 
+class CrawlSpiderWithAsyncCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async def callback"""
+    name = 'crawl_spider_with_async_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async', follow=True),
+    )
+
+    async def parse_async(self, response, foo=None):
+        self.logger.info('[parse_async] status %i (foo: %s)', response.status, foo)
+        return Request(self.mockserver.url("/status?n=202"), self.parse_async, cb_kwargs={"foo": "bar"})
+
+
+class CrawlSpiderWithAsyncGeneratorCallback(CrawlSpiderWithParseMethod):
+    """A CrawlSpider with an async generator callback"""
+    name = 'crawl_spider_with_async_generator_callback'
+    rules = (
+        Rule(LinkExtractor(), callback='parse_async_gen', follow=True),
+    )
+
+    async def parse_async_gen(self, response, foo=None):
+        self.logger.info('[parse_async_gen] status %i (foo: %s)', response.status, foo)
+        yield Request(self.mockserver.url("/status?n=202"), self.parse_async_gen, cb_kwargs={"foo": "bar"})
+
+
 class CrawlSpiderWithErrback(CrawlSpiderWithParseMethod):
     name = 'crawl_spider_with_errback'
     rules = (
diff --git a/tests/test_command_parse.py b/tests/test_command_parse.py
index f21ee971d..8368356e2 100644
--- a/tests/test_command_parse.py
+++ b/tests/test_command_parse.py
@@ -1,6 +1,7 @@
 import os
 import argparse
 from os.path import join, abspath, isfile, exists
+
 from twisted.internet import defer
 from scrapy.commands import parse
 from scrapy.settings import Settings
@@ -28,7 +29,15 @@ class ParseCommandTest(ProcessTest, SiteTest, CommandTest):
 import scrapy
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import CrawlSpider, Rule
+from scrapy.utils.test import get_from_asyncio_queue
 
+class AsyncDefAsyncioSpider(scrapy.Spider):
+
+    name = 'asyncdef{self.spider_name}'
+
+    async def parse(self, response):
+        status = await get_from_asyncio_queue(response.status)
+        return [scrapy.Item(), dict(foo='bar')]
 
 class MySpider(scrapy.Spider):
     name = '{self.spider_name}'
@@ -159,6 +168,13 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
                                            self.url('/html')])
         self.assertIn("INFO: It Works!", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_asyncio_parse_items(self):
+        status, out, stderr = yield self.execute(
+            ['--spider', 'asyncdef' + self.spider_name, '-c', 'parse', self.url('/html')]
+        )
+        self.assertIn("""[{}, {'foo': 'bar'}]""", _textmode(out))
+
     @defer.inlineCallbacks
     def test_parse_items(self):
         status, out, stderr = yield self.execute(
@@ -222,6 +238,11 @@ ITEM_PIPELINES = {{'{self.project_name}.pipelines.MyPipeline': 1}}
         self.assertRegex(_textmode(out), r"""# Scraped Items  -+\n\[\]""")
         self.assertIn("""Cannot find a rule that matches""", _textmode(stderr))
 
+    @defer.inlineCallbacks
+    def test_crawlspider_not_exists_with_not_matched_url(self):
+        status, out, stderr = yield self.execute([self.url('/invalid_url')])
+        self.assertEqual(status, 0)
+
     @defer.inlineCallbacks
     def test_output_flag(self):
         """Checks if a file was created successfully having
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 16c9559b5..33189e9be 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -6,7 +6,7 @@ from twisted.internet import defer
 from scrapy.utils.testsite import SiteTest
 from scrapy.utils.testproc import ProcessTest
 
-from tests import tests_datadir
+from tests import tests_datadir, NON_EXISTING_RESOLVABLE
 
 
 class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@@ -109,6 +109,8 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_dns_failures(self):
+        if NON_EXISTING_RESOLVABLE:
+            raise unittest.SkipTest("Non-existing hosts are resolvable")
         url = 'www.somedomainthatdoesntexi.st'
         errcode, out, err = yield self.execute([url, '-c', 'item'], check_code=False)
         self.assertEqual(errcode, 1, out or err)
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 7cd19b29a..eaca41102 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -689,8 +689,15 @@ class MySpider(scrapy.Spider):
         ])
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_asyncio_enabled_false(self):
+    def test_asyncio_enabled_default(self):
         log = self.get_log(self.debug_log_spider, args=[])
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+
+    def test_asyncio_enabled_false(self):
+        log = self.get_log(self.debug_log_spider, args=[
+            '-s', 'TWISTED_REACTOR=twisted.internet.selectreactor.SelectReactor'
+        ])
+        self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
@@ -770,6 +777,21 @@ class MySpider(scrapy.Spider):
         log = self.get_log(spider_code, args=args)
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
 
+    def test_output_stdout(self):
+        spider_code = """
+import scrapy
+
+class MySpider(scrapy.Spider):
+    name = 'myspider'
+
+    def start_requests(self):
+        self.logger.debug('FEEDS: {}'.format(self.settings.getdict('FEEDS')))
+        return []
+"""
+        args = ['-o', '-:json']
+        log = self.get_log(spider_code, args=args)
+        self.assertIn("[myspider] DEBUG: FEEDS: {'stdout:': {'format': 'json'}}", log)
+
 
 @skipIf(platform.system() != 'Windows', "Windows required for .pyw files")
 class WindowsRunSpiderCommandTest(RunSpiderCommandTest):
@@ -915,3 +937,17 @@ class MySpider(scrapy.Spider):
         args = ['-o', 'example1.json', '-O', 'example2.json']
         log = self.get_log(spider_code, args=args)
         self.assertIn("error: Please use only one of -o/--output and -O/--overwrite-output", log)
+
+
+class HelpMessageTest(CommandTest):
+
+    def setUp(self):
+        super().setUp()
+        self.commands = ["parse", "startproject", "view", "crawl", "edit",
+                         "list", "fetch", "settings", "shell", "runspider",
+                         "version", "genspider", "check", "bench"]
+
+    def test_help_messages(self):
+        for command in self.commands:
+            _, out, _ = self.proc(command, "-h")
+            self.assertIn("Usage", out)
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index d0f4a68c2..136056f50 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -5,11 +5,11 @@ from twisted.python import failure
 from twisted.trial import unittest
 
 from scrapy import FormRequest
-from scrapy.crawler import CrawlerRunner
 from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.http import Request
 from scrapy.item import Item, Field
+from scrapy.utils.test import get_crawler
 from scrapy.contracts import ContractsManager, Contract
 from scrapy.contracts.default import (
     UrlContract,
@@ -398,7 +398,7 @@ class ContractsManagerTest(unittest.TestCase):
             TestSameUrlSpider.parse_first.__doc__ = contract_doc
             TestSameUrlSpider.parse_second.__doc__ = contract_doc
 
-            crawler = CrawlerRunner().create_crawler(TestSameUrlSpider)
+            crawler = get_crawler(TestSameUrlSpider)
             yield crawler.crawl()
 
         self.assertEqual(crawler.spider.visited, 2)
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 7bda3bef2..5383ec652 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -3,6 +3,7 @@ import logging
 from ipaddress import IPv4Address
 from socket import gethostbyname
 from urllib.parse import urlparse
+import unittest
 
 from pytest import mark
 from testfixtures import LogCapture
@@ -17,19 +18,27 @@ from scrapy.exceptions import StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
 from scrapy.utils.python import to_unicode
+from scrapy.utils.test import get_crawler
+from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
+    AsyncDefAsyncioGenExcSpider,
     AsyncDefAsyncioGenLoopSpider,
     AsyncDefAsyncioGenSpider,
     AsyncDefAsyncioReqsReturnSpider,
     AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
+    AsyncDefDeferredDirectSpider,
+    AsyncDefDeferredMaybeWrappedSpider,
+    AsyncDefDeferredWrappedSpider,
     AsyncDefSpider,
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
     BytesReceivedErrbackSpider,
+    CrawlSpiderWithAsyncCallback,
+    CrawlSpiderWithAsyncGeneratorCallback,
     CrawlSpiderWithErrback,
     CrawlSpiderWithParseMethod,
     DelaySpider,
@@ -47,14 +56,13 @@ class CrawlTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_follow_all(self):
-        crawler = self.runner.create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertEqual(len(crawler.spider.urls_visited), 11)  # 10 + start_url
 
@@ -77,7 +85,7 @@ class CrawlTestCase(TestCase):
 
         settings = {"DOWNLOAD_DELAY": delay,
                     'RANDOMIZE_DOWNLOAD_DELAY': randomize}
-        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider, settings)
         yield crawler.crawl(**crawl_kwargs)
         times = crawler.spider.times
         total_time = times[-1] - times[0]
@@ -90,7 +98,7 @@ class CrawlTestCase(TestCase):
         # of ``total`` and ``delay`` values that are too small for the test
         # code above to have any meaning.
         settings["DOWNLOAD_DELAY"] = 0
-        crawler = CrawlerRunner(settings).create_crawler(FollowAllSpider)
+        crawler = get_crawler(FollowAllSpider, settings)
         yield crawler.crawl(**crawl_kwargs)
         times = crawler.spider.times
         total_time = times[-1] - times[0]
@@ -100,7 +108,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_success(self):
-        crawler = self.runner.create_crawler(DelaySpider)
+        crawler = get_crawler(DelaySpider)
         yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 > 0)
@@ -108,7 +116,7 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_timeout_failure(self):
-        crawler = CrawlerRunner({"DOWNLOAD_TIMEOUT": 0.35}).create_crawler(DelaySpider)
+        crawler = get_crawler(DelaySpider, {"DOWNLOAD_TIMEOUT": 0.35})
         yield crawler.crawl(n=0.5, mockserver=self.mockserver)
         self.assertTrue(crawler.spider.t1 > 0)
         self.assertTrue(crawler.spider.t2 == 0)
@@ -123,21 +131,23 @@ class CrawlTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_retry_503(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
         self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_conn_failed(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl("http://localhost:65432/status?n=503", mockserver=self.mockserver)
         self._assert_retried(log)
 
     @defer.inlineCallbacks
     def test_retry_dns_error(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        if NON_EXISTING_RESOLVABLE:
+            raise unittest.SkipTest("Non-existing hosts are resolvable")
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             # try to fetch the homepage of a non-existent domain
             yield crawler.crawl("http://dns.resolution.invalid./", mockserver=self.mockserver)
@@ -146,7 +156,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_before_yield(self):
         with LogCapture('scrapy', level=logging.ERROR) as log:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+            crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_before_yield=1, mockserver=self.mockserver)
 
         self.assertEqual(len(log.records), 1)
@@ -157,7 +167,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_bug_yielding(self):
         with LogCapture('scrapy', level=logging.ERROR) as log:
-            crawler = self.runner.create_crawler(BrokenStartRequestsSpider)
+            crawler = get_crawler(BrokenStartRequestsSpider)
             yield crawler.crawl(fail_yielding=1, mockserver=self.mockserver)
 
         self.assertEqual(len(log.records), 1)
@@ -168,7 +178,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_lazyness(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
+        crawler = get_crawler(BrokenStartRequestsSpider, settings)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(
             crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
@@ -177,7 +187,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_start_requests_dupes(self):
         settings = {"CONCURRENT_REQUESTS": 1}
-        crawler = CrawlerRunner(settings).create_crawler(DuplicateStartRequestsSpider)
+        crawler = get_crawler(DuplicateStartRequestsSpider, settings)
         yield crawler.crawl(dont_filter=True, distinct_urls=2, dupe_factor=3, mockserver=self.mockserver)
         self.assertEqual(crawler.spider.visited, 6)
 
@@ -206,7 +216,7 @@ Connection: close
 foo body
 with multiples lines
 '''})
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url(f"/raw?{query}"), mockserver=self.mockserver)
         self.assertEqual(str(log).count("Got response 200"), 1)
@@ -214,7 +224,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_lost(self):
         # connection lost after receiving data
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=0"), mockserver=self.mockserver)
         self._assert_retried(log)
@@ -222,7 +232,7 @@ with multiples lines
     @defer.inlineCallbacks
     def test_retry_conn_aborted(self):
         # connection lost before receiving data
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/drop?abort=1"), mockserver=self.mockserver)
         self._assert_retried(log)
@@ -241,7 +251,7 @@ with multiples lines
         req0.meta['next'] = req1
         req1.meta['next'] = req2
         req2.meta['next'] = req3
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=req0, mockserver=self.mockserver)
         # basic asserts in case of weird communication errors
         self.assertIn('responses', crawler.spider.meta)
@@ -267,7 +277,7 @@ with multiples lines
         def cb(response):
             est.append(get_engine_status(crawler.engine))
 
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
         self.assertEqual(len(est), 1, est)
         s = dict(est[0])
@@ -282,7 +292,7 @@ with multiples lines
         def cb(response):
             est.append(format_engine_status(crawler.engine))
 
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=self.mockserver.url('/'), callback_func=cb, mockserver=self.mockserver)
         self.assertEqual(len(est), 1, est)
         est = est[0].split("\n")[2:-2]  # remove header & footer
@@ -313,7 +323,7 @@ with multiples lines
             def start_requests(self):
                 raise TestError
 
-        crawler = self.runner.create_crawler(FaultySpider)
+        crawler = get_crawler(FaultySpider)
         yield self.assertFailure(crawler.crawl(mockserver=self.mockserver), TestError)
         self.assertFalse(crawler.crawling)
 
@@ -324,26 +334,28 @@ with multiples lines
                 "tests.pipelines.ZeroDivisionErrorPipeline": 300,
             }
         }
-        crawler = CrawlerRunner(settings).create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider, settings)
         yield self.assertFailure(
-            self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
+            crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver),
             ZeroDivisionError)
         self.assertFalse(crawler.crawling)
 
     @defer.inlineCallbacks
     def test_crawlerrunner_accepts_crawler(self):
-        crawler = self.runner.create_crawler(SimpleSpider)
+        crawler = get_crawler(SimpleSpider)
+        runner = CrawlerRunner()
         with LogCapture() as log:
-            yield self.runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+            yield runner.crawl(crawler, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @defer.inlineCallbacks
     def test_crawl_multiple(self):
-        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
-        self.runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
+        runner = CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
+        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        runner.crawl(SimpleSpider, self.mockserver.url("/status?n=503"), mockserver=self.mockserver)
 
         with LogCapture() as log:
-            yield self.runner.join()
+            yield runner.join()
 
         self._assert_retried(log)
         self.assertIn("Got response 200", str(log))
@@ -354,7 +366,6 @@ class CrawlSpiderTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
@@ -366,7 +377,7 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        crawler = self.runner.create_crawler(spider_cls)
+        crawler = get_crawler(spider_cls)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -374,21 +385,39 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_crawlspider_with_parse(self):
-        self.runner.crawl(CrawlSpiderWithParseMethod, mockserver=self.mockserver)
-
+        crawler = get_crawler(CrawlSpiderWithParseMethod)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(mockserver=self.mockserver)
 
         self.assertIn("[parse] status 200 (foo: None)", str(log))
         self.assertIn("[parse] status 201 (foo: None)", str(log))
         self.assertIn("[parse] status 202 (foo: bar)", str(log))
 
     @defer.inlineCallbacks
-    def test_crawlspider_with_errback(self):
-        self.runner.crawl(CrawlSpiderWithErrback, mockserver=self.mockserver)
-
+    def test_crawlspider_with_async_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncCallback)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async] status 202 (foo: bar)", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_async_generator_callback(self):
+        crawler = get_crawler(CrawlSpiderWithAsyncGeneratorCallback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
+
+        self.assertIn("[parse_async_gen] status 200 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 201 (foo: None)", str(log))
+        self.assertIn("[parse_async_gen] status 202 (foo: bar)", str(log))
+
+    @defer.inlineCallbacks
+    def test_crawlspider_with_errback(self):
+        crawler = get_crawler(CrawlSpiderWithErrback)
+        with LogCapture() as log:
+            yield crawler.crawl(mockserver=self.mockserver)
 
         self.assertIn("[parse] status 200 (foo: None)", str(log))
         self.assertIn("[parse] status 201 (foo: None)", str(log))
@@ -399,18 +428,19 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_async_def_parse(self):
-        self.runner.crawl(AsyncDefSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        crawler = get_crawler(AsyncDefSpider)
         with LogCapture() as log:
-            yield self.runner.join()
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncio_parse(self):
-        runner = CrawlerRunner({"TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"})
-        runner.crawl(AsyncDefAsyncioSpider, self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
+        crawler = get_crawler(AsyncDefAsyncioSpider, {
+            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor"
+        })
         with LogCapture() as log:
-            yield runner.join()
+            yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
         self.assertIn("Got response 200", str(log))
 
     @mark.only_asyncio()
@@ -429,7 +459,7 @@ class CrawlSpiderTestCase(TestCase):
         def _on_item_scraped(item):
             items.append(item)
 
-        crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSingleElementSpider)
+        crawler = get_crawler(AsyncDefAsyncioReturnSingleElementSpider)
         crawler.signals.connect(_on_item_scraped, signals.item_scraped)
         with LogCapture() as log:
             yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
@@ -454,6 +484,18 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(10):
             self.assertIn({'foo': i}, items)
 
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_exc(self):
+        log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
+        log = str(log)
+        self.assertIn("Spider error processing", log)
+        self.assertIn("ValueError", log)
+        itemcount = stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 7)
+        for i in range(7):
+            self.assertIn({'foo': i}, items)
+
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):
@@ -473,16 +515,33 @@ class CrawlSpiderTestCase(TestCase):
         for req_id in range(3):
             self.assertIn(f"Got response 200, req_id {req_id}", str(log))
 
+    @mark.only_not_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_direct(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_wrapped(self):
+        log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @defer.inlineCallbacks
+    def test_async_def_deferred_maybe_wrapped(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
     @defer.inlineCallbacks
     def test_response_ssl_certificate_none(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=False)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta['responses'][0].certificate)
 
     @defer.inlineCallbacks
     def test_response_ssl_certificate(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/echo?body=test", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         cert = crawler.spider.meta['responses'][0].certificate
@@ -493,7 +552,7 @@ class CrawlSpiderTestCase(TestCase):
     @mark.xfail(reason="Responses with no body return early and contain no certificate")
     @defer.inlineCallbacks
     def test_response_ssl_certificate_empty_response(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url("/status?n=200", is_secure=True)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         cert = crawler.spider.meta['responses'][0].certificate
@@ -503,7 +562,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address_none(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url('/status?n=200')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         ip_address = crawler.spider.meta['responses'][0].ip_address
@@ -511,7 +570,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_dns_server_ip_address(self):
-        crawler = self.runner.create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         url = self.mockserver.url('/echo?body=test')
         expected_netloc, _ = urlparse(url).netloc.split(':')
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
@@ -521,7 +580,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_bytes_received_stop_download_callback(self):
-        crawler = self.runner.create_crawler(BytesReceivedCallbackSpider)
+        crawler = get_crawler(BytesReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -530,7 +589,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_bytes_received_stop_download_errback(self):
-        crawler = self.runner.create_crawler(BytesReceivedErrbackSpider)
+        crawler = get_crawler(BytesReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
@@ -545,7 +604,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_callback(self):
-        crawler = self.runner.create_crawler(HeadersReceivedCallbackSpider)
+        crawler = get_crawler(HeadersReceivedCallbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("failure"))
         self.assertIsInstance(crawler.spider.meta["response"], Response)
@@ -553,7 +612,7 @@ class CrawlSpiderTestCase(TestCase):
 
     @defer.inlineCallbacks
     def test_headers_received_stop_download_errback(self):
-        crawler = self.runner.create_crawler(HeadersReceivedErrbackSpider)
+        crawler = get_crawler(HeadersReceivedErrbackSpider)
         yield crawler.crawl(mockserver=self.mockserver)
         self.assertIsNone(crawler.spider.meta.get("response"))
         self.assertIsInstance(crawler.spider.meta["failure"], Failure)
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8f6227109..da6024c2b 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -13,14 +13,18 @@ from twisted.trial import unittest
 
 import scrapy
 from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings, default_settings
 from scrapy.spiderloader import SpiderLoader
 from scrapy.utils.log import configure_logging, get_scrapy_root_handler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.misc import load_object
+from scrapy.utils.test import get_crawler
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.extensions import telnet
 from scrapy.utils.test import get_testenv
+from pkg_resources import parse_version
+from w3lib import __version__ as w3lib_version
 
 from tests.mockserver import MockServer
 
@@ -34,9 +38,6 @@ class BaseCrawlerTest(unittest.TestCase):
 
 class CrawlerTestCase(BaseCrawlerTest):
 
-    def setUp(self):
-        self.crawler = Crawler(DefaultSpider, Settings())
-
     def test_populate_spidercls_settings(self):
         spider_settings = {'TEST1': 'spider', 'TEST2': 'spider'}
         project_settings = {'TEST1': 'project', 'TEST3': 'project'}
@@ -46,7 +47,9 @@ class CrawlerTestCase(BaseCrawlerTest):
 
         settings = Settings()
         settings.setdict(project_settings, priority='project')
-        crawler = Crawler(CustomSettingsSpider, settings)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            crawler = Crawler(CustomSettingsSpider, settings)
 
         self.assertEqual(crawler.settings.get('TEST1'), 'spider')
         self.assertEqual(crawler.settings.get('TEST2'), 'spider')
@@ -56,12 +59,14 @@ class CrawlerTestCase(BaseCrawlerTest):
         self.assertTrue(crawler.settings.frozen)
 
     def test_crawler_accepts_dict(self):
-        crawler = Crawler(DefaultSpider, {'foo': 'bar'})
+        crawler = get_crawler(DefaultSpider, {'foo': 'bar'})
         self.assertEqual(crawler.settings['foo'], 'bar')
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
     def test_crawler_accepts_None(self):
-        crawler = Crawler(DefaultSpider)
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+            crawler = Crawler(DefaultSpider)
         self.assertOptionIsDefault(crawler.settings, 'RETRY_ENABLED')
 
     def test_crawler_rejects_spider_objects(self):
@@ -77,7 +82,7 @@ class SpiderSettingsTestCase(unittest.TestCase):
                 'AUTOTHROTTLE_ENABLED': True
             }
 
-        crawler = Crawler(MySpider, {})
+        crawler = get_crawler(MySpider)
         enabled_exts = [e.__class__ for e in crawler.extensions.middlewares]
         self.assertIn(AutoThrottle, enabled_exts)
 
@@ -91,7 +96,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
         class MySpider(scrapy.Spider):
             name = 'spider'
 
-        Crawler(MySpider, {})
+        get_crawler(MySpider)
         assert get_scrapy_root_handler() is None
 
     def test_spider_custom_settings_log_level(self):
@@ -104,13 +109,14 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             custom_settings = {
                 'LOG_LEVEL': 'INFO',
                 'LOG_FILE': log_file,
-                # disable telnet if not available to avoid an extra warning
+                # settings to avoid extra warnings
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
                 'TELNETCONSOLE_ENABLED': telnet.TWISTED_CONCH_AVAILABLE,
             }
 
         configure_logging()
         self.assertEqual(get_scrapy_root_handler().level, logging.DEBUG)
-        crawler = Crawler(MySpider, {})
+        crawler = get_crawler(MySpider)
         self.assertEqual(get_scrapy_root_handler().level, logging.INFO)
         info_count = crawler.stats.get_value('log_count/INFO')
         logging.debug('debug message')
@@ -147,7 +153,7 @@ class CrawlerLoggingTestCase(unittest.TestCase):
             }
 
         configure_logging()
-        Crawler(MySpider, {})
+        get_crawler(MySpider)
         logging.debug('debug message')
 
         with open(log_file, 'rb') as fo:
@@ -228,22 +234,25 @@ class NoRequestsSpider(scrapy.Spider):
 @mark.usefixtures('reactor_pytest')
 class CrawlerRunnerHasSpider(unittest.TestCase):
 
+    def _runner(self):
+        return CrawlerRunner({'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'})
+
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         self.assertEqual(runner.bootstrap_failed, False)
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_successful_for_several(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
         yield runner.crawl(NoRequestsSpider)
         yield runner.crawl(NoRequestsSpider)
         self.assertEqual(runner.bootstrap_failed, False)
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_failed(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
 
         try:
             yield runner.crawl(ExceptionSpider)
@@ -256,7 +265,7 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
 
     @defer.inlineCallbacks
     def test_crawler_runner_bootstrap_failed_for_several(self):
-        runner = CrawlerRunner()
+        runner = self._runner()
 
         try:
             yield runner.crawl(ExceptionSpider)
@@ -274,12 +283,14 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
         if self.reactor_pytest == 'asyncio':
             CrawlerRunner(settings={
                 "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
             })
         else:
             msg = r"The installed reactor \(.*?\) does not match the requested one \(.*?\)"
             with self.assertRaisesRegex(Exception, msg):
                 runner = CrawlerRunner(settings={
                     "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+                    "REQUEST_FINGERPRINTER_IMPLEMENTATION": "2.7",
                 })
                 yield runner.crawl(NoRequestsSpider)
 
@@ -302,6 +313,63 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    def test_multi(self):
+        log = self.run_script('multi.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_default(self):
+        log = self.run_script('reactor_default.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_default_twisted_reactor_select(self):
+        log = self.run_script('reactor_default_twisted_reactor_select.py')
+        if platform.system() in ['Windows', 'Darwin']:
+            # The goal of this test function is to test that, when a reactor is
+            # installed (the default one here) and a different reactor is
+            # configured (select here), an error raises.
+            #
+            # In Windows the default reactor is the select reactor, so that
+            # error does not raise.
+            #
+            # If that ever becomes the case on more platforms (i.e. if Linux
+            # also starts using the select reactor by default in a future
+            # version of Twisted), then we will need to rethink this test.
+            self.assertIn('Spider closed (finished)', log)
+        else:
+            self.assertNotIn('Spider closed (finished)', log)
+            self.assertIn(
+                (
+                    "does not match the requested one "
+                    "(twisted.internet.selectreactor.SelectReactor)"
+                ),
+                log,
+            )
+
+    def test_reactor_select(self):
+        log = self.run_script('reactor_select.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_select_twisted_reactor_select(self):
+        log = self.run_script('reactor_select_twisted_reactor_select.py')
+        self.assertIn('Spider closed (finished)', log)
+        self.assertNotIn("ReactorAlreadyInstalledError", log)
+
+    def test_reactor_select_subclass_twisted_reactor_select(self):
+        log = self.run_script('reactor_select_subclass_twisted_reactor_select.py')
+        self.assertNotIn('Spider closed (finished)', log)
+        self.assertIn(
+            (
+                "does not match the requested one "
+                "(twisted.internet.selectreactor.SelectReactor)"
+            ),
+            log,
+        )
+
     def test_asyncio_enabled_no_reactor(self):
         log = self.run_script('asyncio_enabled_no_reactor.py')
         self.assertIn('Spider closed (finished)', log)
@@ -312,6 +380,8 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn('Spider closed (finished)', log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
+    @mark.skipif(parse_version(w3lib_version) >= parse_version("2.0.0"),
+                 reason='w3lib 2.0.0 and later do not allow invalid domains.')
     def test_ipv6_default_name_resolver(self):
         log = self.run_script('default_name_resolver.py')
         self.assertIn('Spider closed (finished)', log)
@@ -334,33 +404,33 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             self.assertNotIn("TimeoutError", log)
             self.assertNotIn("twisted.internet.error.DNSLookupError", log)
 
-    def test_reactor_select(self):
+    def test_twisted_reactor_select(self):
         log = self.run_script("twisted_reactor_select.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
 
     @mark.skipif(platform.system() == 'Windows', reason="PollReactor is not supported on Windows")
-    def test_reactor_poll(self):
+    def test_twisted_reactor_poll(self):
         log = self.run_script("twisted_reactor_poll.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.pollreactor.PollReactor", log)
 
-    def test_reactor_asyncio(self):
+    def test_twisted_reactor_asyncio(self):
         log = self.run_script("twisted_reactor_asyncio.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings(self):
+    def test_twisted_reactor_asyncio_custom_settings(self):
         log = self.run_script("twisted_reactor_custom_settings.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings_same(self):
+    def test_twisted_reactor_asyncio_custom_settings_same(self):
         log = self.run_script("twisted_reactor_custom_settings_same.py")
         self.assertIn("Spider closed (finished)", log)
         self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
 
-    def test_reactor_asyncio_custom_settings_conflict(self):
+    def test_twisted_reactor_asyncio_custom_settings_conflict(self):
         log = self.run_script("twisted_reactor_custom_settings_conflict.py")
         self.assertIn("Using reactor: twisted.internet.selectreactor.SelectReactor", log)
         self.assertIn("(twisted.internet.selectreactor.SelectReactor) does not match the requested one", log)
@@ -384,6 +454,29 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
         self.assertIn("Using asyncio event loop: uvloop.Loop", log)
         self.assertIn("async pipeline opened!", log)
 
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    def test_asyncio_enabled_reactor_same_loop(self):
+        log = self.run_script("asyncio_enabled_reactor_same_loop.py")
+        self.assertIn("Spider closed (finished)", log)
+        self.assertIn("Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor", log)
+        self.assertIn("Using asyncio event loop: uvloop.Loop", log)
+
+    @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
+    @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
+    def test_asyncio_enabled_reactor_different_loop(self):
+        log = self.run_script("asyncio_enabled_reactor_different_loop.py")
+        self.assertNotIn("Spider closed (finished)", log)
+        self.assertIn(
+            (
+                "does not match the one specified in the ASYNCIO_EVENT_LOOP "
+                "setting (uvloop.Loop)"
+            ),
+            log,
+        )
+
     def test_default_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py")
         self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 2bb53950d..883960084 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -4,7 +4,7 @@ import shutil
 import sys
 import tempfile
 from typing import Optional, Type
-from unittest import mock
+from unittest import mock, SkipTest
 
 from testfixtures import LogCapture
 from twisted.cred import checkers, credentials, portal
@@ -25,13 +25,14 @@ from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
-from scrapy.http import Headers, Request
+from scrapy.http import Headers, HtmlResponse, Request
 from scrapy.http.response.text import TextResponse
 from scrapy.responsetypes import responsetypes
 from scrapy.spiders import Spider
 from scrapy.utils.misc import create_instance
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler, skip_if_no_boto
+from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver import (
     Echo,
     ForeverTakingResource,
@@ -389,6 +390,23 @@ class HttpTestCase(unittest.TestCase):
         d.addCallback(self.assertEqual, b'159')
         return d
 
+    def _test_response_class(self, filename, body, response_class):
+        def _test(response):
+            self.assertEqual(type(response), response_class)
+
+        request = Request(self.getURL(filename), body=body)
+        return self.download_request(request, Spider('foo')).addCallback(_test)
+
+    def test_response_class_from_url(self):
+        return self._test_response_class('foo.html', b'', HtmlResponse)
+
+    def test_response_class_from_body(self):
+        return self._test_response_class(
+            'foo',
+            b"<!DOCTYPE html>\n<title>.</title>",
+            HtmlResponse,
+        )
+
 
 class Http10TestCase(HttpTestCase):
     """HTTP 1.0 test case"""
@@ -774,6 +792,8 @@ class Http11ProxyTestCase(HttpProxyTestCase):
     @defer.inlineCallbacks
     def test_download_with_proxy_https_timeout(self):
         """ Test TunnelingTCP4ClientEndpoint """
+        if NON_EXISTING_RESOLVABLE:
+            raise SkipTest("Non-existing hosts are resolvable")
         http_proxy = self.getURL('')
         domain = 'https://no-such-domain.nosuch'
         request = Request(
@@ -971,6 +991,12 @@ class BaseFTPTestCase(unittest.TestCase):
     password = "passwd"
     req_meta = {"ftp_user": username, "ftp_password": password}
 
+    test_files = (
+        ('file.txt', b"I have the power!"),
+        ('file with spaces.txt', b"Moooooooooo power!"),
+        ('html-file-without-extension', b"<!DOCTYPE html>\n<title>.</title>"),
+    )
+
     def setUp(self):
         from twisted.protocols.ftp import FTPRealm, FTPFactory
         from scrapy.core.downloader.handlers.ftp import FTPDownloadHandler
@@ -981,8 +1007,8 @@ class BaseFTPTestCase(unittest.TestCase):
         userdir = os.path.join(self.directory, self.username)
         os.mkdir(userdir)
         fp = FilePath(userdir)
-        fp.child('file.txt').setContent(b"I have the power!")
-        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+        for filename, content in self.test_files:
+            fp.child(filename).setContent(content)
 
         # setup server
         realm = FTPRealm(anonymousRoot=self.directory, userHome=self.directory)
@@ -1069,6 +1095,27 @@ class BaseFTPTestCase(unittest.TestCase):
 
         return self._add_test_callbacks(d, _test)
 
+    def _test_response_class(self, filename, response_class):
+        f, local_fname = tempfile.mkstemp()
+        local_fname = to_bytes(local_fname)
+        os.close(f)
+        meta = {}
+        meta.update(self.req_meta)
+        request = Request(url=f"ftp://127.0.0.1:{self.portNum}/{filename}",
+                          meta=meta)
+        d = self.download_handler.download_request(request, None)
+
+        def _test(r):
+            self.assertEqual(type(r), response_class)
+            os.remove(local_fname)
+        return self._add_test_callbacks(d, _test)
+
+    def test_response_class_from_url(self):
+        return self._test_response_class('file.txt', TextResponse)
+
+    def test_response_class_from_body(self):
+        return self._test_response_class('html-file-without-extension', HtmlResponse)
+
 
 class FTPTestCase(BaseFTPTestCase):
 
@@ -1104,8 +1151,8 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
         os.mkdir(self.directory)
 
         fp = FilePath(self.directory)
-        fp.child('file.txt').setContent(b"I have the power!")
-        fp.child('file with spaces.txt').setContent(b"Moooooooooo power!")
+        for filename, content in self.test_files:
+            fp.child(filename).setContent(content)
 
         # setup server for anonymous access
         realm = FTPRealm(anonymousRoot=self.directory)
diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py
index b538a0ed3..38be915f2 100644
--- a/tests/test_downloadermiddleware.py
+++ b/tests/test_downloadermiddleware.py
@@ -1,13 +1,11 @@
 import asyncio
-from unittest import mock, SkipTest
+from unittest import mock
 
 from pytest import mark
-from twisted import version as twisted_version
 from twisted.internet import defer
 from twisted.internet.defer import Deferred
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
-from twisted.python.versions import Version
 
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
@@ -218,16 +216,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
     """Middlewares using asyncio coroutines should work"""
 
     def test_asyncdef(self):
-        if (
-            self.reactor_pytest == 'asyncio'
-            and twisted_version < Version('twisted', 18, 4, 0)
-        ):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'hangs when using AsyncIO and Twisted versions lower than '
-                '18.4.0'
-            )
-
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
@@ -248,12 +236,6 @@ class MiddlewareUsingCoro(ManagerTestCase):
 
     @mark.only_asyncio()
     def test_asyncdef_asyncio(self):
-        if twisted_version < Version('twisted', 18, 4, 0):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'hangs when using Twisted versions lower than 18.4.0'
-            )
-
         resp = Response('http://example.com/index.html')
 
         class CoroMiddleware:
diff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.py
index 0362e2018..b9f3e24a4 100644
--- a/tests/test_downloadermiddleware_httpauth.py
+++ b/tests/test_downloadermiddleware_httpauth.py
@@ -1,7 +1,9 @@
 import unittest
 
+import pytest
 from w3lib.http import basic_auth_header
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware
 from scrapy.spiders import Spider
@@ -30,8 +32,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
         self.spider = TestSpiderLegacy('foo')
 
     def test_auth(self):
-        mw = HttpAuthMiddleware()
-        mw.spider_opened(self.spider)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+            mw = HttpAuthMiddleware()
+            mw.spider_opened(self.spider)
 
         # initial request, sets the domain and sends the header
         req = Request('http://example.com/')
@@ -49,8 +53,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):
         self.assertNotIn('Authorization', req.headers)
 
     def test_auth_already_set(self):
-        mw = HttpAuthMiddleware()
-        mw.spider_opened(self.spider)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Using HttpAuthMiddleware without http_auth_domain is deprecated"):
+            mw = HttpAuthMiddleware()
+            mw.spider_opened(self.spider)
         req = Request('http://example.com/',
                       headers=dict(Authorization='Digest 123'))
         assert mw.process_request(req, self.spider) is None
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index 0c6dcf2aa..928c007f5 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -122,6 +122,21 @@ class DefaultStorageTest(_BaseTest):
             time.sleep(0.5)  # give the chance to expire
             assert storage.retrieve_response(self.spider, self.request)
 
+    def test_storage_no_content_type_header(self):
+        """Test that the response body is used to get the right response class
+        even if there is no Content-Type header"""
+        with self._storage() as storage:
+            assert storage.retrieve_response(self.spider, self.request) is None
+            response = Response(
+                'http://www.example.com',
+                body=b'<!DOCTYPE html>\n<title>.</title>',
+                status=202,
+            )
+            storage.store_response(self.spider, self.request, response)
+            cached_response = storage.retrieve_response(self.spider, self.request)
+            self.assertIsInstance(cached_response, HtmlResponse)
+            self.assertEqualResponse(response, cached_response)
+
 
 class DbmStorageTest(DefaultStorageTest):
 
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7c97bf32a..44434f90e 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,13 +1,13 @@
 import os
-from functools import partial
+
+import pytest
 from twisted.trial.unittest import TestCase
 
 from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.crawler import Crawler
-from scrapy.settings import Settings
+from scrapy.utils.test import get_crawler
 
 spider = Spider('foo')
 
@@ -23,9 +23,9 @@ class TestHttpProxyMiddleware(TestCase):
         os.environ = self._oldenv
 
     def test_not_enabled(self):
-        settings = Settings({'HTTPPROXY_ENABLED': False})
-        crawler = Crawler(Spider, settings)
-        self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+        crawler = get_crawler(Spider, {'HTTPPROXY_ENABLED': False})
+        with pytest.raises(NotConfigured):
+            HttpProxyMiddleware.from_crawler(crawler)
 
     def test_no_environment_proxies(self):
         os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
@@ -65,12 +65,12 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjpwYXNz')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:password@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6cGFzc3dvcmQ=')
 
     def test_proxy_auth_empty_passwd(self):
@@ -78,12 +78,12 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware()
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://username:@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic dXNlcm5hbWU6')
 
     def test_proxy_auth_encoding(self):
@@ -92,26 +92,26 @@ class TestHttpProxyMiddleware(TestCase):
         mw = HttpProxyMiddleware(auth_encoding='utf-8')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
 
         # proxy from request.meta
         req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic w7xzZXI6cGFzcw==')
 
         # default latin-1 encoding
         mw = HttpProxyMiddleware(auth_encoding='latin-1')
         req = Request('http://scrapytest.org')
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
 
         # proxy from request.meta, latin-1 encoding
         req = Request('http://scrapytest.org', meta={'proxy': 'https://\u00FCser:pass@proxy:3128'})
         assert mw.process_request(req, spider) is None
-        self.assertEqual(req.meta, {'proxy': 'https://proxy:3128'})
+        self.assertEqual(req.meta['proxy'], 'https://proxy:3128')
         self.assertEqual(req.headers.get('Proxy-Authorization'), b'Basic /HNlcjpwYXNz')
 
     def test_proxy_already_seted(self):
@@ -152,3 +152,312 @@ class TestHttpProxyMiddleware(TestCase):
         # '/var/run/docker.sock' may be used by the user for
         # no_proxy value but is not parseable and should be skipped
         assert 'no' not in mw.proxies
+
+    def test_add_proxy_without_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request('https://example.com')
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_add_proxy_with_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request('https://example.com')
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://user1:password1@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_remove_proxy_without_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = None
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_remove_proxy_with_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = None
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_add_credentials(self):
+        """If the proxy request meta switches to a proxy URL with the same
+        proxy and adds credentials (there were no credentials before), the new
+        credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_credentials(self):
+        """If the proxy request meta switches to a proxy URL with different
+        credentials, those new credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://user2:password2@example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_remove_credentials(self):
+        """If the proxy request meta switches to a proxy URL with the same
+        proxy but no credentials, the original credentials must be still
+        used.
+
+        To remove credentials while keeping the same proxy URL, users must
+        delete the Proxy-Authorization header.
+        """
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+        request.meta['proxy'] = 'https://example.com'
+        del request.headers[b'Proxy-Authorization']
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_add_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_proxy_keep_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user1:password1@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+        # Make sure, indirectly, that _auth_proxy is updated.
+        request.meta['proxy'] = 'https://example.com'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_change_credentials(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+
+        request.meta['proxy'] = 'https://user2:password2@example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.org')
+        encoded_credentials = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_change_proxy_remove_credentials(self):
+        """If the proxy request meta switches to a proxy URL with a different
+        proxy and no credentials, no credentials must be used."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.org'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_change_proxy_remove_credentials_preremoved_header(self):
+        """Corner case of proxy switch with credentials removal where the
+        credentials have been removed beforehand.
+
+        It ensures that our implementation does not assume that the credentials
+        header exists when trying to remove it.
+        """
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        request.meta['proxy'] = 'https://example.org'
+        del request.headers[b'Proxy-Authorization']
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta, {'proxy': 'https://example.org'})
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_undefined_proxy(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertNotIn('proxy', request.meta)
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_disabled_proxy(self):
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+            meta={'proxy': None},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertIsNone(request.meta['proxy'])
+        self.assertNotIn(b'Proxy-Authorization', request.headers)
+
+    def test_proxy_authentication_header_proxy_without_credentials(self):
+        """As long as the proxy URL in request metadata remains the same, the
+        Proxy-Authorization header is used and kept, and may even be
+        changed."""
+        middleware = HttpProxyMiddleware()
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': 'Basic foo'},
+            meta={'proxy': 'https://example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+        request.headers['Proxy-Authorization'] = b'Basic bar'
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')
+
+    def test_proxy_authentication_header_proxy_with_same_credentials(self):
+        middleware = HttpProxyMiddleware()
+        encoded_credentials = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials},
+            meta={'proxy': 'https://user1:password1@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials,
+        )
+
+    def test_proxy_authentication_header_proxy_with_different_credentials(self):
+        middleware = HttpProxyMiddleware()
+        encoded_credentials1 = middleware._basic_auth_header(
+            'user1',
+            'password1',
+        )
+        request = Request(
+            'https://example.com',
+            headers={'Proxy-Authorization': b'Basic ' + encoded_credentials1},
+            meta={'proxy': 'https://user2:password2@example.com'},
+        )
+        assert middleware.process_request(request, spider) is None
+        self.assertEqual(request.meta['proxy'], 'https://example.com')
+        encoded_credentials2 = middleware._basic_auth_header(
+            'user2',
+            'password2',
+        )
+        self.assertEqual(
+            request.headers['Proxy-Authorization'],
+            b'Basic ' + encoded_credentials2,
+        )
diff --git a/tests/test_downloadermiddleware_stats.py b/tests/test_downloadermiddleware_stats.py
index 9e75f0a50..7d88ba4d2 100644
--- a/tests/test_downloadermiddleware_stats.py
+++ b/tests/test_downloadermiddleware_stats.py
@@ -1,7 +1,9 @@
+import warnings
 from itertools import product
 from unittest import TestCase
 
 from scrapy.downloadermiddlewares.stats import DownloaderStats
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.response import response_httprepr
@@ -54,7 +56,10 @@ class TestDownloaderStats(TestCase):
         for test_response in test_responses:
             self.crawler.stats.set_value('downloader/response_bytes', 0)
             self.mw.process_response(self.req, test_response, self.spider)
-            self.assertStatsEqual('downloader/response_bytes', len(response_httprepr(test_response)))
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+                resp_size = len(response_httprepr(test_response))
+            self.assertStatsEqual('downloader/response_bytes', resp_size)
 
     def test_process_exception(self):
         self.mw.process_exception(self.req, MyException(), self.spider)
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 680bb6dc8..6ebb716b0 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -10,17 +10,25 @@ from scrapy.dupefilters import RFPDupeFilter
 from scrapy.http import Request
 from scrapy.core.scheduler import Scheduler
 from scrapy.utils.python import to_bytes
-from scrapy.utils.job import job_dir
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
 
 
+def _get_dupefilter(*, crawler=None, settings=None, open=True):
+    if crawler is None:
+        crawler = get_crawler(settings_dict=settings)
+    scheduler = Scheduler.from_crawler(crawler)
+    dupefilter = scheduler.df
+    if open:
+        dupefilter.open()
+    return dupefilter
+
+
 class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
     def from_crawler(cls, crawler):
-        debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
-        df = cls(job_dir(crawler.settings), debug)
+        df = super().from_crawler(crawler)
         df.method = 'from_crawler'
         return df
 
@@ -28,9 +36,8 @@ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
 class FromSettingsRFPDupeFilter(RFPDupeFilter):
 
     @classmethod
-    def from_settings(cls, settings):
-        debug = settings.getbool('DUPEFILTER_DEBUG')
-        df = cls(job_dir(settings), debug)
+    def from_settings(cls, settings, *, fingerprinter=None):
+        df = super().from_settings(settings, fingerprinter=fingerprinter)
         df.method = 'from_settings'
         return df
 
@@ -43,7 +50,8 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_crawler_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                    'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
@@ -51,22 +59,22 @@ class RFPDupeFilterTest(unittest.TestCase):
 
     def test_df_from_settings_scheduler(self):
         settings = {'DUPEFILTER_DEBUG': True,
-                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter}
+                    'DUPEFILTER_CLASS': FromSettingsRFPDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertTrue(scheduler.df.debug)
         self.assertEqual(scheduler.df.method, 'from_settings')
 
     def test_df_direct_scheduler(self):
-        settings = {'DUPEFILTER_CLASS': DirectDupeFilter}
+        settings = {'DUPEFILTER_CLASS': DirectDupeFilter,
+                    'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
         crawler = get_crawler(settings_dict=settings)
         scheduler = Scheduler.from_crawler(crawler)
         self.assertEqual(scheduler.df.method, 'n/a')
 
     def test_filter(self):
-        dupefilter = RFPDupeFilter()
-        dupefilter.open()
-
+        dupefilter = _get_dupefilter()
         r1 = Request('http://scrapytest.org/1')
         r2 = Request('http://scrapytest.org/2')
         r3 = Request('http://scrapytest.org/2')
@@ -85,7 +93,7 @@ class RFPDupeFilterTest(unittest.TestCase):
 
         path = tempfile.mkdtemp()
         try:
-            df = RFPDupeFilter(path)
+            df = _get_dupefilter(settings={'JOBDIR': path}, open=False)
             try:
                 df.open()
                 assert not df.request_seen(r1)
@@ -93,7 +101,8 @@ class RFPDupeFilterTest(unittest.TestCase):
             finally:
                 df.close('finished')
 
-            df2 = RFPDupeFilter(path)
+            df2 = _get_dupefilter(settings={'JOBDIR': path}, open=False)
+            assert df != df2
             try:
                 df2.open()
                 assert df2.request_seen(r1)
@@ -109,26 +118,24 @@ class RFPDupeFilterTest(unittest.TestCase):
         output of request_seen.
 
         """
+        dupefilter = _get_dupefilter()
         r1 = Request('http://scrapytest.org/index.html')
         r2 = Request('http://scrapytest.org/INDEX.html')
 
-        dupefilter = RFPDupeFilter()
-        dupefilter.open()
-
         assert not dupefilter.request_seen(r1)
         assert not dupefilter.request_seen(r2)
 
         dupefilter.close('finished')
 
-        class CaseInsensitiveRFPDupeFilter(RFPDupeFilter):
+        class RequestFingerprinter:
 
-            def request_fingerprint(self, request):
+            def fingerprint(self, request):
                 fp = hashlib.sha1()
                 fp.update(to_bytes(request.url.lower()))
-                return fp.hexdigest()
+                return fp.digest()
 
-        case_insensitive_dupefilter = CaseInsensitiveRFPDupeFilter()
-        case_insensitive_dupefilter.open()
+        settings = {'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter}
+        case_insensitive_dupefilter = _get_dupefilter(settings=settings)
 
         assert not case_insensitive_dupefilter.request_seen(r1)
         assert case_insensitive_dupefilter.request_seen(r2)
@@ -142,8 +149,10 @@ class RFPDupeFilterTest(unittest.TestCase):
         r1 = Request('http://scrapytest.org/1')
 
         path = tempfile.mkdtemp()
+        crawler = get_crawler(settings_dict={'JOBDIR': path})
         try:
-            df = RFPDupeFilter(path)
+            scheduler = Scheduler.from_crawler(crawler)
+            df = scheduler.df
             df.open()
             df.request_seen(r1)
             df.close('finished')
@@ -162,13 +171,11 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': False,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
-
-            dupefilter = scheduler.df
-            dupefilter.open()
+            dupefilter = _get_dupefilter(crawler=crawler)
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html')
@@ -191,13 +198,45 @@ class RFPDupeFilterTest(unittest.TestCase):
     def test_log_debug(self):
         with LogCapture() as log:
             settings = {'DUPEFILTER_DEBUG': True,
-                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter}
+                        'DUPEFILTER_CLASS': FromCrawlerRFPDupeFilter,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
             crawler = get_crawler(SimpleSpider, settings_dict=settings)
-            scheduler = Scheduler.from_crawler(crawler)
             spider = SimpleSpider.from_crawler(crawler)
-
-            dupefilter = scheduler.df
-            dupefilter.open()
+            dupefilter = _get_dupefilter(crawler=crawler)
+
+            r1 = Request('http://scrapytest.org/index.html')
+            r2 = Request('http://scrapytest.org/index.html',
+                         headers={'Referer': 'http://scrapytest.org/INDEX.html'})
+
+            dupefilter.log(r1, spider)
+            dupefilter.log(r2, spider)
+
+            assert crawler.stats.get_value('dupefilter/filtered') == 2
+            log.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html> (referer: None)'
+                )
+            )
+            log.check_present(
+                (
+                    'scrapy.dupefilters',
+                    'DEBUG',
+                    'Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+                    ' (referer: http://scrapytest.org/INDEX.html)'
+                )
+            )
+
+            dupefilter.close('finished')
+
+    def test_log_debug_default_dupefilter(self):
+        with LogCapture() as log:
+            settings = {'DUPEFILTER_DEBUG': True,
+                        'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7'}
+            crawler = get_crawler(SimpleSpider, settings_dict=settings)
+            spider = SimpleSpider.from_crawler(crawler)
+            dupefilter = _get_dupefilter(crawler=crawler)
 
             r1 = Request('http://scrapytest.org/index.html')
             r2 = Request('http://scrapytest.org/index.html',
diff --git a/tests/test_engine.py b/tests/test_engine.py
index fa7d0c8d4..5677052f6 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -12,11 +12,14 @@ module with the ``runserver`` argument::
 
 import os
 import re
+import subprocess
 import sys
-import warnings
 from collections import defaultdict
+from threading import Timer
 from urllib.parse import urlparse
+from dataclasses import dataclass
 
+import pytest
 import attr
 from itemadapter import ItemAdapter
 from pydispatch import dispatcher
@@ -50,6 +53,13 @@ class AttrsItem:
     price = attr.ib(default=0)
 
 
+@dataclass
+class DataClassItem:
+    name: str = ""
+    url: str = ""
+    price: int = 0
+
+
 class TestSpider(Spider):
     name = "scrapytest.org"
     allowed_domains = ["scrapytest.org", "localhost"]
@@ -92,17 +102,8 @@ class AttrsItemsSpider(TestSpider):
     item_cls = AttrsItem
 
 
-try:
-    from dataclasses import make_dataclass
-except ImportError:
-    DataClassItemsSpider = None
-else:
-    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("url", str), ("price", int)])
-
-    class DataClassItemsSpider(DictItemsSpider):  # type: ignore[no-redef]
-        def parse_item(self, response):
-            item = super().parse_item(response)
-            return TestDataClass(**item)
+class DataClassItemsSpider(TestSpider):
+    item_cls = DataClassItem
 
 
 class ItemZeroDivisionErrorSpider(TestSpider):
@@ -188,7 +189,7 @@ class CrawlerRun:
         return self.deferred
 
     def stop(self):
-        self.port.stopListening()
+        self.port.stopListening()  # FIXME: wait for this Deferred
         for name, signal in vars(signals).items():
             if not name.startswith('_'):
                 disconnect_all(signal)
@@ -239,79 +240,77 @@ class EngineTest(unittest.TestCase):
     def test_crawler(self):
 
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = CrawlerRun(spider)
-            yield self.run.run()
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(count=9)
-            self._assert_downloaded_responses(count=9)
-            self._assert_scraped_items()
-            self._assert_signals_caught()
-            self._assert_bytes_received()
+            run = CrawlerRun(spider)
+            yield run.run()
+            self._assert_visited_urls(run)
+            self._assert_scheduled_requests(run, count=9)
+            self._assert_downloaded_responses(run, count=9)
+            self._assert_scraped_items(run)
+            self._assert_signals_caught(run)
+            self._assert_bytes_received(run)
 
     @defer.inlineCallbacks
     def test_crawler_dupefilter(self):
-        self.run = CrawlerRun(TestDupeFilterSpider)
-        yield self.run.run()
-        self._assert_scheduled_requests(count=8)
-        self._assert_dropped_requests()
+        run = CrawlerRun(TestDupeFilterSpider)
+        yield run.run()
+        self._assert_scheduled_requests(run, count=8)
+        self._assert_dropped_requests(run)
 
     @defer.inlineCallbacks
     def test_crawler_itemerror(self):
-        self.run = CrawlerRun(ItemZeroDivisionErrorSpider)
-        yield self.run.run()
-        self._assert_items_error()
+        run = CrawlerRun(ItemZeroDivisionErrorSpider)
+        yield run.run()
+        self._assert_items_error(run)
 
     @defer.inlineCallbacks
     def test_crawler_change_close_reason_on_idle(self):
-        self.run = CrawlerRun(ChangeCloseReasonSpider)
-        yield self.run.run()
-        self.assertEqual({'spider': self.run.spider, 'reason': 'custom_reason'},
-                         self.run.signals_caught[signals.spider_closed])
+        run = CrawlerRun(ChangeCloseReasonSpider)
+        yield run.run()
+        self.assertEqual({'spider': run.spider, 'reason': 'custom_reason'},
+                         run.signals_caught[signals.spider_closed])
 
-    def _assert_visited_urls(self):
+    def _assert_visited_urls(self, run: CrawlerRun):
         must_be_visited = ["/", "/redirect", "/redirected",
                            "/item1.html", "/item2.html", "/item999.html"]
-        urls_visited = {rp[0].url for rp in self.run.respplug}
-        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
 
-    def _assert_scheduled_requests(self, count=None):
-        self.assertEqual(count, len(self.run.reqplug))
+    def _assert_scheduled_requests(self, run: CrawlerRun, count=None):
+        self.assertEqual(count, len(run.reqplug))
 
         paths_expected = ['/item999.html', '/item2.html', '/item1.html']
 
-        urls_requested = {rq[0].url for rq in self.run.reqplug}
-        urls_expected = {self.run.geturl(p) for p in paths_expected}
+        urls_requested = {rq[0].url for rq in run.reqplug}
+        urls_expected = {run.geturl(p) for p in paths_expected}
         assert urls_expected <= urls_requested
-        scheduled_requests_count = len(self.run.reqplug)
-        dropped_requests_count = len(self.run.reqdropped)
-        responses_count = len(self.run.respplug)
+        scheduled_requests_count = len(run.reqplug)
+        dropped_requests_count = len(run.reqdropped)
+        responses_count = len(run.respplug)
         self.assertEqual(scheduled_requests_count,
                          dropped_requests_count + responses_count)
-        self.assertEqual(len(self.run.reqreached),
+        self.assertEqual(len(run.reqreached),
                          responses_count)
 
-    def _assert_dropped_requests(self):
-        self.assertEqual(len(self.run.reqdropped), 1)
+    def _assert_dropped_requests(self, run: CrawlerRun):
+        self.assertEqual(len(run.reqdropped), 1)
 
-    def _assert_downloaded_responses(self, count):
+    def _assert_downloaded_responses(self, run: CrawlerRun, count):
         # response tests
-        self.assertEqual(count, len(self.run.respplug))
-        self.assertEqual(count, len(self.run.reqreached))
+        self.assertEqual(count, len(run.respplug))
+        self.assertEqual(count, len(run.reqreached))
 
-        for response, _ in self.run.respplug:
-            if self.run.getpath(response.url) == '/item999.html':
+        for response, _ in run.respplug:
+            if run.getpath(response.url) == '/item999.html':
                 self.assertEqual(404, response.status)
-            if self.run.getpath(response.url) == '/redirect':
+            if run.getpath(response.url) == '/redirect':
                 self.assertEqual(302, response.status)
 
-    def _assert_items_error(self):
-        self.assertEqual(2, len(self.run.itemerror))
-        for item, response, spider, failure in self.run.itemerror:
+    def _assert_items_error(self, run: CrawlerRun):
+        self.assertEqual(2, len(run.itemerror))
+        for item, response, spider, failure in run.itemerror:
             self.assertEqual(failure.value.__class__, ZeroDivisionError)
-            self.assertEqual(spider, self.run.spider)
+            self.assertEqual(spider, run.spider)
 
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
@@ -321,9 +320,9 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_scraped_items(self):
-        self.assertEqual(2, len(self.run.itemresp))
-        for item, response in self.run.itemresp:
+    def _assert_scraped_items(self, run: CrawlerRun):
+        self.assertEqual(2, len(run.itemresp))
+        for item, response in run.itemresp:
             item = ItemAdapter(item)
             self.assertEqual(item['url'], response.url)
             if 'item1.html' in item['url']:
@@ -333,26 +332,26 @@ class EngineTest(unittest.TestCase):
                 self.assertEqual('Item 2 name', item['name'])
                 self.assertEqual('200', item['price'])
 
-    def _assert_headers_received(self):
-        for headers in self.run.headers.values():
+    def _assert_headers_received(self, run: CrawlerRun):
+        for headers in run.headers.values():
             self.assertIn(b"Server", headers)
             self.assertIn(b"TwistedWeb", headers[b"Server"])
             self.assertIn(b"Date", headers)
             self.assertIn(b"Content-Type", headers)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(9, len(run.bytes))
+        for request, data in run.bytes.items():
             joined_data = b"".join(data)
-            if self.run.getpath(request.url) == "/":
+            if run.getpath(request.url) == "/":
                 self.assertEqual(joined_data, get_testdata("test_site", "index.html"))
-            elif self.run.getpath(request.url) == "/item1.html":
+            elif run.getpath(request.url) == "/item1.html":
                 self.assertEqual(joined_data, get_testdata("test_site", "item1.html"))
-            elif self.run.getpath(request.url) == "/item2.html":
+            elif run.getpath(request.url) == "/item2.html":
                 self.assertEqual(joined_data, get_testdata("test_site", "item2.html"))
-            elif self.run.getpath(request.url) == "/redirected":
+            elif run.getpath(request.url) == "/redirected":
                 self.assertEqual(joined_data, b"Redirected here")
-            elif self.run.getpath(request.url) == '/redirect':
+            elif run.getpath(request.url) == '/redirect':
                 self.assertEqual(
                     joined_data,
                     b"\n<html>\n"
@@ -364,7 +363,7 @@ class EngineTest(unittest.TestCase):
                     b"    </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/tem999.html":
+            elif run.getpath(request.url) == "/tem999.html":
                 self.assertEqual(
                     joined_data,
                     b"\n<html>\n"
@@ -375,27 +374,27 @@ class EngineTest(unittest.TestCase):
                     b"  </body>\n"
                     b"</html>\n"
                 )
-            elif self.run.getpath(request.url) == "/numbers":
+            elif run.getpath(request.url) == "/numbers":
                 # signal was fired multiple times
                 self.assertTrue(len(data) > 1)
                 # bytes were received in order
                 numbers = [str(x).encode("utf8") for x in range(2**18)]
                 self.assertEqual(joined_data, b"".join(numbers))
 
-    def _assert_signals_caught(self):
-        assert signals.engine_started in self.run.signals_caught
-        assert signals.engine_stopped in self.run.signals_caught
-        assert signals.spider_opened in self.run.signals_caught
-        assert signals.spider_idle in self.run.signals_caught
-        assert signals.spider_closed in self.run.signals_caught
-        assert signals.headers_received in self.run.signals_caught
+    def _assert_signals_caught(self, run: CrawlerRun):
+        assert signals.engine_started in run.signals_caught
+        assert signals.engine_stopped in run.signals_caught
+        assert signals.spider_opened in run.signals_caught
+        assert signals.spider_idle in run.signals_caught
+        assert signals.spider_closed in run.signals_caught
+        assert signals.headers_received in run.signals_caught
 
-        self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_caught[signals.spider_opened])
-        self.assertEqual({'spider': self.run.spider},
-                         self.run.signals_caught[signals.spider_idle])
-        self.assertEqual({'spider': self.run.spider, 'reason': 'finished'},
-                         self.run.signals_caught[signals.spider_closed])
+        self.assertEqual({'spider': run.spider},
+                         run.signals_caught[signals.spider_opened])
+        self.assertEqual({'spider': run.spider},
+                         run.signals_caught[signals.spider_idle])
+        self.assertEqual({'spider': run.spider, 'reason': 'finished'},
+                         run.signals_caught[signals.spider_closed])
 
     @defer.inlineCallbacks
     def test_close_downloader(self):
@@ -407,28 +406,29 @@ class EngineTest(unittest.TestCase):
         e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
         yield e.open_spider(TestSpider(), [])
         e.start()
-        yield self.assertFailure(e.start(), RuntimeError).addBoth(
-            lambda exc: self.assertEqual(str(exc), "Engine already running")
-        )
-        yield e.stop()
+        try:
+            yield self.assertFailure(e.start(), RuntimeError).addBoth(
+                lambda exc: self.assertEqual(str(exc), "Engine already running")
+            )
+        finally:
+            yield e.stop()
 
     @defer.inlineCallbacks
     def test_close_spiders_downloader(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.open_spiders is deprecated, "
+                                "please use ExecutionEngine.spider instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             self.assertEqual(len(e.open_spiders), 1)
             yield e.close()
             self.assertEqual(len(e.open_spiders), 0)
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
-            )
 
     @defer.inlineCallbacks
     def test_close_engine_spiders_downloader(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.open_spiders is deprecated, "
+                                "please use ExecutionEngine.spider instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             yield e.open_spider(TestSpider(), [])
             e.start()
@@ -436,61 +436,47 @@ class EngineTest(unittest.TestCase):
             yield e.close()
             self.assertFalse(e.running)
             self.assertEqual(len(e.open_spiders), 0)
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.open_spiders is deprecated, please use ExecutionEngine.spider instead",
-            )
 
     @defer.inlineCallbacks
     def test_crawl_deprecated_spider_arg(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Passing a 'spider' argument to "
+                                "ExecutionEngine.crawl is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.crawl(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "Passing a 'spider' argument to ExecutionEngine.crawl is deprecated",
-            )
 
     @defer.inlineCallbacks
     def test_download_deprecated_spider_arg(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Passing a 'spider' argument to "
+                                "ExecutionEngine.download is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.download(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
-            )
 
     @defer.inlineCallbacks
     def test_deprecated_schedule(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.schedule is deprecated, please use "
+                                "ExecutionEngine.crawl or ExecutionEngine.download instead"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             spider = TestSpider()
             yield e.open_spider(spider, [])
             e.start()
             e.schedule(Request("data:,"), spider)
             yield e.close()
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(
-                str(warning_list[0].message),
-                "ExecutionEngine.schedule is deprecated, please use "
-                "ExecutionEngine.crawl or ExecutionEngine.download instead",
-            )
 
     @defer.inlineCallbacks
     def test_deprecated_has_capacity(self):
-        with warnings.catch_warnings(record=True) as warning_list:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="ExecutionEngine.has_capacity is deprecated"):
             e = ExecutionEngine(get_crawler(TestSpider), lambda _: None)
             self.assertTrue(e.has_capacity())
             spider = TestSpider()
@@ -499,8 +485,37 @@ class EngineTest(unittest.TestCase):
             e.start()
             yield e.close()
             self.assertTrue(e.has_capacity())
-            self.assertEqual(warning_list[0].category, ScrapyDeprecationWarning)
-            self.assertEqual(str(warning_list[0].message), "ExecutionEngine.has_capacity is deprecated")
+
+    def test_short_timeout(self):
+        args = (
+            sys.executable,
+            '-m',
+            'scrapy.cmdline',
+            'fetch',
+            '-s',
+            'CLOSESPIDER_TIMEOUT=0.001',
+            '-s',
+            'LOG_LEVEL=DEBUG',
+            'http://toscrape.com',
+        )
+        p = subprocess.Popen(
+            args,
+            stderr=subprocess.PIPE,
+        )
+
+        def kill_proc():
+            p.kill()
+            p.communicate()
+            assert False, 'Command took too much time to complete'
+
+        timer = Timer(15, kill_proc)
+        try:
+            timer.start()
+            _, stderr = p.communicate()
+        finally:
+            timer.cancel()
+
+        self.assertNotIn(b'Traceback', stderr)
 
 
 if __name__ == "__main__":
diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py
index 0ba69e096..933e4067d 100644
--- a/tests/test_engine_stop_download_bytes.py
+++ b/tests/test_engine_stop_download_bytes.py
@@ -23,36 +23,34 @@ class BytesReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = BytesReceivedCrawlerRun(spider)
+            run = BytesReceivedCrawlerRun(spider)
             with LogCapture() as log:
-                yield self.run.run()
+                yield run.run()
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> "
+                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> "
                                    "from signal handler BytesReceivedCrawlerRun.bytes_received"))
-            self._assert_visited_urls()
-            self._assert_scheduled_requests(count=9)
-            self._assert_downloaded_responses(count=9)
-            self._assert_signals_caught()
-            self._assert_headers_received()
-            self._assert_bytes_received()
+            self._assert_visited_urls(run)
+            self._assert_scheduled_requests(run, count=9)
+            self._assert_downloaded_responses(run, count=9)
+            self._assert_signals_caught(run)
+            self._assert_headers_received(run)
+            self._assert_bytes_received(run)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(9, len(self.run.bytes))
-        for request, data in self.run.bytes.items():
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(9, len(run.bytes))
+        for request, data in run.bytes.items():
             joined_data = b"".join(data)
             self.assertTrue(len(data) == 1)  # signal was fired only once
-            if self.run.getpath(request.url) == "/numbers":
+            if run.getpath(request.url) == "/numbers":
                 # Received bytes are not the complete response. The exact amount depends
                 # on the buffer size, which can vary, so we only check that the amount
                 # of received bytes is strictly less than the full response.
diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py
index fad6643ad..8975d0e3f 100644
--- a/tests/test_engine_stop_download_headers.py
+++ b/tests/test_engine_stop_download_headers.py
@@ -23,34 +23,32 @@ class HeadersReceivedEngineTest(EngineTest):
     @defer.inlineCallbacks
     def test_crawler(self):
         for spider in (TestSpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider):
-            if spider is None:
-                continue
-            self.run = HeadersReceivedCrawlerRun(spider)
+            run = HeadersReceivedCrawlerRun(spider)
             with LogCapture() as log:
-                yield self.run.run()
+                yield run.run()
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/redirected> from"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/redirected> from"
                                    " signal handler HeadersReceivedCrawlerRun.headers_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/> from signal"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/> from signal"
                                    " handler HeadersReceivedCrawlerRun.headers_received"))
                 log.check_present(("scrapy.core.downloader.handlers.http11",
                                    "DEBUG",
-                                   f"Download stopped for <GET http://localhost:{self.run.portno}/numbers> from"
+                                   f"Download stopped for <GET http://localhost:{run.portno}/numbers> from"
                                    " signal handler HeadersReceivedCrawlerRun.headers_received"))
-            self._assert_visited_urls()
-            self._assert_downloaded_responses(count=6)
-            self._assert_signals_caught()
-            self._assert_bytes_received()
-            self._assert_headers_received()
+            self._assert_visited_urls(run)
+            self._assert_downloaded_responses(run, count=6)
+            self._assert_signals_caught(run)
+            self._assert_bytes_received(run)
+            self._assert_headers_received(run)
 
-    def _assert_bytes_received(self):
-        self.assertEqual(0, len(self.run.bytes))
+    def _assert_bytes_received(self, run: CrawlerRun):
+        self.assertEqual(0, len(run.bytes))
 
-    def _assert_visited_urls(self):
+    def _assert_visited_urls(self, run: CrawlerRun):
         must_be_visited = ["/", "/redirect", "/redirected"]
-        urls_visited = {rp[0].url for rp in self.run.respplug}
-        urls_expected = {self.run.geturl(p) for p in must_be_visited}
+        urls_visited = {rp[0].url for rp in run.respplug}
+        urls_expected = {run.geturl(p) for p in must_be_visited}
         assert urls_expected <= urls_visited, f"URLs not visited: {list(urls_expected - urls_visited)}"
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b263b3475..69ac928c3 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,6 +4,7 @@ import marshal
 import pickle
 import tempfile
 import unittest
+import dataclasses
 from io import BytesIO
 from datetime import datetime
 from warnings import catch_warnings, filterwarnings
@@ -21,31 +22,30 @@ from scrapy.exporters import (
 )
 
 
+def custom_serializer(value):
+    return str(int(value) + 2)
+
+
 class TestItem(Item):
     name = Field()
     age = Field()
 
 
-def custom_serializer(value):
-    return str(int(value) + 2)
-
-
 class CustomFieldItem(Item):
     name = Field()
     age = Field(serializer=custom_serializer)
 
 
-try:
-    from dataclasses import make_dataclass, field
-except ImportError:
-    TestDataClass = None
-    CustomFieldDataclass = None
-else:
-    TestDataClass = make_dataclass("TestDataClass", [("name", str), ("age", int)])
-    CustomFieldDataclass = make_dataclass(
-        "CustomFieldDataclass",
-        [("name", str), ("age", int, field(metadata={"serializer": custom_serializer}))]
-    )
+@dataclasses.dataclass
+class TestDataClass:
+    name: str
+    age: int
+
+
+@dataclasses.dataclass
+class CustomFieldDataclass:
+    name: str
+    age: int = dataclasses.field(metadata={"serializer": custom_serializer})
 
 
 class BaseItemExporterTest(unittest.TestCase):
@@ -54,8 +54,6 @@ class BaseItemExporterTest(unittest.TestCase):
     custom_field_item_class = CustomFieldItem
 
     def setUp(self):
-        if self.item_class is None:
-            raise unittest.SkipTest("item class is None")
         self.i = self.item_class(name='John\xa3', age='22')
         self.output = BytesIO()
         self.ie = self._get_exporter()
@@ -112,6 +110,14 @@ class BaseItemExporterTest(unittest.TestCase):
         assert isinstance(name, str)
         self.assertEqual(name, 'John\xa3')
 
+        ie = self._get_exporter(
+            fields_to_export={'name': '名稱'}
+        )
+        self.assertEqual(
+            list(ie._get_serialized_fields(self.i)),
+            [('名稱', 'John\xa3')]
+        )
+
     def test_field_custom_serializer(self):
         i = self.custom_field_item_class(name='John\xa3', age='22')
         a = ItemAdapter(i)
@@ -272,6 +278,7 @@ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
 
 class CsvItemExporterTest(BaseItemExporterTest):
     def _get_exporter(self, **kwargs):
+        self.output = tempfile.TemporaryFile()
         return CsvItemExporter(self.output, **kwargs)
 
     def assertCsvEqual(self, first, second, msg=None):
@@ -283,7 +290,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
         return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
 
     def _check_output(self):
-        self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
+        self.output.seek(0)
+        self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
 
     def assertExportResult(self, item, expected, **kwargs):
         fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index f0acf1941..ad2383018 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -7,6 +7,7 @@ import os
 import random
 import shutil
 import string
+import sys
 import tempfile
 import warnings
 from abc import ABC, abstractmethod
@@ -21,6 +22,7 @@ from urllib.parse import urljoin, quote
 from urllib.request import pathname2url
 
 import lxml.etree
+import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial import unittest
@@ -29,7 +31,6 @@ from zope.interface import implementer
 from zope.interface.verify import verifyObject
 
 import scrapy
-from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.exporters import CsvItemExporter
 from scrapy.extensions.feedexport import (
@@ -655,8 +656,8 @@ class FeedExportTestBase(ABC, unittest.TestCase):
         return data
 
     @defer.inlineCallbacks
-    def assertExported(self, items, header, rows, settings=None, ordered=True):
-        yield self.assertExportedCsv(items, header, rows, settings, ordered)
+    def assertExported(self, items, header, rows, settings=None):
+        yield self.assertExportedCsv(items, header, rows, settings)
         yield self.assertExportedJsonLines(items, rows, settings)
         yield self.assertExportedXml(items, rows, settings)
         yield self.assertExportedPickle(items, rows, settings)
@@ -696,9 +697,9 @@ class FeedExportTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for file_path, feed_options in FEEDS.items():
                 if not os.path.exists(str(file_path)):
@@ -717,7 +718,7 @@ class FeedExportTest(FeedExportTestBase):
         return content
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
@@ -725,15 +726,9 @@ class FeedExportTest(FeedExportTestBase):
             },
         })
         data = yield self.exported_data(items, settings)
-
         reader = csv.DictReader(to_unicode(data['csv']).splitlines())
-        got_rows = list(reader)
-        if ordered:
-            self.assertEqual(reader.fieldnames, header)
-        else:
-            self.assertEqual(set(reader.fieldnames), set(header))
-
-        self.assertEqual(rows, got_rows)
+        self.assertEqual(reader.fieldnames, list(header))
+        self.assertEqual(rows, list(reader))
 
     @defer.inlineCallbacks
     def assertExportedJsonLines(self, items, rows, settings=None):
@@ -884,7 +879,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
         ]
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, ordered=False)
+        yield self.assertExported(items, header, rows)
 
     @defer.inlineCallbacks
     def test_export_no_items_not_store_empty(self):
@@ -956,25 +951,72 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam4', 'foo': '', 'baz': ''},
         ]
         rows_jl = [dict(row) for row in items]
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, header, rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
-        # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+    @defer.inlineCallbacks
+    def test_export_items_empty_field_list(self):
+        # FEED_EXPORT_FIELDS==[] means the same as default None
+        items = [{'foo': 'bar'}]
+        header = ["foo"]
+        rows = [{'foo': 'bar'}]
         settings = {'FEED_EXPORT_FIELDS': []}
-        yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
-        yield self.assertExportedJsonLines(items, rows_jl, settings)
+        yield self.assertExportedCsv(items, header, rows)
+        yield self.assertExportedJsonLines(items, rows, settings)
 
-        # it is possible to override fields using FEED_EXPORT_FIELDS
-        header = ["foo", "baz", "hello"]
+    @defer.inlineCallbacks
+    def test_export_items_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
         settings = {'FEED_EXPORT_FIELDS': header}
-        rows = [
-            {'foo': 'bar1', 'baz': '', 'hello': ''},
-            {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
-            {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
-            {'foo': '', 'baz': '', 'hello': 'world4'},
-        ]
-        yield self.assertExported(items, header, rows,
-                                  settings=settings, ordered=True)
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_comma_separated_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
+        settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_list(self):
+        items = [{'foo': 'bar'}]
+        header = ["foo", "baz"]
+        rows = [{'foo': 'bar', 'baz': ''}]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        yield self.assertExported(items, header, rows, settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {'foo': 'Foo'}
+        rows = [{'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_dict_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {
+            'baz': 'Baz',
+            'foo': 'Foo',
+        }
+        rows = [{'Baz': '', 'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': header}
+        yield self.assertExported(items, ['Baz', 'Foo'], rows,
+                                  settings=settings)
+
+    @defer.inlineCallbacks
+    def test_export_items_json_field_names(self):
+        items = [{'foo': 'bar'}]
+        header = {'foo': 'Foo'}
+        rows = [{'Foo': 'bar'}]
+        settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+        yield self.assertExported(items, list(header.values()), rows,
+                                  settings=settings)
 
     @defer.inlineCallbacks
     def test_export_based_on_item_classes(self):
@@ -1097,7 +1139,7 @@ class FeedExportTest(FeedExportTestBase):
             {'egg': 'spam', 'foo': 'bar'}
         ]
         rows_jl = items
-        yield self.assertExportedCsv(items, ['egg', 'foo'], rows_csv, ordered=False)
+        yield self.assertExportedCsv(items, ['foo', 'egg'], rows_csv)
         yield self.assertExportedJsonLines(items, rows_jl)
 
     @defer.inlineCallbacks
@@ -1118,7 +1160,7 @@ class FeedExportTest(FeedExportTestBase):
                 {'egg': 'spam2', 'foo': 'bar2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['foo', 'baz', 'egg'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
             # export a subset of columns
             settings = {'FEED_EXPORT_FIELDS': 'egg,baz'}
@@ -1127,7 +1169,7 @@ class FeedExportTest(FeedExportTestBase):
                 {'egg': 'spam2', 'baz': 'quux2'}
             ]
             yield self.assertExported(items, ['egg', 'baz'], rows,
-                                      settings=settings, ordered=True)
+                                      settings=settings)
 
     @defer.inlineCallbacks
     def test_export_encoding(self):
@@ -1512,9 +1554,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
         content = {}
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for file_path, feed_options in FEEDS.items():
                 if not os.path.exists(str(file_path)):
@@ -1769,7 +1811,6 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):
 
     @defer.inlineCallbacks
     def test_lzma_plugin_filters(self):
-        import sys
         if "PyPy" in sys.version:
             # https://foss.heptapod.net/pypy/pypy/-/issues/3527
             raise unittest.SkipTest("lzma filters doesn't work in PyPy")
@@ -1985,9 +2026,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
         content = defaultdict(list)
         try:
             with MockServer() as s:
-                runner = CrawlerRunner(Settings(settings))
                 spider_cls.start_urls = [s.url('/')]
-                yield runner.crawl(spider_cls)
+                crawler = get_crawler(spider_cls, settings)
+                yield crawler.crawl()
 
             for path, feed in FEEDS.items():
                 dir_name = os.path.dirname(path)
@@ -2007,7 +2048,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['jl']:
@@ -2016,14 +2057,14 @@ class BatchDeliveriesTest(FeedExportTestBase):
             self.assertEqual(expected_batch, got_batch)
 
     @defer.inlineCallbacks
-    def assertExportedCsv(self, items, header, rows, settings=None, ordered=True):
+    def assertExportedCsv(self, items, header, rows, settings=None):
         settings = settings or {}
         settings.update({
             'FEEDS': {
                 os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         data = yield self.exported_data(items, settings)
         for batch in data['csv']:
             got_batch = csv.DictReader(to_unicode(batch).splitlines())
@@ -2039,7 +2080,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         for batch in data['xml']:
@@ -2057,7 +2098,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         # XML
@@ -2082,7 +2123,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import pickle
@@ -2099,7 +2140,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
                 os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},
             },
         })
-        batch_size = settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT')
+        batch_size = Settings(settings).getint('FEED_EXPORT_BATCH_ITEM_COUNT')
         rows = [{k: v for k, v in row.items() if v} for row in rows]
         data = yield self.exported_data(items, settings)
         import marshal
@@ -2125,7 +2166,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_EXPORT_BATCH_ITEM_COUNT': 2
         }
         header = self.MyItem.fields.keys()
-        yield self.assertExported(items, header, rows, settings=Settings(settings))
+        yield self.assertExported(items, header, rows, settings=settings)
 
     def test_wrong_path(self):
         """ If path is without %(batch_time)s and %(batch_id) an exception must be raised """
@@ -2244,6 +2285,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             for expected_batch, got_batch in zip(expected, data[fmt]):
                 self.assertEqual(expected_batch, got_batch)
 
+    @pytest.mark.skipif(sys.platform == 'win32', reason='Odd behaviour on file creation/output')
     @defer.inlineCallbacks
     def test_batch_path_differ(self):
         """
@@ -2264,7 +2306,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
             'FEED_EXPORT_BATCH_ITEM_COUNT': 1,
         }
         data = yield self.exported_data(items, settings)
-        self.assertEqual(len(items) + 1, len(data['json']))
+        self.assertEqual(len(items), len([_ for _ in data['json'] if _]))
 
     @defer.inlineCallbacks
     def test_stats_batch_file_success(self):
@@ -2341,9 +2383,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
                     yield item
 
         with MockServer() as server:
-            runner = CrawlerRunner(Settings(settings))
             TestSpider.start_urls = [server.url('/')]
-            yield runner.crawl(TestSpider)
+            crawler = get_crawler(TestSpider, settings)
+            yield crawler.crawl()
 
         self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)
         for stub in CustomS3FeedStorage.stubs[:-1]:
@@ -2393,25 +2435,16 @@ class StdoutFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': StdoutFeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
-        with warnings.catch_warnings(record=True) as w:
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="StdoutFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "StdoutFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class FileFeedStorageWithoutFeedOptions(FileFeedStorage):
@@ -2435,25 +2468,16 @@ class FileFeedStoragePreFeedOptionsTest(unittest.TestCase):
                     'file': FileFeedStorageWithoutFeedOptions
                 },
             }
-            crawler = get_crawler(settings_dict=settings_dict)
-            feed_exporter = FeedExporter.from_crawler(crawler)
+            with pytest.warns(ScrapyDeprecationWarning,
+                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+                crawler = get_crawler(settings_dict=settings_dict)
+                feed_exporter = FeedExporter.from_crawler(crawler)
         spider = scrapy.Spider("default")
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FileFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FileFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class S3FeedStorageWithoutFeedOptions(S3FeedStorage):
@@ -2483,26 +2507,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': S3FeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="S3FeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "S3FeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
     def test_from_crawler(self):
         settings_dict = {
@@ -2511,26 +2527,18 @@ class S3FeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': S3FeedStorageWithoutFeedOptionsWithFromCrawler
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "S3FeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
-                        "does not support the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class FTPFeedStorageWithoutFeedOptions(FTPFeedStorage):
@@ -2560,26 +2568,18 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': FTPFeedStorageWithoutFeedOptions
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FTPFeedStorageWithoutFeedOptions does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FTPFeedStorageWithoutFeedOptions does not support "
-                        "the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
     def test_from_crawler(self):
         settings_dict = {
@@ -2588,50 +2588,50 @@ class FTPFeedStoragePreFeedOptionsTest(unittest.TestCase):
                 'file': FTPFeedStorageWithoutFeedOptionsWithFromCrawler
             },
         }
-        crawler = get_crawler(settings_dict=settings_dict)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+            crawler = get_crawler(settings_dict=settings_dict)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+
         spider = scrapy.Spider("default")
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler does not support "
+                                "the 'feed_options' keyword argument."):
             feed_exporter.open_spider(spider)
-            messages = tuple(str(item.message) for item in w
-                             if item.category is ScrapyDeprecationWarning)
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        "FTPFeedStorageWithoutFeedOptionsWithFromCrawler.from_crawler "
-                        "does not support the 'feed_options' keyword argument. Add a "
-                        "'feed_options' parameter to its signature to remove "
-                        "this warning. This parameter will become mandatory "
-                        "in a future version of Scrapy."
-                    ),
-                )
-            )
 
 
 class URIParamsTest:
 
     spider_name = "uri_params_spider"
+    deprecated_options = False
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         raise NotImplementedError
 
+    def _crawler_feed_exporter(self, settings):
+        if self.deprecated_options:
+            with pytest.warns(ScrapyDeprecationWarning,
+                              match="The `FEED_URI` and `FEED_FORMAT` settings have been deprecated"):
+                crawler = get_crawler(settings_dict=settings)
+                feed_exporter = FeedExporter.from_crawler(crawler)
+        else:
+            crawler = get_crawler(settings_dict=settings)
+            feed_exporter = FeedExporter.from_crawler(crawler)
+        return crawler, feed_exporter
+
     def test_default(self):
         settings = self.build_settings(
             uri='file:///tmp/%(name)s',
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2646,28 +2646,13 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with pytest.warns(ScrapyDeprecationWarning,
+                          match="Modifying the params dictionary in-place"):
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(
-                messages,
-                (
-                    (
-                        'Modifying the params dictionary in-place in the '
-                        'function defined in the FEED_URI_PARAMS setting or '
-                        'in the uri_params key of the FEEDS setting is '
-                        'deprecated. The function must return a new '
-                        'dictionary instead.'
-                    ),
-                )
-            )
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2682,18 +2667,14 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             with self.assertRaises(KeyError):
                 feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
     def test_params_as_is(self):
         def uri_params(params, spider):
@@ -2703,17 +2684,12 @@ class URIParamsTest:
             uri='file:///tmp/%(name)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2728,17 +2704,12 @@ class URIParamsTest:
             uri='file:///tmp/%(foo)s',
             uri_params=uri_params,
         )
-        crawler = get_crawler(settings_dict=settings)
-        feed_exporter = FeedExporter.from_crawler(crawler)
+        crawler, feed_exporter = self._crawler_feed_exporter(settings)
         spider = scrapy.Spider(self.spider_name)
         spider.crawler = crawler
-        with warnings.catch_warnings(record=True) as w:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", ScrapyDeprecationWarning)
             feed_exporter.open_spider(spider)
-            messages = tuple(
-                str(item.message) for item in w
-                if item.category is ScrapyDeprecationWarning
-            )
-            self.assertEqual(messages, tuple())
 
         self.assertEqual(
             feed_exporter.slots[0].uri,
@@ -2747,6 +2718,7 @@ class URIParamsTest:
 
 
 class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
+    deprecated_options = True
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         extra_settings = {}
@@ -2759,6 +2731,7 @@ class URIParamsSettingTest(URIParamsTest, unittest.TestCase):
 
 
 class URIParamsFeedOptionTest(URIParamsTest, unittest.TestCase):
+    deprecated_options = False
 
     def build_settings(self, uri='file:///tmp/foobar', uri_params=None):
         options = {
diff --git a/tests/test_http_headers.py b/tests/test_http_headers.py
index 64ff7a73d..1ca936247 100644
--- a/tests/test_http_headers.py
+++ b/tests/test_http_headers.py
@@ -38,6 +38,12 @@ class HeadersTest(unittest.TestCase):
         self.assertEqual(h.getlist('X-Forwarded-For'), [b'ip1', b'ip2'])
         assert h.getlist('X-Forwarded-For') is not hlist
 
+    def test_multivalue_for_one_header(self):
+        h = Headers((("a", "b"), ("a", "c")))
+        self.assertEqual(h["a"], b"c")
+        self.assertEqual(h.get("a"), b"c")
+        self.assertEqual(h.getlist("a"), [b"b", b"c"])
+
     def test_encode_utf8(self):
         h = Headers({'key': '\xa3'}, encoding='utf-8')
         key, val = dict(h).popitem()
diff --git a/tests/test_http_response.py b/tests/test_http_response.py
index 2986f884f..b42c95045 100644
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@@ -1,3 +1,4 @@
+import codecs
 import unittest
 from unittest import mock
 
@@ -358,6 +359,8 @@ class TextResponseTest(BaseResponseTest):
                                  headers={"Content-type": ["text/html; charset=gb2312"]})
         r7 = self.response_class("http://www.example.com", body=b"\xa8D",
                                  headers={"Content-type": ["text/html; charset=gbk"]})
+        r8 = self.response_class("http://www.example.com", body=codecs.BOM_UTF8 + b"\xc2\xa3",
+                                 headers={"Content-type": ["text/html; charset=cp1251"]})
 
         self.assertEqual(r1._headers_encoding(), "utf-8")
         self.assertEqual(r2._headers_encoding(), None)
@@ -367,7 +370,10 @@ class TextResponseTest(BaseResponseTest):
         self.assertEqual(r3._declared_encoding(), "cp1252")
         self.assertEqual(r4._headers_encoding(), None)
         self.assertEqual(r5._headers_encoding(), None)
+        self.assertEqual(r8._headers_encoding(), "cp1251")
+        self.assertEqual(r8._declared_encoding(), "utf-8")
         self._assert_response_encoding(r5, "utf-8")
+        self._assert_response_encoding(r8, "utf-8")
         assert r4._body_inferred_encoding() is not None and r4._body_inferred_encoding() != 'ascii'
         self._assert_response_values(r1, 'utf-8', "\xa3")
         self._assert_response_values(r2, 'utf-8', "\xa3")
@@ -699,7 +705,8 @@ class HtmlResponseTest(TextResponseTest):
 
     def test_html_encoding(self):
 
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
         </head><body>Price: \xa3100</body></html>'
         """
         r1 = self.response_class("http://www.example.com", body=body)
@@ -713,7 +720,8 @@ class HtmlResponseTest(TextResponseTest):
         self._assert_response_values(r2, 'iso-8859-1', body)
 
         # for conflicting declarations headers must take precedence
-        body = b"""<html><head><title>Some page</title><meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+        body = b"""<html><head><title>Some page</title>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
         </head><body>Price: \xa3100</body></html>'
         """
         r3 = self.response_class("http://www.example.com", body=body,
diff --git a/tests/test_loader.py b/tests/test_loader.py
index f7ab1f236..b3e44d36b 100644
--- a/tests/test_loader.py
+++ b/tests/test_loader.py
@@ -1,4 +1,5 @@
 import unittest
+import dataclasses
 
 import attr
 from itemadapter import ItemAdapter
@@ -10,13 +11,6 @@ from scrapy.loader import ItemLoader
 from scrapy.selector import Selector
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 # test items
 class NameItem(Item):
     name = Field()
@@ -41,6 +35,11 @@ class AttrsNameItem:
     name = attr.ib(default="")
 
 
+@dataclasses.dataclass
+class TestDataClass:
+    name: list = dataclasses.field(default_factory=list)
+
+
 # test item loaders
 class NameItemLoader(ItemLoader):
     default_item_class = TestItem
@@ -187,16 +186,8 @@ class InitializationFromAttrsItemTest(InitializationTestMixin, unittest.TestCase
     item_class = AttrsNameItem
 
 
-@unittest.skipIf(not make_dataclass, "dataclasses module is not available")
 class InitializationFromDataClassTest(InitializationTestMixin, unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "TestDataClass",
-                [("name", list, dataclass_field(default_factory=list))],
-            )
+    item_class = TestDataClass
 
 
 class BaseNoInputReprocessingLoader(ItemLoader):
@@ -304,7 +295,7 @@ class SelectortemLoaderTest(unittest.TestCase):
 
         l.add_css('name', 'div::text')
         self.assertEqual(l.get_output_value('name'), ['Marta'])
-    
+
     def test_init_method_with_base_response(self):
         """Selector should be None after initialization"""
         response = Response("https://scrapy.org")
diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py
index 6381f895b..f3bb23bda 100644
--- a/tests/test_logformatter.py
+++ b/tests/test_logformatter.py
@@ -5,8 +5,8 @@ from twisted.internet import defer
 from twisted.python.failure import Failure
 from twisted.trial.unittest import TestCase as TwistedTestCase
 
-from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import DropItem
+from scrapy.utils.test import get_crawler
 from scrapy.http import Request, Response
 from scrapy.item import Item, Field
 from scrapy.logformatter import LogFormatter
@@ -202,7 +202,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
 
     @defer.inlineCallbacks
     def test_show_messages(self):
-        crawler = CrawlerRunner(self.base_settings).create_crawler(ItemSpider)
+        crawler = get_crawler(ItemSpider, self.base_settings)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertIn("Scraped from <200 http://127.0.0.1:", str(lc))
@@ -213,7 +213,7 @@ class ShowOrSkipMessagesTestCase(TwistedTestCase):
     def test_skip_messages(self):
         settings = self.base_settings.copy()
         settings['LOG_FORMATTER'] = SkipMessagesLogFormatter
-        crawler = CrawlerRunner(settings).create_crawler(ItemSpider)
+        crawler = get_crawler(ItemSpider, settings)
         with LogCapture() as lc:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertNotIn("Scraped from <200 http://127.0.0.1:", str(lc))
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index f49fda701..0e174cd34 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -64,6 +64,7 @@ class FileDownloadCrawlTestCase(TestCase):
         self.tmpmediastore = self.mktemp()
         os.mkdir(self.tmpmediastore)
         self.settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
             'ITEM_PIPELINES': {self.pipeline_class: 1},
             self.store_setting_key: self.tmpmediastore,
         }
@@ -78,8 +79,10 @@ class FileDownloadCrawlTestCase(TestCase):
     def _on_item_scraped(self, item):
         self.items.append(item)
 
-    def _create_crawler(self, spider_class, **kwargs):
-        crawler = self.runner.create_crawler(spider_class, **kwargs)
+    def _create_crawler(self, spider_class, runner=None, **kwargs):
+        if runner is None:
+            runner = self.runner
+        crawler = runner.create_crawler(spider_class, **kwargs)
         crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
         return crawler
 
@@ -167,9 +170,8 @@ class FileDownloadCrawlTestCase(TestCase):
     def test_download_media_redirected_allowed(self):
         settings = dict(self.settings)
         settings.update({'MEDIA_ALLOW_REDIRECTS': True})
-        self.runner = CrawlerRunner(settings)
-
-        crawler = self._create_crawler(RedirectedMediaDownloadSpider)
+        runner = CrawlerRunner(settings)
+        crawler = self._create_crawler(RedirectedMediaDownloadSpider, runner=runner)
         with LogCapture() as log:
             yield crawler.crawl(
                 self.mockserver.url("/files/images/"),
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 4e1b90787..d641e7a43 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -5,8 +5,9 @@ from datetime import datetime
 from io import BytesIO
 from shutil import rmtree
 from tempfile import mkdtemp
-from unittest import mock, skipIf
+from unittest import mock
 from urllib.parse import urlparse
+import dataclasses
 
 import attr
 from itemadapter import ItemAdapter
@@ -25,19 +26,13 @@ from scrapy.pipelines.files import (
 from scrapy.settings import Settings
 from scrapy.utils.test import (
     assert_gcs_environ,
+    get_crawler,
     get_ftp_content_and_delete,
     get_gcs_content_and_delete,
     skip_if_no_boto,
 )
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -47,7 +42,9 @@ class FilesPipelineTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tempdir = mkdtemp()
-        self.pipeline = FilesPipeline.from_settings(Settings({'FILES_STORE': self.tempdir}))
+        settings_dict = {'FILES_STORE': self.tempdir}
+        crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+        self.pipeline = FilesPipeline.from_crawler(crawler)
         self.pipeline.download_func = _mocked_download_func
         self.pipeline.open_spider(None)
 
@@ -223,24 +220,19 @@ class FilesPipelineTestCaseFieldsItem(FilesPipelineTestCaseFieldsMixin, unittest
     item_class = FilesPipelineTestItem
 
 
-@skipIf(not make_dataclass, "dataclasses module is not available")
-class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+@dataclasses.dataclass
+class FilesPipelineTestDataClass:
+    name: str
+    # default fields
+    file_urls: list = dataclasses.field(default_factory=list)
+    files: list = dataclasses.field(default_factory=list)
+    # overridden fields
+    custom_file_urls: list = dataclasses.field(default_factory=list)
+    custom_files: list = dataclasses.field(default_factory=list)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "FilesPipelineTestDataClass",
-                [
-                    ("name", str),
-                    # default fields
-                    ("file_urls", list, dataclass_field(default_factory=list)),
-                    ("files", list, dataclass_field(default_factory=list)),
-                    # overridden fields
-                    ("custom_file_urls", list, dataclass_field(default_factory=list)),
-                    ("custom_files", list, dataclass_field(default_factory=list)),
-                ],
-            )
+
+class FilesPipelineTestCaseFieldsDataClass(FilesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = FilesPipelineTestDataClass
 
 
 @attr.s
@@ -525,6 +517,29 @@ class TestGCSFilesStore(unittest.TestCase):
         self.assertEqual(blob.content_type, 'application/octet-stream')
         self.assertIn(expected_policy, acl)
 
+    @defer.inlineCallbacks
+    def test_blob_path_consistency(self):
+        """Test to make sure that paths used to store files is the same as the one used to get
+        already uploaded files.
+        """
+        assert_gcs_environ()
+        try:
+            import google.cloud.storage # noqa
+        except ModuleNotFoundError:
+            raise unittest.SkipTest("google-cloud-storage is not installed")
+        else:
+            with mock.patch('google.cloud.storage') as _:
+                with mock.patch('scrapy.pipelines.files.time') as _:
+                    uri = 'gs://my_bucket/my_prefix/'
+                    store = GCSFilesStore(uri)
+                    store.bucket = mock.Mock()
+                    path = 'full/my_data.txt'
+                    yield store.persist_file(path, mock.Mock(), info=None, meta=None, headers=None)
+                    yield store.stat_file(path, info=None)
+                    expected_blob_path = store.prefix + path
+                    store.bucket.blob.assert_called_with(expected_blob_path)
+                    store.bucket.get_blob.assert_called_with(expected_blob_path)
+
 
 class TestFTPFileStore(unittest.TestCase):
     @defer.inlineCallbacks
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 613190f9c..c4ce2736f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -1,3 +1,4 @@
+import dataclasses
 import hashlib
 import io
 import random
@@ -18,13 +19,6 @@ from scrapy.settings import Settings
 from scrapy.utils.python import to_bytes
 
 
-try:
-    from dataclasses import make_dataclass, field as dataclass_field
-except ImportError:
-    make_dataclass = None
-    dataclass_field = None
-
-
 try:
     from PIL import Image
 except ImportError:
@@ -95,6 +89,22 @@ class ImagesPipelineTestCase(unittest.TestCase):
                                     info=object()),
                          'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
 
+    def test_thumbnail_name_from_item(self):
+        """
+        Custom thumbnail name based on item data, overriding default implementation
+        """
+
+        class CustomImagesPipeline(ImagesPipeline):
+            def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+                return f"thumb/{thumb_id}/{item.get('path')}"
+
+        thumb_path = CustomImagesPipeline.from_settings(Settings(
+            {'IMAGES_STORE': self.tempdir}
+        )).thumb_path
+        item = dict(path='path-to-store-file')
+        request = Request("http://example.com")
+        self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+
     def test_convert_image(self):
         SIZE = (100, 100)
         # straigh forward case: RGB and JPEG
@@ -189,25 +199,19 @@ class ImagesPipelineTestCaseFieldsItem(ImagesPipelineTestCaseFieldsMixin, unitte
     item_class = ImagesPipelineTestItem
 
 
-@skipIf(not make_dataclass, "dataclasses module is not available")
-class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
-    item_class = None
+@dataclasses.dataclass
+class ImagesPipelineTestDataClass:
+    name: str
+    # default fields
+    image_urls: list = dataclasses.field(default_factory=list)
+    images: list = dataclasses.field(default_factory=list)
+    # overridden fields
+    custom_image_urls: list = dataclasses.field(default_factory=list)
+    custom_images: list = dataclasses.field(default_factory=list)
 
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        if make_dataclass:
-            self.item_class = make_dataclass(
-                "FilesPipelineTestDataClass",
-                [
-                    ("name", str),
-                    # default fields
-                    ("image_urls", list, dataclass_field(default_factory=list)),
-                    ("images", list, dataclass_field(default_factory=list)),
-                    # overridden fields
-                    ("custom_image_urls", list, dataclass_field(default_factory=list)),
-                    ("custom_images", list, dataclass_field(default_factory=list)),
-                ],
-            )
+
+class ImagesPipelineTestCaseFieldsDataClass(ImagesPipelineTestCaseFieldsMixin, unittest.TestCase):
+    item_class = ImagesPipelineTestDataClass
 
 
 @attr.s
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 893d43052..84e867660 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,4 +1,5 @@
 from typing import Optional
+import io
 
 from testfixtures import LogCapture
 from twisted.trial import unittest
@@ -6,17 +7,17 @@ from twisted.python.failure import Failure
 from twisted.internet import reactor
 from twisted.internet.defer import Deferred, inlineCallbacks
 
+from scrapy import signals
 from scrapy.http import Request, Response
 from scrapy.settings import Settings
 from scrapy.spiders import Spider
-from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
+from scrapy.pipelines.files import FileException
 from scrapy.pipelines.images import ImagesPipeline
 from scrapy.pipelines.media import MediaPipeline
-from scrapy.pipelines.files import FileException
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
-from scrapy import signals
+from scrapy.utils.test import get_crawler
 
 
 try:
@@ -38,11 +39,14 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
     settings = None
 
     def setUp(self):
-        self.spider = Spider('media.com')
-        self.pipe = self.pipeline_class(download_func=_mocked_download_func,
-                                        settings=Settings(self.settings))
+        spider_cls = Spider
+        self.spider = spider_cls('media.com')
+        crawler = get_crawler(spider_cls, self.settings)
+        self.pipe = self.pipeline_class.from_crawler(crawler)
+        self.pipe.download_func = _mocked_download_func
         self.pipe.open_spider(self.spider)
         self.info = self.pipe.spiderinfo
+        self.fingerprint = crawler.request_fingerprinter.fingerprint
 
     def tearDown(self):
         for name, signal in vars(signals).items():
@@ -155,7 +159,7 @@ class BaseMediaPipelineTestCase(unittest.TestCase):
         self.assertEqual(failure.value.__context__, def_gen_return_exc)
 
         # Let's calculate the request fingerprint and fake some runtime data...
-        fp = request_fingerprint(request)
+        fp = self.fingerprint(request)
         info = self.pipe.spiderinfo
         info.downloading.add(fp)
         info.waiting[fp] = []
@@ -272,7 +276,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req)  # pass a single item
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
-        assert request_fingerprint(req) in self.info.downloaded
+        self.assertIn(self.fingerprint(req), self.info.downloaded)
 
         # returns iterable of Requests
         req1 = Request('http://url1')
@@ -280,8 +284,8 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=iter([req1, req2]))
         new_item = yield self.pipe.process_item(item, self.spider)
         assert new_item is item
-        assert request_fingerprint(req1) in self.info.downloaded
-        assert request_fingerprint(req2) in self.info.downloaded
+        assert self.fingerprint(req1) in self.info.downloaded
+        assert self.fingerprint(req2) in self.info.downloaded
 
     @inlineCallbacks
     def test_results_are_cached_across_multiple_items(self):
@@ -297,7 +301,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
         item = dict(requests=req2)
         new_item = yield self.pipe.process_item(item, self.spider)
         self.assertTrue(new_item is item)
-        self.assertEqual(request_fingerprint(req1), request_fingerprint(req2))
+        self.assertEqual(self.fingerprint(req1), self.fingerprint(req2))
         self.assertEqual(new_item['results'], [(True, rsp1)])
 
     @inlineCallbacks
@@ -313,7 +317,7 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
     @inlineCallbacks
     def test_wait_if_request_is_downloading(self):
         def _check_downloading(response):
-            fp = request_fingerprint(req1)
+            fp = self.fingerprint(req1)
             self.assertTrue(fp in self.info.downloading)
             self.assertTrue(fp in self.info.waiting)
             self.assertTrue(fp not in self.info.downloaded)
@@ -350,14 +354,17 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
 class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def __init__(self, *args, **kwargs):
-        super(MockedMediaPipelineDeprecatedMethods, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self._mockcalled = []
 
     def get_media_requests(self, item, info):
         item_url = item['image_urls'][0]
+        output_img = io.BytesIO()
+        img = Image.new('RGB', (60, 30), color='red')
+        img.save(output_img, format='JPEG')
         return Request(
             item_url,
-            meta={'response': Response(item_url, status=200, body=b'data')}
+            meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
         )
 
     def inc_stats(self, *args, **kwargs):
@@ -365,34 +372,44 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
     def media_to_download(self, request, info):
         self._mockcalled.append('media_to_download')
-        return super(MockedMediaPipelineDeprecatedMethods, self).media_to_download(request, info)
+        return super().media_to_download(request, info)
 
     def media_downloaded(self, response, request, info):
         self._mockcalled.append('media_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).media_downloaded(response, request, info)
+        return super().media_downloaded(response, request, info)
 
     def file_downloaded(self, response, request, info):
         self._mockcalled.append('file_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).file_downloaded(response, request, info)
+        return super().file_downloaded(response, request, info)
 
     def file_path(self, request, response=None, info=None):
         self._mockcalled.append('file_path')
-        return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+        return super().file_path(request, response, info)
+
+    def thumb_path(self, request, thumb_id, response=None, info=None):
+        self._mockcalled.append('thumb_path')
+        return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
 
     def get_images(self, response, request, info):
         self._mockcalled.append('get_images')
-        return []
+        return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
 
     def image_downloaded(self, response, request, info):
         self._mockcalled.append('image_downloaded')
-        return super(MockedMediaPipelineDeprecatedMethods, self).image_downloaded(response, request, info)
+        return super().image_downloaded(response, request, info)
 
 
 class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
     skip = skip_pillow
 
     def setUp(self):
-        self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
+        settings_dict = {
+            'IMAGES_STORE': 'store-uri',
+            'IMAGES_THUMBS': {'small': (50, 50)},
+        }
+        crawler = get_crawler(spidercls=None, settings_dict=settings_dict)
+        self.pipe = MockedMediaPipelineDeprecatedMethods.from_crawler(crawler)
+        self.pipe.download_func = _mocked_download_func
         self.pipe.open_spider(None)
         self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
 
@@ -444,6 +461,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
         )
         self._assert_method_called_with_warnings('file_path', message, warnings)
 
+    @inlineCallbacks
+    def test_thumb_path_called(self):
+        yield self.pipe.process_item(self.item, None)
+        warnings = self.flushWarnings([MediaPipeline._compatible])
+        message = (
+            'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
+            'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
+        )
+        self._assert_method_called_with_warnings('thumb_path', message, warnings)
+
     @inlineCallbacks
     def test_get_images_called(self):
         yield self.pipe.process_item(self.item, None)
diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py
index 25d9657d5..0406d906f 100644
--- a/tests/test_request_attribute_binding.py
+++ b/tests/test_request_attribute_binding.py
@@ -2,8 +2,8 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
 from scrapy import Request, signals
-from scrapy.crawler import CrawlerRunner
 from scrapy.http.response import Response
+from scrapy.utils.test import get_crawler
 
 from testfixtures import LogCapture
 
@@ -71,7 +71,7 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_response_200(self):
         url = self.mockserver.url("/status?n=200")
-        crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+        crawler = get_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.request.url, url)
@@ -80,7 +80,7 @@ class CrawlTestCase(TestCase):
     def test_response_error(self):
         for status in ("404", "500"):
             url = self.mockserver.url(f"/status?n={status}")
-            crawler = CrawlerRunner().create_crawler(SingleRequestSpider)
+            crawler = get_crawler(SingleRequestSpider)
             yield crawler.crawl(seed=url, mockserver=self.mockserver)
             failure = crawler.spider.meta["failure"]
             response = failure.value.response
@@ -90,12 +90,11 @@ class CrawlTestCase(TestCase):
     @defer.inlineCallbacks
     def test_downloader_middleware_raise_exception(self):
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         failure = crawler.spider.meta["failure"]
         self.assertEqual(failure.request.url, url)
@@ -117,12 +116,11 @@ class CrawlTestCase(TestCase):
             signal_params["request"] = request
 
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 ProcessResponseMiddleware: 595,
             }
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         crawler.signals.connect(signal_handler, signal=signals.response_received)
 
         with LogCapture() as log:
@@ -147,13 +145,12 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the overridden response.request
         """
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
                 CatchExceptionOverrideRequestMiddleware: 595,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -168,13 +165,12 @@ class CrawlTestCase(TestCase):
         The spider callback should receive the original response.request
         """
         url = self.mockserver.url("/status?n=200")
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(SingleRequestSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 RaiseExceptionRequestMiddleware: 590,
                 CatchExceptionDoNotOverrideRequestMiddleware: 595,
             },
         })
-        crawler = runner.create_crawler(SingleRequestSpider)
         yield crawler.crawl(seed=url, mockserver=self.mockserver)
         response = crawler.spider.meta["responses"][0]
         self.assertEqual(response.body, b"Caught ZeroDivisionError")
@@ -186,12 +182,11 @@ class CrawlTestCase(TestCase):
         Downloader middleware which returns a response with a
         specific 'request' attribute, with an alternative callback
         """
-        runner = CrawlerRunner(settings={
+        crawler = get_crawler(AlternativeCallbacksSpider, {
             "DOWNLOADER_MIDDLEWARES": {
                 AlternativeCallbacksMiddleware: 595,
             }
         })
-        crawler = runner.create_crawler(AlternativeCallbacksSpider)
 
         with LogCapture() as log:
             url = self.mockserver.url("/status?n=200")
diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py
index 473a93e69..002a04358 100644
--- a/tests/test_request_cb_kwargs.py
+++ b/tests/test_request_cb_kwargs.py
@@ -3,7 +3,7 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 
 from scrapy.http import Request
-from scrapy.crawler import CrawlerRunner
+from scrapy.utils.test import get_crawler
 from tests.spiders import MockServerSpider
 from tests.mockserver import MockServer
 
@@ -140,14 +140,13 @@ class CallbackKeywordArgumentsTestCase(TestCase):
     def setUp(self):
         self.mockserver = MockServer()
         self.mockserver.__enter__()
-        self.runner = CrawlerRunner()
 
     def tearDown(self):
         self.mockserver.__exit__(None, None, None)
 
     @defer.inlineCallbacks
     def test_callback_kwargs(self):
-        crawler = self.runner.create_crawler(KeywordArgumentsSpider)
+        crawler = get_crawler(KeywordArgumentsSpider)
         with LogCapture() as log:
             yield crawler.crawl(mockserver=self.mockserver)
         self.assertTrue(all(crawler.spider.checks))
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index c07d3a99c..4b4095fb0 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -54,6 +54,8 @@ class ResponseTypesTest(unittest.TestCase):
             (b'\x03\x02\xdf\xdd\x23', Response),
             (b'Some plain text\ndata with tabs\t and null bytes\0', TextResponse),
             (b'<html><head><title>Hello</title></head>', HtmlResponse),
+            # https://codersblock.com/blog/the-smallest-valid-html5-page/
+            (b'<!DOCTYPE html>\n<title>.</title>', HtmlResponse),
             (b'<?xml version="1.0" encoding="utf-8"', XmlResponse),
         ]
         for source, cls in mappings:
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index 2d4bfa165..50a7755c1 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -52,6 +52,7 @@ class MockCrawler(Crawler):
             SCHEDULER_PRIORITY_QUEUE=priority_queue_cls,
             JOBDIR=jobdir,
             DUPEFILTER_CLASS='scrapy.dupefilters.BaseDupeFilter',
+            REQUEST_FINGERPRINTER_IMPLEMENTATION='2.7',
         )
         super().__init__(Spider, settings)
         self.engine = MockEngine(downloader=MockDownloader())
@@ -334,7 +335,7 @@ class TestIncompatibility(unittest.TestCase):
             SCHEDULER_PRIORITY_QUEUE='scrapy.pqueues.DownloaderAwarePriorityQueue',
             CONCURRENT_REQUESTS_PER_IP=1,
         )
-        crawler = Crawler(Spider, settings)
+        crawler = get_crawler(Spider, settings)
         scheduler = Scheduler.from_crawler(crawler)
         spider = Spider(name='spider')
         scheduler.open(spider)
diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py
index bf90b4320..fc234a83d 100644
--- a/tests/test_scheduler_base.py
+++ b/tests/test_scheduler_base.py
@@ -7,10 +7,10 @@ from twisted.internet import defer
 from twisted.trial.unittest import TestCase as TwistedTestCase
 
 from scrapy.core.scheduler import BaseScheduler
-from scrapy.crawler import CrawlerRunner
 from scrapy.http import Request
 from scrapy.spiders import Spider
-from scrapy.utils.request import request_fingerprint
+from scrapy.utils.request import fingerprint
+from scrapy.utils.test import get_crawler
 
 from tests.mockserver import MockServer
 
@@ -21,13 +21,13 @@ URLS = [urljoin("https://example.org", p) for p in PATHS]
 
 class MinimalScheduler:
     def __init__(self) -> None:
-        self.requests: Dict[str, Request] = {}
+        self.requests: Dict[bytes, Request] = {}
 
     def has_pending_requests(self) -> bool:
         return bool(self.requests)
 
     def enqueue_request(self, request: Request) -> bool:
-        fp = request_fingerprint(request)
+        fp = fingerprint(request)
         if fp not in self.requests:
             self.requests[fp] = request
             return True
@@ -147,9 +147,12 @@ class MinimalSchedulerCrawlTest(TwistedTestCase):
     @defer.inlineCallbacks
     def test_crawl(self):
         with MockServer() as mockserver:
-            settings = {"SCHEDULER": self.scheduler_cls}
+            settings = {
+                "SCHEDULER": self.scheduler_cls,
+            }
             with LogCapture() as log:
-                yield CrawlerRunner(settings).crawl(TestSpider, mockserver)
+                crawler = get_crawler(TestSpider, settings)
+                yield crawler.crawl(mockserver)
             for path in PATHS:
                 self.assertIn(f"{{'path': '{path}'}}", str(log))
             self.assertIn(f"'item_scraped_count': {len(PATHS)}", str(log))
diff --git a/tests/test_spider.py b/tests/test_spider.py
index 689349999..e1527620f 100644
--- a/tests/test_spider.py
+++ b/tests/test_spider.py
@@ -22,6 +22,7 @@ from scrapy.spiders import (
 from scrapy.linkextractors import LinkExtractor
 from scrapy.utils.test import get_crawler
 from tests import get_testdata
+from w3lib.url import safe_url_string
 
 
 class SpiderTest(unittest.TestCase):
@@ -361,9 +362,9 @@ class CrawlSpiderTest(SpiderTest):
         self.assertEqual(len(output), 3)
         self.assertTrue(all(map(lambda r: isinstance(r, Request), output)))
         self.assertEqual([r.url for r in output],
-                         ['http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML',
-                          'http://EXAMPLE.ORG/ABOUT.HTML',
-                          'http://EXAMPLE.ORG/NOFOLLOW.HTML'])
+                         [safe_url_string('http://EXAMPLE.ORG/SOMEPAGE/ITEM/12.HTML'),
+                          safe_url_string('http://EXAMPLE.ORG/ABOUT.HTML'),
+                          safe_url_string('http://EXAMPLE.ORG/NOFOLLOW.HTML')])
 
     def test_process_request_instance_method_with_response(self):
 
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 8a35e9fd7..7a590f96c 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -96,7 +96,10 @@ class SpiderLoaderTest(unittest.TestCase):
 
     def test_crawler_runner_loading(self):
         module = 'tests.test_spiderloader.test_spiders.spider1'
-        runner = CrawlerRunner({'SPIDER_MODULES': [module]})
+        runner = CrawlerRunner({
+            'SPIDER_MODULES': [module],
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
+        })
 
         self.assertRaisesRegex(KeyError, 'Spider not found',
                                runner.create_crawler, 'spider2')
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index b39576996..edde6f682 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,11 +1,17 @@
+import collections.abc
+from typing import Optional
 from unittest import mock
 
+from testfixtures import LogCapture
+from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
+from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
@@ -101,3 +107,427 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)
         self.assertIsInstance(result.value, ZeroDivisionError)
+
+
+class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
+    """ Helpers for testing sync, async and mixed middlewares.
+
+    Should work for process_spider_output and, when it's supported, process_start_requests.
+    """
+
+    RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
+
+    @staticmethod
+    def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None):
+        if start_index is None:
+            start_index = 10
+        return {i: c for c, i in enumerate(mw_classes, start=start_index)}
+
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    @defer.inlineCallbacks
+    def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+        self.assertIsInstance(result, collections.abc.Iterable)
+        result_list = list(result)
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
+        self.assertIsInstance(result, collections.abc.AsyncIterator)
+        result_list = yield deferred_from_coro(collect_asyncgen(result))
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
+
+
+class ProcessSpiderOutputSimpleMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            yield r
+
+
+class ProcessSpiderOutputAsyncGenMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        async for r in result:
+            yield r
+
+
+class ProcessSpiderOutputUniversalMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            yield r
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result:
+            yield r
+
+
+class ProcessSpiderExceptionSimpleIterableMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+
+class ProcessSpiderExceptionAsyncIterableMiddleware:
+    async def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        d = defer.Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, None)
+        await maybe_deferred_to_future(d)
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+
+class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_spider_output tests for simple callbacks"""
+
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    def test_simple(self):
+        """ Simple mw """
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_asyncgen(self):
+        """ Asyncgen mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE)
+
+    def test_asyncgen_simple(self):
+        """ Asyncgen mw -> simple mw; upgrade then downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_ASYNCGEN,
+                                      downgrade=True)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE)
+
+    def test_universal_asyncgen(self):
+        """ Universal mw -> asyncgen mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_UNIVERSAL)
+
+    def test_asyncgen_universal(self):
+        """ Asyncgen mw -> universal mw; upgrade """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL,
+                                        self.MW_ASYNCGEN)
+
+
+class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
+    """ process_spider_output tests for async generator callbacks """
+
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_simple(self):
+        """ Simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      downgrade=True)
+
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw; downgrade then upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE,
+                                        downgrade=True)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL,
+                                      downgrade=True)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw; downgrade """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE,
+                                      downgrade=True)
+
+
+class ProcessSpiderOutputNonIterableMiddleware:
+    def process_spider_output(self, response, result, spider):
+        return
+
+
+class ProcessSpiderOutputCoroutineMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        results = []
+        for r in result:
+            results.append(r)
+        return results
+
+
+class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
+
+    @defer.inlineCallbacks
+    def test_non_iterable(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            (
+                r"\.process_spider_output must return an iterable, got <class "
+                r"'NoneType'>"
+            ),
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputNonIterableMiddleware,
+            )
+
+    @defer.inlineCallbacks
+    def test_coroutine(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            r"\.process_spider_output must be an asynchronous generator",
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputCoroutineMiddleware,
+            )
+
+
+class ProcessStartRequestsSimpleMiddleware:
+    def process_start_requests(self, start_requests, spider):
+        for r in start_requests:
+            yield r
+
+
+class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_start_requests tests for simple start_requests"""
+
+    ITEM_TYPE = Request
+    MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
+
+    def _start_requests(self):
+        for i in range(3):
+            yield Request(f'https://example.com/{i}', dont_filter=True)
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+        start_requests = iter(self._start_requests())
+        results = yield self.mwman.process_start_requests(start_requests, self.spider)
+        return results
+
+    def test_simple(self):
+        """ Simple mw """
+        return self._test_simple_base(self.MW_SIMPLE)
+
+
+class UniversalMiddlewareNoSync:
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothSync:
+    def process_spider_output(self, response, result, spider):
+        yield
+
+    def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothAsync:
+    async def process_spider_output(self, response, result, spider):
+        yield
+
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareManagerTest(TestCase):
+    def setUp(self):
+        self.mwman = SpiderMiddlewareManager()
+
+    def test_simple_mw(self):
+        mw = ProcessSpiderOutputSimpleMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_async_mw(self):
+        mw = ProcessSpiderOutputAsyncGenMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw(self):
+        mw = ProcessSpiderOutputUniversalMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0],
+                         (mw.process_spider_output, mw.process_spider_output_async))
+
+    def test_universal_mw_no_sync(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareNoSync)
+        self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"
+                      " without process_spider_output", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+    def test_universal_mw_both_sync(self):
+        mw = UniversalMiddlewareBothSync
+        with LogCapture() as log:
+            self.mwman._add_middleware(mw)
+        self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async "
+                      "is not an async generator function", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw_both_async(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareBothAsync)
+        self.assertIn("UniversalMiddlewareBothAsync.process_spider_output "
+                      "is an async generator function while process_spider_output_async exists",
+                      str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+
+class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    def test_just_builtin(self):
+        return self._test_simple_base()
+
+    def test_builtin_simple(self):
+        return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
+
+    def test_builtin_async(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_async_builtin(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+
+class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_just_builtin(self):
+        return self._test_asyncgen_base()
+
+    def test_builtin_simple(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
+
+    def test_builtin_async(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
+
+    def test_async_builtin(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+
+class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+    MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
+    MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        1 / 0
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_nodowngrade(self, *mw_classes):
+        with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"):
+            yield self._get_middleware_result(*mw_classes)
+
+    def test_exc_simple(self):
+        """ Simple exc mw """
+        return self._test_simple_base(self.MW_EXC_SIMPLE)
+
+    def test_exc_async(self):
+        """ Async exc mw """
+        return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_simple(self):
+        """ Simple exc mw -> simple output mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_EXC_SIMPLE)
+
+    def test_exc_async_async(self):
+        """ Async exc mw -> async output mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_async(self):
+        """ Simple exc mw -> async output mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_SIMPLE)
+
+    def test_exc_async_simple(self):
+        """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """
+        return self._test_asyncgen_nodowngrade(self.MW_SIMPLE,
+                                               self.MW_EXC_ASYNCGEN)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 029bf8bd6..dac246fb6 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -28,6 +28,7 @@ class RecoveryMiddleware:
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
     custom_settings = {
+        'SPIDER_MIDDLEWARES_BASE': {},
         'SPIDER_MIDDLEWARES': {
             RecoveryMiddleware: 10,
         },
@@ -43,6 +44,14 @@ class RecoverySpider(Spider):
             raise TabError()
 
 
+class RecoveryAsyncGenSpider(RecoverySpider):
+    name = 'RecoveryAsyncGenSpider'
+
+    async def parse(self, response):
+        for r in super().parse(response):
+            yield r
+
+
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
@@ -99,6 +108,13 @@ class GeneratorCallbackSpider(Spider):
         raise ImportError()
 
 
+class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
+    async def parse(self, response):
+        yield {'test': 1}
+        yield {'test': 2}
+        raise ImportError()
+
+
 # ================================================================================
 # (2.1) exceptions from a spider callback (generator, middleware right after callback)
 class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
@@ -307,6 +323,16 @@ class TestSpiderMiddleware(TestCase):
         self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
         self.assertIn("'item_scraped_count': 3", str(log))
 
+    @defer.inlineCallbacks
+    def test_recovery_asyncgen(self):
+        """
+        Same as test_recovery but with an async callback.
+        """
+        log = yield self.crawl_log(RecoveryAsyncGenSpider)
+        self.assertIn("Middleware: TabError exception caught", str(log))
+        self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
+        self.assertIn("'item_scraped_count': 3", str(log))
+
     @defer.inlineCallbacks
     def test_process_spider_input_without_errback(self):
         """
@@ -342,6 +368,15 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("Middleware: ImportError exception caught", str(log2))
         self.assertIn("'item_scraped_count': 2", str(log2))
 
+    @defer.inlineCallbacks
+    def test_async_generator_callback(self):
+        """
+        Same as test_generator_callback but with an async callback.
+        """
+        log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
+
     @defer.inlineCallbacks
     def test_generator_callback_right_after_callback(self):
         """
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
new file mode 100644
index 000000000..9ae66c57c
--- /dev/null
+++ b/tests/test_utils_asyncgen.py
@@ -0,0 +1,20 @@
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+
+
+class AsyncgenUtilsTest(unittest.TestCase):
+    @deferred_f_from_coro_f
+    async def test_as_async_generator(self):
+        ag = as_async_generator(range(42))
+        results = []
+        async for i in ag:
+            results.append(i)
+        self.assertEqual(results, list(range(42)))
+
+    @deferred_f_from_coro_f
+    async def test_collect_asyncgen(self):
+        ag = as_async_generator(range(42))
+        results = await collect_asyncgen(ag)
+        self.assertEqual(results, list(range(42)))
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 032dbc8c5..d39de7430 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,12 +1,18 @@
+import random
+
 from pytest import mark
 from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
+from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator
 from scrapy.utils.defer import (
+    aiter_errback,
     deferred_f_from_coro_f,
     iter_errback,
+    maybe_deferred_to_future,
     mustbe_deferred,
+    parallel_async,
     process_chain,
     process_chain_both,
     process_parallel,
@@ -121,6 +127,34 @@ class IterErrbackTest(unittest.TestCase):
         self.assertIsInstance(errors[0].value, ZeroDivisionError)
 
 
+class AiterErrbackTest(unittest.TestCase):
+
+    @deferred_f_from_coro_f
+    async def test_aiter_errback_good(self):
+        async def itergood():
+            for x in range(10):
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
+        self.assertEqual(out, list(range(10)))
+        self.assertFalse(errors)
+
+    @deferred_f_from_coro_f
+    async def test_iter_errback_bad(self):
+        async def iterbad():
+            for x in range(10):
+                if x == 5:
+                    1 / 0
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
+        self.assertEqual(out, [0, 1, 2, 3, 4])
+        self.assertEqual(len(errors), 1)
+        self.assertIsInstance(errors[0].value, ZeroDivisionError)
+
+
 class AsyncDefTestsuiteTest(unittest.TestCase):
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f(self):
@@ -134,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f_xfail(self):
         raise Exception("This is expected to be raised")
+
+
+class AsyncCooperatorTest(unittest.TestCase):
+    """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+
+    parallel_async is called with the results of a callback (so an iterable of items, requests and None,
+    with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
+    (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and
+    None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting.
+
+    We want to test different concurrency values compared to the iterable length.
+    We also want to simulate the real usage, with arbitrary delays between getting the values
+    from the iterable. We also want to simulate sync and async results from the callable.
+    """
+    CONCURRENT_ITEMS = 50
+
+    @staticmethod
+    def callable(o, results):
+        if random.random() < 0.4:
+            # simulate async processing
+            dfd = defer.Deferred()
+            dfd.addCallback(lambda _: results.append(o))
+            delay = random.random() / 8
+            reactor.callLater(delay, dfd.callback, None)
+            return dfd
+        else:
+            # simulate trivial sync processing
+            results.append(o)
+
+    @staticmethod
+    def get_async_iterable(length):
+        # simulate a simple callback without delays between results
+        return as_async_generator(range(length))
+
+    @staticmethod
+    async def get_async_iterable_with_delays(length):
+        # simulate a callback with delays between some of the results
+        for i in range(length):
+            if random.random() < 0.1:
+                dfd = defer.Deferred()
+                delay = random.random() / 20
+                reactor.callLater(delay, dfd.callback, None)
+                await maybe_deferred_to_future(dfd)
+            yield i
+
+    @defer.inlineCallbacks
+    def test_simple(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))
+
+    @defer.inlineCallbacks
+    def test_delays(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable_with_delays(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))
diff --git a/tests/test_utils_misc/test_return_with_argument_inside_generator.py b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
index 1c85ca353..72277d701 100644
--- a/tests/test_utils_misc/test_return_with_argument_inside_generator.py
+++ b/tests/test_utils_misc/test_return_with_argument_inside_generator.py
@@ -165,6 +165,89 @@ https://example.org
             warn_on_generator_with_return_value(None, l2)
             self.assertEqual(len(w), 0)
 
+    def test_generators_return_none_with_decorator(self):
+        def decorator(func):
+            def inner_func():
+                func()
+            return inner_func
+
+        @decorator
+        def f3():
+            yield 1
+            return None
+
+        @decorator
+        def g3():
+            yield 1
+            return
+
+        @decorator
+        def h3():
+            yield 1
+
+        @decorator
+        def i3():
+            yield 1
+            yield from generator_that_returns_stuff()
+
+        @decorator
+        def j3():
+            yield 1
+
+            def helper():
+                return 0
+
+            yield helper()
+
+        @decorator
+        def k3():
+            """
+docstring
+            """
+            url = """
+https://example.org
+        """
+            yield url
+            return
+
+        @decorator
+        def l3():
+            return
+
+        assert not is_generator_with_return_value(top_level_return_none)
+        assert not is_generator_with_return_value(f3)
+        assert not is_generator_with_return_value(g3)
+        assert not is_generator_with_return_value(h3)
+        assert not is_generator_with_return_value(i3)
+        assert not is_generator_with_return_value(j3)  # not recursive
+        assert not is_generator_with_return_value(k3)  # not recursive
+        assert not is_generator_with_return_value(l3)
+
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, top_level_return_none)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, f3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, g3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, h3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, i3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, j3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, k3)
+            self.assertEqual(len(w), 0)
+        with warnings.catch_warnings(record=True) as w:
+            warn_on_generator_with_return_value(None, l3)
+            self.assertEqual(len(w), 0)
+
     @mock.patch("scrapy.utils.misc.is_generator_with_return_value", new=_indentation_error)
     def test_indentation_error(self):
         with warnings.catch_warnings(record=True) as w:
diff --git a/tests/test_utils_project.py b/tests/test_utils_project.py
index 1ef4eeb14..46452415a 100644
--- a/tests/test_utils_project.py
+++ b/tests/test_utils_project.py
@@ -3,6 +3,7 @@ import os
 import tempfile
 import shutil
 import contextlib
+import warnings
 
 from pytest import warns
 
@@ -68,20 +69,21 @@ class GetProjectSettingsTestCase(unittest.TestCase):
         envvars = {
             'SCRAPY_SETTINGS_MODULE': value,
         }
-        with set_env(**envvars), warns(None) as warnings:
-            settings = get_project_settings()
-        assert not warnings
+        with warnings.catch_warnings():
+            warnings.simplefilter("error")
+            with set_env(**envvars):
+                settings = get_project_settings()
+
         assert settings.get('SETTINGS_MODULE') == value
 
     def test_invalid_envvar(self):
         envvars = {
             'SCRAPY_FOO': 'bar',
         }
-        with set_env(**envvars), warns(None) as warnings:
-            get_project_settings()
-        assert len(warnings) == 1
-        assert warnings[0].category == ScrapyDeprecationWarning
-        assert str(warnings[0].message).endswith(': FOO')
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                get_project_settings()
+        assert len(record) == 1
 
     def test_valid_and_invalid_envvars(self):
         value = 'tests.test_cmdline.settings'
@@ -89,9 +91,8 @@ class GetProjectSettingsTestCase(unittest.TestCase):
             'SCRAPY_FOO': 'bar',
             'SCRAPY_SETTINGS_MODULE': value,
         }
-        with set_env(**envvars), warns(None) as warnings:
-            settings = get_project_settings()
-        assert len(warnings) == 1
-        assert warnings[0].category == ScrapyDeprecationWarning
-        assert str(warnings[0].message).endswith(': FOO')
+        with warns(ScrapyDeprecationWarning, match=': FOO') as record:
+            with set_env(**envvars):
+                settings = get_project_settings()
+        assert len(record) == 1
         assert settings.get('SETTINGS_MODULE') == value
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 4b3964154..b1a8fdc04 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,16 +2,18 @@ import functools
 import gc
 import operator
 import platform
-import unittest
-from datetime import datetime
 from itertools import count
 from warnings import catch_warnings, filterwarnings
 
+from twisted.trial import unittest
+
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, get_func_args, to_bytes, to_unicode,
-    without_none_values, MutableChain)
+    without_none_values, MutableChain, MutableAsyncChain)
 
 
 __doctests__ = ['scrapy.utils.python']
@@ -33,6 +35,57 @@ class MutableChainTest(unittest.TestCase):
         self.assertEqual(list(m), list(range(3, 13)))
 
 
+class MutableAsyncChainTest(unittest.TestCase):
+    @staticmethod
+    async def g1():
+        for i in range(3):
+            yield i
+
+    @staticmethod
+    async def g2():
+        return
+        yield
+
+    @staticmethod
+    async def g3():
+        for i in range(7, 10):
+            yield i
+
+    @staticmethod
+    async def g4():
+        for i in range(3, 5):
+            yield i
+        1 / 0
+        for i in range(5, 7):
+            yield i
+
+    @staticmethod
+    async def collect_asyncgen_exc(asyncgen):
+        results = []
+        async for x in asyncgen:
+            results.append(x)
+        return results
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain(self):
+        m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
+        m.extend(self.g2())
+        m.extend(self.g3())
+
+        self.assertEqual(await m.__anext__(), 0)
+        results = await collect_asyncgen(m)
+        self.assertEqual(results, list(range(1, 10)))
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain_exc(self):
+        m = MutableAsyncChain(self.g1())
+        m.extend(self.g4())
+        m.extend(self.g3())
+
+        results = await collect_asyncgen(aiter_errback(m, lambda _: None))
+        self.assertEqual(results, list(range(5)))
+
+
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
         self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')
@@ -224,12 +277,7 @@ class UtilsPythonTestCase(unittest.TestCase):
         elif platform.python_implementation() == 'PyPy':
             self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])
             self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])
-
-            build_date = datetime.strptime(platform.python_build()[1], '%b %d %Y')
-            if build_date >= datetime(2020, 4, 7):  # PyPy 3.6-v7.3.1
-                self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
-            else:
-                self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])
+            self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])
 
     def test_without_none_values(self):
         self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 7e0049b1d..a92d9a0ac 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -1,73 +1,29 @@
 import unittest
+import warnings
+from hashlib import sha1
+from typing import Dict, Mapping, Optional, Tuple, Union
+from weakref import WeakKeyDictionary
+
+import pytest
+from w3lib.url import canonicalize_url
+
 from scrapy.http import Request
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
+from scrapy.utils.python import to_bytes
 from scrapy.utils.request import (
+    _deprecated_fingerprint_cache,
     _fingerprint_cache,
+    _request_fingerprint_as_bytes,
+    fingerprint,
     request_authenticate,
     request_fingerprint,
     request_httprepr,
 )
+from scrapy.utils.test import get_crawler
 
 
 class UtilsRequestTest(unittest.TestCase):
 
-    def test_request_fingerprint(self):
-        r1 = Request("http://www.example.com/query?id=111&cat=222")
-        r2 = Request("http://www.example.com/query?cat=222&id=111")
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1))
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
-        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        # make sure caching is working
-        self.assertEqual(request_fingerprint(r1), _fingerprint_cache[r1][(None, False)])
-
-        r1 = Request("http://www.example.com/members/offers.html")
-        r2 = Request("http://www.example.com/members/offers.html")
-        r2.headers['SESSIONID'] = b"somehash"
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-
-        r1 = Request("http://www.example.com/")
-        r2 = Request("http://www.example.com/")
-        r2.headers['Accept-Language'] = b'en'
-        r3 = Request("http://www.example.com/")
-        r3.headers['Accept-Language'] = b'en'
-        r3.headers['SESSIONID'] = b"somehash"
-
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2), request_fingerprint(r3))
-
-        self.assertEqual(request_fingerprint(r1),
-                         request_fingerprint(r1, include_headers=['Accept-Language']))
-
-        self.assertNotEqual(
-            request_fingerprint(r1),
-            request_fingerprint(r2, include_headers=['Accept-Language']))
-
-        self.assertEqual(request_fingerprint(r3, include_headers=['accept-language', 'sessionid']),
-                         request_fingerprint(r3, include_headers=['SESSIONID', 'Accept-Language']))
-
-        r1 = Request("http://www.example.com/test.html")
-        r2 = Request("http://www.example.com/test.html#fragment")
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r2))
-        self.assertEqual(request_fingerprint(r1), request_fingerprint(r1, keep_fragments=True))
-        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r2, keep_fragments=True))
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2, keep_fragments=True))
-
-        r1 = Request("http://www.example.com")
-        r2 = Request("http://www.example.com", method='POST')
-        r3 = Request("http://www.example.com", method='POST', body=b'request body')
-
-        self.assertNotEqual(request_fingerprint(r1), request_fingerprint(r2))
-        self.assertNotEqual(request_fingerprint(r2), request_fingerprint(r3))
-
-        # cached fingerprint must be cleared on request copy
-        r1 = Request("http://www.example.com")
-        fp1 = request_fingerprint(r1)
-        r2 = r1.replace(url="http://www.example.com/other")
-        fp2 = request_fingerprint(r2)
-        self.assertNotEqual(fp1, fp2)
-
     def test_request_authenticate(self):
         r = Request("http://www.example.com")
         request_authenticate(r, 'someuser', 'somepass')
@@ -93,5 +49,641 @@ class UtilsRequestTest(unittest.TestCase):
         request_httprepr(Request("ftp://localhost/tmp/foo.txt"))
 
 
+class FingerprintTest(unittest.TestCase):
+    maxDiff = None
+
+    function: staticmethod = staticmethod(fingerprint)
+    cache: Union[
+        "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
+        "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
+    ] = _fingerprint_cache
+    default_cache_key = (None, False)
+    known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+        (
+            Request("http://example.org"),
+            b'xs\xd7\x0c3uj\x15\xfe\xd7d\x9b\xa9\t\xe0d\xbf\x9cXD',
+            {},
+        ),
+        (
+            Request("https://example.org"),
+            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            {},
+        ),
+        (
+            Request("https://example.org?a"),
+            b'G\xad\xb8Ck\x19\x1c\xed\x838,\x01\xc4\xde;\xee\xa5\x94a\x0c',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b"),
+            b'\x024MYb\x8a\xc2\x1e\xbc>\xd6\xac*\xda\x9cF\xc1r\x7f\x17',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a"),
+            b't+\xe8*\xfb\x84\xe3v\x1a}\x88p\xc0\xccB\xd7\x9d\xfez\x96',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a=c"),
+            b'\xda\x1ec\xd0\x9c\x08s`\xb4\x9b\xe2\xb6R\xf8k\xef\xeaQG\xef',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST'),
+            b'\x9d\xcdA\x0fT\x02:\xca\xa0}\x90\xda\x05B\xded\x8aN7\x1d',
+            {},
+        ),
+        (
+            Request("https://example.org", body=b'a'),
+            b'\xc34z>\xd8\x99\x8b\xda7\x05r\x99I\xa8\xa0x;\xa41_',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST', body=b'a'),
+            b'5`\xe2y4\xd0\x9d\xee\xe0\xbatw\x87Q\xe8O\xd78\xfc\xe7',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'\xc04\x85P,\xaa\x91\x06\xf8t\xb4\xbd*\xd9\xe9\x8a:m\xc3l',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b']\xc7\x1f\xf2\xafG2\xbc\xa4\xfa\x99\n33\xda\x18\x94\x81U.',
+            {'include_headers': ['A']},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'<\x1a\xeb\x85y\xdeW\xfb\xdcq\x88\xee\xaf\x17\xdd\x0c\xbfH\x18\x1f',
+            {'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            b'\xc1\xef~\x94\x9bS\xc1\x83\t\xdcz8\x9f\xdc{\x11\x16I.\x11',
+            {'include_headers': ['A'], 'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org/ab"),
+            b'N\xe5l\xb8\x12@iw\xe2\xf3\x1bp\xea\xffp!u\xe2\x8a\xc6',
+            {},
+        ),
+        (
+            Request("https://example.org/a", body=b'b'),
+            b'_NOv\xbco$6\xfcW\x9f\xb24g\x9f\xbb\xdd\xa82\xc5',
+            {},
+        ),
+    )
+
+    def test_query_string_key_order(self):
+        r1 = Request("http://www.example.com/query?id=111&cat=222")
+        r2 = Request("http://www.example.com/query?cat=222&id=111")
+        self.assertEqual(self.function(r1), self.function(r1))
+        self.assertEqual(self.function(r1), self.function(r2))
+
+    def test_query_string_key_without_value(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78132,199')
+        r2 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertNotEqual(self.function(r1), self.function(r2))
+
+    def test_caching(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertEqual(
+            self.function(r1),
+            self.cache[r1][self.default_cache_key]
+        )
+
+    def test_header(self):
+        r1 = Request("http://www.example.com/members/offers.html")
+        r2 = Request("http://www.example.com/members/offers.html")
+        r2.headers['SESSIONID'] = b"somehash"
+        self.assertEqual(self.function(r1), self.function(r2))
+
+    def test_headers(self):
+        r1 = Request("http://www.example.com/")
+        r2 = Request("http://www.example.com/")
+        r2.headers['Accept-Language'] = b'en'
+        r3 = Request("http://www.example.com/")
+        r3.headers['Accept-Language'] = b'en'
+        r3.headers['SESSIONID'] = b"somehash"
+
+        self.assertEqual(self.function(r1), self.function(r2), self.function(r3))
+
+        self.assertEqual(self.function(r1),
+                         self.function(r1, include_headers=['Accept-Language']))
+
+        self.assertNotEqual(
+            self.function(r1),
+            self.function(r2, include_headers=['Accept-Language']))
+
+        self.assertEqual(self.function(r3, include_headers=['accept-language', 'sessionid']),
+                         self.function(r3, include_headers=['SESSIONID', 'Accept-Language']))
+
+    def test_fragment(self):
+        r1 = Request("http://www.example.com/test.html")
+        r2 = Request("http://www.example.com/test.html#fragment")
+        self.assertEqual(self.function(r1), self.function(r2))
+        self.assertEqual(self.function(r1), self.function(r1, keep_fragments=True))
+        self.assertNotEqual(self.function(r2), self.function(r2, keep_fragments=True))
+        self.assertNotEqual(self.function(r1), self.function(r2, keep_fragments=True))
+
+    def test_method_and_body(self):
+        r1 = Request("http://www.example.com")
+        r2 = Request("http://www.example.com", method='POST')
+        r3 = Request("http://www.example.com", method='POST', body=b'request body')
+
+        self.assertNotEqual(self.function(r1), self.function(r2))
+        self.assertNotEqual(self.function(r2), self.function(r3))
+
+    def test_request_replace(self):
+        # cached fingerprint must be cleared on request copy
+        r1 = Request("http://www.example.com")
+        fp1 = self.function(r1)
+        r2 = r1.replace(url="http://www.example.com/other")
+        fp2 = self.function(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_part_separation(self):
+        # An old implementation used to serialize request data in a way that
+        # would put the body right after the URL.
+        r1 = Request("http://www.example.com/foo")
+        fp1 = self.function(r1)
+        r2 = Request("http://www.example.com/f", body=b'oo')
+        fp2 = self.function(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_hashes(self):
+        """Test hardcoded hashes, to make sure future changes to not introduce
+        backward incompatibilities."""
+        actual = [
+            self.function(request, **kwargs)
+            for request, _, kwargs in self.known_hashes
+        ]
+        expected = [
+            _fingerprint
+            for _, _fingerprint, _ in self.known_hashes
+        ]
+        self.assertEqual(actual, expected)
+
+
+class RequestFingerprintTest(FingerprintTest):
+    function = staticmethod(request_fingerprint)
+    cache = _deprecated_fingerprint_cache
+    known_hashes: Tuple[Tuple[Request, Union[bytes, str], Dict], ...] = (
+        (
+            Request("http://example.org"),
+            'b2e5245ef826fd9576c93bd6e392fce3133fab62',
+            {},
+        ),
+        (
+            Request("https://example.org"),
+            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            {},
+        ),
+        (
+            Request("https://example.org?a"),
+            '2fb7d48ae02f04b749f40caa969c0bc3c43204ce',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b"),
+            '42e5fe149b147476e3f67ad0670c57b4cc57856a',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a"),
+            'd23a9787cb56c6375c2cae4453c5a8c634526942',
+            {},
+        ),
+        (
+            Request("https://example.org?a=b&a=c"),
+            '9a18a7a8552a9182b7f1e05d33876409e421e5c5',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST'),
+            'ba20a80cb5c5ca460021ceefb3c2467b2bfd1bc6',
+            {},
+        ),
+        (
+            Request("https://example.org", body=b'a'),
+            '4bb136e54e715a4ea7a9dd1101831765d33f2d60',
+            {},
+        ),
+        (
+            Request("https://example.org", method='POST', body=b'a'),
+            '6c6595374a304b293be762f7b7be3f54e9947c65',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            'bd10a0a89ea32cdee77917320f1309b0da87e892',
+            {},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            '515b633cb3ca502a33a9d8c890e889ec1e425e65',
+            {'include_headers': ['A']},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            '505c96e7da675920dfef58725e8c957dfdb38f47',
+            {'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org#a", headers={'A': b'B'}),
+            'd6f673cdcb661b7970c2b9a00ee63e87d1e2e5da',
+            {'include_headers': ['A'], 'keep_fragments': True},
+        ),
+        (
+            Request("https://example.org/ab"),
+            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            {},
+        ),
+        (
+            Request("https://example.org/a", body=b'b'),
+            '4e2870fee58582d6f81755e9b8fdefe3cba0c951',
+            {},
+        ),
+    )
+
+    def setUp(self) -> None:
+        warnings.simplefilter("ignore", ScrapyDeprecationWarning)
+
+    def tearDown(self) -> None:
+        warnings.simplefilter("default", ScrapyDeprecationWarning)
+
+    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    def test_part_separation(self):
+        super().test_part_separation()
+
+
+class RequestFingerprintDeprecationTest(unittest.TestCase):
+
+    def test_deprecation_default_parameters(self):
+        with pytest.warns(ScrapyDeprecationWarning) as warnings:
+            request_fingerprint(Request("http://www.example.com"))
+        messages = [str(warning.message) for warning in warnings]
+        self.assertTrue(
+            any(
+                'Call to deprecated function' in message
+                for message in messages
+            )
+        )
+        self.assertFalse(any('non-default' in message for message in messages))
+
+    def test_deprecation_non_default_parameters(self):
+        with pytest.warns(ScrapyDeprecationWarning) as warnings:
+            request_fingerprint(Request("http://www.example.com"), keep_fragments=True)
+        messages = [str(warning.message) for warning in warnings]
+        self.assertTrue(
+            any(
+                'Call to deprecated function' in message
+                for message in messages
+            )
+        )
+        self.assertTrue(any('non-default' in message for message in messages))
+
+
+class RequestFingerprintAsBytesTest(FingerprintTest):
+    function = staticmethod(_request_fingerprint_as_bytes)
+    cache = _deprecated_fingerprint_cache
+    known_hashes = RequestFingerprintTest.known_hashes
+
+    def test_caching(self):
+        r1 = Request('http://www.example.com/hnnoticiaj1.aspx?78160,199')
+        self.assertEqual(
+            self.function(r1),
+            bytes.fromhex(self.cache[r1][self.default_cache_key])
+        )
+
+    @pytest.mark.xfail(reason='known bug kept for backward compatibility', strict=True)
+    def test_part_separation(self):
+        super().test_part_separation()
+
+    def test_hashes(self):
+        actual = [
+            self.function(request, **kwargs)
+            for request, _, kwargs in self.known_hashes
+        ]
+        expected = [
+            bytes.fromhex(_fingerprint)
+            for _, _fingerprint, _ in self.known_hashes
+        ]
+        self.assertEqual(actual, expected)
+
+
+_fingerprint_cache_2_6: Mapping[Request, Tuple[None, bool]] = WeakKeyDictionary()
+
+
+def request_fingerprint_2_6(request, include_headers=None, keep_fragments=False):
+    if include_headers:
+        include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+    cache = _fingerprint_cache_2_6.setdefault(request, {})
+    cache_key = (include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+        fp.update(request.body or b'')
+        if include_headers:
+            for hdr in include_headers:
+                if hdr in request.headers:
+                    fp.update(hdr)
+                    for v in request.headers.getlist(hdr):
+                        fp.update(v)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
+REQUEST_OBJECTS_TO_TEST = (
+    Request("http://www.example.com/"),
+    Request("http://www.example.com/query?id=111&cat=222"),
+    Request("http://www.example.com/query?cat=222&id=111"),
+    Request('http://www.example.com/hnnoticiaj1.aspx?78132,199'),
+    Request('http://www.example.com/hnnoticiaj1.aspx?78160,199'),
+    Request("http://www.example.com/members/offers.html"),
+    Request(
+        "http://www.example.com/members/offers.html",
+        headers={'SESSIONID': b"somehash"},
+    ),
+    Request(
+        "http://www.example.com/",
+        headers={'Accept-Language': b"en"},
+    ),
+    Request(
+        "http://www.example.com/",
+        headers={
+            'Accept-Language': b"en",
+            'SESSIONID': b"somehash",
+        },
+    ),
+    Request("http://www.example.com/test.html"),
+    Request("http://www.example.com/test.html#fragment"),
+    Request("http://www.example.com", method='POST'),
+    Request("http://www.example.com", method='POST', body=b'request body'),
+)
+
+
+class BackwardCompatibilityTestCase(unittest.TestCase):
+
+    def test_function_backward_compatibility(self):
+        include_headers_to_test = (
+            None,
+            ['Accept-Language'],
+            ['accept-language', 'sessionid'],
+            ['SESSIONID', 'Accept-Language'],
+        )
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            for include_headers in include_headers_to_test:
+                for keep_fragments in (False, True):
+                    with warnings.catch_warnings():
+                        warnings.simplefilter("ignore")
+                        fp = request_fingerprint(
+                            request_object,
+                            include_headers=include_headers,
+                            keep_fragments=keep_fragments,
+                        )
+                    old_fp = request_fingerprint_2_6(
+                        request_object,
+                        include_headers=include_headers,
+                        keep_fragments=keep_fragments,
+                    )
+                    self.assertEqual(fp, old_fp)
+
+    def test_component_backward_compatibility(self):
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                crawler = get_crawler(prevent_warnings=False)
+                fp = crawler.request_fingerprinter.fingerprint(request_object)
+            old_fp = request_fingerprint_2_6(request_object)
+            self.assertEqual(fp.hex(), old_fp)
+
+    def test_custom_component_backward_compatibility(self):
+        """Tests that the backward-compatible request fingerprinting class featured
+        in the documentation is indeed backward compatible and does not cause a
+        warning to be logged."""
+
+        class RequestFingerprinter:
+
+            cache = WeakKeyDictionary()
+
+            def fingerprint(self, request):
+                if request not in self.cache:
+                    fp = sha1()
+                    fp.update(to_bytes(request.method))
+                    fp.update(to_bytes(canonicalize_url(request.url)))
+                    fp.update(request.body or b'')
+                    self.cache[request] = fp.digest()
+                return self.cache[request]
+
+        for request_object in REQUEST_OBJECTS_TO_TEST:
+            with warnings.catch_warnings() as logged_warnings:
+                settings = {
+                    'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+                }
+                crawler = get_crawler(settings_dict=settings)
+                fp = crawler.request_fingerprinter.fingerprint(request_object)
+            old_fp = request_fingerprint_2_6(request_object)
+            self.assertEqual(fp.hex(), old_fp)
+            self.assertFalse(logged_warnings)
+
+
+class RequestFingerprinterTestCase(unittest.TestCase):
+
+    def test_default_implementation(self):
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(prevent_warnings=False)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            _request_fingerprint_as_bytes(request),
+        )
+        self.assertTrue(logged_warnings)
+
+    def test_deprecated_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.6',
+        }
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(settings_dict=settings)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            _request_fingerprint_as_bytes(request),
+        )
+        self.assertTrue(logged_warnings)
+
+    def test_recommended_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.7',
+        }
+        with warnings.catch_warnings(record=True) as logged_warnings:
+            crawler = get_crawler(settings_dict=settings)
+        request = Request('https://example.com')
+        self.assertEqual(
+            crawler.request_fingerprinter.fingerprint(request),
+            fingerprint(request),
+        )
+        self.assertFalse(logged_warnings)
+
+    def test_unknown_implementation(self):
+        settings = {
+            'REQUEST_FINGERPRINTER_IMPLEMENTATION': '2.5',
+        }
+        with self.assertRaises(ValueError):
+            get_crawler(settings_dict=settings)
+
+
+class CustomRequestFingerprinterTestCase(unittest.TestCase):
+
+    def test_include_headers(self):
+
+        class RequestFingerprinter:
+
+            def fingerprint(self, request):
+                return fingerprint(request, include_headers=['X-ID'])
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com", headers={'X-ID': '1'})
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com", headers={'X-ID': '2'})
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_dont_canonicalize(self):
+
+        class RequestFingerprinter:
+            cache = WeakKeyDictionary()
+
+            def fingerprint(self, request):
+                if request not in self.cache:
+                    fp = sha1()
+                    fp.update(to_bytes(request.url))
+                    self.cache[request] = fp.digest()
+                return self.cache[request]
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com?a=1&a=2")
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com?a=2&a=1")
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        self.assertNotEqual(fp1, fp2)
+
+    def test_meta(self):
+
+        class RequestFingerprinter:
+
+            def fingerprint(self, request):
+                if 'fingerprint' in request.meta:
+                    return request.meta['fingerprint']
+                return fingerprint(request)
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        r1 = Request("http://www.example.com")
+        fp1 = crawler.request_fingerprinter.fingerprint(r1)
+        r2 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        fp2 = crawler.request_fingerprinter.fingerprint(r2)
+        r3 = Request("http://www.example.com", meta={'fingerprint': 'a'})
+        fp3 = crawler.request_fingerprinter.fingerprint(r3)
+        r4 = Request("http://www.example.com", meta={'fingerprint': 'b'})
+        fp4 = crawler.request_fingerprinter.fingerprint(r4)
+        self.assertNotEqual(fp1, fp2)
+        self.assertNotEqual(fp1, fp4)
+        self.assertNotEqual(fp2, fp4)
+        self.assertEqual(fp2, fp3)
+
+    def test_from_crawler(self):
+
+        class RequestFingerprinter:
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler)
+
+            def __init__(self, crawler):
+                self._fingerprint = crawler.settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+    def test_from_settings(self):
+
+        class RequestFingerprinter:
+
+            @classmethod
+            def from_settings(cls, settings):
+                return cls(settings)
+
+            def __init__(self, settings):
+                self._fingerprint = settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+    def test_from_crawler_and_settings(self):
+
+        class RequestFingerprinter:
+
+            # This method is ignored due to the presence of from_crawler
+            @classmethod
+            def from_settings(cls, settings):
+                return cls(settings)
+
+            @classmethod
+            def from_crawler(cls, crawler):
+                return cls(crawler)
+
+            def __init__(self, crawler):
+                self._fingerprint = crawler.settings['FINGERPRINT']
+
+            def fingerprint(self, request):
+                return self._fingerprint
+
+        settings = {
+            'REQUEST_FINGERPRINTER_CLASS': RequestFingerprinter,
+            'FINGERPRINT': b'fingerprint',
+        }
+        crawler = get_crawler(settings_dict=settings)
+
+        request = Request("http://www.example.com")
+        fingerprint = crawler.request_fingerprinter.fingerprint(request)
+        self.assertEqual(fingerprint, settings['FINGERPRINT'])
+
+
 if __name__ == "__main__":
     unittest.main()
diff --git a/tests/test_utils_response.py b/tests/test_utils_response.py
index 0a09f6109..d20852e62 100644
--- a/tests/test_utils_response.py
+++ b/tests/test_utils_response.py
@@ -1,7 +1,9 @@
 import os
 import unittest
+import warnings
 from urllib.parse import urlparse
 
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.http import Response, TextResponse, HtmlResponse
 from scrapy.utils.python import to_bytes
 from scrapy.utils.response import (response_httprepr, open_in_browser,
@@ -15,14 +17,21 @@ class ResponseUtilsTest(unittest.TestCase):
     dummy_response = TextResponse(url='http://example.org/', body=b'dummy_response')
 
     def test_response_httprepr(self):
-        r1 = Response("http://www.example.com")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", ScrapyDeprecationWarning)
 
-        r1 = Response("http://www.example.com", status=404, headers={"Content-type": "text/html"}, body=b"Some body")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response("http://www.example.com")
+            self.assertEqual(response_httprepr(r1), b'HTTP/1.1 200 OK\r\n\r\n')
 
-        r1 = Response("http://www.example.com", status=6666, headers={"Content-type": "text/html"}, body=b"Some body")
-        self.assertEqual(response_httprepr(r1), b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
+            r1 = Response("http://www.example.com", status=404,
+                          headers={"Content-type": "text/html"}, body=b"Some body")
+            self.assertEqual(response_httprepr(r1),
+                             b'HTTP/1.1 404 Not Found\r\nContent-Type: text/html\r\n\r\nSome body')
+
+            r1 = Response("http://www.example.com", status=6666,
+                          headers={"Content-type": "text/html"}, body=b"Some body")
+            self.assertEqual(response_httprepr(r1),
+                             b'HTTP/1.1 6666 \r\nContent-Type: text/html\r\n\r\nSome body')
 
     def test_open_in_browser(self):
         url = "http:///www.example.com/some/page.html"
diff --git a/tests/test_utils_serialize.py b/tests/test_utils_serialize.py
index daf022aee..a51de1877 100644
--- a/tests/test_utils_serialize.py
+++ b/tests/test_utils_serialize.py
@@ -1,6 +1,7 @@
 import datetime
 import json
 import unittest
+import dataclasses
 from decimal import Decimal
 
 import attr
@@ -10,12 +11,6 @@ from scrapy.http import Request, Response
 from scrapy.utils.serialize import ScrapyJSONEncoder
 
 
-try:
-    from dataclasses import make_dataclass
-except ImportError:
-    make_dataclass = None
-
-
 class JsonEncoderTestCase(unittest.TestCase):
 
     def setUp(self):
@@ -56,12 +51,13 @@ class JsonEncoderTestCase(unittest.TestCase):
         self.assertIn(r.url, rs)
         self.assertIn(str(r.status), rs)
 
-    @unittest.skipIf(not make_dataclass, "No dataclass support")
     def test_encode_dataclass_item(self):
-        TestDataClass = make_dataclass(
-            "TestDataClass",
-            [("name", str), ("url", str), ("price", int)],
-        )
+        @dataclasses.dataclass
+        class TestDataClass:
+            name: str
+            url: str
+            price: int
+
         item = TestDataClass(name="Product", url="http://product.org", price=1)
         encoded = self.encoder.encode(item)
         self.assertEqual(
diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py
index ad7394232..a36e7bc97 100644
--- a/tests/test_utils_signal.py
+++ b/tests/test_utils_signal.py
@@ -1,13 +1,10 @@
 import asyncio
-from unittest import SkipTest
 
 from pydispatch import dispatcher
 from pytest import mark
 from testfixtures import LogCapture
-from twisted import version as twisted_version
 from twisted.internet import defer, reactor
 from twisted.python.failure import Failure
-from twisted.python.versions import Version
 from twisted.trial import unittest
 
 from scrapy.utils.signal import send_catch_log, send_catch_log_deferred
@@ -81,16 +78,6 @@ class SendCatchLogDeferredAsyncDefTest(SendCatchLogDeferredTest):
         return "OK"
 
     def test_send_catch_log(self):
-        if (
-            self.reactor_pytest == 'asyncio'
-            and twisted_version < Version('twisted', 18, 4, 0)
-        ):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'fails due to a timeout when using AsyncIO and Twisted '
-                'versions lower than 18.4.0'
-            )
-
         return super().test_send_catch_log()
 
 
@@ -104,13 +91,6 @@ class SendCatchLogDeferredAsyncioTest(SendCatchLogDeferredTest):
         return await get_from_asyncio_queue("OK")
 
     def test_send_catch_log(self):
-        if twisted_version < Version('twisted', 18, 4, 0):
-            raise SkipTest(
-                'Due to https://twistedmatrix.com/trac/ticket/9390, this test '
-                'fails due to a timeout when using Twisted versions lower '
-                'than 18.4.0'
-            )
-
         return super().test_send_catch_log()
 
 
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 144c7bd76..58e2be622 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,6 +1,8 @@
 import unittest
 
+from scrapy.linkextractors import IGNORED_EXTENSIONS
 from scrapy.spiders import Spider
+from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.url import (
     add_http_if_no_scheme,
     guess_scheme,
@@ -8,9 +10,9 @@ from scrapy.utils.url import (
     strip_url,
     url_is_from_any_domain,
     url_is_from_spider,
+    url_has_any_extension,
 )
 
-
 __doctests__ = ['scrapy.utils.url']
 
 
@@ -81,6 +83,15 @@ class UrlUtilsTest(unittest.TestCase):
         self.assertTrue(url_is_from_spider('http://www.example.net/some/page.html', MySpider))
         self.assertFalse(url_is_from_spider('http://www.example.us/some/page.html', MySpider))
 
+    def test_url_has_any_extension(self):
+        deny_extensions = {'.' + e for e in arg_to_iter(IGNORED_EXTENSIONS)}
+        self.assertTrue(url_has_any_extension("http://www.example.com/archive.tar.gz", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.doc", deny_extensions))
+        self.assertTrue(url_has_any_extension("http://www.example.com/page.pdf", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.htm", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/", deny_extensions))
+        self.assertFalse(url_has_any_extension("http://www.example.com/page.doc.html", deny_extensions))
+
 
 class AddHttpIfNoScheme(unittest.TestCase):
 
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index a6d55cb38..0d5827339 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,10 +4,7 @@ Tests borrowed from the twisted.web.client tests.
 """
 import os
 import shutil
-import sys
-from pkg_resources import parse_version
 
-import cryptography
 import OpenSSL.SSL
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -417,8 +414,6 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):
         ).addCallback(self.assertEqual, to_bytes(s))
 
     def testPayloadDisabledCipher(self):
-        if sys.implementation.name == "pypy" and parse_version(cryptography.__version__) <= parse_version("2.3.1"):
-            self.skipTest("This test expects a failure, but the code does work in PyPy with cryptography<=2.3.1")
         s = "0123456789" * 10
         settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})
         client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)
diff --git a/tox.ini b/tox.ini
index db151f215..eee99cb2d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,14 +11,13 @@ minversion = 1.7.0
 deps =
     -rtests/requirements.txt
     # mitmproxy does not support PyPy
-    # mitmproxy does not support Windows when running Python < 3.7
-    # Python 3.9+ requires https://github.com/mitmproxy/mitmproxy/commit/8e5e43de24c9bc93092b63efc67fbec029a9e7fe
+    # Python 3.9+ requires mitmproxy >= 5.3.0
     # mitmproxy >= 5.3.0 requires h2 >= 4.0, Twisted 21.2 requires h2 < 4.0
     #mitmproxy >= 5.3.0; python_version >= '3.9' and implementation_name != 'pypy'
-    mitmproxy >= 4.0.4; python_version >= '3.7' and python_version < '3.9' and implementation_name != 'pypy'
-    mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
+    # The tests hang with mitmproxy 8.0.0: https://github.com/scrapy/scrapy/issues/5454
+    mitmproxy >= 4.0.4, < 8; python_version < '3.9' and implementation_name != 'pypy'
     # newer markupsafe is incompatible with deps of old mitmproxy (which we get on Python 3.7 and lower)
-    markupsafe < 2.1.0; python_version >= '3.6' and python_version < '3.8' and implementation_name != 'pypy'
+    markupsafe < 2.1.0; python_version < '3.8' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
 passenv =
@@ -39,17 +38,17 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.910
-    types-pyOpenSSL==20.0.3
+    mypy==0.982
+    types-attrs==19.1.0
+    types-pyOpenSSL==21.0.0
     types-setuptools==57.0.0
 commands =
-    pip install types-dataclasses  # remove once py36 support is dropped
     mypy --show-error-codes {posargs: scrapy tests}
 
 [testenv:security]
 basepython = python3
 deps =
-    bandit==1.7.3
+    bandit==1.7.4
 commands =
     bandit -r -c .bandit.yml {posargs:scrapy}
 
@@ -59,33 +58,42 @@ deps =
     {[testenv]deps}
     # Twisted[http2] is required to import some files
     Twisted[http2]>=17.9.0
-    pytest-flake8
-    flake8==3.9.2  # https://github.com/tholo/pytest-flake8/issues/81
+    flake8==5.0.4
 commands =
-    pytest --flake8 {posargs:docs scrapy tests}
+    flake8 {posargs:docs scrapy tests}
 
 [testenv:pylint]
-basepython = python3
+# reppy does not support Python 3.9+
+basepython = python3.8
 deps =
     {[testenv:extra-deps]deps}
-    pylint==2.12.2
+    pylint==2.15.3
 commands =
     pylint conftest.py docs extras scrapy setup.py tests
 
+[testenv:twinecheck]
+basepython = python3
+deps =
+    twine==4.0.1
+commands =
+    python setup.py sdist
+    twine check dist/*
+
 [pinned]
 deps =
-    cryptography==2.0
+    cryptography==3.3
     cssselect==0.9.1
     h2==3.0
     itemadapter==0.1.0
     parsel==1.5.0
     Protego==0.1.15
-    pyOpenSSL==16.2.0
+    pyOpenSSL==21.0.0
     queuelib==1.4.2
-    service_identity==16.0.0
-    Twisted[http2]==17.9.0
+    service_identity==18.1.0
+    Twisted[http2]==18.9.0
     w3lib==1.17.0
-    zope.interface==4.1.3
+    zope.interface==5.1.0
+    lxml==4.3.0
     -rtests/requirements.txt
 
     # mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
@@ -94,7 +102,7 @@ deps =
     # Extras
     botocore==1.4.87
     google-cloud-storage==1.29.0
-    Pillow==4.0.0
+    Pillow==7.1.0
 setenv =
     _SCRAPY_PINNED=true
 install_command =
@@ -103,7 +111,6 @@ install_command =
 [testenv:pinned]
 deps =
     {[pinned]deps}
-    lxml==3.5.0
     PyDispatcher==2.0.5
 install_command = {[pinned]install_command}
 setenv =
@@ -113,25 +120,25 @@ setenv =
 basepython = python3
 deps =
     {[pinned]deps}
-    # First lxml version that includes a Windows wheel for Python 3.6, so we do
-    # not need to build lxml from sources in a CI Windows job:
-    lxml==3.8.0
     PyDispatcher==2.0.5
 install_command = {[pinned]install_command}
 setenv =
     {[pinned]setenv}
 
 [testenv:extra-deps]
+# reppy does not support Python 3.9+
+basepython = python3.8
 deps =
     {[testenv]deps}
     boto
+    google-cloud-storage
+    # Twisted[http2] currently forces old mitmproxy because of h2 version
+    # restrictions in their deps, so we need to pin old markupsafe here too.
+    markupsafe < 2.1.0
     reppy
     robotexclusionrulesparser
     Pillow>=4.0.0
     Twisted[http2]>=17.9.0
-    # Twisted[http2] currently forces old mitmproxy because of h2 version restrictions in their deps,
-    # so we need to pin old markupsafe here too
-    markupsafe < 2.1.0
 
 [testenv:asyncio]
 commands =
@@ -153,7 +160,6 @@ commands =
 basepython = {[testenv:pypy3]basepython}
 deps =
     {[pinned]deps}
-    lxml==4.0.0
     PyPyDispatcher==2.1.0
 commands = {[testenv:pypy3]commands}
 install_command = {[pinned]install_command}