Merge branch 'master' into deprecate-NoimagesDrop

2022-10-27 15:46:32 +05:00 · 2022-10-27 15:46:32 +05:00 · 5780ccba55
parent cc16af35af 9fbd819b0e
commit 5780ccba55
163 changed files with 5208 additions and 1444 deletions
--- a/.bumpversion.cfg
+++ b/.bumpversion.cfg
@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.6.1
+current_version = 2.7.0
 commit = True
 tag = True
 tag_name = {new_version}
--- a/.flake8
+++ b/.flake8
@ -4,16 +4,19 @@ max-line-length = 119
 ignore = W503

 exclude =
+    docs/conf.py
+
+per-file-ignores =
 # Exclude files that are meant to provide top-level imports
 # E402: Module level import not at top of file
 # F401: Module imported but unused
-    scrapy/__init__.py E402
-    scrapy/core/downloader/handlers/http.py F401
-    scrapy/http/__init__.py F401
-    scrapy/linkextractors/__init__.py E402 F401
-    scrapy/selector/__init__.py F401
-    scrapy/spiders/__init__.py E402 F401
+    scrapy/__init__.py:E402
+    scrapy/core/downloader/handlers/http.py:F401
+    scrapy/http/__init__.py:F401
+    scrapy/linkextractors/__init__.py:E402,F401
+    scrapy/selector/__init__.py:F401
+    scrapy/spiders/__init__.py:E402,F401

    # Issues pending a review:
-    scrapy/utils/url.py F403 F405
-    tests/test_loader.py E741
+    scrapy/utils/url.py:F403,F405
+    tests/test_loader.py:E741
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@ -3,7 +3,7 @@ on: [push, pull_request]

 jobs:
  checks:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
@ -19,18 +19,21 @@ jobs:
        - python-version: 3.8
          env:
            TOXENV: pylint
-        - python-version: 3.6
+        - python-version: 3.7
          env:
            TOXENV: typing
        - python-version: "3.10"  # Keep in sync with .readthedocs.yml
          env:
            TOXENV: docs
+        - python-version: "3.10"
+          env:
+            TOXENV: twinecheck

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@ -3,14 +3,14 @@ on: [push]

 jobs:
  publish:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    if: startsWith(github.event.ref, 'refs/tags/')

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3

    - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
      with:
        python-version: "3.10"

--- a/.github/workflows/tests-macos.yml
+++ b/.github/workflows/tests-macos.yml
@ -3,17 +3,17 @@ on: [push, pull_request]

 jobs:
  tests:
-    runs-on: macos-10.15
+    runs-on: macos-11
    strategy:
      fail-fast: false
      matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@ -3,14 +3,11 @@ on: [push, pull_request]

 jobs:
  tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
-        - python-version: 3.7
-          env:
-            TOXENV: py
        - python-version: 3.8
          env:
            TOXENV: py
@ -23,22 +20,26 @@ jobs:
        - python-version: "3.10"
          env:
            TOXENV: asyncio
-        - python-version: pypy3
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: py
+        - python-version: "3.11.0-rc.2"
+          env:
+            TOXENV: asyncio
+        - python-version: pypy3.9
          env:
            TOXENV: pypy3
-            PYPY_VERSION: 3.6-v7.3.3

        # pinned deps
-        - python-version: 3.6.12
+        - python-version: 3.7.13
          env:
            TOXENV: pinned
-        - python-version: 3.6.12
+        - python-version: 3.7.13
          env:
            TOXENV: asyncio-pinned
-        - python-version: pypy3
+        - python-version: pypy3.7
          env:
            TOXENV: pypy3-pinned
-            PYPY_VERSION: 3.6-v7.2.0

        # extras
        # extra-deps includes reppy, which does not support Python 3.9
@ -48,30 +49,22 @@ jobs:
            TOXENV: extra-deps

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install system libraries
-      if: matrix.python-version == 'pypy3' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.10.0-beta.4'
+      if: matrix.python-version == 'pypy3.9' || contains(matrix.env.TOXENV, 'pinned') || matrix.python-version == '3.11.0-rc.2'
      run: |
        sudo apt-get update
-        # libxml2 2.9.12 from ondrej/php PPA breaks lxml so we pin it to the bionic-updates repo version
-        sudo apt-get install libxml2-dev/bionic-updates libxslt-dev
+        sudo apt-get install libxml2-dev libxslt-dev

    - name: Run tests
      env: ${{ matrix.env }}
      run: |
-        if [[ ! -z "$PYPY_VERSION" ]]; then
-          export PYPY_VERSION="pypy$PYPY_VERSION-linux64"
-          wget "https://downloads.python.org/pypy/${PYPY_VERSION}.tar.bz2"
-          tar -jxf ${PYPY_VERSION}.tar.bz2
-          $PYPY_VERSION/bin/pypy3 -m venv "$HOME/virtualenvs/$PYPY_VERSION"
-          source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"
-        fi
        pip install -U tox
        tox

--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@ -8,12 +8,9 @@ jobs:
      fail-fast: false
      matrix:
        include:
-        - python-version: 3.6
-          env:
-            TOXENV: windows-pinned
        - python-version: 3.7
          env:
-            TOXENV: py
+            TOXENV: windows-pinned
        - python-version: 3.8
          env:
            TOXENV: py
@ -28,10 +25,10 @@ jobs:
            TOXENV: asyncio

    steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3

    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

--- a/.gitignore
+++ b/.gitignore
@ -23,3 +23,6 @@ test-output.*

 # Windows
 Thumbs.db
+
+# OSX miscellaneous
+.DS_Store
--- a/4
+++ b/4
@ -1,4 +0,0 @@
-For information about installing Scrapy see:
-
-* docs/intro/install.rst (local file)
-* https://docs.scrapy.org/en/latest/intro/install.html (online version)
--- a/INSTALL.md
+++ b/INSTALL.md
@ -0,0 +1,4 @@
+For information about installing Scrapy see:
+
+* [Local docs](docs/intro/install.rst)
+* [Online docs](https://docs.scrapy.org/en/latest/intro/install.html)
--- a/README.rst
+++ b/README.rst
@ -1,4 +1,5 @@
 .. image:: https://scrapy.org/img/scrapylogo.png
+   :target: https://scrapy.org/
   
 ======
 Scrapy
@ -57,13 +58,15 @@ including a list of features.
 Requirements
 ============

-* Python 3.6+
+* Python 3.7+
 * Works on Linux, Windows, macOS, BSD

 Install
 =======

-The quick way::
+The quick way:
+
+.. code:: bash

    pip install scrapy

@ -94,8 +97,7 @@ See https://docs.scrapy.org/en/master/contributing.html for details.
 Code of Conduct
 ---------------

-Please note that this project is released with a Contributor Code of Conduct
-(see https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md).
+Please note that this project is released with a Contributor `Code of Conduct <https://github.com/scrapy/scrapy/blob/master/CODE_OF_CONDUCT.md>`_.

 By participating in this project you agree to abide by its terms.
 Please report unacceptable behavior to opensource@zyte.com.
--- a/conftest.py
+++ b/conftest.py
@ -42,16 +42,6 @@ def chdir(tmpdir):
    tmpdir.chdir()


-def pytest_collection_modifyitems(session, config, items):
-    # Avoid executing tests when executing `--flake8` flag (pytest-flake8)
-    try:
-        from pytest_flake8 import Flake8Item
-        if config.getoption('--flake8'):
-            items[:] = [item for item in items if isinstance(item, Flake8Item)]
-    except ImportError:
-        pass
-
-
 def pytest_addoption(parser):
    parser.addoption(
        "--reactor",
--- a/docs/README.rst
+++ b/docs/README.rst
@ -43,7 +43,7 @@ This command will fire up your default browser and open the main page of your
 Start over
 ----------

-To cleanup all generated documentation files and start from scratch run::
+To clean up all generated documentation files and start from scratch run::

    make clean

--- a/docs/_ext/scrapydocs.py
+++ b/docs/_ext/scrapydocs.py
@ -15,7 +15,7 @@ class SettingsListDirective(Directive):


 def is_setting_index(node):
-    if node.tagname == 'index':
+    if node.tagname == 'index' and node['entries']:
        # index entries for setting directives look like:
        # [('pair', 'SETTING_NAME; setting', 'std:setting-SETTING_NAME', '')]
        entry_type, info, refid = node['entries'][0][:3]
@ -80,24 +80,24 @@ def replace_settingslist_nodes(app, doctree, fromdocname):

 def setup(app):
    app.add_crossref_type(
-        directivename = "setting",
-        rolename      = "setting",
-        indextemplate = "pair: %s; setting",
+        directivename="setting",
+        rolename="setting",
+        indextemplate="pair: %s; setting",
    )
    app.add_crossref_type(
-        directivename = "signal",
-        rolename      = "signal",
-        indextemplate = "pair: %s; signal",
+        directivename="signal",
+        rolename="signal",
+        indextemplate="pair: %s; signal",
    )
    app.add_crossref_type(
-        directivename = "command",
-        rolename      = "command",
-        indextemplate = "pair: %s; command",
+        directivename="command",
+        rolename="command",
+        indextemplate="pair: %s; command",
    )
    app.add_crossref_type(
-        directivename = "reqmeta",
-        rolename      = "reqmeta",
-        indextemplate = "pair: %s; reqmeta",
+        directivename="reqmeta",
+        rolename="reqmeta",
+        indextemplate="pair: %s; reqmeta",
    )
    app.add_role('source', source_role)
    app.add_role('commit', commit_role)
--- a/docs/_static/selectors-sample1.html
+++ b/docs/_static/selectors-sample1.html
@ -1,16 +1,17 @@
-<html>
- <head>
-  <base href='http://example.com/' />
-  <title>Example website</title>
- </head>
- <body>
-  <div id='images'>
-   <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' /></a>
-   <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' /></a>
-   <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' /></a>
-   <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' /></a>
-   <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' /></a>
-  </div>
- </body>
-</html>
+<!DOCTYPE html>

+<html>
+  <head>
+    <base href='http://example.com/' />
+    <title>Example website</title>
+  </head>
+  <body>
+    <div id='images'>
+      <a href='image1.html'>Name: My image 1 <br /><img src='image1_thumb.jpg' alt='image1'/></a>
+      <a href='image2.html'>Name: My image 2 <br /><img src='image2_thumb.jpg' alt='image2'/></a>
+      <a href='image3.html'>Name: My image 3 <br /><img src='image3_thumb.jpg' alt='image3'/></a>
+      <a href='image4.html'>Name: My image 4 <br /><img src='image4_thumb.jpg' alt='image4'/></a>
+      <a href='image5.html'>Name: My image 5 <br /><img src='image5_thumb.jpg' alt='image5'/></a>
+    </div>
+  </body>
+</html>
--- a/docs/conf.py
+++ b/docs/conf.py
@ -291,10 +291,12 @@ intersphinx_mapping = {
    'pytest': ('https://docs.pytest.org/en/latest', None),
    'python': ('https://docs.python.org/3', None),
    'sphinx': ('https://www.sphinx-doc.org/en/master', None),
-    'tox': ('https://tox.readthedocs.io/en/latest', None),
-    'twisted': ('https://twistedmatrix.com/documents/current', None),
-    'twistedapi': ('https://twistedmatrix.com/documents/current/api', None),
+    'tox': ('https://tox.wiki/en/latest/', None),
+    'twisted': ('https://docs.twisted.org/en/stable/', None),
+    'twistedapi': ('https://docs.twisted.org/en/stable/api/', None),
+    'w3lib': ('https://w3lib.readthedocs.io/en/latest', None),
 }
+intersphinx_disabled_reftypes = []


 # Options for sphinx-hoverxref options
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@ -214,7 +214,7 @@ Tests
 =====

 Tests are implemented using the :doc:`Twisted unit-testing framework
-<twisted:core/development/policy/test-standard>`. Running tests requires
+<twisted:development/test-standard>`. Running tests requires
 :doc:`tox <tox:index>`.

 .. _running-tests:
@ -232,15 +232,15 @@ To run a specific test (say ``tests/test_loader.py``) use:

 To run the tests on a specific :doc:`tox <tox:index>` environment, use
 ``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.6 use::
+the tests with Python 3.7 use::

-    tox -e py36
+    tox -e py37

 You can also specify a comma-separated list of environments, and use :ref:`tox’s
 parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
 parallel::

-    tox -e py36,py38 -p auto
+    tox -e py37,py38 -p auto

 To pass command-line options to :doc:`pytest <pytest:index>`, add them after
 ``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@ -250,9 +250,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
    tox -- scrapy tests -x  # stop after first failure

 You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.6 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.7 :doc:`tox <tox:index>` environment using all your CPU cores::

-    tox -e py36 -- scrapy tests -n auto
+    tox -e py37 -- scrapy tests -n auto

 To see coverage report install :doc:`coverage <coverage:index>`
 (``pip install coverage``) and run:
--- a/docs/index.rst
+++ b/docs/index.rst
@ -130,7 +130,6 @@ Built-in services
   topics/stats
   topics/email
   topics/telnetconsole
-   topics/webservice

 :doc:`topics/logging`
    Learn how to use Python's builtin logging on Scrapy.
@ -144,9 +143,6 @@ Built-in services
 :doc:`topics/telnetconsole`
    Inspect a running crawler using a built-in Python console.

-:doc:`topics/webservice`
-    Monitor and control a crawler using a web service.
-

 Solving specific problems
 =========================
@ -229,10 +225,11 @@ Extending Scrapy
   topics/downloader-middleware
   topics/spider-middleware
   topics/extensions
-   topics/api
   topics/signals
   topics/scheduler
   topics/exporters
+   topics/components
+   topics/api


 :doc:`topics/architecture`
@ -247,9 +244,6 @@ Extending Scrapy
 :doc:`topics/extensions`
    Extend Scrapy with your custom functionality

-:doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
-
 :doc:`topics/signals`
    See all available signals and how to work with them.

@ -259,6 +253,13 @@ Extending Scrapy
 :doc:`topics/exporters`
    Quickly export your scraped items to a file (XML, CSV, etc).

+:doc:`topics/components`
+    Learn the common API and some good practices when building custom Scrapy
+    components.
+
+:doc:`topics/api`
+    Use it on extensions and middlewares to extend Scrapy functionality.
+

 All the rest
 ============
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@ -9,8 +9,8 @@ Installation guide
 Supported Python versions
 =========================

-Scrapy requires Python 3.6+, either the CPython implementation (default) or
-the PyPy 7.2.0+ implementation (see :ref:`python:implementations`).
+Scrapy requires Python 3.7+, either the CPython implementation (default) or
+the PyPy implementation (see :ref:`python:implementations`).

 .. _intro-install-scrapy:

@ -52,17 +52,7 @@ Scrapy is written in pure Python and depends on a few key Python packages (among
 * `twisted`_, an asynchronous networking framework
 * `cryptography`_ and `pyOpenSSL`_, to deal with various network-level security needs

-The minimal versions which Scrapy is tested against are:
-
-* Twisted 14.0
-* lxml 3.4
-* pyOpenSSL 0.14
-
-Scrapy may work with older versions of these packages
-but it is not guaranteed it will continue working
-because it’s not being tested against them.
-
-Some of these packages themselves depends on non-Python packages
+Some of these packages themselves depend on non-Python packages
 that might require additional installation steps depending on your platform.
 Please check :ref:`platform-specific guides below <intro-install-platform-notes>`.

@ -197,7 +187,7 @@ solutions:
  * Install `homebrew`_ following the instructions in https://brew.sh/

  * Update your ``PATH`` variable to state that homebrew packages should be
-    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordantly
+    used before system packages (Change ``.bashrc`` to ``.zshrc`` accordingly
    if you're using `zsh`_ as default shell)::

      echo "export PATH=/usr/local/bin:/usr/local/sbin:$PATH" >> ~/.bashrc
@ -229,7 +219,7 @@ After any of these workarounds you should be able to install Scrapy::
 PyPy
 ----

-We recommend using the latest PyPy version. The version tested is 5.9.0.
+We recommend using the latest PyPy version.
 For PyPy3, only Linux installation was tested.

 Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.
--- a/docs/intro/overview.rst
+++ b/docs/intro/overview.rst
@ -45,9 +45,9 @@ https://quotes.toscrape.com, following the pagination::
 Put this in a text file, name it to something like ``quotes_spider.py``
 and run the spider using the :command:`runspider` command::

-    scrapy runspider quotes_spider.py -o quotes.jl
+    scrapy runspider quotes_spider.py -o quotes.jsonl

-When this finishes you will have in the ``quotes.jl`` file a list of the
+When this finishes you will have in the ``quotes.jsonl`` file a list of the
 quotes in JSON Lines format, containing text and author, looking like this::

    {"author": "Jane Austen", "text": "\u201cThe person, be it gentleman or lady, who has not pleasure in a good novel, must be intolerably stupid.\u201d"}
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@ -379,7 +379,7 @@ like this:
 Let's open up scrapy shell and play a bit to find out how to extract the data
 we want::

-    $ scrapy shell 'https://quotes.toscrape.com'
+    scrapy shell 'https://quotes.toscrape.com'

 We get a list of selectors for the quote HTML elements with:

@ -482,7 +482,7 @@ to append new content to any existing file. However, appending to a JSON file
 makes the file contents invalid JSON. When appending to a file, consider
 using a different serialization format, such as `JSON Lines`_::

-    scrapy crawl quotes -o quotes.jl
+    scrapy crawl quotes -o quotes.jsonl

 The `JSON Lines`_ format is useful because it's stream-like, you can easily
 append new records to it. It doesn't have the same problem of JSON when you run
--- a/docs/news.rst
+++ b/docs/news.rst
@ -3,6 +3,293 @@
 Release notes
 =============

+.. _release-2.7.0:
+
+Scrapy 2.7.0 (2022-10-17)
+-----------------------------
+
+Highlights:
+
+-   Added Python 3.11 support, dropped Python 3.6 support
+-   Improved support for :ref:`asynchronous callbacks <topics-coroutines>`
+-   :ref:`Asyncio support <using-asyncio>` is enabled by default on new
+    projects
+-   Output names of item fields can now be arbitrary strings
+-   Centralized :ref:`request fingerprinting <request-fingerprints>`
+    configuration is now possible
+
+Modified requirements
+~~~~~~~~~~~~~~~~~~~~~
+
+Python 3.7 or greater is now required; support for Python 3.6 has been dropped.
+Support for the upcoming Python 3.11 has been added.
+
+The minimum required version of some dependencies has changed as well:
+
+-   lxml_: 3.5.0 → 4.3.0
+
+-   Pillow_ (:ref:`images pipeline <images-pipeline>`): 4.0.0 → 7.1.0
+
+-   zope.interface_: 5.0.0 → 5.1.0
+
+(:issue:`5512`, :issue:`5514`, :issue:`5524`, :issue:`5563`, :issue:`5664`,
+:issue:`5670`, :issue:`5678`)
+
+
+Deprecations
+~~~~~~~~~~~~
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` must now accept an
+    ``item`` parameter (:issue:`5504`, :issue:`5508`).
+
+-   The ``scrapy.downloadermiddlewares.decompression`` module is now
+    deprecated (:issue:`5546`, :issue:`5547`).
+
+
+New features
+~~~~~~~~~~~~
+
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>` can now be
+    defined as an :term:`asynchronous generator` (:issue:`4978`).
+
+-   The output of :class:`~scrapy.Request` callbacks defined as
+    :ref:`coroutines <topics-coroutines>` is now processed asynchronously
+    (:issue:`4978`).
+
+-   :class:`~scrapy.spiders.crawl.CrawlSpider` now supports :ref:`asynchronous
+    callbacks <topics-coroutines>` (:issue:`5657`).
+
+-   New projects created with the :command:`startproject` command have
+    :ref:`asyncio support <using-asyncio>` enabled by default (:issue:`5590`,
+    :issue:`5679`).
+
+-   The :setting:`FEED_EXPORT_FIELDS` setting can now be defined as a
+    dictionary to customize the output name of item fields, lifting the
+    restriction that required output names to be valid Python identifiers, e.g.
+    preventing them to have whitespace (:issue:`1008`, :issue:`3266`,
+    :issue:`3696`).
+
+-   You can now customize :ref:`request fingerprinting <request-fingerprints>`
+    through the new :setting:`REQUEST_FINGERPRINTER_CLASS` setting, instead of
+    having to change it on every Scrapy component that relies on request
+    fingerprinting (:issue:`900`, :issue:`3420`, :issue:`4113`, :issue:`4762`,
+    :issue:`4524`).
+
+-   ``jsonl`` is now supported and encouraged as a file extension for `JSON
+    Lines`_ files (:issue:`4848`).
+
+    .. _JSON Lines: https://jsonlines.org/
+
+-   :meth:`ImagesPipeline.thumb_path
+    <scrapy.pipelines.images.ImagesPipeline.thumb_path>` now receives the
+    source :ref:`item <topics-items>` (:issue:`5504`, :issue:`5508`).
+
+
+Bug fixes
+~~~~~~~~~
+
+-   When using Google Cloud Storage with a :ref:`media pipeline
+    <topics-media-pipeline>`, :setting:`FILES_EXPIRES` now also works when
+    :setting:`FILES_STORE` does not point at the root of your Google Cloud
+    Storage bucket (:issue:`5317`, :issue:`5318`).
+
+-   The :command:`parse` command now supports :ref:`asynchronous callbacks
+    <topics-coroutines>` (:issue:`5424`, :issue:`5577`).
+
+-   When using the :command:`parse` command with a URL for which there is no
+    available spider, an exception is no longer raised (:issue:`3264`,
+    :issue:`3265`, :issue:`5375`, :issue:`5376`, :issue:`5497`).
+
+-   :class:`~scrapy.http.TextResponse` now gives higher priority to the `byte
+    order mark`_ when determining the text encoding of the response body,
+    following the `HTML living standard`_ (:issue:`5601`, :issue:`5611`).
+
+    .. _byte order mark: https://en.wikipedia.org/wiki/Byte_order_mark
+    .. _HTML living standard: https://html.spec.whatwg.org/multipage/parsing.html#determining-the-character-encoding
+
+-   MIME sniffing takes the response body into account in FTP and HTTP/1.0
+    requests, as well as in cached requests (:issue:`4873`).
+
+-   MIME sniffing now detects valid HTML 5 documents even if the ``html`` tag
+    is missing (:issue:`4873`).
+
+-   An exception is now raised if :setting:`ASYNCIO_EVENT_LOOP` has a value
+    that does not match the asyncio event loop actually installed
+    (:issue:`5529`).
+
+-   Fixed :meth:`Headers.getlist <scrapy.http.headers.Headers.getlist>`
+    returning only the last header (:issue:`5515`, :issue:`5526`).
+
+-   Fixed :class:`LinkExtractor
+    <scrapy.linkextractors.lxmlhtml.LxmlLinkExtractor>` not ignoring the
+    ``tar.gz`` file extension by default (:issue:`1837`, :issue:`2067`,
+    :issue:`4066`)
+
+
+Documentation
+~~~~~~~~~~~~~
+
+-   Clarified the return type of :meth:`Spider.parse <scrapy.Spider.parse>`
+    (:issue:`5602`, :issue:`5608`).
+
+-   To enable
+    :class:`~scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware`
+    to do `brotli compression`_, installing brotli_ is now recommended instead
+    of installing brotlipy_, as the former provides a more recent version of
+    brotli.
+
+    .. _brotli: https://github.com/google/brotli
+    .. _brotli compression: https://www.ietf.org/rfc/rfc7932.txt
+
+-   :ref:`Signal documentation <topics-signals>` now mentions :ref:`coroutine
+    support <topics-coroutines>` and uses it in code examples (:issue:`4852`,
+    :issue:`5358`).
+
+-   :ref:`bans` now recommends `Common Crawl`_ instead of `Google cache`_
+    (:issue:`3582`, :issue:`5432`).
+
+    .. _Common Crawl: https://commoncrawl.org/
+    .. _Google cache: http://www.googleguide.com/cached_pages.html
+
+-   The new :ref:`topics-components` topic covers enforcing requirements on
+    Scrapy components, like :ref:`downloader middlewares
+    <topics-downloader-middleware>`, :ref:`extensions <topics-extensions>`,
+    :ref:`item pipelines <topics-item-pipeline>`, :ref:`spider middlewares
+    <topics-spider-middleware>`, and more; :ref:`enforce-asyncio-requirement`
+    has also been added (:issue:`4978`).
+
+-   :ref:`topics-settings` now indicates that setting values must be
+    :ref:`picklable <pickle-picklable>` (:issue:`5607`, :issue:`5629`).
+
+-   Removed outdated documentation (:issue:`5446`, :issue:`5373`,
+    :issue:`5369`, :issue:`5370`, :issue:`5554`).
+
+-   Fixed typos (:issue:`5442`, :issue:`5455`, :issue:`5457`, :issue:`5461`,
+    :issue:`5538`, :issue:`5553`, :issue:`5558`, :issue:`5624`, :issue:`5631`).
+
+-   Fixed other issues (:issue:`5283`, :issue:`5284`, :issue:`5559`,
+    :issue:`5567`, :issue:`5648`, :issue:`5659`, :issue:`5665`).
+
+
+Quality assurance
+~~~~~~~~~~~~~~~~~
+
+-   Added a continuous integration job to run `twine check`_ (:issue:`5655`,
+    :issue:`5656`).
+
+    .. _twine check: https://twine.readthedocs.io/en/stable/#twine-check
+
+-   Addressed test issues and warnings (:issue:`5560`, :issue:`5561`,
+    :issue:`5612`, :issue:`5617`, :issue:`5639`, :issue:`5645`, :issue:`5662`,
+    :issue:`5671`, :issue:`5675`).
+
+-   Cleaned up code (:issue:`4991`, :issue:`4995`, :issue:`5451`,
+    :issue:`5487`, :issue:`5542`, :issue:`5667`, :issue:`5668`, :issue:`5672`).
+
+-   Applied minor code improvements (:issue:`5661`).
+
+
+.. _release-2.6.3:
+
+Scrapy 2.6.3 (2022-09-27)
+-------------------------
+
+-   Added support for pyOpenSSL_ 22.1.0, removing support for SSLv3
+    (:issue:`5634`, :issue:`5635`, :issue:`5636`).
+
+-   Upgraded the minimum versions of the following dependencies:
+
+    -   cryptography_: 2.0 → 3.3
+
+    -   pyOpenSSL_: 16.2.0 → 21.0.0
+
+    -   service_identity_: 16.0.0 → 18.1.0
+
+    -   Twisted_: 17.9.0 → 18.9.0
+
+    -   zope.interface_: 4.1.3 → 5.0.0
+
+    (:issue:`5621`, :issue:`5632`)
+
+-   Fixes test and documentation issues (:issue:`5612`, :issue:`5617`,
+    :issue:`5631`).
+
+
+.. _release-2.6.2:
+
+Scrapy 2.6.2 (2022-07-25)
+-------------------------
+
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+Also fixes the following regressions introduced in 2.6.0:
+
+-   :class:`~scrapy.crawler.CrawlerProcess` supports again crawling multiple
+    spiders (:issue:`5435`, :issue:`5436`)
+
+-   Installing a Twisted reactor before Scrapy does (e.g. importing
+    :mod:`twisted.internet.reactor` somewhere at the module level) no longer
+    prevents Scrapy from starting, as long as a different reactor is not
+    specified in :setting:`TWISTED_REACTOR` (:issue:`5525`, :issue:`5528`)
+
+-   Fixed an exception that was being logged after the spider finished under
+    certain conditions (:issue:`5437`, :issue:`5440`)
+
+-   The ``--output``/``-o`` command-line parameter supports again a value
+    starting with a hyphen (:issue:`5444`, :issue:`5445`)
+
+-   The ``scrapy parse -h`` command no longer throws an error (:issue:`5481`,
+    :issue:`5482`)
+
+
 .. _release-2.6.1:

 Scrapy 2.6.1 (2022-03-01)
@ -113,6 +400,9 @@ Backward-incompatible changes
    meet expectations, :exc:`TypeError` is now raised at startup time. Before,
    other exceptions would be raised at run time. (:issue:`3559`)

+-   The ``_encoding`` field of serialized :class:`~scrapy.http.Request` objects
+    is now named ``encoding``, in line with all other fields (:issue:`5130`)
+

 Deprecation removals
 ~~~~~~~~~~~~~~~~~~~~
@ -1643,7 +1933,7 @@ New features
    :issue:`4370`)

 *   A new ``keep_fragments`` parameter of
-    :func:`scrapy.utils.request.request_fingerprint` allows to generate
+    ``scrapy.utils.request.request_fingerprint`` allows to generate
    different fingerprints for requests with different fragments in their URL
    (:issue:`4104`)

@ -1897,6 +2187,59 @@ affect subclasses:
 (:issue:`3884`)


+.. _release-1.8.3:
+
+Scrapy 1.8.3 (2022-07-25)
+-------------------------
+
+**Security bug fix:**
+
+-   When :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware`
+    processes a request with :reqmeta:`proxy` metadata, and that
+    :reqmeta:`proxy` metadata includes proxy credentials,
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` sets
+    the ``Proxy-Authentication`` header, but only if that header is not already
+    set.
+
+    There are third-party proxy-rotation downloader middlewares that set
+    different :reqmeta:`proxy` metadata every time they process a request.
+
+    Because of request retries and redirects, the same request can be processed
+    by downloader middlewares more than once, including both
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` and
+    any third-party proxy-rotation downloader middleware.
+
+    These third-party proxy-rotation downloader middlewares could change the
+    :reqmeta:`proxy` metadata of a request to a new value, but fail to remove
+    the ``Proxy-Authentication`` header from the previous value of the
+    :reqmeta:`proxy` metadata, causing the credentials of one proxy to be sent
+    to a different proxy.
+
+    To prevent the unintended leaking of proxy credentials, the behavior of
+    :class:`~scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware` is now
+    as follows when processing a request:
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata that
+        includes credentials, the ``Proxy-Authorization`` header is always
+        updated to feature those credentials.
+
+    -   If the request being processed defines :reqmeta:`proxy` metadata
+        without credentials, the ``Proxy-Authorization`` header is removed
+        *unless* it was originally defined for the same proxy URL.
+
+        To remove proxy credentials while keeping the same proxy URL, remove
+        the ``Proxy-Authorization`` header.
+
+    -   If the request has no :reqmeta:`proxy` metadata, or that metadata is a
+        falsy value (e.g. ``None``), the ``Proxy-Authorization`` header is
+        removed.
+
+        It is no longer possible to set a proxy URL through the
+        :reqmeta:`proxy` metadata but set the credentials through the
+        ``Proxy-Authorization`` header. Set proxy credentials through the
+        :reqmeta:`proxy` metadata instead.
+
+
 .. _release-1.8.2:

 Scrapy 1.8.2 (2022-03-01)
@ -2985,7 +3328,7 @@ New Features
 ~~~~~~~~~~~~

 - Accept proxy credentials in :reqmeta:`proxy` request meta key (:issue:`2526`)
- Support `brotli`_-compressed content; requires optional `brotlipy`_
+- Support `brotli-compressed`_ content; requires optional `brotlipy`_
  (:issue:`2535`)
 - New :ref:`response.follow <response-follow-example>` shortcut
  for creating requests (:issue:`1940`)
@ -3022,7 +3365,7 @@ New Features
 - ``python -m scrapy`` as a more explicit alternative to ``scrapy`` command
  (:issue:`2740`)

-.. _brotli: https://github.com/google/brotli
+.. _brotli-compressed: https://www.ietf.org/rfc/rfc7932.txt
 .. _brotlipy: https://github.com/python-hyper/brotlipy/

 Bug fixes
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@ -1,4 +1,4 @@
-Sphinx>=3.0
-sphinx-hoverxref>=0.2b1
-sphinx-notfound-page>=0.4
-sphinx-rtd-theme>=0.5.2
+sphinx==5.0.2
+sphinx-hoverxref==1.1.1
+sphinx-notfound-page==0.8
+sphinx-rtd-theme==1.0.0
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@ -32,6 +32,13 @@ how you :ref:`configure the downloader middlewares
    :class:`scrapy.Spider` subclass and a
    :class:`scrapy.settings.Settings` object.

+    .. attribute:: request_fingerprinter
+
+        The request fingerprint builder of this crawler.
+
+        This is used from extensions and middlewares to build short, unique
+        identifiers for requests. See :ref:`request-fingerprints`.
+
    .. attribute:: settings

        The settings manager of this crawler.
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this:
         down to Scrapy 2.0 (earlier versions do not support
         :mod:`asyncio`), you can copy the implementation of these functions
         into your own code.
+
+
+.. _enforce-asyncio-requirement:
+
+Enforcing asyncio as a requirement
+==================================
+
+If you are writing a :ref:`component <topics-components>` that requires asyncio
+to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
+:ref:`enforce it as a requirement <enforce-component-requirements>`. For
+example::
+
+    from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    class MyComponent:
+
+        def __init__(self):
+            if not is_asyncio_reactor_installed():
+                raise ValueError(
+                    f"{MyComponent.__qualname__} requires the asyncio Twisted "
+                    f"reactor. Make sure you have it configured in the "
+                    f"TWISTED_REACTOR setting. See the asyncio documentation "
+                    f"of Scrapy for more information."
+                )
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@ -0,0 +1,84 @@
+.. _topics-components:
+
+==========
+Components
+==========
+
+A Scrapy component is any class whose objects are created using
+:func:`scrapy.utils.misc.create_instance`.
+
+That includes the classes that you may assign to the following settings:
+
+-   :setting:`DNS_RESOLVER`
+
+-   :setting:`DOWNLOAD_HANDLERS`
+
+-   :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
+
+-   :setting:`DOWNLOADER_MIDDLEWARES`
+
+-   :setting:`DUPEFILTER_CLASS`
+
+-   :setting:`EXTENSIONS`
+
+-   :setting:`FEED_EXPORTERS`
+
+-   :setting:`FEED_STORAGES`
+
+-   :setting:`ITEM_PIPELINES`
+
+-   :setting:`SCHEDULER`
+
+-   :setting:`SCHEDULER_DISK_QUEUE`
+
+-   :setting:`SCHEDULER_MEMORY_QUEUE`
+
+-   :setting:`SCHEDULER_PRIORITY_QUEUE`
+
+-   :setting:`SPIDER_MIDDLEWARES`
+
+Third-party Scrapy components may also let you define additional Scrapy
+components, usually configurable through :ref:`settings <topics-settings>`, to
+modify their behavior.
+
+.. _enforce-component-requirements:
+
+Enforcing component requirements
+================================
+
+Sometimes, your components may only be intended to work under certain
+conditions. For example, they may require a minimum version of Scrapy to work as
+intended, or they may require certain settings to have specific values.
+
+In addition to describing those conditions in the documentation of your
+component, it is a good practice to raise an exception from the ``__init__``
+method of your component if those conditions are not met at run time.
+
+In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
+:ref:`extensions <topics-extensions>`, :ref:`item pipelines
+<topics-item-pipeline>`, and :ref:`spider middlewares
+<topics-spider-middleware>`, you should raise
+:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
+parameter to the exception so that it is printed in the logs, for the user to
+see. For other components, feel free to raise whatever other exception feels
+right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
+version mismatch, while :exc:`ValueError` may be better if the issue is the
+value of a setting.
+
+If your requirement is a minimum Scrapy version, you may use
+:attr:`scrapy.__version__` to enforce your requirement. For example::
+
+    from pkg_resources import parse_version
+
+    import scrapy
+
+    class MyComponent:
+
+        def __init__(self):
+            if parse_version(scrapy.__version__) < parse_version('2.7'):
+                raise RuntimeError(
+                    f"{MyComponent.__qualname__} requires Scrapy 2.7 or "
+                    f"later, which allow defining the process_spider_output "
+                    f"method of spider middlewares as an asynchronous "
+                    f"generator."
+                )
--- a/docs/topics/contracts.rst
+++ b/docs/topics/contracts.rst
@ -102,7 +102,7 @@ override three methods:
    .. method:: Contract.post_process(output)

        This allows processing the output of the callback. Iterators are
-        converted listified before being passed to this hook.
+        converted to lists before being passed to this hook.

 Raise :class:`~scrapy.exceptions.ContractFail` from
 :class:`~scrapy.contracts.Contract.pre_process` or
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@ -1,3 +1,5 @@
+.. _topics-coroutines:
+
 ==========
 Coroutines
 ==========
@ -17,14 +19,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):

 -   :class:`~scrapy.Request` callbacks.

-    .. note:: The callback output is not processed until the whole callback
-        finishes.
+    If you are using any custom or third-party :ref:`spider middleware
+    <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.

-        As a side effect, if the callback raises an exception, none of its
-        output is processed.
-
-        This is a known caveat of the current implementation that we aim to
-        address in a future version of Scrapy.
+    .. versionchanged:: 2.7
+       Output of async callbacks is now processed asynchronously instead of
+       collecting all of it first.

 -   The :meth:`process_item` method of
    :ref:`item pipelines <topics-item-pipeline>`.
@ -39,12 +39,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):

 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.

-Usage
-=====
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>`.

-There are several use cases for coroutines in Scrapy. Code that would
-return Deferreds when written for previous Scrapy versions, such as downloader
-middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+    It must be defined as an :term:`asynchronous generator`. The input
+    ``result`` parameter is an :term:`asynchronous iterable`.
+
+    See also :ref:`sync-async-spider-middleware` and
+    :ref:`universal-spider-middleware`.
+
+    .. versionadded:: 2.7
+
+General usage
+=============
+
+There are several use cases for coroutines in Scrapy.
+
+Code that would return Deferreds when written for previous Scrapy versions,
+such as downloader middlewares and signal handlers, can be rewritten to be
+shorter and cleaner::

    from itemadapter import ItemAdapter

@ -104,7 +118,100 @@ Common use cases for asynchronous code include:
 * storing data in databases (in pipelines and middlewares);
 * delaying the spider initialization until some external event (in the
  :signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
-  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
+  (see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).

 .. _aio-libs: https://github.com/aio-libs
+
+
+.. _sync-async-spider-middleware:
+
+Mixing synchronous and asynchronous spider middlewares
+======================================================
+
+.. versionadded:: 2.7
+
+The output of a :class:`~scrapy.Request` callback is passed as the ``result``
+parameter to the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of the first :ref:`spider middleware <topics-spider-middleware>` from the
+:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
+Then the output of that ``process_spider_output`` method is passed to the
+``process_spider_output`` method of the next spider middleware, and so on for
+every active spider middleware.
+
+Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
+in this chain of calls.
+
+However, if any of the ``process_spider_output`` methods is defined as a
+synchronous method, and the previous ``Request`` callback or
+``process_spider_output`` method is a coroutine, there are some drawbacks to
+the asynchronous-to-synchronous conversion that Scrapy does so that the
+synchronous ``process_spider_output`` method gets a synchronous iterable as its
+``result`` parameter:
+
+-   The whole output of the previous ``Request`` callback or
+    ``process_spider_output`` method is awaited at this point.
+
+-   If an exception raises while awaiting the output of the previous
+    ``Request`` callback or ``process_spider_output`` method, none of that
+    output will be processed.
+
+    This contrasts with the regular behavior, where all items yielded before
+    an exception raises are processed.
+
+Asynchronous-to-synchronous conversions are supported for backward
+compatibility, but they are deprecated and will stop working in a future
+version of Scrapy.
+
+To avoid asynchronous-to-synchronous conversions, when defining ``Request``
+callbacks as coroutine methods or when using spider middlewares whose
+``process_spider_output`` method is an :term:`asynchronous generator`, all
+active spider middlewares must either have their ``process_spider_output``
+method defined as an asynchronous generator or :ref:`define a
+process_spider_output_async method <universal-spider-middleware>`.
+
+.. note:: When using third-party spider middlewares that only define a
+          synchronous ``process_spider_output`` method, consider
+          :ref:`making them universal <universal-spider-middleware>` through
+          :ref:`subclassing <tut-inheritance>`.
+
+
+.. _universal-spider-middleware:
+
+Universal spider middlewares
+============================
+
+.. versionadded:: 2.7
+
+To allow writing a spider middleware that supports asynchronous execution of
+its ``process_spider_output`` method in Scrapy 2.7 and later (avoiding
+:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
+while maintaining support for older Scrapy versions, you may define
+``process_spider_output`` as a synchronous method and define an
+:term:`asynchronous generator` version of that method with an alternative name:
+``process_spider_output_async``.
+
+For example::
+
+    class UniversalSpiderMiddleware:
+        def process_spider_output(self, response, result, spider):
+            for r in result:
+                # ... do something with r
+                yield r
+
+        async def process_spider_output_async(self, response, result, spider):
+            async for r in result:
+                # ... do something with r
+                yield r
+
+.. note:: This is an interim measure to allow, for a time, to write code that
+          works in Scrapy 2.7 and later without requiring
+          asynchronous-to-synchronous conversions, and works in earlier Scrapy
+          versions as well.
+
+          In some future version of Scrapy, however, this feature will be
+          deprecated and, eventually, in a later version of Scrapy, this
+          feature will be removed, and all spider middlewares will be expected
+          to define their ``process_spider_output`` method as an asynchronous
+          generator.
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@ -955,7 +955,7 @@ default because HTTP specs say so.
 .. setting:: RETRY_PRIORITY_ADJUST

 RETRY_PRIORITY_ADJUST
---------------------
+^^^^^^^^^^^^^^^^^^^^^

 Default: ``-1``

@ -1119,7 +1119,7 @@ In order to use this parser:
 .. _support-for-new-robots-parser:

 Implementing support for a new parser
-------------------------------------
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 You can implement support for a new robots.txt_ parser by subclassing
 the abstract base class :class:`~scrapy.robotstxt.RobotParser` and
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@ -117,7 +117,7 @@ after your custom code.

 Example::

-      from scrapy.exporter import XmlItemExporter
+      from scrapy.exporters import XmlItemExporter

      class ProductXmlExporter(XmlItemExporter):

@ -195,17 +195,25 @@ BaseItemExporter

   .. attribute:: fields_to_export

-      A list with the name of the fields that will be exported, or ``None`` if
-      you want to export all fields. Defaults to ``None``.
+      Fields to export, their order [1]_ and their output names.

-      Some exporters (like :class:`CsvItemExporter`) respect the order of the
-      fields defined in this attribute.
+      Possible values are:

-      When using :ref:`item objects <item-types>` that do not expose all their
-      possible fields, exporters that do not support exporting a different
-      subset of fields per item will only export the fields found in the first
-      item exported. Use ``fields_to_export`` to define all the fields to be
-      exported.
+      -   ``None`` (all fields [2]_, default)
+
+      -   A list of fields::
+
+              ['field1', 'field2']
+
+      -   A dict where keys are fields and values are output names::
+
+              {'field1': 'Field 1', 'field2': 'Field 2'}
+
+      .. [1] Not all exporters respect the specified field order.
+      .. [2] When using :ref:`item objects <item-types>` that do not expose
+             all their possible fields, exporters that do not support exporting
+             a different subset of fields per item will only export the fields 
+             found in the first item exported.

   .. attribute:: export_empty_fields

@ -297,8 +305,8 @@ CsvItemExporter

   Exports items in CSV format to the given file-like object. If the
   :attr:`fields_to_export` attribute is set, it will be used to define the
-   CSV columns and their order. The :attr:`export_empty_fields` attribute has
-   no effect on this exporter.
+   CSV columns, their order and their column names. The
+   :attr:`export_empty_fields` attribute has no effect on this exporter.

   :param file: the file-like object to use for exporting the data. Its ``write`` method should
                accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
--- a/docs/topics/extensions.rst
+++ b/docs/topics/extensions.rst
@ -17,7 +17,7 @@ settings, just like any other Scrapy code.

 It is customary for extensions to prefix their settings with their own name, to
 avoid collision with existing (and future) extensions. For example, a
-hypothetic extension to handle `Google Sitemaps`_ would use settings like
+hypothetical extension to handle `Google Sitemaps`_ would use settings like
 ``GOOGLESITEMAP_ENABLED``, ``GOOGLESITEMAP_DEPTH``, and so on.

 .. _Google Sitemaps: https://en.wikipedia.org/wiki/Sitemaps
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@ -58,7 +58,7 @@ CSV

 -   Exporter used: :class:`~scrapy.exporters.CsvItemExporter`

-   To specify columns to export and their order use
+-   To specify columns to export, their order and their column names, use
    :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
    option, but it is important for CSV because unlike many other export
    formats CSV uses a fixed header.
@ -522,18 +522,9 @@ FEED_EXPORT_FIELDS

 Default: ``None``

-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses the fields
-defined in :ref:`item objects <topics-items>` yielded by your spider.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.

 .. setting:: FEED_EXPORT_INDENT

@ -638,6 +629,7 @@ Default::
    {
        'json': 'scrapy.exporters.JsonItemExporter',
        'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+        'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
        'jl': 'scrapy.exporters.JsonLinesItemExporter',
        'csv': 'scrapy.exporters.CsvItemExporter',
        'xml': 'scrapy.exporters.XmlItemExporter',
@ -763,7 +755,7 @@ source spider in the feed URI:

 #.  Use ``%(spider_name)s`` in your feed URI::

-        scrapy crawl <spider_name> -o "%(spider_name)s.jl"
+        scrapy crawl <spider_name> -o "%(spider_name)s.jsonl"


 .. _URIs: https://en.wikipedia.org/wiki/Uniform_Resource_Identifier
--- a/docs/topics/item-pipeline.rst
+++ b/docs/topics/item-pipeline.rst
@ -60,9 +60,9 @@ Additionally, they may also implement the following methods:
   :param spider: the spider which was closed
   :type spider: :class:`~scrapy.Spider` object

-.. method:: from_crawler(cls, crawler)
+.. classmethod:: from_crawler(cls, crawler)

-   If present, this classmethod is called to create a pipeline instance
+   If present, this class method is called to create a pipeline instance
   from a :class:`~scrapy.crawler.Crawler`. It must return a new instance
   of the pipeline. Crawler object provides access to all Scrapy core
   components like settings and signals; it is a way for pipeline to
@ -99,11 +99,11 @@ contain a price::
                raise DropItem(f"Missing price in {item}")


-Write items to a JSON file
--------------------------
+Write items to a JSON lines file
+--------------------------------

 The following pipeline stores all scraped items (from all spiders) into a
-single ``items.jl`` file, containing one item per line serialized in JSON
+single ``items.jsonl`` file, containing one item per line serialized in JSON
 format::

   import json
@ -113,7 +113,7 @@ format::
   class JsonWriterPipeline:

       def open_spider(self, spider):
-           self.file = open('items.jl', 'w')
+           self.file = open('items.jsonl', 'w')

       def close_spider(self, spider):
           self.file.close()
--- a/docs/topics/items.rst
+++ b/docs/topics/items.rst
@ -102,11 +102,6 @@ Additionally, ``dataclass`` items also allow to:
 * define custom field metadata through :func:`dataclasses.field`, which can be used to
  :ref:`customize serialization <topics-exporters-field-serialization>`.

-They work natively in Python 3.7 or later, or using the `dataclasses
-backport`_ in Python 3.6.
-
-.. _dataclasses backport: https://pypi.org/project/dataclasses/
-
 Example::

    from dataclasses import dataclass
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@ -154,7 +154,7 @@ Too many spiders?
 If your project has too many spiders executed in parallel,
 the output of :func:`prefs()` can be difficult to read.
 For this reason, that function has a ``ignore`` argument which can be used to
-ignore a particular class (and all its subclases). For
+ignore a particular class (and all its subclasses). For
 example, this won't show any live references to spiders:

 >>> from scrapy.spiders import Spider
--- a/docs/topics/media-pipeline.rst
+++ b/docs/topics/media-pipeline.rst
@ -70,7 +70,7 @@ The advantage of using the :class:`ImagesPipeline` for image files is that you
 can configure some extra functions like generating thumbnails and filtering
 the images based on their size.

-The Images Pipeline requires Pillow_ 4.0.0 or greater. It is used for
+The Images Pipeline requires Pillow_ 7.1.0 or greater. It is used for
 thumbnailing and normalizing images to JPEG/RGB format.

 .. _Pillow: https://github.com/python-pillow/Pillow
@ -656,6 +656,26 @@ See here the methods that you can override in your custom Images Pipeline:
      .. versionadded:: 2.4
         The *item* parameter.

+   .. method:: ImagesPipeline.thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)
+
+      This method is called for every item of  :setting:`IMAGES_THUMBS` per downloaded item. It returns the
+      thumbnail download path of the image originating from the specified
+      :class:`response <scrapy.http.Response>`.
+
+      In addition to ``response``, this method receives the original
+      :class:`request <scrapy.Request>`,
+      ``thumb_id``,
+      :class:`info <scrapy.pipelines.media.MediaPipeline.SpiderInfo>` and
+      :class:`item <scrapy.Item>`.
+
+      You can override this method to customize the thumbnail download path of each image.
+      You can use the ``item`` to determine the file path based on some item
+      property.
+
+      By default the :meth:`thumb_path` method returns
+      ``thumbs/<size name>/<request URL hash>.<extension>``.
+
+
   .. method:: ImagesPipeline.get_media_requests(item, info)

      Works the same way as :meth:`FilesPipeline.get_media_requests` method,
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@ -339,6 +339,7 @@ errors if needed::
                request = failure.request
                self.logger.error('TimeoutError on %s', request.url)

+
 .. _errback-cb_kwargs:

 Accessing additional data in errback functions
@ -364,6 +365,278 @@ achieve this by using ``Failure.request.cb_kwargs``::
            main_url=failure.request.cb_kwargs['main_url'],
        )

+
+.. _request-fingerprints:
+
+Request fingerprints
+--------------------
+
+There are some aspects of scraping, such as filtering out duplicate requests
+(see :setting:`DUPEFILTER_CLASS`) or caching responses (see
+:setting:`HTTPCACHE_POLICY`), where you need the ability to generate a short,
+unique identifier from a :class:`~scrapy.http.Request` object: a request
+fingerprint.
+
+You often do not need to worry about request fingerprints, the default request
+fingerprinter works for most projects.
+
+However, there is no universal way to generate a unique identifier from a
+request, because different situations require comparing requests differently.
+For example, sometimes you may need to compare URLs case-insensitively, include
+URL fragments, exclude certain URL query parameters, include some or all
+headers, etc.
+
+To change how request fingerprints are built for your requests, use the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.
+
+.. setting:: REQUEST_FINGERPRINTER_CLASS
+
+REQUEST_FINGERPRINTER_CLASS
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.7
+
+Default: :class:`scrapy.utils.request.RequestFingerprinter`
+
+A :ref:`request fingerprinter class <custom-request-fingerprinter>` or its
+import path.
+
+.. autoclass:: scrapy.utils.request.RequestFingerprinter
+
+
+.. setting:: REQUEST_FINGERPRINTER_IMPLEMENTATION
+
+REQUEST_FINGERPRINTER_IMPLEMENTATION
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 2.7
+
+Default: ``'2.6'``
+
+Determines which request fingerprinting algorithm is used by the default
+request fingerprinter class (see :setting:`REQUEST_FINGERPRINTER_CLASS`).
+
+Possible values are:
+
+-   ``'2.6'`` (default)
+
+    This implementation uses the same request fingerprinting algorithm as
+    Scrapy 2.6 and earlier versions.
+
+    Even though this is the default value for backward compatibility reasons,
+    it is a deprecated value.
+
+-   ``'2.7'``
+
+    This implementation was introduced in Scrapy 2.7 to fix an issue of the
+    previous implementation.
+
+    New projects should use this value. The :command:`startproject` command
+    sets this value in the generated ``settings.py`` file.
+
+If you are using the default value (``'2.6'``) for this setting, and you are
+using Scrapy components where changing the request fingerprinting algorithm
+would cause undesired results, you need to carefully decide when to change the
+value of this setting, or switch the :setting:`REQUEST_FINGERPRINTER_CLASS`
+setting to a custom request fingerprinter class that implements the 2.6 request
+fingerprinting algorithm and does not log this warning (
+:ref:`2.6-request-fingerprinter` includes an example implementation of such a
+class).
+
+Scenarios where changing the request fingerprinting algorithm may cause
+undesired results include, for example, using the HTTP cache middleware (see
+:class:`~scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware`).
+Changing the request fingerprinting algorithm would invalidate the current
+cache, requiring you to redownload all requests again.
+
+Otherwise, set :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` to ``'2.7'`` in
+your settings to switch already to the request fingerprinting implementation
+that will be the only request fingerprinting implementation available in a
+future version of Scrapy, and remove the deprecation warning triggered by using
+the default value (``'2.6'``).
+
+
+.. _2.6-request-fingerprinter:
+.. _custom-request-fingerprinter:
+
+Writing your own request fingerprinter
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+A request fingerprinter is a class that must implement the following method:
+
+.. currentmodule:: None
+
+.. method:: fingerprint(self, request)
+
+   Return a :class:`bytes` object that uniquely identifies *request*.
+
+   See also :ref:`request-fingerprint-restrictions`.
+
+   :param request: request to fingerprint
+   :type request: scrapy.http.Request
+
+Additionally, it may also implement the following methods:
+
+.. classmethod:: from_crawler(cls, crawler)
+   :noindex:
+
+   If present, this class method is called to create a request fingerprinter
+   instance from a :class:`~scrapy.crawler.Crawler` object. It must return a
+   new instance of the request fingerprinter.
+
+   *crawler* provides access to all Scrapy core components like settings and
+   signals; it is a way for the request fingerprinter to access them and hook
+   its functionality into Scrapy.
+
+   :param crawler: crawler that uses this request fingerprinter
+   :type crawler: :class:`~scrapy.crawler.Crawler` object
+
+.. classmethod:: from_settings(cls, settings)
+
+   If present, and ``from_crawler`` is not defined, this class method is called
+   to create a request fingerprinter instance from a
+   :class:`~scrapy.settings.Settings` object. It must return a new instance of
+   the request fingerprinter.
+
+.. currentmodule:: scrapy.http
+
+The :meth:`fingerprint` method of the default request fingerprinter,
+:class:`scrapy.utils.request.RequestFingerprinter`, uses
+:func:`scrapy.utils.request.fingerprint` with its default parameters. For some
+common use cases you can use :func:`scrapy.utils.request.fingerprint` as well
+in your :meth:`fingerprint` method implementation:
+
+.. autofunction:: scrapy.utils.request.fingerprint
+
+For example, to take the value of a request header named ``X-ID`` into
+account::
+
+    # my_project/settings.py
+    REQUEST_FINGERPRINTER_CLASS = 'my_project.utils.RequestFingerprinter'
+
+    # my_project/utils.py
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            return fingerprint(request, include_headers=['X-ID'])
+
+You can also write your own fingerprinting logic from scratch.
+
+However, if you do not use :func:`scrapy.utils.request.fingerprint`, make sure
+you use :class:`~weakref.WeakKeyDictionary` to cache request fingerprints:
+
+-   Caching saves CPU by ensuring that fingerprints are calculated only once
+    per request, and not once per Scrapy component that needs the fingerprint
+    of a request.
+
+-   Using :class:`~weakref.WeakKeyDictionary` saves memory by ensuring that
+    request objects do not stay in memory forever just because you have
+    references to them in your cache dictionary.
+
+For example, to take into account only the URL of a request, without any prior
+URL canonicalization or taking the request method or body into account::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.url))
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+If you need to be able to override the request fingerprinting for arbitrary
+requests from your spider callbacks, you may implement a request fingerprinter
+that reads fingerprints from :attr:`request.meta <scrapy.http.Request.meta>`
+when available, and then falls back to
+:func:`scrapy.utils.request.fingerprint`. For example::
+
+    from scrapy.utils.request import fingerprint
+
+    class RequestFingerprinter:
+
+        def fingerprint(self, request):
+            if 'fingerprint' in request.meta:
+                return request.meta['fingerprint']
+            return fingerprint(request)
+
+If you need to reproduce the same fingerprinting algorithm as Scrapy 2.6
+without using the deprecated ``'2.6'`` value of the
+:setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION` setting, use the following
+request fingerprinter::
+
+    from hashlib import sha1
+    from weakref import WeakKeyDictionary
+
+    from scrapy.utils.python import to_bytes
+    from w3lib.url import canonicalize_url
+
+    class RequestFingerprinter:
+
+        cache = WeakKeyDictionary()
+
+        def fingerprint(self, request):
+            if request not in self.cache:
+                fp = sha1()
+                fp.update(to_bytes(request.method))
+                fp.update(to_bytes(canonicalize_url(request.url)))
+                fp.update(request.body or b'')
+                self.cache[request] = fp.digest()
+            return self.cache[request]
+
+
+.. _request-fingerprint-restrictions:
+
+Request fingerprint restrictions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Scrapy components that use request fingerprints may impose additional
+restrictions on the format of the fingerprints that your :ref:`request
+fingerprinter <custom-request-fingerprinter>` generates.
+
+The following built-in Scrapy components have such restrictions:
+
+-   :class:`scrapy.extensions.httpcache.FilesystemCacheStorage` (default
+    value of :setting:`HTTPCACHE_STORAGE`)
+
+    Request fingerprints must be at least 1 byte long.
+
+    Path and filename length limits of the file system of
+    :setting:`HTTPCACHE_DIR` also apply. Inside :setting:`HTTPCACHE_DIR`,
+    the following directory structure is created:
+
+    -   :attr:`Spider.name <scrapy.spiders.Spider.name>`
+
+        -   first byte of a request fingerprint as hexadecimal
+
+            -   fingerprint as hexadecimal
+
+                -   filenames up to 16 characters long
+
+    For example, if a request fingerprint is made of 20 bytes (default),
+    :setting:`HTTPCACHE_DIR` is ``'/home/user/project/.scrapy/httpcache'``,
+    and the name of your spider is ``'my_spider'`` your file system must
+    support a file path like::
+
+        /home/user/project/.scrapy/httpcache/my_spider/01/0123456789abcdef0123456789abcdef01234567/response_headers
+
+-   :class:`scrapy.extensions.httpcache.DbmCacheStorage`
+
+    The underlying DBM implementation must support keys as long as twice
+    the number of bytes of a request fingerprint, plus 5. For example,
+    if a request fingerprint is made of 20 bytes (default),
+    45-character-long keys must be supported.
+
+
 .. _topics-request-meta:

 Request.meta special keys
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@ -98,6 +98,10 @@ class.
 The global defaults are located in the ``scrapy.settings.default_settings``
 module and documented in the :ref:`topics-settings-ref` section.

+Compatibility with pickle
+=========================
+
+Setting values must be :ref:`picklable <pickle-picklable>`.

 Import paths and classes
 ========================
@ -560,7 +564,6 @@ This setting must be one of these string values:
  set this if you want the behavior of Scrapy<1.1
 - ``'TLSv1.1'``: forces TLS version 1.1
 - ``'TLSv1.2'``: forces TLS version 1.2
- ``'SSLv3'``: forces SSL version 3 (**not recommended**)


 .. setting:: DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING
@ -825,12 +828,8 @@ Default: ``'scrapy.dupefilters.RFPDupeFilter'``

 The class used to detect and filter duplicate requests.

-The default (``RFPDupeFilter``) filters based on request fingerprint using
-the ``scrapy.utils.request.request_fingerprint`` function. In order to change
-the way duplicates are checked you could subclass ``RFPDupeFilter`` and
-override its ``request_fingerprint`` method. This method should accept
-scrapy :class:`~scrapy.Request` object and return its fingerprint
-(a string).
+The default (``RFPDupeFilter``) filters based on the
+:setting:`REQUEST_FINGERPRINTER_CLASS` setting.

 You can disable filtering of duplicate requests by setting
 :setting:`DUPEFILTER_CLASS` to ``'scrapy.dupefilters.BaseDupeFilter'``.
@ -1638,9 +1637,15 @@ which raises :exc:`Exception`, becomes::


 The default value of the :setting:`TWISTED_REACTOR` setting is ``None``, which
-means that Scrapy will install the default reactor defined by Twisted for the
-current platform. This is to maintain backward compatibility and avoid possible
-problems caused by using a non-default reactor.
+means that Scrapy will use the existing reactor if one is already installed, or
+install the default reactor defined by Twisted for the current platform. This
+is to maintain backward compatibility and avoid possible problems caused by
+using a non-default reactor.
+
+.. versionchanged:: 2.7
+   The :command:`startproject` command now sets this setting to
+   ``twisted.internet.asyncioreactor.AsyncioSelectorReactor`` in the generated
+   ``settings.py`` file.

 For additional information, see :doc:`core/howto/choosing-reactor`.

@ -1656,14 +1661,14 @@ Scope: ``spidermiddlewares.urllength``

 The maximum URL length to allow for crawled URLs.

-This setting can act as a stopping condition in case of URLs of ever-increasing 
-length, which may be caused for example by a programming error either in the 
-target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and 
+This setting can act as a stopping condition in case of URLs of ever-increasing
+length, which may be caused for example by a programming error either in the
+target server or in your code. See also :setting:`REDIRECT_MAX_TIMES` and
 :setting:`DEPTH_LIMIT`.

 Use ``0`` to allow URLs of any length.

-The default value is copied from the `Microsoft Internet Explorer maximum URL 
+The default value is copied from the `Microsoft Internet Explorer maximum URL
 length`_, even though this setting exists for different reasons.

 .. _Microsoft Internet Explorer maximum URL length: https://support.microsoft.com/en-us/topic/maximum-url-length-is-2-083-characters-in-internet-explorer-174e7c8a-6666-f4e0-6fd6-908b53c12246
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@ -51,12 +51,12 @@ Deferred signal handlers
 ========================

 Some signals support returning :class:`~twisted.internet.defer.Deferred`
-objects from their handlers, allowing you to run asynchronous code that
-does not block Scrapy. If a signal handler returns a
-:class:`~twisted.internet.defer.Deferred`, Scrapy waits for that
-:class:`~twisted.internet.defer.Deferred` to fire.
+or :term:`awaitable objects <awaitable>` from their handlers, allowing
+you to run asynchronous code that does not block Scrapy. If a signal
+handler returns one of these objects, Scrapy waits for that asynchronous
+operation to finish.

-Let's take an example::
+Let's take an example using :ref:`coroutines <topics-coroutines>`::

    class SignalSpider(scrapy.Spider):
        name = 'signals'
@ -68,17 +68,15 @@ Let's take an example::
            crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
            return spider

-        def item_scraped(self, item):
+        async def item_scraped(self, item):
            # Send the scraped item to the server
-            d = treq.post(
+            response = await treq.post(
                'http://example.com/post',
                json.dumps(item).encode('ascii'),
                headers={b'Content-Type': [b'application/json']}
            )

-            # The next item will be scraped only after
-            # deferred (d) is fired
-            return d
+            return response

        def parse(self, response):
            for quote in response.css('div.quote'):
@ -89,7 +87,7 @@ Let's take an example::
                }

 See the :ref:`topics-signals-ref` below to know which signals support
-:class:`~twisted.internet.defer.Deferred`.
+:class:`~twisted.internet.defer.Deferred` and :term:`awaitable objects <awaitable>`.

 .. _topics-signals-ref:

--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@ -102,27 +102,47 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
        it has processed the response.

        :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.Request` objects and :ref:`item object
+        :class:`~scrapy.Request` objects and :ref:`item objects
        <topics-items>`.

+        .. versionchanged:: 2.7
+           This method may be defined as an :term:`asynchronous generator`, in
+           which case ``result`` is an :term:`asynchronous iterable`.
+
+        Consider defining this method as an :term:`asynchronous generator`,
+        which will be a requirement in a future version of Scrapy. However, if
+        you plan on sharing your spider middleware with other people, consider
+        either :ref:`enforcing Scrapy 2.7 <enforce-component-requirements>`
+        as a minimum requirement of your spider middleware, or :ref:`making
+        your spider middleware universal <universal-spider-middleware>` so that
+        it works with Scrapy versions earlier than Scrapy 2.7.
+
        :param response: the response which generated this output from the
          spider
        :type response: :class:`~scrapy.http.Response` object

        :param result: the result returned by the spider
        :type result: an iterable of :class:`~scrapy.Request` objects and
-          :ref:`item object <topics-items>`
+          :ref:`item objects <topics-items>`

        :param spider: the spider whose result is being processed
        :type spider: :class:`~scrapy.Spider` object

+    .. method:: process_spider_output_async(response, result, spider)
+
+        .. versionadded:: 2.7
+
+        If defined, this method must be an :term:`asynchronous generator`,
+        which will be called instead of :meth:`process_spider_output` if
+        ``result`` is an :term:`asynchronous iterable`.
+
    .. method:: process_spider_exception(response, exception, spider)

        This method is called when a spider or :meth:`process_spider_output`
        method (from a previous spider middleware) raises an exception.

        :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>` 
+        iterable of :class:`~scrapy.Request` or :ref:`item <topics-items>`
        objects.

        If it returns ``None``, Scrapy will continue processing this exception,
--- a/docs/topics/spiders.rst
+++ b/docs/topics/spiders.rst
@ -181,9 +181,10 @@ scrapy.Spider
       scraped data and/or more URLs to follow. Other Requests callbacks have
       the same requirements as the :class:`Spider` class.

-       This method, as well as any other Request callback, must return an
-       iterable of :class:`~scrapy.Request` and/or :ref:`item objects
-       <topics-items>`.
+       This method, as well as any other Request callback, must return a
+       :class:`~scrapy.Request` object, an :ref:`item object <topics-items>`, an
+       iterable of :class:`~scrapy.Request` objects and/or :ref:`item objects
+       <topics-items>`, or ``None``.

       :param response: the response to parse
       :type response: :class:`~scrapy.http.Response`
--- a/docs/topics/webservice.rst
+++ b/docs/topics/webservice.rst
@ -1,11 +0,0 @@
-.. _topics-webservice:
-
-===========
-Web Service
-===========
-
-webservice has been moved into a separate project.
-
-It is hosted at:
-
-    https://github.com/scrapy-plugins/scrapy-jsonrpc
--- a/7
+++ b/7
@ -9,11 +9,9 @@ disable=abstract-method,
        arguments-renamed,
        attribute-defined-outside-init,
        bad-classmethod-argument,
-        bad-continuation,
        bad-indentation,
        bad-mcs-classmethod-argument,
        bad-super-call,
-        bad-whitespace,
        bare-except,
        blacklisted-name,
        broad-except,
@ -49,10 +47,10 @@ disable=abstract-method,
        keyword-arg-before-vararg,
        line-too-long,
        logging-format-interpolation,
+        logging-fstring-interpolation,
        logging-not-lazy,
        lost-exception,
        method-hidden,
-        misplaced-comparison-constant,
        missing-docstring,
        missing-final-newline,
        multiple-imports,
@ -60,12 +58,10 @@ disable=abstract-method,
        no-else-continue,
        no-else-raise,
        no-else-return,
-        no-init,
        no-member,
        no-method-argument,
        no-name-in-module,
        no-self-argument,
-        no-self-use,
        no-value-for-parameter,
        not-an-iterable,
        not-callable,
@ -102,6 +98,7 @@ disable=abstract-method,
        ungrouped-imports,
        unidiomatic-typecheck,
        unnecessary-comprehension,
+        unnecessary-dunder-call,
        unnecessary-lambda,
        unnecessary-pass,
        unreachable,
--- a/pytest.ini
+++ b/pytest.ini
@ -21,3 +21,6 @@ addopts =
 markers =
    only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
    only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
+filterwarnings =
+    ignore:scrapy.downloadermiddlewares.decompression is deprecated
+    ignore:Module scrapy.utils.reqser is deprecated
--- a/scrapy/VERSION
+++ b/scrapy/VERSION
@ -1 +1 @@
-2.6.1
+2.7.0
--- a/scrapy/init.py
+++ b/scrapy/init.py
@ -28,8 +28,8 @@ twisted_version = (_txv.major, _txv.minor, _txv.micro)


 # Check minimum required Python version
-if sys.version_info < (3, 6):
-    print(f"Scrapy {__version__} requires Python 3.6+")
+if sys.version_info < (3, 7):
+    print(f"Scrapy {__version__} requires Python 3.7+")
    sys.exit(1)


--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@ -14,6 +14,15 @@ from scrapy.utils.project import inside_project, get_project_settings
 from scrapy.utils.python import garbage_collect


+class ScrapyArgumentParser(argparse.ArgumentParser):
+    def _parse_optional(self, arg_string):
+        # if starts with -: it means that is a parameter not a argument
+        if arg_string[:2] == '-:':
+            return None
+
+        return super()._parse_optional(arg_string)
+
+
 def _iter_command_classes(module_name):
    # TODO: add `name` attribute to commands and and merge this function with
    # scrapy.utils.spider.iter_spider_classes
@ -131,10 +140,10 @@ def execute(argv=None, settings=None):
        sys.exit(2)

    cmd = cmds[cmdname]
-    parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter,
-                                     usage=f"scrapy {cmdname} {cmd.syntax()}",
-                                     conflict_handler='resolve',
-                                     description=cmd.long_desc())
+    parser = ScrapyArgumentParser(formatter_class=ScrapyHelpFormatter,
+                                  usage=f"scrapy {cmdname} {cmd.syntax()}",
+                                  conflict_handler='resolve',
+                                  description=cmd.long_desc())
    settings.setdict(cmd.default_settings, priority='command')
    cmd.settings = settings
    cmd.add_options(parser)
--- a/scrapy/commands/parse.py
+++ b/scrapy/commands/parse.py
@ -5,6 +5,8 @@ from typing import Dict
 from itemadapter import is_item, ItemAdapter
 from w3lib.url import is_url

+from twisted.internet.defer import maybeDeferred
+
 from scrapy.commands import BaseRunSpiderCommand
 from scrapy.http import Request
 from scrapy.utils import display
@ -51,7 +53,7 @@ class Command(BaseRunSpiderCommand):
        parser.add_argument("--cbkwargs", dest="cbkwargs",
                            help="inject extra callback kwargs into the Request, it must be a valid raw json string")
        parser.add_argument("-d", "--depth", dest="depth", type=int, default=1,
-                            help="maximum depth for parsing requests [default: %default]")
+                            help="maximum depth for parsing requests [default: %(default)s]")
        parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                            help="print each depth level one by one")

@ -110,16 +112,19 @@ class Command(BaseRunSpiderCommand):
            if not opts.nolinks:
                self.print_requests(colour=colour)

-    def run_callback(self, response, callback, cb_kwargs=None):
-        cb_kwargs = cb_kwargs or {}
+    def _get_items_and_requests(self, spider_output, opts, depth, spider, callback):
        items, requests = [], []
-
-        for x in iterate_spider_output(callback(response, **cb_kwargs)):
+        for x in spider_output:
            if is_item(x):
                items.append(x)
            elif isinstance(x, Request):
                requests.append(x)
-        return items, requests
+        return items, requests, opts, depth, spider, callback
+
+    def run_callback(self, response, callback, cb_kwargs=None):
+        cb_kwargs = cb_kwargs or {}
+        d = maybeDeferred(iterate_spider_output, callback(response, **cb_kwargs))
+        return d

    def get_callback_from_rules(self, spider, response):
        if getattr(spider, 'rules', None):
@ -146,7 +151,8 @@ class Command(BaseRunSpiderCommand):

        def _start_requests(spider):
            yield self.prepare_request(spider, Request(url), opts)
-        self.spidercls.start_requests = _start_requests
+        if self.spidercls:
+            self.spidercls.start_requests = _start_requests

    def start_parsing(self, url, opts):
        self.crawler_process.crawl(self.spidercls, **opts.spargs)
@ -157,6 +163,25 @@ class Command(BaseRunSpiderCommand):
            logger.error('No response downloaded for: %(url)s',
                         {'url': url})

+    def scraped_data(self, args):
+        items, requests, opts, depth, spider, callback = args
+        if opts.pipelines:
+            itemproc = self.pcrawler.engine.scraper.itemproc
+            for item in items:
+                itemproc.process_item(item, spider)
+        self.add_items(depth, items)
+        self.add_requests(depth, requests)
+
+        scraped_data = items if opts.output else []
+        if depth < opts.depth:
+            for req in requests:
+                req.meta['_depth'] = depth + 1
+                req.meta['_callback'] = req.callback
+                req.callback = callback
+            scraped_data += requests
+
+        return scraped_data
+
    def prepare_request(self, spider, request, opts):
        def callback(response, **cb_kwargs):
            # memorize first request
@ -190,23 +215,10 @@ class Command(BaseRunSpiderCommand):
            # parse items and requests
            depth = response.meta['_depth']

-            items, requests = self.run_callback(response, cb, cb_kwargs)
-            if opts.pipelines:
-                itemproc = self.pcrawler.engine.scraper.itemproc
-                for item in items:
-                    itemproc.process_item(item, spider)
-            self.add_items(depth, items)
-            self.add_requests(depth, requests)
-
-            scraped_data = items if opts.output else []
-            if depth < opts.depth:
-                for req in requests:
-                    req.meta['_depth'] = depth + 1
-                    req.meta['_callback'] = req.callback
-                    req.callback = callback
-                scraped_data += requests
-
-            return scraped_data
+            d = self.run_callback(response, cb, cb_kwargs)
+            d.addCallback(self._get_items_and_requests, opts, depth, spider, callback)
+            d.addCallback(self.scraped_data)
+            return d

        # update request meta if any extra meta was passed through the --meta/-m opts.
        if opts.meta:
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@ -21,7 +21,7 @@ class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
    which allows TLS protocol negotiation

    'A TLS/SSL connection established with [this method] may
-     understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
+     understand the TLSv1, TLSv1.1 and TLSv1.2 protocols.'
    """

    def __init__(self, method=SSL.SSLv23_METHOD, tls_verbose_logging=False, tls_ciphers=None, *args, **kwargs):
--- a/scrapy/core/downloader/handlers/ftp.py
+++ b/scrapy/core/downloader/handlers/ftp.py
@ -102,11 +102,11 @@ class FTPDownloadHandler:

    def _build_response(self, result, request, protocol):
        self.result = result
-        respcls = responsetypes.from_args(url=request.url)
        protocol.close()
-        body = protocol.filename or protocol.body.read()
        headers = {"local filename": protocol.filename or '', "size": protocol.size}
-        return respcls(url=request.url, status=200, body=to_bytes(body), headers=headers)
+        body = to_bytes(protocol.filename or protocol.body.read())
+        respcls = responsetypes.from_args(url=request.url, body=body)
+        return respcls(url=request.url, status=200, body=body, headers=headers)

    def _failed(self, result, request):
        message = result.getErrorMessage()
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@ -384,8 +384,7 @@ class ScrapyAgent:
                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                             {"request": request, "handler": handler.__qualname__})
                txresponse._transport.stopProducing()
-                with suppress(AttributeError):
-                    txresponse._transport._producer.loseConnection()
+                txresponse._transport.loseConnection()
                return {
                    "txresponse": txresponse,
                    "body": b"",
@ -417,7 +416,7 @@ class ScrapyAgent:

            logger.warning(warning_msg, warning_args)

-            txresponse._transport._producer.loseConnection()
+            txresponse._transport.loseConnection()
            raise defer.CancelledError(warning_msg % warning_args)

        if warnsize and expected_size > warnsize:
@ -543,7 +542,7 @@ class _ResponseReader(protocol.Protocol):
                logger.debug("Download stopped for %(request)s from signal handler %(handler)s",
                             {"request": self._request, "handler": handler.__qualname__})
                self.transport.stopProducing()
-                self.transport._producer.loseConnection()
+                self.transport.loseConnection()
                failure = result if result.value.fail else None
                self._finish_response(flags=["download_stopped"], failure=failure)

--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@ -11,7 +11,6 @@ from scrapy.utils.ssl import x509name_to_string, get_temp_key_info
 logger = logging.getLogger(__name__)


-METHOD_SSLv3 = 'SSLv3'
 METHOD_TLS = 'TLS'
 METHOD_TLSv10 = 'TLSv1.0'
 METHOD_TLSv11 = 'TLSv1.1'
@ -20,7 +19,6 @@ METHOD_TLSv12 = 'TLSv1.2'

 openssl_methods = {
    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
-    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@ -112,7 +112,7 @@ class ScrapyHTTPClientFactory(ClientFactory):
        request.meta['download_latency'] = self.headers_time - self.start_time
        status = int(self.status)
        headers = Headers(self.response_headers)
-        respcls = responsetypes.from_args(headers=headers, url=self._url)
+        respcls = responsetypes.from_args(headers=headers, url=self._url, body=body)
        return respcls(url=self._url, status=status, headers=headers, body=body, protocol=to_unicode(self.version))

    def _set_connection_attributes(self, request):
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@ -136,7 +136,9 @@ class ExecutionEngine:
        self.paused = False

    def _next_request(self) -> None:
-        assert self.slot is not None  # typing
+        if self.slot is None:
+            return
+
        assert self.spider is not None  # typing

        if self.paused:
@ -184,7 +186,8 @@ class ExecutionEngine:
        d.addErrback(lambda f: logger.info('Error while removing request from slot',
                                           exc_info=failure_to_exc_info(f),
                                           extra={'spider': self.spider}))
-        d.addBoth(lambda _: self.slot.nextcall.schedule())
+        slot = self.slot
+        d.addBoth(lambda _: slot.nextcall.schedule())
        d.addErrback(lambda f: logger.info('Error while scheduling new request',
                                           exc_info=failure_to_exc_info(f),
                                           extra={'spider': self.spider}))
@ -254,9 +257,7 @@ class ExecutionEngine:

    def download(self, request: Request, spider: Optional[Spider] = None) -> Deferred:
        """Return a Deferred which fires with a Response as result, only downloader middlewares are applied"""
-        if spider is None:
-            spider = self.spider
-        else:
+        if spider is not None:
            warnings.warn(
                "Passing a 'spider' argument to ExecutionEngine.download is deprecated",
                category=ScrapyDeprecationWarning,
@ -264,7 +265,7 @@ class ExecutionEngine:
            )
            if spider is not self.spider:
                logger.warning("The spider '%s' does not match the open spider", spider.name)
-        if spider is None:
+        if self.spider is None:
            raise RuntimeError(f"No open spider to crawl: {request}")
        return self._download(request, spider).addBoth(self._downloaded, request, spider)

@ -275,11 +276,14 @@ class ExecutionEngine:
        self.slot.remove_request(request)
        return self.download(result, spider) if isinstance(result, Request) else result

-    def _download(self, request: Request, spider: Spider) -> Deferred:
+    def _download(self, request: Request, spider: Optional[Spider]) -> Deferred:
        assert self.slot is not None  # typing

        self.slot.add_request(request)

+        if spider is None:
+            spider = self.spider
+
        def _on_success(result: Union[Response, Request]) -> Union[Response, Request]:
            if not isinstance(result, (Response, Request)):
                raise TypeError(f"Incorrect type: expected Response or Request, got {type(result)}: {result!r}")
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@ -1,9 +1,8 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-
 import logging
 from collections import deque
-from typing import Any, Deque, Iterable, Optional, Set, Tuple, Union
+from typing import Any, AsyncGenerator, AsyncIterable, Deque, Generator, Iterable, Optional, Set, Tuple, Union

 from itemadapter import is_item
 from twisted.internet.defer import Deferred, inlineCallbacks
@ -13,7 +12,15 @@ from scrapy import signals, Spider
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import (
+    aiter_errback,
+    defer_fail,
+    defer_succeed,
+    iter_errback,
+    parallel,
+    parallel_async,
+)
+
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
@ -185,12 +192,19 @@ class Scraper:
            spider=spider
        )

-    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
+    def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
+                             response: Response, spider: Spider) -> Deferred:
        if not result:
            return defer_succeed(None)
-        it = iter_errback(result, self.handle_spider_error, request, response, spider)
-        dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
-                       request, response, spider)
+        it: Union[Generator, AsyncGenerator]
+        if isinstance(result, AsyncIterable):
+            it = aiter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                 request, response, spider)
+        else:
+            it = iter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+                           request, response, spider)
        return dfd

    def _process_spidermw_output(self, output: Any, request: Request, response: Response,
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@ -3,32 +3,42 @@ Spider Middleware manager

 See documentation in docs/topics/spider-middleware.rst
 """
+import logging
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, cast
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast

-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure

 from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.python import MutableChain
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.python import MutableAsyncChain, MutableChain
+
+
+logger = logging.getLogger(__name__)


 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]


 def _isiterable(o) -> bool:
-    return isinstance(o, Iterable)
+    return isinstance(o, (Iterable, AsyncIterable))


 class SpiderMiddlewareManager(MiddlewareManager):

    component_name = 'spider middleware'

+    def __init__(self, *middlewares):
+        super().__init__(*middlewares)
+        self.downgrade_warning_done = False
+
    @classmethod
    def _get_mwlist_from_settings(cls, settings):
        return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@ -39,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
            self.methods['process_spider_input'].append(mw.process_spider_input)
        if hasattr(mw, 'process_start_requests'):
            self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        process_spider_output = getattr(mw, 'process_spider_output', None)
+        process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
        self.methods['process_spider_output'].appendleft(process_spider_output)
        process_spider_exception = getattr(mw, 'process_spider_exception', None)
        self.methods['process_spider_exception'].appendleft(process_spider_exception)
@ -51,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
            try:
                result = method(response=response, spider=spider)
                if result is not None:
-                    msg = (f"Middleware {method.__qualname__} must return None "
+                    msg = (f"{method.__qualname__} must return None "
                           f"or raise an exception, got {type(result)}")
                    raise _InvalidOutput(msg)
            except _InvalidOutput:
@ -60,17 +70,35 @@ class SpiderMiddlewareManager(MiddlewareManager):
                return scrape_func(Failure(), request, spider)
        return scrape_func(response, request, spider)

-    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
-                           exception_processor_index: int, recover_to: MutableChain) -> Generator:
-        try:
-            for r in iterable:
-                yield r
-        except Exception as ex:
-            exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                              exception_processor_index)
-            if isinstance(exception_result, Failure):
-                raise
-            recover_to.extend(exception_result)
+    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
+                           exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
+                           ) -> Union[Generator, AsyncGenerator]:
+
+        def process_sync(iterable: Iterable):
+            try:
+                for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        async def process_async(iterable: AsyncIterable):
+            try:
+                async for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        if isinstance(iterable, AsyncIterable):
+            return process_async(iterable)
+        return process_sync(iterable)

    def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
                                  start_index: int = 0) -> Union[Failure, MutableChain]:
@ -82,30 +110,83 @@ class SpiderMiddlewareManager(MiddlewareManager):
        for method_index, method in enumerate(method_list, start=start_index):
            if method is None:
                continue
+            method = cast(Callable, method)
            result = method(response=response, exception=exception, spider=spider)
            if _isiterable(result):
                # stop exception handling by handing control over to the
                # process_spider_output chain if an iterable has been returned
-                return self._process_spider_output(response, spider, result, method_index + 1)
+                dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
+                # _process_spider_output() returns a Deferred only because of downgrading so this can be
+                # simplified when downgrading is removed.
+                if dfd.called:
+                    # the result is available immediately if _process_spider_output didn't do downgrading
+                    return dfd.result
+                else:
+                    # we forbid waiting here because otherwise we would need to return a deferred from
+                    # _process_spider_exception too, which complicates the architecture
+                    msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
+                    raise _InvalidOutput(msg)
            elif result is None:
                continue
            else:
-                msg = (f"Middleware {method.__qualname__} must return None "
+                msg = (f"{method.__qualname__} must return None "
                       f"or an iterable, got {type(result)}")
                raise _InvalidOutput(msg)
        return _failure

+    # This method cannot be made async def, as _process_spider_exception relies on the Deferred result
+    # being available immediately which doesn't work when it's a wrapped coroutine.
+    # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
+    @inlineCallbacks
    def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Iterable, start_index: int = 0) -> MutableChain:
+                               result: Union[Iterable, AsyncIterable], start_index: int = 0
+                               ) -> Deferred:
        # items in this iterable do not need to go through the process_spider_output
        # chain, they went through it already from the process_spider_exception method
-        recovered = MutableChain()
+        recovered: Union[MutableChain, MutableAsyncChain]
+        last_result_is_async = isinstance(result, AsyncIterable)
+        if last_result_is_async:
+            recovered = MutableAsyncChain()
+        else:
+            recovered = MutableChain()
+
+        # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
+        # 1. def foo. Sync iterables are passed as is, async ones are downgraded.
+        # 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
+        # 3. def foo + async def foo_async. Iterables are passed to the respective method.
+        # Storing methods and method tuples in the same list is weird but we should be able to roll this back
+        # when we drop this compatibility feature.

        method_list = islice(self.methods['process_spider_output'], start_index, None)
-        for method_index, method in enumerate(method_list, start=start_index):
-            if method is None:
+        for method_index, method_pair in enumerate(method_list, start=start_index):
+            if method_pair is None:
                continue
+            need_upgrade = need_downgrade = False
+            if isinstance(method_pair, tuple):
+                # This tuple handling is only needed until _async compatibility methods are removed.
+                method_sync, method_async = method_pair
+                method = method_async if last_result_is_async else method_sync
+            else:
+                method = method_pair
+                if not last_result_is_async and isasyncgenfunction(method):
+                    need_upgrade = True
+                elif last_result_is_async and not isasyncgenfunction(method):
+                    need_downgrade = True
            try:
+                if need_upgrade:
+                    # Iterable -> AsyncIterable
+                    result = as_async_generator(result)
+                elif need_downgrade:
+                    if not self.downgrade_warning_done:
+                        logger.warning(f"Async iterable passed to {method.__qualname__} "
+                                       f"was downgraded to a non-async one")
+                        self.downgrade_warning_done = True
+                    assert isinstance(result, AsyncIterable)
+                    # AsyncIterable -> Iterable
+                    result = yield deferred_from_coro(collect_asyncgen(result))
+                    if isinstance(recovered, AsyncIterable):
+                        recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
+                        recovered = MutableChain(recovered_collected)
                # might fail directly if the output value is not a generator
                result = method(response=response, result=result, spider=spider)
            except Exception as ex:
@ -116,28 +197,77 @@ class SpiderMiddlewareManager(MiddlewareManager):
            if _isiterable(result):
                result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
            else:
-                msg = (f"Middleware {method.__qualname__} must return an "
-                       f"iterable, got {type(result)}")
+                if iscoroutine(result):
+                    result.close()  # Silence warning about not awaiting
+                    msg = (
+                        f"{method.__qualname__} must be an asynchronous "
+                        f"generator (i.e. use yield)"
+                    )
+                else:
+                    msg = (
+                        f"{method.__qualname__} must return an iterable, got "
+                        f"{type(result)}"
+                    )
                raise _InvalidOutput(msg)
+            last_result_is_async = isinstance(result, AsyncIterable)

-        return MutableChain(result, recovered)
+        if last_result_is_async:
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)  # type: ignore[arg-type]

-    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
-        recovered = MutableChain()
+    async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                       ) -> Union[MutableChain, MutableAsyncChain]:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        if isinstance(result, AsyncIterable):
+            recovered = MutableAsyncChain()
+        else:
+            recovered = MutableChain()
        result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        return MutableChain(self._process_spider_output(response, spider, result), recovered)
+        result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
+        if isinstance(result, AsyncIterable):
+            return MutableAsyncChain(result, recovered)
+        else:
+            if isinstance(recovered, AsyncIterable):
+                recovered_collected = await collect_asyncgen(recovered)
+                recovered = MutableChain(recovered_collected)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]

    def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                        spider: Spider) -> Deferred:
-        def process_callback_output(result: Iterable) -> MutableChain:
-            return self._process_callback_output(response, spider, result)
+        async def process_callback_output(result: Union[Iterable, AsyncIterable]
+                                          ) -> Union[MutableChain, MutableAsyncChain]:
+            return await self._process_callback_output(response, spider, result)

        def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
            return self._process_spider_exception(response, spider, _failure)

        dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
-        dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
+        dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
        return dfd

    def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
        return self._process_chain('process_start_requests', start_requests, spider)
+
+    # This method is only needed until _async compatibility methods are removed.
+    @staticmethod
+    def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
+        normal_method = getattr(mw, methodname, None)
+        methodname_async = methodname + "_async"
+        async_method = getattr(mw, methodname_async, None)
+        if not async_method:
+            return normal_method
+        if not normal_method:
+            logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
+                         f"without {methodname}, skipping this method.")
+            return None
+        if not isasyncgenfunction(async_method):
+            logger.error(f"{async_method.__qualname__} is not "
+                         f"an async generator function, skipping this method.")
+            return normal_method
+        if isasyncgenfunction(normal_method):
+            logger.error(f"{normal_method.__qualname__} is an async "
+                         f"generator function while {methodname_async} exists, "
+                         f"skipping both methods.")
+            return None
+        return normal_method, async_method
--- a/scrapy/crawler.py
+++ b/scrapy/crawler.py
@ -31,7 +31,12 @@ from scrapy.utils.log import (
 )
 from scrapy.utils.misc import create_instance, load_object
 from scrapy.utils.ossignal import install_shutdown_handlers, signal_names
-from scrapy.utils.reactor import install_reactor, verify_installed_reactor
+from scrapy.utils.reactor import (
+    install_reactor,
+    is_asyncio_reactor_installed,
+    verify_installed_asyncio_event_loop,
+    verify_installed_reactor,
+)


 logger = logging.getLogger(__name__)
@ -51,6 +56,7 @@ class Crawler:
        self.spidercls.update_settings(self.settings)

        self.signals = SignalManager(self)
+
        self.stats = load_object(self.settings['STATS_CLASS'])(self)

        handler = LogCounterHandler(self, level=self.settings.get('LOG_LEVEL'))
@ -71,18 +77,26 @@ class Crawler:
        lf_cls = load_object(self.settings['LOG_FORMATTER'])
        self.logformatter = lf_cls.from_crawler(self)

-        reactor_class = self.settings.get("TWISTED_REACTOR")
+        self.request_fingerprinter = create_instance(
+            load_object(self.settings['REQUEST_FINGERPRINTER_CLASS']),
+            settings=self.settings,
+            crawler=self,
+        )
+
+        reactor_class = self.settings["TWISTED_REACTOR"]
+        event_loop = self.settings["ASYNCIO_EVENT_LOOP"]
        if init_reactor:
            # this needs to be done after the spider settings are merged,
            # but before something imports twisted.internet.reactor
            if reactor_class:
-                install_reactor(reactor_class, self.settings["ASYNCIO_EVENT_LOOP"])
+                install_reactor(reactor_class, event_loop)
            else:
-                from twisted.internet import default
-                default.install()
+                from twisted.internet import reactor  # noqa: F401
            log_reactor_info()
        if reactor_class:
            verify_installed_reactor(reactor_class)
+            if is_asyncio_reactor_installed() and event_loop:
+                verify_installed_asyncio_event_loop(event_loop)

        self.extensions = ExtensionManager.from_crawler(self)

@ -290,6 +304,7 @@ class CrawlerProcess(CrawlerRunner):
        super().__init__(settings)
        configure_logging(self.settings, install_root_handler)
        log_scrapy_info(self.settings)
+        self._initialized_reactor = False

    def _signal_shutdown(self, signum, _):
        from twisted.internet import reactor
@ -310,7 +325,9 @@ class CrawlerProcess(CrawlerRunner):
    def _create_crawler(self, spidercls):
        if isinstance(spidercls, str):
            spidercls = self.spider_loader.load(spidercls)
-        return Crawler(spidercls, self.settings, init_reactor=True)
+        init_reactor = not self._initialized_reactor
+        self._initialized_reactor = True
+        return Crawler(spidercls, self.settings, init_reactor=init_reactor)

    def start(self, stop_after_crawl=True, install_signal_handlers=True):
        """
--- a/scrapy/downloadermiddlewares/cookies.py
+++ b/scrapy/downloadermiddlewares/cookies.py
@ -104,8 +104,8 @@ class CookiesMiddleware:
        for key in ("name", "value", "path", "domain"):
            if cookie.get(key) is None:
                if key in ("name", "value"):
-                    msg = "Invalid cookie found in request {}: {} ('{}' is missing)"
-                    logger.warning(msg.format(request, cookie, key))
+                    msg = f"Invalid cookie found in request {request}: {cookie} ('{key}' is missing)"
+                    logger.warning(msg)
                    return
                continue
            if isinstance(cookie[key], (bool, float, int, str)):
--- a/scrapy/downloadermiddlewares/decompression.py
+++ b/scrapy/downloadermiddlewares/decompression.py
@ -9,10 +9,19 @@ import tarfile
 import zipfile
 from io import BytesIO
 from tempfile import mktemp
+from warnings import warn

+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.responsetypes import responsetypes


+warn(
+    'scrapy.downloadermiddlewares.decompression is deprecated',
+    ScrapyDeprecationWarning,
+    stacklevel=2,
+)
+
+
 logger = logging.getLogger(__name__)


--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@ -45,31 +45,40 @@ class HttpProxyMiddleware:
        return creds, proxy_url

    def process_request(self, request, spider):
-        # ignore if proxy is already set
+        creds, proxy_url = None, None
        if 'proxy' in request.meta:
-            if request.meta['proxy'] is None:
-                return
-            # extract credentials if present
-            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+            if request.meta['proxy'] is not None:
+                creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+        elif self.proxies:
+            parsed = urlparse_cached(request)
+            scheme = parsed.scheme
+            if (
+                (
+                    # 'no_proxy' is only supported by http schemes
+                    scheme not in ('http', 'https')
+                    or not proxy_bypass(parsed.hostname)
+                )
+                and scheme in self.proxies
+            ):
+                creds, proxy_url = self.proxies[scheme]
+
+        self._set_proxy_and_creds(request, proxy_url, creds)
+
+    def _set_proxy_and_creds(self, request, proxy_url, creds):
+        if proxy_url:
            request.meta['proxy'] = proxy_url
-            if creds and not request.headers.get('Proxy-Authorization'):
-                request.headers['Proxy-Authorization'] = b'Basic ' + creds
-            return
-        elif not self.proxies:
-            return
-
-        parsed = urlparse_cached(request)
-        scheme = parsed.scheme
-
-        # 'no_proxy' is only supported by http schemes
-        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
-            return
-
-        if scheme in self.proxies:
-            self._set_proxy(request, scheme)
-
-    def _set_proxy(self, request, scheme):
-        creds, proxy = self.proxies[scheme]
-        request.meta['proxy'] = proxy
+        elif request.meta.get('proxy') is not None:
+            request.meta['proxy'] = None
        if creds:
-            request.headers['Proxy-Authorization'] = b'Basic ' + creds
+            request.headers[b'Proxy-Authorization'] = b'Basic ' + creds
+            request.meta['_auth_proxy'] = proxy_url
+        elif '_auth_proxy' in request.meta:
+            if proxy_url != request.meta['_auth_proxy']:
+                if b'Proxy-Authorization' in request.headers:
+                    del request.headers[b'Proxy-Authorization']
+                del request.meta['_auth_proxy']
+        elif b'Proxy-Authorization' in request.headers:
+            if proxy_url:
+                request.meta['_auth_proxy'] = proxy_url
+            else:
+                del request.headers[b'Proxy-Authorization']
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@ -1,14 +1,16 @@
 import logging
 import os
 from typing import Optional, Set, Type, TypeVar
+from warnings import warn

 from twisted.internet.defer import Deferred

 from scrapy.http.request import Request
 from scrapy.settings import BaseSettings
 from scrapy.spiders import Spider
+from scrapy.utils.deprecate import ScrapyDeprecationWarning
 from scrapy.utils.job import job_dir
-from scrapy.utils.request import referer_str, request_fingerprint
+from scrapy.utils.request import referer_str, RequestFingerprinter


 BaseDupeFilterTV = TypeVar("BaseDupeFilterTV", bound="BaseDupeFilter")
@ -39,8 +41,15 @@ RFPDupeFilterTV = TypeVar("RFPDupeFilterTV", bound="RFPDupeFilter")
 class RFPDupeFilter(BaseDupeFilter):
    """Request Fingerprint duplicates filter"""

-    def __init__(self, path: Optional[str] = None, debug: bool = False) -> None:
+    def __init__(
+        self,
+        path: Optional[str] = None,
+        debug: bool = False,
+        *,
+        fingerprinter=None,
+    ) -> None:
        self.file = None
+        self.fingerprinter = fingerprinter or RequestFingerprinter()
        self.fingerprints: Set[str] = set()
        self.logdupes = True
        self.debug = debug
@ -51,9 +60,39 @@ class RFPDupeFilter(BaseDupeFilter):
            self.fingerprints.update(x.rstrip() for x in self.file)

    @classmethod
-    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings) -> RFPDupeFilterTV:
+    def from_settings(cls: Type[RFPDupeFilterTV], settings: BaseSettings, *, fingerprinter=None) -> RFPDupeFilterTV:
        debug = settings.getbool('DUPEFILTER_DEBUG')
-        return cls(job_dir(settings), debug)
+        try:
+            return cls(job_dir(settings), debug, fingerprinter=fingerprinter)
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their '__init__' "
+                "method to support a 'fingerprinter' parameter or reimplement "
+                "the 'from_settings' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls(job_dir(settings), debug)
+            result.fingerprinter = fingerprinter
+            return result
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        try:
+            return cls.from_settings(
+                crawler.settings,
+                fingerprinter=crawler.request_fingerprinter,
+            )
+        except TypeError:
+            warn(
+                "RFPDupeFilter subclasses must either modify their overridden "
+                "'__init__' method and 'from_settings' class method to "
+                "support a 'fingerprinter' parameter, or reimplement the "
+                "'from_crawler' class method.",
+                ScrapyDeprecationWarning,
+            )
+            result = cls.from_settings(crawler.settings)
+            result.fingerprinter = crawler.request_fingerprinter
+            return result

    def request_seen(self, request: Request) -> bool:
        fp = self.request_fingerprint(request)
@ -65,7 +104,7 @@ class RFPDupeFilter(BaseDupeFilter):
        return False

    def request_fingerprint(self, request: Request) -> str:
-        return request_fingerprint(request)
+        return self.fingerprinter.fingerprint(request).hex()

    def close(self, reason: str) -> None:
        if self.file:
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@ -8,6 +8,7 @@ import marshal
 import pickle
 import pprint
 import warnings
+from collections.abc import Mapping
 from xml.sax.saxutils import XMLGenerator

 from itemadapter import is_item, ItemAdapter
@ -68,6 +69,14 @@ class BaseItemExporter:
                field_iter = item.field_names()
            else:
                field_iter = item.keys()
+        elif isinstance(self.fields_to_export, Mapping):
+            if include_empty:
+                field_iter = self.fields_to_export.items()
+            else:
+                field_iter = (
+                    (x, y) for x, y in self.fields_to_export.items()
+                    if x in item
+                )
        else:
            if include_empty:
                field_iter = self.fields_to_export
@ -75,13 +84,17 @@ class BaseItemExporter:
                field_iter = (x for x in self.fields_to_export if x in item)

        for field_name in field_iter:
-            if field_name in item:
-                field_meta = item.get_field_meta(field_name)
-                value = self.serialize_field(field_meta, field_name, item[field_name])
+            if isinstance(field_name, str):
+                item_field, output_field = field_name, field_name
+            else:
+                item_field, output_field = field_name
+            if item_field in item:
+                field_meta = item.get_field_meta(item_field)
+                value = self.serialize_field(field_meta, output_field, item[item_field])
            else:
                value = default_value

-            yield field_name, value
+            yield output_field, value


 class JsonLinesItemExporter(BaseItemExporter):
@ -246,7 +259,11 @@ class CsvItemExporter(BaseItemExporter):
            if not self.fields_to_export:
                # use declared field names, or keys if the item is a dict
                self.fields_to_export = ItemAdapter(item).field_names()
-            row = list(self._build_row(self.fields_to_export))
+            if isinstance(self.fields_to_export, Mapping):
+                fields = self.fields_to_export.values()
+            else:
+                fields = self.fields_to_export
+            row = list(self._build_row(fields))
            self.csv_writer.writerow(row)


--- a/scrapy/extensions/httpcache.py
+++ b/scrapy/extensions/httpcache.py
@ -14,7 +14,6 @@ from scrapy.responsetypes import responsetypes
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.project import data_path
 from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.request import request_fingerprint


 logger = logging.getLogger(__name__)
@ -228,6 +227,8 @@ class DbmCacheStorage:

        logger.debug("Using DBM cache storage in %(cachepath)s", {'cachepath': dbpath}, extra={'spider': spider})

+        self._fingerprinter = spider.crawler.request_fingerprinter
+
    def close_spider(self, spider):
        self.db.close()

@ -239,12 +240,12 @@ class DbmCacheStorage:
        status = data['status']
        headers = Headers(data['headers'])
        body = data['body']
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
        response = respcls(url=url, headers=headers, status=status, body=body)
        return response

    def store_response(self, spider, request, response):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
        data = {
            'status': response.status,
            'url': response.url,
@ -255,7 +256,7 @@ class DbmCacheStorage:
        self.db[f'{key}_time'] = str(time())

    def _read_data(self, spider, request):
-        key = self._request_key(request)
+        key = self._fingerprinter.fingerprint(request).hex()
        db = self.db
        tkey = f'{key}_time'
        if tkey not in db:
@ -267,9 +268,6 @@ class DbmCacheStorage:

        return pickle.loads(db[f'{key}_data'])

-    def _request_key(self, request):
-        return request_fingerprint(request)
-

 class FilesystemCacheStorage:

@ -283,6 +281,8 @@ class FilesystemCacheStorage:
        logger.debug("Using filesystem cache storage in %(cachedir)s", {'cachedir': self.cachedir},
                     extra={'spider': spider})

+        self._fingerprinter = spider.crawler.request_fingerprinter
+
    def close_spider(self, spider):
        pass

@ -299,7 +299,7 @@ class FilesystemCacheStorage:
        url = metadata.get('response_url')
        status = metadata['status']
        headers = Headers(headers_raw_to_dict(rawheaders))
-        respcls = responsetypes.from_args(headers=headers, url=url)
+        respcls = responsetypes.from_args(headers=headers, url=url, body=body)
        response = respcls(url=url, headers=headers, status=status, body=body)
        return response

@ -329,7 +329,7 @@ class FilesystemCacheStorage:
            f.write(request.body)

    def _get_request_path(self, spider, request):
-        key = request_fingerprint(request)
+        key = self._fingerprinter.fingerprint(request).hex()
        return os.path.join(self.cachedir, spider.name, key[0:2], key)

    def _read_meta(self, spider, request):
--- a/scrapy/extensions/memusage.py
+++ b/scrapy/extensions/memusage.py
@ -33,8 +33,8 @@ class MemoryUsage:
        self.crawler = crawler
        self.warned = False
        self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
-        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB')*1024*1024
-        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB')*1024*1024
+        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
+        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
        self.mail = MailSender.from_settings(crawler.settings)
        crawler.signals.connect(self.engine_started, signal=signals.engine_started)
@ -77,7 +77,7 @@ class MemoryUsage:
    def _check_limit(self):
        if self.get_virtual_size() > self.limit:
            self.crawler.stats.set_value('memusage/limit_reached', 1)
-            mem = self.limit/1024/1024
+            mem = self.limit / 1024 / 1024
            logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
                         {'memusage': mem}, extra={'crawler': self.crawler})
            if self.notify_mails:
@ -94,11 +94,11 @@ class MemoryUsage:
                self.crawler.stop()

    def _check_warning(self):
-        if self.warned: # warn only once
+        if self.warned:  # warn only once
            return
        if self.get_virtual_size() > self.warning:
            self.crawler.stats.set_value('memusage/warning_reached', 1)
-            mem = self.warning/1024/1024
+            mem = self.warning / 1024 / 1024
            logger.warning("Memory usage reached %(memusage)dM",
                           {'memusage': mem}, extra={'crawler': self.crawler})
            if self.notify_mails:
--- a/scrapy/extensions/statsmailer.py
+++ b/scrapy/extensions/statsmailer.py
@ -8,6 +8,7 @@ from scrapy import signals
 from scrapy.mail import MailSender
 from scrapy.exceptions import NotConfigured

+
 class StatsMailer:

    def __init__(self, stats, recipients, mail):
--- a/scrapy/http/headers.py
+++ b/scrapy/http/headers.py
@ -1,3 +1,5 @@
+from collections.abc import Mapping
+
 from w3lib.http import headers_dict_to_raw
 from scrapy.utils.datatypes import CaselessDict
 from scrapy.utils.python import to_unicode
@ -10,6 +12,13 @@ class Headers(CaselessDict):
        self.encoding = encoding
        super().__init__(seq)

+    def update(self, seq):
+        seq = seq.items() if isinstance(seq, Mapping) else seq
+        iseq = {}
+        for k, v in seq:
+            iseq.setdefault(self.normkey(k), []).extend(self.normvalue(v))
+        super().update(iseq)
+
    def normkey(self, key):
        """Normalize key to bytes"""
        return self._tobytes(key.title())
@ -86,4 +95,5 @@ class Headers(CaselessDict):

    def __copy__(self):
        return self.__class__(self)
+
    copy = __copy__
--- a/scrapy/http/response/text.py
+++ b/scrapy/http/response/text.py
@ -11,8 +11,13 @@ from typing import Generator, Tuple
 from urllib.parse import urljoin

 import parsel
-from w3lib.encoding import (html_body_declared_encoding, html_to_unicode,
-                            http_content_type_encoding, resolve_encoding)
+from w3lib.encoding import (
+    html_body_declared_encoding,
+    html_to_unicode,
+    http_content_type_encoding,
+    resolve_encoding,
+    read_bom,
+)
 from w3lib.html import strip_html5_whitespace

 from scrapy.http import Request
@ -60,6 +65,7 @@ class TextResponse(Response):
    def _declared_encoding(self):
        return (
            self._encoding
+            or self._bom_encoding()
            or self._headers_encoding()
            or self._body_declared_encoding()
        )
@ -117,6 +123,10 @@ class TextResponse(Response):
    def _body_declared_encoding(self):
        return html_body_declared_encoding(self.body)

+    @memoizemethod_noargs
+    def _bom_encoding(self):
+        return read_bom(self.body)[0]
+
    @property
    def selector(self):
        from scrapy.selector import Selector
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Optional, cast, Iterable
+from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast

 from twisted.internet.defer import Deferred

@ -21,8 +21,9 @@ class MiddlewareManager:

    def __init__(self, *middlewares):
        self.middlewares = middlewares
-        # Optional because process_spider_output and process_spider_exception can be None
-        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
+        # Only process_spider_output and process_spider_exception can be None.
+        # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
+        self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
        for mw in middlewares:
            self._add_middleware(mw)

--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@ -222,8 +222,8 @@ class GCSFilesStore:
                return {'checksum': checksum, 'last_modified': last_modified}
            else:
                return {}
-
-        return threads.deferToThread(self.bucket.get_blob, path).addCallback(_onsuccess)
+        blob_path = self._get_blob_path(path)
+        return threads.deferToThread(self.bucket.get_blob, blob_path).addCallback(_onsuccess)

    def _get_content_type(self, headers):
        if headers and 'Content-Type' in headers:
@ -231,8 +231,12 @@ class GCSFilesStore:
        else:
            return 'application/octet-stream'

+    def _get_blob_path(self, path):
+        return self.prefix + path
+
    def persist_file(self, path, buf, info, meta=None, headers=None):
-        blob = self.bucket.blob(self.prefix + path)
+        blob_path = self._get_blob_path(path)
+        blob = self.bucket.blob(blob_path)
        blob.cache_control = self.CACHE_CONTROL
        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
        return threads.deferToThread(
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@ -146,7 +146,7 @@ class ImagesPipeline(FilesPipeline):
        yield path, image, buf

        for thumb_id, size in self.thumbs.items():
-            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
+            thumb_path = self.thumb_path(request, thumb_id, response=response, info=info, item=item)
            thumb_image, thumb_buf = self.convert_image(image, size)
            yield thumb_path, thumb_image, thumb_buf

@ -165,7 +165,14 @@ class ImagesPipeline(FilesPipeline):

        if size:
            image = image.copy()
-            image.thumbnail(size, self._Image.ANTIALIAS)
+            try:
+                # Image.Resampling.LANCZOS was added in Pillow 9.1.0
+                # remove this try except block,
+                # when updating the minimum requirements for Pillow.
+                resampling_filter = self._Image.Resampling.LANCZOS
+            except AttributeError:
+                resampling_filter = self._Image.ANTIALIAS
+            image.thumbnail(size, resampling_filter)

        buf = BytesIO()
        image.save(buf, 'JPEG')
@ -184,6 +191,6 @@ class ImagesPipeline(FilesPipeline):
        image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
        return f'full/{image_guid}.jpg'

-    def thumb_path(self, request, thumb_id, response=None, info=None):
+    def thumb_path(self, request, thumb_id, response=None, info=None, *, item=None):
        thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
        return f'thumbs/{thumb_id}/{thumb_guid}.jpg'
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@ -11,7 +11,6 @@ from scrapy.settings import Settings
 from scrapy.utils.datatypes import SequenceExclude
 from scrapy.utils.defer import mustbe_deferred, defer_result
 from scrapy.utils.deprecate import ScrapyDeprecationWarning
-from scrapy.utils.request import request_fingerprint
 from scrapy.utils.misc import arg_to_iter
 from scrapy.utils.log import failure_to_exc_info

@ -77,6 +76,7 @@ class MediaPipeline:
        except AttributeError:
            pipe = cls()
        pipe.crawler = crawler
+        pipe._fingerprinter = crawler.request_fingerprinter
        return pipe

    def open_spider(self, spider):
@ -90,7 +90,7 @@ class MediaPipeline:
        return dfd.addCallback(self.item_completed, item, info)

    def _process_request(self, request, info, item):
-        fp = request_fingerprint(request)
+        fp = self._fingerprinter.fingerprint(request)
        cb = request.callback or (lambda _: _)
        eb = request.errback
        request.callback = None
@ -121,7 +121,7 @@ class MediaPipeline:
    def _make_compatible(self):
        """Make overridable methods of MediaPipeline and subclasses backwards compatible"""
        methods = [
-            "file_path", "media_to_download", "media_downloaded",
+            "file_path", "thumb_path", "media_to_download", "media_downloaded",
            "file_downloaded", "image_downloaded", "get_images"
        ]

--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@ -95,12 +95,14 @@ class ResponseTypes:
        chunk = to_bytes(chunk)
        if not binary_is_text(chunk):
            return self.from_mimetype('application/octet-stream')
-        elif b"<html>" in chunk.lower():
+        lowercase_chunk = chunk.lower()
+        if b"<html>" in lowercase_chunk:
            return self.from_mimetype('text/html')
-        elif b"<?xml" in chunk.lower():
+        if b"<?xml" in lowercase_chunk:
            return self.from_mimetype('text/xml')
-        else:
-            return self.from_mimetype('text')
+        if b'<!doctype html>' in lowercase_chunk:
+            return self.from_mimetype('text/html')
+        return self.from_mimetype('text')

    def from_args(self, headers=None, url=None, filename=None, body=None):
        """Guess the most appropriate Response class based on
--- a/scrapy/settings/init.py
+++ b/scrapy/settings/init.py
@ -197,6 +197,38 @@ class BaseSettings(MutableMapping):
            value = json.loads(value)
        return dict(value)

+    def getdictorlist(self, name, default=None):
+        """Get a setting value as either a :class:`dict` or a :class:`list`.
+
+        If the setting is already a dict or a list, a copy of it will be
+        returned.
+
+        If it is a string it will be evaluated as JSON, or as a comma-separated
+        list of strings as a fallback.
+
+        For example, settings populated from the command line will return:
+
+        -   ``{'key1': 'value1', 'key2': 'value2'}`` if set to
+            ``'{"key1": "value1", "key2": "value2"}'``
+
+        -   ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+        :param name: the setting name
+        :type name: string
+
+        :param default: the value to return if no setting is found
+        :type default: any
+        """
+        value = self.get(name, default)
+        if value is None:
+            return {}
+        if isinstance(value, str):
+            try:
+                return json.loads(value)
+            except ValueError:
+                return value.split(',')
+        return copy.deepcopy(value)
+
    def getwithbase(self, name):
        """Get a composition of a dictionary-like setting and its `_BASE`
        counterpart.
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@ -154,6 +154,7 @@ FEED_EXPORTERS = {}
 FEED_EXPORTERS_BASE = {
    'json': 'scrapy.exporters.JsonItemExporter',
    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
+    'jsonl': 'scrapy.exporters.JsonLinesItemExporter',
    'jl': 'scrapy.exporters.JsonLinesItemExporter',
    'csv': 'scrapy.exporters.CsvItemExporter',
    'xml': 'scrapy.exporters.XmlItemExporter',
@ -246,6 +247,9 @@ REDIRECT_PRIORITY_ADJUST = +2
 REFERER_ENABLED = True
 REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'

+REQUEST_FINGERPRINTER_CLASS = 'scrapy.utils.request.RequestFingerprinter'
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.6'
+
 RETRY_ENABLED = True
 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
 RETRY_HTTP_CODES = [500, 502, 503, 504, 522, 524, 408, 429]
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@ -28,31 +28,39 @@ class DepthMiddleware:
        return cls(maxdepth, crawler.stats, verbose, prio)

    def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request):
-                depth = response.meta['depth'] + 1
-                request.meta['depth'] = depth
-                if self.prio:
-                    request.priority -= depth * self.prio
-                if self.maxdepth and depth > self.maxdepth:
-                    logger.debug(
-                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                        {'maxdepth': self.maxdepth, 'requrl': request.url},
-                        extra={'spider': spider}
-                    )
-                    return False
-                else:
-                    if self.verbose_stats:
-                        self.stats.inc_value(f'request_depth_count/{depth}',
-                                             spider=spider)
-                    self.stats.max_value('request_depth_max', depth,
-                                         spider=spider)
-            return True
+        self._init_depth(response, spider)
+        return (r for r in result or () if self._filter(r, response, spider))

+    async def process_spider_output_async(self, response, result, spider):
+        self._init_depth(response, spider)
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _init_depth(self, response, spider):
        # base case (depth=0)
        if 'depth' not in response.meta:
            response.meta['depth'] = 0
            if self.verbose_stats:
                self.stats.inc_value('request_depth_count/0', spider=spider)

-        return (r for r in result or () if _filter(r))
+    def _filter(self, request, response, spider):
+        if not isinstance(request, Request):
+            return True
+        depth = response.meta['depth'] + 1
+        request.meta['depth'] = depth
+        if self.prio:
+            request.priority -= depth * self.prio
+        if self.maxdepth and depth > self.maxdepth:
+            logger.debug(
+                "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                {'maxdepth': self.maxdepth, 'requrl': request.url},
+                extra={'spider': spider}
+            )
+            return False
+        if self.verbose_stats:
+            self.stats.inc_value(f'request_depth_count/{depth}',
+                                 spider=spider)
+        self.stats.max_value('request_depth_max', depth,
+                             spider=spider)
+        return True
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@ -26,21 +26,27 @@ class OffsiteMiddleware:
        return o

    def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
-                else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+        return (r for r in result or () if self._filter(r, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider) -> bool:
+        if not isinstance(request, Request):
+            return True
+        if request.dont_filter or self.should_follow(request, spider):
+            return True
+        domain = urlparse_cached(request).hostname
+        if domain and domain not in self.domains_seen:
+            self.domains_seen.add(domain)
+            logger.debug(
+                "Filtered offsite request to %(domain)r: %(request)s",
+                {'domain': domain, 'request': request}, extra={'spider': spider})
+            self.stats.inc_value('offsite/domains', spider=spider)
+        self.stats.inc_value('offsite/filtered', spider=spider)
+        return False

    def should_follow(self, request, spider):
        regex = self.host_regex
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@ -333,13 +333,18 @@ class RefererMiddleware:
        return cls() if cls else self.default_policy()

    def process_spider_output(self, response, result, spider):
-        def _set_referer(r):
-            if isinstance(r, Request):
-                referrer = self.policy(response, r).referrer(response.url, r.url)
-                if referrer is not None:
-                    r.headers.setdefault('Referer', referrer)
-            return r
-        return (_set_referer(r) for r in result or ())
+        return (self._set_referer(r, response) for r in result or ())
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            yield self._set_referer(r, response)
+
+    def _set_referer(self, r, response):
+        if isinstance(r, Request):
+            referrer = self.policy(response, r).referrer(response.url, r.url)
+            if referrer is not None:
+                r.headers.setdefault('Referer', referrer)
+        return r

    def request_scheduled(self, request, spider):
        # check redirected request to patch "Referer" header if necessary
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@ -25,16 +25,20 @@ class UrlLengthMiddleware:
        return cls(maxlength)

    def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.info(
-                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
-                    {'maxlength': self.maxlength, 'url': request.url},
-                    extra={'spider': spider}
-                )
-                spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
-                return False
-            else:
-                return True
+        return (r for r in result or () if self._filter(r, spider))

-        return (r for r in result or () if _filter(r))
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider):
+        if isinstance(request, Request) and len(request.url) > self.maxlength:
+            logger.info(
+                "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                {'maxlength': self.maxlength, 'url': request.url},
+                extra={'spider': spider}
+            )
+            spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
+            return False
+        return True
--- a/scrapy/spiders/crawl.py
+++ b/scrapy/spiders/crawl.py
@ -6,11 +6,12 @@ See documentation in docs/topics/spiders.rst
 """

 import copy
-from typing import Sequence
+from typing import AsyncIterable, Awaitable, Sequence

-from scrapy.http import Request, HtmlResponse
+from scrapy.http import Request, Response, HtmlResponse
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.spider import iterate_spider_output


@ -78,7 +79,7 @@ class CrawlSpider(Spider):
    def parse_start_url(self, response, **kwargs):
        return []

-    def process_results(self, response, results):
+    def process_results(self, response: Response, results: list):
        return results

    def _build_request(self, rule_index, link):
@ -109,9 +110,13 @@ class CrawlSpider(Spider):
        rule = self._rules[failure.request.meta['rule']]
        return self._handle_failure(failure, rule.errback)

-    def _parse_response(self, response, callback, cb_kwargs, follow=True):
+    async def _parse_response(self, response, callback, cb_kwargs, follow=True):
        if callback:
            cb_res = callback(response, **cb_kwargs) or ()
+            if isinstance(cb_res, AsyncIterable):
+                cb_res = await collect_asyncgen(cb_res)
+            elif isinstance(cb_res, Awaitable):
+                cb_res = await cb_res
            cb_res = self.process_results(response, cb_res)
            for request_or_item in iterate_spider_output(cb_res):
                yield request_or_item
--- a/scrapy/templates/project/module/settings.py.tmpl
+++ b/scrapy/templates/project/module/settings.py.tmpl
@ -86,3 +86,7 @@ ROBOTSTXT_OBEY = True
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
+
+# Set settings whose default value is deprecated to a future-proof value
+REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
+TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@ -1,8 +1,18 @@
-from collections.abc import AsyncIterable
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union


-async def collect_asyncgen(result: AsyncIterable):
+async def collect_asyncgen(result: AsyncIterable) -> list:
    results = []
    async for x in result:
        results.append(x)
    return results
+
+
+async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    """ Wraps an iterable (sync or async) into an async generator. """
+    if isinstance(it, AsyncIterable):
+        async for r in it:
+            yield r
+    else:
+        for r in it:
+            yield r
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@ -118,7 +118,7 @@ def feed_complete_default_values_from_settings(feed, settings):
    out = feed.copy()
    out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
    out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
-    out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+    out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
    out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
    out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
    out.setdefault("item_export_kwargs", {})
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@ -7,10 +7,15 @@ from asyncio import Future
 from functools import wraps
 from typing import (
    Any,
+    AsyncGenerator,
+    AsyncIterable,
    Callable,
    Coroutine,
    Generator,
    Iterable,
+    Iterator,
+    List,
+    Optional,
    Union
 )

@ -87,6 +92,109 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
    return DeferredList([coop.coiterate(work) for _ in range(count)])


+class _AsyncCooperatorAdapter(Iterator):
+    """ A class that wraps an async iterable into a normal iterator suitable
+    for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
+    it calls the callable directly in the callback, instead of providing a more
+    generic interface.
+
+    On the outside, this class behaves as an iterator that yields Deferreds.
+    Each Deferred is fired with the result of the callable which was called on
+    the next result from aiterator. It raises StopIteration when aiterator is
+    exhausted, as expected.
+
+    Cooperator calls __next__() multiple times and waits on the Deferreds
+    returned from it. As async generators (since Python 3.8) don't support
+    awaiting on __anext__() several times in parallel, we need to serialize
+    this. It's done by storing the Deferreds returned from __next__() and
+    firing the oldest one when a result from __anext__() is available.
+
+    The workflow:
+    1. When __next__() is called for the first time, it creates a Deferred, stores it
+    in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
+    for self.aiterator.__anext__() and puts it into self.anext_deferred.
+    2. If __next__() is called again before self.anext_deferred fires, more Deferreds
+    are added to self.waiting_deferreds.
+    3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
+    clear self.anext_deferred.
+    3.1. _callback() calls the callable passing the result value that it takes, pops a
+    Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
+    chains those Deferreds so that the waiting Deferred will fire when the result
+    Deferred does, otherwise it fires it directly. This causes one awaiting task to
+    receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
+    called and self.anext_deferred is populated.
+    3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
+    self.aiterator is exhausted and so it sets self.finished and fires all
+    self.waiting_deferreds. Other exceptions are propagated.
+    4. If __next__() is called after __anext__() was handled, then if self.finished is
+    True, it raises StopIteration, otherwise it acts like in step 2, but if
+    self.anext_deferred is now empty is also populates it with a new __anext__().
+
+    Note that CooperativeTask ignores the value returned from the Deferred that it waits
+    for, so we fire them with None when needed.
+
+    It may be possible to write an async iterator-aware replacement for
+    Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
+    goal.
+    """
+    def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterable.__aiter__()
+        self.callable = callable
+        self.callable_args = callable_args
+        self.callable_kwargs = callable_kwargs
+        self.finished = False
+        self.waiting_deferreds: List[Deferred] = []
+        self.anext_deferred: Optional[Deferred] = None
+
+    def _callback(self, result: Any) -> None:
+        # This gets called when the result from aiterator.__anext__() is available.
+        # It calls the callable on it and sends the result to the oldest waiting Deferred
+        # (by chaining if the result is a Deferred too or by firing if not).
+        self.anext_deferred = None
+        result = self.callable(result, *self.callable_args, **self.callable_kwargs)
+        d = self.waiting_deferreds.pop(0)
+        if isinstance(result, Deferred):
+            result.chainDeferred(d)
+        else:
+            d.callback(None)
+        if self.waiting_deferreds:
+            self._call_anext()
+
+    def _errback(self, failure: Failure) -> None:
+        # This gets called on any exceptions in aiterator.__anext__().
+        # It handles StopAsyncIteration by stopping the iteration and reraises all others.
+        self.anext_deferred = None
+        failure.trap(StopAsyncIteration)
+        self.finished = True
+        for d in self.waiting_deferreds:
+            d.callback(None)
+
+    def _call_anext(self) -> None:
+        # This starts waiting for the next result from aiterator.
+        # If aiterator is exhausted, _errback will be called.
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred.addCallbacks(self._callback, self._errback)
+
+    def __next__(self) -> Deferred:
+        # This puts a new Deferred into self.waiting_deferreds and returns it.
+        # It also calls __anext__() if needed.
+        if self.finished:
+            raise StopIteration
+        d = Deferred()
+        self.waiting_deferreds.append(d)
+        if not self.anext_deferred:
+            self._call_anext()
+        return d
+
+
+def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:
+    """ Like parallel but for async iterators """
+    coop = Cooperator()
+    work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
+    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
+    return dl
+
+
 def process_chain(callbacks: Iterable[Callable], input, *a, **kw) -> Deferred:
    """Return a Deferred built by chaining the given callbacks"""
    d = Deferred()
@ -136,6 +244,20 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
            errback(failure.Failure(), *a, **kw)


+async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:
+    """Wraps an async iterable calling an errback if an error is caught while
+    iterating it. Similar to scrapy.utils.defer.iter_errback()
+    """
+    it = aiterable.__aiter__()
+    while True:
+        try:
+            yield await it.__anext__()
+        except StopAsyncIteration:
+            break
+        except Exception:
+            errback(failure.Failure(), *a, **kw)
+
+
 def deferred_from_coro(o) -> Any:
    """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
    if isinstance(o, Deferred):
--- a/scrapy/utils/display.py
+++ b/scrapy/utils/display.py
@ -5,7 +5,7 @@ pprint and pformat wrappers with colorization support
 import ctypes
 import platform
 import sys
-from distutils.version import LooseVersion as parse_version
+from packaging.version import Version as parse_version
 from pprint import pformat as pformat_


--- a/scrapy/utils/misc.py
+++ b/scrapy/utils/misc.py
@ -226,7 +226,14 @@ def is_generator_with_return_value(callable):
        return value is None or isinstance(value, ast.NameConstant) and value.value is None

    if inspect.isgeneratorfunction(callable):
-        code = re.sub(r"^[\t ]+", "", inspect.getsource(callable))
+        src = inspect.getsource(callable)
+        pattern = re.compile(r"(^[\t ]+)")
+        code = pattern.sub("", src)
+
+        match = pattern.match(src)  # finds indentation
+        if match:
+            code = re.sub(f"\n{match.group(0)}", "\n", code)  # remove indentation
+
        tree = ast.parse(code)
        for node in walk_callable(tree):
            if isinstance(node, ast.Return) and not returns_none(node):
--- a/scrapy/utils/py36.py
+++ b/scrapy/utils/py36.py
@ -1,11 +0,0 @@
-import warnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncgen import collect_asyncgen  # noqa: F401
-
-
-warnings.warn(
-    "Module `scrapy.utils.py36` is deprecated, please import from `scrapy.utils.asyncgen` instead.",
-    category=ScrapyDeprecationWarning,
-    stacklevel=2,
-)
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@ -8,11 +8,12 @@ import re
 import sys
 import warnings
 import weakref
-from collections.abc import Iterable
 from functools import partial, wraps
 from itertools import chain
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union

 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.decorators import deprecated


@ -179,23 +180,6 @@ def binary_is_text(data):
    return all(c not in _BINARYCHARS for c in data)


-def _getargspec_py23(func):
-    """_getargspec_py23(function) -> named tuple ArgSpec(args, varargs, keywords,
-                                                        defaults)
-
-    Was identical to inspect.getargspec() in python2, but uses
-    inspect.getfullargspec() for python3 behind the scenes to avoid
-    DeprecationWarning.
-
-    >>> def f(a, b=2, *ar, **kw):
-    ...     pass
-
-    >>> _getargspec_py23(f)
-    ArgSpec(args=['a', 'b'], varargs='ar', keywords='kw', defaults=(2,))
-    """
-    return inspect.ArgSpec(*inspect.getfullargspec(func)[:4])
-
-
 def get_func_args(func, stripself=False):
    """Return the argument name list of a callable"""
    if inspect.isfunction(func):
@ -247,9 +231,9 @@ def get_spec(func):
    """

    if inspect.isfunction(func) or inspect.ismethod(func):
-        spec = _getargspec_py23(func)
+        spec = inspect.getfullargspec(func)
    elif hasattr(func, '__call__'):
-        spec = _getargspec_py23(func.__call__)
+        spec = inspect.getfullargspec(func.__call__)
    else:
        raise TypeError(f'{type(func)} is not callable')

@ -344,7 +328,7 @@ class MutableChain(Iterable):
    def __init__(self, *args: Iterable):
        self.data = chain.from_iterable(args)

-    def extend(self, *iterables: Iterable):
+    def extend(self, *iterables: Iterable) -> None:
        self.data = chain(self.data, chain.from_iterable(iterables))

    def __iter__(self):
@ -356,3 +340,27 @@ class MutableChain(Iterable):
    @deprecated("scrapy.utils.python.MutableChain.__next__")
    def next(self):
        return self.__next__()
+
+
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    for it in iterables:
+        async for o in as_async_generator(it):
+            yield o
+
+
+class MutableAsyncChain(AsyncIterable):
+    """
+    Similar to MutableChain but for async iterables
+    """
+
+    def __init__(self, *args: Union[Iterable, AsyncIterable]):
+        self.data = _async_chain(*args)
+
+    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+        self.data = _async_chain(self.data, _async_chain(*iterables))
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        return await self.data.__anext__()
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@ -83,13 +83,31 @@ def verify_installed_reactor(reactor_path):
    path."""
    from twisted.internet import reactor
    reactor_class = load_object(reactor_path)
-    if not isinstance(reactor, reactor_class):
+    if not reactor.__class__ == reactor_class:
        msg = ("The installed reactor "
               f"({reactor.__module__}.{reactor.__class__.__name__}) does not "
               f"match the requested one ({reactor_path})")
        raise Exception(msg)


+def verify_installed_asyncio_event_loop(loop_path):
+    from twisted.internet import reactor
+    loop_class = load_object(loop_path)
+    if isinstance(reactor._asyncioEventloop, loop_class):
+        return
+    installed = (
+        f"{reactor._asyncioEventloop.__class__.__module__}"
+        f".{reactor._asyncioEventloop.__class__.__qualname__}"
+    )
+    specified = f"{loop_class.__module__}.{loop_class.__qualname__}"
+    raise Exception(
+        "Scrapy found an asyncio Twisted reactor already "
+        f"installed, and its event loop class ({installed}) does "
+        "not match the one specified in the ASYNCIO_EVENT_LOOP "
+        f"setting ({specified})"
+    )
+
+
 def is_asyncio_reactor_installed():
    from twisted.internet import reactor
    return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@ -4,7 +4,9 @@ scrapy.http.Request objects
 """

 import hashlib
-from typing import Dict, Iterable, Optional, Tuple, Union
+import json
+import warnings
+from typing import Dict, Iterable, List, Optional, Tuple, Union
 from urllib.parse import urlunparse
 from weakref import WeakKeyDictionary

@ -12,13 +14,22 @@ from w3lib.http import basic_auth_header
 from w3lib.url import canonicalize_url

 from scrapy import Request, Spider
+from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode


-_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
-_fingerprint_cache = WeakKeyDictionary()
+_deprecated_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]"
+_deprecated_fingerprint_cache = WeakKeyDictionary()
+
+
+def _serialize_headers(headers, request):
+    for header in headers:
+        if header in request.headers:
+            yield header
+            for value in request.headers.getlist(header):
+                yield value


 def request_fingerprint(
@ -26,6 +37,123 @@ def request_fingerprint(
    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
    keep_fragments: bool = False,
 ) -> str:
+    """
+    Return the request fingerprint as an hexadecimal string.
+
+    The request fingerprint is a hash that uniquely identifies the resource the
+    request points to. For example, take the following two urls:
+
+    http://www.example.com/query?id=111&cat=222
+    http://www.example.com/query?cat=222&id=111
+
+    Even though those are two different URLs both point to the same resource
+    and are equivalent (i.e. they should return the same response).
+
+    Another example are cookies used to store session ids. Suppose the
+    following page is only accessible to authenticated users:
+
+    http://www.example.com/members/offers.html
+
+    Lots of sites use a cookie to store the session id, which adds a random
+    component to the HTTP Request and thus should be ignored when calculating
+    the fingerprint.
+
+    For this reason, request headers are ignored by default when calculating
+    the fingerprint. If you want to include specific headers use the
+    include_headers argument, which is a list of Request headers to include.
+
+    Also, servers usually ignore fragments in urls when handling requests,
+    so they are also ignored by default when calculating the fingerprint.
+    If you want to include them, set the keep_fragments argument to True
+    (for instance when handling requests with a headless browser).
+    """
+    if include_headers or keep_fragments:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component because you '
+            'need a non-default fingerprinting algorithm, and you are OK '
+            'with that non-default fingerprinting algorithm being used by '
+            'all Scrapy components and not just the one calling this '
+            'function, use crawler.request_fingerprinter.fingerprint() '
+            'instead in your Scrapy component (you can get the crawler '
+            'object from the \'from_crawler\' class method), and use the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting to configure your '
+            'non-default fingerprinting algorithm.\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'If you switch to \'fingerprint()\', or assign the '
+            '\'REQUEST_FINGERPRINTER_CLASS\' setting a class that uses '
+            '\'fingerprint()\', the generated fingerprints will not only be '
+            'bytes instead of a string, but they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    else:
+        message = (
+            'Call to deprecated function '
+            'scrapy.utils.request.request_fingerprint().\n'
+            '\n'
+            'If you are using this function in a Scrapy component, and you '
+            'are OK with users of your component changing the fingerprinting '
+            'algorithm through settings, use '
+            'crawler.request_fingerprinter.fingerprint() instead in your '
+            'Scrapy component (you can get the crawler object from the '
+            '\'from_crawler\' class method).\n'
+            '\n'
+            'Otherwise, consider using the '
+            'scrapy.utils.request.fingerprint() function instead.\n'
+            '\n'
+            'Either way, the resulting fingerprints will be returned as '
+            'bytes, not as a string, and they will also be different from '
+            'those generated by \'request_fingerprint()\'. Before you switch, '
+            'make sure that you understand the consequences of this (e.g. '
+            'cache invalidation) and are OK with them; otherwise, consider '
+            'implementing your own function which returns the same '
+            'fingerprints as the deprecated \'request_fingerprint()\' function.'
+        )
+    warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
+    if include_headers:
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
+    cache = _deprecated_fingerprint_cache.setdefault(request, {})
+    cache_key = (processed_include_headers, keep_fragments)
+    if cache_key not in cache:
+        fp = hashlib.sha1()
+        fp.update(to_bytes(request.method))
+        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
+        fp.update(request.body or b'')
+        if processed_include_headers:
+            for part in _serialize_headers(processed_include_headers, request):
+                fp.update(part)
+        cache[cache_key] = fp.hexdigest()
+    return cache[cache_key]
+
+
+def _request_fingerprint_as_bytes(*args, **kwargs):
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return bytes.fromhex(request_fingerprint(*args, **kwargs))
+
+
+_fingerprint_cache: "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]"
+_fingerprint_cache = WeakKeyDictionary()
+
+
+def fingerprint(
+    request: Request,
+    *,
+    include_headers: Optional[Iterable[Union[bytes, str]]] = None,
+    keep_fragments: bool = False,
+) -> bytes:
    """
    Return the request fingerprint.

@ -43,7 +171,7 @@ def request_fingerprint(

    http://www.example.com/members/offers.html

-    Lot of sites use a cookie to store the session id, which adds a random
+    Lots of sites use a cookie to store the session id, which adds a random
    component to the HTTP Request and thus should be ignored when calculating
    the fingerprint.

@ -55,29 +183,96 @@ def request_fingerprint(
    so they are also ignored by default when calculating the fingerprint.
    If you want to include them, set the keep_fragments argument to True
    (for instance when handling requests with a headless browser).
-
    """
-    headers: Optional[Tuple[bytes, ...]] = None
+    processed_include_headers: Optional[Tuple[bytes, ...]] = None
    if include_headers:
-        headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))
+        processed_include_headers = tuple(
+            to_bytes(h.lower()) for h in sorted(include_headers)
+        )
    cache = _fingerprint_cache.setdefault(request, {})
-    cache_key = (headers, keep_fragments)
+    cache_key = (processed_include_headers, keep_fragments)
    if cache_key not in cache:
-        fp = hashlib.sha1()
-        fp.update(to_bytes(request.method))
-        fp.update(to_bytes(canonicalize_url(request.url, keep_fragments=keep_fragments)))
-        fp.update(request.body or b'')
-        if headers:
-            for hdr in headers:
-                if hdr in request.headers:
-                    fp.update(hdr)
-                    for v in request.headers.getlist(hdr):
-                        fp.update(v)
-        cache[cache_key] = fp.hexdigest()
+        # To decode bytes reliably (JSON does not support bytes), regardless of
+        # character encoding, we use bytes.hex()
+        headers: Dict[str, List[str]] = {}
+        if processed_include_headers:
+            for header in processed_include_headers:
+                if header in request.headers:
+                    headers[header.hex()] = [
+                        header_value.hex()
+                        for header_value in request.headers.getlist(header)
+                    ]
+        fingerprint_data = {
+            'method': to_unicode(request.method),
+            'url': canonicalize_url(request.url, keep_fragments=keep_fragments),
+            'body': (request.body or b'').hex(),
+            'headers': headers,
+        }
+        fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
+        cache[cache_key] = hashlib.sha1(fingerprint_json.encode()).digest()
    return cache[cache_key]


-def request_authenticate(request: Request, username: str, password: str) -> None:
+class RequestFingerprinter:
+    """Default fingerprinter.
+
+    It takes into account a canonical version
+    (:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
+    <scrapy.http.Request.url>` and the values of :attr:`request.method
+    <scrapy.http.Request.method>` and :attr:`request.body
+    <scrapy.http.Request.body>`. It then generates an `SHA1
+    <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+
+    .. seealso:: :setting:`REQUEST_FINGERPRINTER_IMPLEMENTATION`.
+    """
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        return cls(crawler)
+
+    def __init__(self, crawler=None):
+        if crawler:
+            implementation = crawler.settings.get(
+                'REQUEST_FINGERPRINTER_IMPLEMENTATION'
+            )
+        else:
+            implementation = '2.6'
+        if implementation == '2.6':
+            message = (
+                '\'2.6\' is a deprecated value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting.\n'
+                '\n'
+                'It is also the default value. In other words, it is normal '
+                'to get this warning if you have not defined a value for the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting. This is so '
+                'for backward compatibility reasons, but it will change in a '
+                'future version of Scrapy.\n'
+                '\n'
+                'See the documentation of the '
+                '\'REQUEST_FINGERPRINTER_IMPLEMENTATION\' setting for '
+                'information on how to handle this deprecation.'
+            )
+            warnings.warn(message, category=ScrapyDeprecationWarning, stacklevel=2)
+            self._fingerprint = _request_fingerprint_as_bytes
+        elif implementation == '2.7':
+            self._fingerprint = fingerprint
+        else:
+            raise ValueError(
+                f'Got an invalid value on setting '
+                f'\'REQUEST_FINGERPRINTER_IMPLEMENTATION\': '
+                f'{implementation!r}. Valid values are \'2.6\' (deprecated) '
+                f'and \'2.7\'.'
+            )
+
+    def fingerprint(self, request):
+        return self._fingerprint(request)
+
+
+def request_authenticate(
+    request: Request,
+    username: str,
+    password: str,
+) -> None:
    """Authenticate the given request (in place) using the HTTP basic access
    authentication mechanism (RFC 2617) and the given username and password
    """
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@ -4,7 +4,6 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.asyncgen import collect_asyncgen


 logger = logging.getLogger(__name__)
@ -12,14 +11,13 @@ logger = logging.getLogger(__name__)

 def iterate_spider_output(result):
    if inspect.isasyncgen(result):
-        d = deferred_from_coro(collect_asyncgen(result))
-        d.addCallback(iterate_spider_output)
-        return d
+        return result
    elif inspect.iscoroutine(result):
        d = deferred_from_coro(result)
        d.addCallback(iterate_spider_output)
        return d
-    return arg_to_iter(result)
+    else:
+        return arg_to_iter(deferred_from_coro(result))


 def iter_spider_classes(module):
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@ -54,7 +54,7 @@ def get_ftp_content_and_delete(
    return "".join(ftp_data)


-def get_crawler(spidercls=None, settings_dict=None):
+def get_crawler(spidercls=None, settings_dict=None, prevent_warnings=True):
    """Return an unconfigured Crawler object. If settings_dict is given, it
    will be used to populate the crawler settings with a project level
    priority.
@ -62,7 +62,12 @@ def get_crawler(spidercls=None, settings_dict=None):
    from scrapy.crawler import CrawlerRunner
    from scrapy.spiders import Spider

-    runner = CrawlerRunner(settings_dict)
+    # Set by default settings that prevent deprecation warnings.
+    settings = {}
+    if prevent_warnings:
+        settings['REQUEST_FINGERPRINTER_IMPLEMENTATION'] = '2.7'
+    settings.update(settings_dict or {})
+    runner = CrawlerRunner(settings)
    return runner.create_crawler(spidercls or Spider)


@ -110,3 +115,10 @@ def mock_google_cloud_storage():
    bucket_mock.blob.return_value = blob_mock

    return (client_mock, bucket_mock, blob_mock)
+
+
+def get_web_client_agent_req(url):
+    from twisted.internet import reactor
+    from twisted.web.client import Agent  # imports twisted.internet.reactor
+    agent = Agent(reactor)
+    return agent.request(b'GET', url.encode('utf-8'))
--- a/scrapy/utils/testsite.py
+++ b/scrapy/utils/testsite.py
@ -23,7 +23,7 @@ class NoMetaRefreshRedirect(util.Redirect):
    def render(self, request):
        content = util.Redirect.render(self, request)
        return content.replace(b'http-equiv=\"refresh\"',
-            b'http-no-equiv=\"do-not-refresh-me\"')
+                               b'http-no-equiv=\"do-not-refresh-me\"')


 def test_site():
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@ -5,7 +5,6 @@ library.
 Some of the functions that used to be imported from this module have been moved
 to the w3lib.url module. Always import those from there instead.
 """
-import posixpath
 import re
 from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse

@ -31,7 +30,9 @@ def url_is_from_spider(url, spider):


 def url_has_any_extension(url, extensions):
-    return posixpath.splitext(parse_url(url).path)[1].lower() in extensions
+    """Return True if the url ends with one of the extensions provided"""
+    lowercase_path = parse_url(url).path.lower()
+    return any(lowercase_path.endswith(ext) for ext in extensions)


 def parse_url(url, encoding=None):
--- a/sep/sep-014.rst
+++ b/sep/sep-014.rst
@ -590,11 +590,11 @@ Request Generator

       def generate_requests(self, response):
           """
-           Extract and process new requets from response
+           Extract and process new requests from response
           """
           requests = []
           for ext in self._request_extractors:
-               requets.extend(ext.extract_requests(response))
+               requests.extend(ext.extract_requests(response))

           for proc in self._request_processors:
               requests = proc(requests)
--- a/setup.py
+++ b/setup.py
@ -19,35 +19,30 @@ def has_environment_marker_platform_impl_support():


 install_requires = [
-    'Twisted>=17.9.0',
-    'cryptography>=2.0',
+    'Twisted>=18.9.0',
+    'cryptography>=3.3',
    'cssselect>=0.9.1',
    'itemloaders>=1.0.1',
    'parsel>=1.5.0',
-    'pyOpenSSL>=16.2.0',
+    'pyOpenSSL>=21.0.0',
    'queuelib>=1.4.2',
-    'service_identity>=16.0.0',
+    'service_identity>=18.1.0',
    'w3lib>=1.17.0',
-    'zope.interface>=4.1.3',
+    'zope.interface>=5.1.0',
    'protego>=0.1.15',
    'itemadapter>=0.1.0',
    'setuptools',
+    'packaging',
    'tldextract',
+    'lxml>=4.3.0',
 ]
 extras_require = {}
 cpython_dependencies = [
-    'lxml>=3.5.0',
    'PyDispatcher>=2.0.5',
 ]
 if has_environment_marker_platform_impl_support():
    extras_require[':platform_python_implementation == "CPython"'] = cpython_dependencies
    extras_require[':platform_python_implementation == "PyPy"'] = [
-        # Earlier lxml versions are affected by
-        # https://foss.heptapod.net/pypy/pypy/-/issues/2498,
-        # which was fixed in Cython 0.26, released on 2017-06-19, and used to
-        # generate the C headers of lxml release tarballs published since then, the
-        # first of which was:
-        'lxml>=4.0.0',
        'PyPyDispatcher>=2.1.0',
    ]
 else:
@ -84,18 +79,18 @@ setup(
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
        'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
        'Programming Language :: Python :: Implementation :: CPython',
        'Programming Language :: Python :: Implementation :: PyPy',
        'Topic :: Internet :: WWW/HTTP',
        'Topic :: Software Development :: Libraries :: Application Frameworks',
        'Topic :: Software Development :: Libraries :: Python Modules',
    ],
-    python_requires='>=3.6',
+    python_requires='>=3.7',
    install_requires=install_requires,
    extras_require=extras_require,
 )
--- a/tests/CrawlerProcess/asyncio_deferred_signal.py
+++ b/tests/CrawlerProcess/asyncio_deferred_signal.py
@ -5,7 +5,6 @@ from typing import Optional
 from scrapy import Spider
 from scrapy.crawler import CrawlerProcess
 from scrapy.utils.defer import deferred_from_coro
-from twisted.internet.defer import Deferred


 class UppercasePipeline:
--- a/tests/CrawlerProcess/asyncio_enabled_reactor.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py
@ -6,8 +6,8 @@ if sys.version_info >= (3, 8) and sys.platform == "win32":
    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
 asyncioreactor.install(asyncio.get_event_loop())

-import scrapy
-from scrapy.crawler import CrawlerProcess
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402


 class NoRequestsSpider(scrapy.Spider):
--- a/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_different_loop.py
@ -0,0 +1,25 @@
+import asyncio
+import sys
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
--- a/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
+++ b/tests/CrawlerProcess/asyncio_enabled_reactor_same_loop.py
@ -0,0 +1,28 @@
+import asyncio
+import sys
+
+from uvloop import Loop
+
+from twisted.internet import asyncioreactor
+if sys.version_info >= (3, 8) and sys.platform == "win32":
+    asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
+asyncio.set_event_loop(Loop())
+asyncioreactor.install(asyncio.get_event_loop())
+
+import scrapy  # noqa: E402
+from scrapy.crawler import CrawlerProcess  # noqa: E402
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={
+    "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+    "ASYNCIO_EVENT_LOOP": "uvloop.Loop",
+})
+process.crawl(NoRequestsSpider)
+process.start()
--- a/tests/CrawlerProcess/multi.py
+++ b/tests/CrawlerProcess/multi.py
@ -0,0 +1,16 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+    name = 'no_request'
+
+    def start_requests(self):
+        return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.crawl(NoRequestsSpider)
+process.start()
--- a/Show More
+++ b/Show More
 @ -1 +1 @@
 .6.1
 .7.0