From 76abcedaf4f31a7c76de9e19680dd7499d9eccf4 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 13:36:25 +0500
Subject: [PATCH 01/41] Add as_async_generator.

---
 scrapy/utils/asyncgen.py     | 12 ++++++++++++
 tests/test_utils_asyncgen.py | 20 ++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 tests/test_utils_asyncgen.py

diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 7f697af5f..db2173f85 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,5 +1,17 @@
+import collections
+
+
 async def collect_asyncgen(result):
     results = []
     async for x in result:
         results.append(x)
     return results
+
+
+async def as_async_generator(it):
+    if isinstance(it, collections.abc.AsyncIterator):
+        async for r in it:
+            yield r
+    else:
+        for r in it:
+            yield r
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
new file mode 100644
index 000000000..9ae66c57c
--- /dev/null
+++ b/tests/test_utils_asyncgen.py
@@ -0,0 +1,20 @@
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+
+
+class AsyncgenUtilsTest(unittest.TestCase):
+    @deferred_f_from_coro_f
+    async def test_as_async_generator(self):
+        ag = as_async_generator(range(42))
+        results = []
+        async for i in ag:
+            results.append(i)
+        self.assertEqual(results, list(range(42)))
+
+    @deferred_f_from_coro_f
+    async def test_collect_asyncgen(self):
+        ag = as_async_generator(range(42))
+        results = await collect_asyncgen(ag)
+        self.assertEqual(results, list(range(42)))

From acff1eb4960940eb360f3cf00d5b33ed71acc351 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 14:36:38 +0500
Subject: [PATCH 02/41] Add aiter_errback.

---
 scrapy/utils/defer.py     | 14 ++++++++++++++
 tests/test_utils_defer.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 6db9cc117..2d02c0621 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -124,6 +124,20 @@ def iter_errback(iterable, errback, *a, **kw):
             errback(failure.Failure(), *a, **kw)
 
 
+async def aiter_errback(aiterable, errback, *a, **kw):
+    """Wraps an async iterable calling an errback if an error is caught while
+    iterating it. Similar to scrapy.utils.defer.iter_errback()
+    """
+    it = aiterable.__aiter__()
+    while True:
+        try:
+            yield await it.__anext__()
+        except StopAsyncIteration:
+            break
+        except Exception:
+            errback(failure.Failure(), *a, **kw)
+
+
 def deferred_from_coro(o):
     """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
     if isinstance(o, defer.Deferred):
diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index e60242a3b..06d91c574 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -2,12 +2,15 @@ from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
+from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import (
     iter_errback,
+    aiter_errback,
     mustbe_deferred,
     process_chain,
     process_chain_both,
     process_parallel,
+    deferred_f_from_coro_f,
 )
 
 
@@ -117,3 +120,31 @@ class IterErrbackTest(unittest.TestCase):
         self.assertEqual(out, [0, 1, 2, 3, 4])
         self.assertEqual(len(errors), 1)
         self.assertIsInstance(errors[0].value, ZeroDivisionError)
+
+
+class AiterErrbackTest(unittest.TestCase):
+
+    @deferred_f_from_coro_f
+    async def test_aiter_errback_good(self):
+        async def itergood():
+            for x in range(10):
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(itergood(), errors.append))
+        self.assertEqual(out, list(range(10)))
+        self.assertFalse(errors)
+
+    @deferred_f_from_coro_f
+    async def test_iter_errback_bad(self):
+        async def iterbad():
+            for x in range(10):
+                if x == 5:
+                    1 / 0
+                yield x
+
+        errors = []
+        out = await collect_asyncgen(aiter_errback(iterbad(), errors.append))
+        self.assertEqual(out, [0, 1, 2, 3, 4])
+        self.assertEqual(len(errors), 1)
+        self.assertIsInstance(errors[0].value, ZeroDivisionError)

From 7e9f498e00fd36c76ac139eb285526c9e70b4054 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 14:37:27 +0500
Subject: [PATCH 03/41] Add MutableAsyncChain.

---
 scrapy/utils/python.py     | 25 ++++++++++++++++
 tests/test_utils_python.py | 58 ++++++++++++++++++++++++++++++++++++--
 2 files changed, 81 insertions(+), 2 deletions(-)

diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 5703fd4c3..0bf9bff70 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -12,6 +12,7 @@ from functools import partial, wraps
 from itertools import chain
 
 from scrapy.exceptions import ScrapyDeprecationWarning
+from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.decorators import deprecated
 
 
@@ -355,3 +356,27 @@ class MutableChain:
     @deprecated("scrapy.utils.python.MutableChain.__next__")
     def next(self):
         return self.__next__()
+
+
+async def _async_chain(*iterables):
+    for it in iterables:
+        async for o in as_async_generator(it):
+            yield o
+
+
+class MutableAsyncChain:
+    """
+    Similar to MutableChain but for async iterables
+    """
+
+    def __init__(self, *args):
+        self.data = _async_chain(*args)
+
+    def extend(self, *aiterables):
+        self.data = _async_chain(self.data, _async_chain(*aiterables))
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self):
+        return await self.data.__anext__()
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3115cc92f..58b384591 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,15 +2,18 @@ import functools
 import gc
 import operator
 import platform
-import unittest
 from datetime import datetime
 from itertools import count
 from warnings import catch_warnings
 
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f, aiter_errback
 from scrapy.utils.python import (
     memoizemethod_noargs, binary_is_text, equal_attributes,
     WeakKeyCache, get_func_args, to_bytes, to_unicode,
-    without_none_values, MutableChain)
+    without_none_values, MutableChain, MutableAsyncChain)
 
 
 __doctests__ = ['scrapy.utils.python']
@@ -32,6 +35,57 @@ class MutableChainTest(unittest.TestCase):
         self.assertEqual(list(m), list(range(3, 13)))
 
 
+class MutableAsyncChainTest(unittest.TestCase):
+    @staticmethod
+    async def g1():
+        for i in range(3):
+            yield i
+
+    @staticmethod
+    async def g2():
+        return
+        yield
+
+    @staticmethod
+    async def g3():
+        for i in range(7, 10):
+            yield i
+
+    @staticmethod
+    async def g4():
+        for i in range(3, 5):
+            yield i
+        1 / 0
+        for i in range(5, 7):
+            yield i
+
+    @staticmethod
+    async def collect_asyncgen_exc(asyncgen):
+        results = []
+        async for x in asyncgen:
+            results.append(x)
+        return results
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain(self):
+        m = MutableAsyncChain(self.g1(), as_async_generator(range(3, 7)))
+        m.extend(self.g2())
+        m.extend(self.g3())
+
+        self.assertEqual(await m.__anext__(), 0)
+        results = await collect_asyncgen(m)
+        self.assertEqual(results, list(range(1, 10)))
+
+    @deferred_f_from_coro_f
+    async def test_mutableasyncchain_exc(self):
+        m = MutableAsyncChain(self.g1())
+        m.extend(self.g4())
+        m.extend(self.g3())
+
+        results = await collect_asyncgen(aiter_errback(m, lambda _: None))
+        self.assertEqual(results, list(range(5)))
+
+
 class ToUnicodeTest(unittest.TestCase):
     def test_converting_an_utf8_encoded_string_to_unicode(self):
         self.assertEqual(to_unicode(b'lel\xc3\xb1e'), 'lel\xf1e')

From d658552f232547cc227e597b0d8489c4762eb9d2 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:20:04 +0500
Subject: [PATCH 04/41] Add only_not_asyncio.

---
 conftest.py | 6 ++++++
 pytest.ini  | 1 +
 2 files changed, 7 insertions(+)

diff --git a/conftest.py b/conftest.py
index 68b855c08..407bf9e62 100644
--- a/conftest.py
+++ b/conftest.py
@@ -55,5 +55,11 @@ def only_asyncio(request, reactor_pytest):
         pytest.skip('This test is only run with --reactor=asyncio')
 
 
+@pytest.fixture(autouse=True)
+def only_not_asyncio(request, reactor_pytest):
+    if request.node.get_closest_marker('only_not_asyncio') and reactor_pytest == 'asyncio':
+        pytest.skip('This test is only run without --reactor=asyncio')
+
+
 # Generate localhost certificate files, needed by some tests
 generate_keys()
diff --git a/pytest.ini b/pytest.ini
index d4deeb57c..416b228f9 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -21,6 +21,7 @@ addopts =
 twisted = 1
 markers =
     only_asyncio: marks tests as only enabled when --reactor=asyncio is passed
+    only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed
 flake8-max-line-length = 119
 flake8-ignore =
     W503

From d66d52d3ed8aab0b4f126169c2855d00f4053907 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:21:12 +0500
Subject: [PATCH 05/41] Add process_iterable_helper.

---
 scrapy/utils/middlewares.py     | 35 +++++++++++++
 tests/test_utils_middlewares.py | 87 +++++++++++++++++++++++++++++++++
 2 files changed, 122 insertions(+)
 create mode 100644 scrapy/utils/middlewares.py
 create mode 100644 tests/test_utils_middlewares.py

diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py
new file mode 100644
index 000000000..da28e0ddf
--- /dev/null
+++ b/scrapy/utils/middlewares.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+import inspect
+
+
+def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    for o in it:
+        if in_predicate and not in_predicate(o):
+            continue
+        if processor is not None:
+            o = processor(o)
+        if out_predicate and not out_predicate(o):
+            continue
+        yield o
+
+
+async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    async for o in it:
+        if in_predicate and not in_predicate(o):
+            continue
+        if processor is not None:
+            o = processor(o)
+        if out_predicate and not out_predicate(o):
+            continue
+        yield o
+
+
+def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
+    """
+    For each item in the iterable: skips it if in_predicate is False, applies processor,
+    skips the result if out_predicate is False, else yields it.
+    """
+    if inspect.isasyncgen(it):
+        return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
+    else:
+        return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)
diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py
new file mode 100644
index 000000000..d395ba1a9
--- /dev/null
+++ b/tests/test_utils_middlewares.py
@@ -0,0 +1,87 @@
+import collections
+
+from twisted.trial import unittest
+
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.middlewares import process_iterable_helper
+
+
+def predicate1(o):
+    return bool(o % 2)
+
+
+def predicate2(o):
+    return o < 10
+
+
+def processor(o):
+    return o * 2
+
+
+class ProcessIterableHelperNormalTest(unittest.TestCase):
+
+    def test_normal_in_predicate(self):
+        iterable1 = iter([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [1, 3])
+
+    def test_normal_out_predicate(self):
+        iterable1 = iter([1, 2, 10, 3, 15])
+        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [1, 2, 3])
+
+    def test_normal_processor(self):
+        iterable1 = iter([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [2, 4, 6])
+
+    def test_normal_combined(self):
+        iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
+                                            out_predicate=predicate2, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.Iterable)
+        list2 = list(iterable2)
+        self.assertEqual(list2, [2, 6])
+
+
+class ProcessIterableHelperAsyncTest(unittest.TestCase):
+
+    @deferred_f_from_coro_f
+    async def test_async_in_predicate(self):
+        iterable1 = as_async_generator([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [1, 3])
+
+    @deferred_f_from_coro_f
+    async def test_async_out_predicate(self):
+        iterable1 = as_async_generator([1, 2, 10, 3, 15])
+        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [1, 2, 3])
+
+    @deferred_f_from_coro_f
+    async def test_async_processor(self):
+        iterable1 = as_async_generator([1, 2, 3])
+        iterable2 = process_iterable_helper(iterable1, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_async_combined(self):
+        iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])
+        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
+                                            out_predicate=predicate2, processor=processor)
+        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
+        list2 = await collect_asyncgen(iterable2)
+        self.assertEqual(list2, [2, 6])

From 92f2c9e308a5eda361229a8e74f74a21b9ff770a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 2 Feb 2021 15:22:20 +0500
Subject: [PATCH 06/41] Move spider middlewares to process_iterable_helper.

---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 ++++++++++++++-------------
 scrapy/spidermiddlewares/referer.py   |  3 ++-
 scrapy/spidermiddlewares/urllength.py |  3 ++-
 4 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..73079bca9 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
-
 import logging
 
 from scrapy.http import Request
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..e7f481269 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,6 +10,7 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -26,21 +27,22 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
-                else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+        def in_predicate(x):
+            if not isinstance(x, Request):
+                return True
+            if x.dont_filter or self.should_follow(x, spider):
+                return True
+            domain = urlparse_cached(x).hostname
+            if domain and domain not in self.domains_seen:
+                self.domains_seen.add(domain)
+                logger.debug(
+                    "Filtered offsite request to %(domain)r: %(request)s",
+                    {'domain': domain, 'request': x}, extra={'spider': spider})
+                self.stats.inc_value('offsite/domains', spider=spider)
+            self.stats.inc_value('offsite/filtered', spider=spider)
+            return False
+
+        return process_iterable_helper(result or (), in_predicate=in_predicate)
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..91c8727e1 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,6 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -337,7 +338,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return (_set_referer(r) for r in result or ())
+        return process_iterable_helper(result or (), processor=_set_referer)
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..c7359fecd 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,6 +8,7 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -34,4 +35,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return (r for r in result or () if _filter(r))
+        return process_iterable_helper(result or (), in_predicate=_filter)

From 2a1e9359caa0e16b336fdb2944ab7f4a69549896 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:16:29 +0500
Subject: [PATCH 07/41] Add parallel_async.

---
 scrapy/core/scraper.py |  24 +++++++--
 scrapy/utils/defer.py  | 110 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 129 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 0d3e3450f..3eae71af7 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,6 +1,6 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-
+import collections
 import logging
 from collections import deque
 
@@ -12,7 +12,16 @@ from scrapy import signals
 from scrapy.core.spidermw import SpiderMiddlewareManager
 from scrapy.exceptions import CloseSpider, DropItem, IgnoreRequest
 from scrapy.http import Request, Response
-from scrapy.utils.defer import defer_fail, defer_succeed, iter_errback, parallel
+from scrapy.utils.defer import (
+    aiter_errback,
+    defer_fail,
+    defer_succeed,
+    deferred_from_coro,
+    iter_errback,
+    parallel,
+    parallel_async,
+)
+
 from scrapy.utils.log import failure_to_exc_info, logformatter_adapter
 from scrapy.utils.misc import load_object, warn_on_generator_with_return_value
 from scrapy.utils.spider import iterate_spider_output
@@ -180,9 +189,14 @@ class Scraper:
     def handle_spider_output(self, result, request, response, spider):
         if not result:
             return defer_succeed(None)
-        it = iter_errback(result, self.handle_spider_error, request, response, spider)
-        dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
-                       request, response, spider)
+        if isinstance(result, collections.abc.AsyncIterable):
+            it = aiter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                     request, response, spider))
+        else:
+            it = iter_errback(result, self.handle_spider_error, request, response, spider)
+            dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,
+                           request, response, spider)
         return dfd
 
     def _process_spidermw_output(self, output, request, response, spider):
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 2d02c0621..bd5f9c8fc 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -75,6 +75,116 @@ def parallel(iterable, count, callable, *args, **named):
     return defer.DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
+class _AsyncCooperatorAdapter:
+    """ A class that wraps an async iterator into a normal iterator suitable
+    for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
+    it calls the callable directly in the callback, instead of providing a more
+    generic interface.
+
+    On the outside, this class behaves as an iterator that yields Deferreds.
+    Each Deferred is fired with the result of the callable which was called on
+    the next result from aiterator. It raises StopIteration when aiterator is
+    exhausted, as expected.
+
+    Cooperator calls __next__() multiple times and waits on the Deferreds
+    returned from it. As async generators (since Python 3.8) don't support
+    awaiting on __anext__() several times in parallel, we need to serialize
+    this. It's done by storing the Deferreds returned from __next__() and
+    firing the oldest one when a result from __anext__() is available.
+
+    The workflow:
+    1. When __next__() is called for the first time, it creates a Deferred, stores it
+    in self.waiting_deferreds and returns it. It also makes a Deferred that will wait
+    for self.aiterator.__anext__() and puts it into self.anext_deferred.
+    2. If __next__() is called again before self.anext_deferred fires, more Deferreds
+    are added to self.waiting_deferreds.
+    3. When self.anext_deferred fires, it either calls _callback() or _errback(). Both
+    clear self.anext_deferred.
+    3.1. _callback() calls the callable passing the result value that it takes, pops a
+    Deferred from self.waiting_deferreds, and if the callable result was a Deferred, it
+    chains those Deferreds so that the waiting Deferred will fire when the result
+    Deferred does, otherwise it fires it directly. This causes one awaiting task to
+    receive a result. If self.waiting_deferreds is still not empty, new __anext__() is
+    called and self.anext_deferred is populated.
+    3.2. _errback() checks the exception class. If it's StopAsyncIteration it means
+    self.aiterator is exhausted and so it sets self.finished and fires all
+    self.waiting_deferreds. Other exceptions are propagated.
+    4. If __next__() is called after __anext__() was handled, then if self.finished is
+    True, it raises StopIteration, otherwise it acts like in step 2, but if
+    self.anext_deferred is now empty is also populates it with a new __anext__().
+
+    Note that CooperativeTask ignores the value returned from the Deferred that it waits
+    for, so we fire them with None when needed.
+
+    It may be possible to write an async iterator-aware replacement for
+    Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
+    goal.
+    """
+    def __init__(self, aiterator, callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterator
+        self.callable = callable
+        self.callable_args = callable_args
+        self.callable_kwargs = callable_kwargs
+        self.finished = False
+        self.waiting_deferreds = []
+        self.anext_deferred = None
+
+    def _callback(self, result):
+        # This gets called when the result from aiterator.__anext__() is available.
+        # It calls the callable on it and sends the result to the oldest waiting Deferred
+        # (by chaining if the result is a Deferred too or by firing if not).
+        self.anext_deferred = None
+        result = self.callable(result, *self.callable_args, **self.callable_kwargs)
+        d = self.waiting_deferreds.pop(0)
+        if d.called:
+            raise ValueError('Deferred in waiting_deferreds already called')
+        if isinstance(result, defer.Deferred):
+            result.chainDeferred(d)
+        else:
+            d.callback(None)
+        if self.waiting_deferreds:
+            self._call_anext()
+
+    def _errback(self, failure):
+        # This gets called on any exceptions in aiterator.__anext__().
+        # It handles StopAsyncIteration by stopping the iteration and reraises all others.
+        self.anext_deferred = None
+        failure.trap(StopAsyncIteration)
+        self.finished = True
+        for d in self.waiting_deferreds:
+            if d.called:
+                raise ValueError('Deferred in waiting_deferreds already called')
+            d.callback(None)
+
+    def _call_anext(self):
+        # This starts waiting for the next result from aiterator.
+        # If aiterator is exhausted, _errback will be called.
+        self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
+        self.anext_deferred.addCallbacks(self._callback, self._errback)
+
+    def __iter__(self):
+        return self
+
+    def __next__(self):
+        # This puts a new Deferred into self.waiting_deferreds and returns it.
+        # It also calls __anext__() if needed.
+        if self.finished:
+            raise StopIteration
+        d = defer.Deferred()
+        self.waiting_deferreds.append(d)
+        if not self.anext_deferred:
+            self._call_anext()
+        return d
+
+
+def parallel_async(async_iterable, count, callable, *args, **named):
+    """ Like parallel but for async iterables """
+    coop = task.Cooperator()
+    work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
+    dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+    return dl
+
+
 def process_chain(callbacks, input, *a, **kw):
     """Return a Deferred built by chaining the given callbacks"""
     d = defer.Deferred()

From 2152a2a50898b91a44e6b0c282b69b7bfb8d18f0 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:19:52 +0500
Subject: [PATCH 08/41] Add main infrastructure for async callbacks.

---
 scrapy/core/spidermw.py                     | 45 ++++++++++++++++-----
 scrapy/utils/defer.py                       | 19 +++++++++
 scrapy/utils/spider.py                      |  8 ++--
 scrapy/utils/test.py                        |  7 ++++
 tests/spiders.py                            | 38 ++++++++++++++++-
 tests/test_crawl.py                         | 33 +++++++++++++++
 tests/test_spidermiddleware_output_chain.py | 27 +++++++++++++
 7 files changed, 161 insertions(+), 16 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 763e0cdf6..961606f29 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,6 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import inspect
 from itertools import islice
 
 from twisted.python.failure import Failure
@@ -11,11 +12,11 @@ from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
-from scrapy.utils.python import MutableChain
+from scrapy.utils.python import MutableAsyncChain, MutableChain
 
 
 def _isiterable(possible_iterator):
-    return hasattr(possible_iterator, '__iter__')
+    return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
 
 
 def _fname(f):
@@ -58,15 +59,31 @@ class SpiderMiddlewareManager(MiddlewareManager):
             return scrape_func(response, request, spider)
 
         def _evaluate_iterable(iterable, exception_processor_index, recover_to):
-            try:
-                for r in iterable:
-                    yield r
-            except Exception as ex:
+            def _process_exception(ex):
                 exception_result = process_spider_exception(Failure(ex), exception_processor_index)
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
 
+            def _evaluate_normal_iterable(iterable):
+                try:
+                    for r in iterable:
+                        yield r
+                except Exception as ex:
+                    _process_exception(ex)
+
+            async def _evaluate_async_iterable(iterable):
+                try:
+                    async for r in iterable:
+                        yield r
+                except Exception as ex:
+                    _process_exception(ex)
+
+            if inspect.isasyncgen(iterable):
+                return _evaluate_async_iterable(iterable)
+            else:
+                return _evaluate_normal_iterable(iterable)
+
         def process_spider_exception(_failure, start_index=0):
             exception = _failure.value
             # don't handle _InvalidOutput exception
@@ -92,7 +109,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
         def process_spider_output(result, start_index=0):
             # items in this iterable do not need to go through the process_spider_output
             # chain, they went through it already from the process_spider_exception method
-            recovered = MutableChain()
+            if inspect.isasyncgen(result):
+                iter_class = MutableAsyncChain
+            else:
+                iter_class = MutableChain
+            recovered = iter_class()
 
             method_list = islice(self.methods['process_spider_output'], start_index, None)
             for method_index, method in enumerate(method_list, start=start_index):
@@ -113,12 +134,16 @@ class SpiderMiddlewareManager(MiddlewareManager):
                            f"iterable, got {type(result)}")
                     raise _InvalidOutput(msg)
 
-            return MutableChain(result, recovered)
+            return iter_class(result, recovered)
 
         def process_callback_output(result):
-            recovered = MutableChain()
+            if inspect.isasyncgen(result):
+                iter_class = MutableAsyncChain
+            else:
+                iter_class = MutableChain
+            recovered = iter_class()
             result = _evaluate_iterable(result, 0, recovered)
-            return MutableChain(process_spider_output(result), recovered)
+            return iter_class(process_spider_output(result), recovered)
 
         dfd = mustbe_deferred(process_spider_input, response)
         dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index bd5f9c8fc..554edc38c 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -290,3 +290,22 @@ def maybeDeferred_coro(f, *args, **kw):
         return defer.fail(result)
     else:
         return defer.succeed(result)
+
+
+def deferred_to_future(d):
+    """ Wraps a Deferred into a Future. Requires the asyncio reactor.
+    """
+    return d.asFuture(asyncio.get_event_loop())
+
+
+def maybe_deferred_to_future(d):
+    """ Converts a Deferred to something that can be awaited in a callback or other user coroutine.
+
+    If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be
+    awaited inside them. Otherwise, coroutines are wrapped into Deferreds and Deferreds can be awaited
+    directly inside them.
+    """
+    if not is_asyncio_reactor_installed():
+        return d
+    else:
+        return deferred_to_future(d)
diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py
index 59fc9202f..d0fd1757d 100644
--- a/scrapy/utils/spider.py
+++ b/scrapy/utils/spider.py
@@ -4,7 +4,6 @@ import logging
 from scrapy.spiders import Spider
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.misc import arg_to_iter
-from scrapy.utils.asyncgen import collect_asyncgen
 
 
 logger = logging.getLogger(__name__)
@@ -12,14 +11,13 @@ logger = logging.getLogger(__name__)
 
 def iterate_spider_output(result):
     if inspect.isasyncgen(result):
-        d = deferred_from_coro(collect_asyncgen(result))
-        d.addCallback(iterate_spider_output)
-        return d
+        return result
     elif inspect.iscoroutine(result):
         d = deferred_from_coro(result)
         d.addCallback(iterate_spider_output)
         return d
-    return arg_to_iter(result)
+    else:
+        return arg_to_iter(deferred_from_coro(result))
 
 
 def iter_spider_classes(module):
diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py
index 24c38283a..d8fc25094 100644
--- a/scrapy/utils/test.py
+++ b/scrapy/utils/test.py
@@ -110,3 +110,10 @@ def mock_google_cloud_storage():
     bucket_mock.blob.return_value = blob_mock
 
     return (client_mock, bucket_mock, blob_mock)
+
+
+def get_web_client_agent_req(url):
+    from twisted.internet import reactor
+    from twisted.web.client import Agent  # imports twisted.internet.reactor
+    agent = Agent(reactor)
+    return agent.request(b'GET', url.encode('utf-8'))
diff --git a/tests/spiders.py b/tests/spiders.py
index 106392ea6..3e0ec001b 100644
--- a/tests/spiders.py
+++ b/tests/spiders.py
@@ -14,7 +14,8 @@ from scrapy.item import Item
 from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.spiders.crawl import CrawlSpider, Rule
-from scrapy.utils.test import get_from_asyncio_queue
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
+from scrapy.utils.test import get_from_asyncio_queue, get_web_client_agent_req
 
 
 class MockServerSpider(Spider):
@@ -148,6 +149,41 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
         return reqs
 
 
+class AsyncDefAsyncioGenExcSpider(SimpleSpider):
+    name = 'asyncdef_asyncio_gen_exc'
+
+    async def parse(self, response):
+        for i in range(10):
+            await asyncio.sleep(0.1)
+            yield {'foo': i}
+            if i > 5:
+                raise ValueError("Stopping the processing")
+
+
+class AsyncDefDeferredDirectSpider(SimpleSpider):
+    name = 'asyncdef_deferred_direct'
+
+    async def parse(self, response):
+        resp = await get_web_client_agent_req(self.mockserver.url("/status?n=200"))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
+class AsyncDefDeferredMaybeWrappedSpider(SimpleSpider):
+    name = 'asyncdef_deferred_wrapped'
+
+    async def parse(self, response):
+        resp = await maybe_deferred_to_future(get_web_client_agent_req(self.mockserver.url("/status?n=200")))
+        yield {'code': resp.code}
+
+
 class AsyncDefAsyncioGenSpider(SimpleSpider):
 
     name = 'asyncdef_asyncio_gen'
diff --git a/tests/test_crawl.py b/tests/test_crawl.py
index 1083c1678..cda52f0d4 100644
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@@ -20,12 +20,16 @@ from scrapy.utils.python import to_unicode
 from tests.mockserver import MockServer
 from tests.spiders import (
     AsyncDefAsyncioGenComplexSpider,
+    AsyncDefAsyncioGenExcSpider,
     AsyncDefAsyncioGenLoopSpider,
     AsyncDefAsyncioGenSpider,
     AsyncDefAsyncioReqsReturnSpider,
     AsyncDefAsyncioReturnSingleElementSpider,
     AsyncDefAsyncioReturnSpider,
     AsyncDefAsyncioSpider,
+    AsyncDefDeferredDirectSpider,
+    AsyncDefDeferredMaybeWrappedSpider,
+    AsyncDefDeferredWrappedSpider,
     AsyncDefSpider,
     BrokenStartRequestsSpider,
     BytesReceivedCallbackSpider,
@@ -430,6 +434,18 @@ class CrawlSpiderTestCase(TestCase):
         for i in range(10):
             self.assertIn({'foo': i}, items)
 
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_asyncgen_parse_exc(self):
+        log, items, stats = yield self._run_spider(AsyncDefAsyncioGenExcSpider)
+        log = str(log)
+        self.assertIn("Spider error processing", log)
+        self.assertIn("ValueError", log)
+        itemcount = stats.get_value('item_scraped_count')
+        self.assertEqual(itemcount, 7)
+        for i in range(7):
+            self.assertIn({'foo': i}, items)
+
     @mark.only_asyncio()
     @defer.inlineCallbacks
     def test_async_def_asyncgen_parse_complex(self):
@@ -449,6 +465,23 @@ class CrawlSpiderTestCase(TestCase):
         for req_id in range(3):
             self.assertIn(f"Got response 200, req_id {req_id}", str(log))
 
+    @mark.only_not_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_direct(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredDirectSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @mark.only_asyncio()
+    @defer.inlineCallbacks
+    def test_async_def_deferred_wrapped(self):
+        log, items, _ = yield self._run_spider(AsyncDefDeferredWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
+    @defer.inlineCallbacks
+    def test_async_def_deferred_maybe_wrapped(self):
+        _, items, _ = yield self._run_spider(AsyncDefDeferredMaybeWrappedSpider)
+        self.assertEqual(items, [{'code': 200}])
+
     @defer.inlineCallbacks
     def test_response_ssl_certificate_none(self):
         crawler = self.runner.create_crawler(SingleRequestSpider)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 029bf8bd6..4d1a7fcb0 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -43,6 +43,23 @@ class RecoverySpider(Spider):
             raise TabError()
 
 
+class RecoveryAsyncGenSpider(RecoverySpider):
+    name = 'RecoveryAsyncGenSpider'
+
+    async def parse(self, response):
+        for r in super().parse(response):
+            yield r
+
+
+class RecoveryMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
+        return [
+            {'from': 'process_spider_exception'},
+            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
+        ]
+
+
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:
@@ -307,6 +324,16 @@ class TestSpiderMiddleware(TestCase):
         self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
         self.assertIn("'item_scraped_count': 3", str(log))
 
+    @defer.inlineCallbacks
+    def test_recovery_asyncgen(self):
+        """
+        Same as test_recovery but with an async callback.
+        """
+        log = yield self.crawl_log(RecoveryAsyncGenSpider)
+        self.assertIn("Middleware: TabError exception caught", str(log))
+        self.assertEqual(str(log).count("Middleware: TabError exception caught"), 1)
+        self.assertIn("'item_scraped_count': 3", str(log))
+
     @defer.inlineCallbacks
     def test_process_spider_input_without_errback(self):
         """

From 58f848130145649a0191e98b182977278e2b9b6c Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 16:20:32 +0500
Subject: [PATCH 09/41] Update docs.

---
 docs/topics/asyncio.rst    | 24 ++++++++++++++++++++++++
 docs/topics/coroutines.rst | 12 +++---------
 2 files changed, 27 insertions(+), 9 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 91e1cca0d..4addaa178 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -39,4 +39,28 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 use it instead of the default asyncio event loop.
 
 
+.. _asyncio-await-dfd:
 
+Awaiting on Deferreds
+=====================
+
+When the asyncio reactor isn't installed, you can await on Deferreds in the
+coroutines directly. When it is installed, this is not possible anymore, due to
+specifics of the Scrapy coroutine integration (the coroutines are wrapped into
+asyncio Futures, not into Deferreds directly), and you need to wrap them into
+Futures. Scrapy provides two helpers for this:
+
+.. autofunction:: scrapy.utils.defer.deferred_to_future
+.. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
+
+If you want to write universal code that works on any reactors,
+you should use ``maybe_deferred_to_future`` on all Deferreds::
+
+    from scrapy.utils.defer import maybe_deferred_to_future
+
+    class MySpider(Spider):
+        # ...
+        async def parse_with_deferred(self, response):
+            additional_response = await maybe_deferred_to_future(treq.get('https://additional.url'))
+            additional_data = await maybe_deferred_to_future(treq.content(additional_response))
+            # ... use response and additional_data to yield items and requests
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 3b1549bd3..279632653 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,15 +17,6 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.http.Request` callbacks.
 
-    .. note:: The callback output is not processed until the whole callback
-        finishes.
-
-        As a side effect, if the callback raises an exception, none of its
-        output is processed.
-
-        This is a known caveat of the current implementation that we aim to
-        address in a future version of Scrapy.
-
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
 
@@ -92,6 +83,9 @@ This means you can use many useful Python libraries providing such code::
           :mod:`asyncio` loop and to use them you need to
           :doc:`enable asyncio support in Scrapy<asyncio>`.
 
+.. note:: If you want to ``await`` on Deferreds, you may need to
+          :ref:`wrap them<asyncio-await-dfd>`.
+
 Common use cases for asynchronous code include:
 
 * requesting data from websites, databases and other services (in callbacks,

From 5cf403295d44bd2531ee3fe2f07057cf155e9804 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 5 Feb 2021 19:40:14 +0500
Subject: [PATCH 10/41] Remove a duplicate definition.

---
 tests/test_spidermiddleware_output_chain.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 4d1a7fcb0..088c14ca8 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -51,15 +51,6 @@ class RecoveryAsyncGenSpider(RecoverySpider):
             yield r
 
 
-class RecoveryMiddleware:
-    def process_spider_exception(self, response, exception, spider):
-        spider.logger.info('Middleware: %s exception caught', exception.__class__.__name__)
-        return [
-            {'from': 'process_spider_exception'},
-            Request(response.url, meta={'dont_fail': True}, dont_filter=True),
-        ]
-
-
 # ================================================================================
 # (1) exceptions from a spider middleware's process_spider_input method
 class FailProcessSpiderInputMiddleware:

From 67cff0e8a949a83220ae6d7d2c2ca40e6c8a8207 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 10 Feb 2021 22:44:14 +0500
Subject: [PATCH 11/41] Silence pylint "naked raise" error.

---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 96392aae7..d3d7e5f8c 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             exception_result = self._process_spider_exception(response, spider, Failure(ex),
                                                               exception_processor_index)
             if isinstance(exception_result, Failure):
-                raise
+                raise  # pylint: disable=E0704
             recover_to.extend(exception_result)
 
         def _evaluate_normal_iterable(iterable):

From 40eab1d473a78369702490a901b5d304a40baf3b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 18 Feb 2021 19:56:12 +0500
Subject: [PATCH 12/41] Drop a duplicate import.

---
 tests/test_utils_defer.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index d220f969b..62f6ff194 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -5,14 +5,13 @@ from twisted.python.failure import Failure
 
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.defer import (
+    aiter_errback,
     deferred_f_from_coro_f,
     iter_errback,
-    aiter_errback,
     mustbe_deferred,
     process_chain,
     process_chain_both,
     process_parallel,
-    deferred_f_from_coro_f,
 )
 
 

From f9a5385146c741825d8c0b089cf8e8318dca9403 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 19 Feb 2021 21:19:21 +0500
Subject: [PATCH 13/41] Revert "Move spider middlewares to
 process_iterable_helper."

This reverts commit 92f2c9e308a5eda361229a8e74f74a21b9ff770a.
---
 scrapy/spidermiddlewares/depth.py     |  4 ++--
 scrapy/spidermiddlewares/offsite.py   | 32 +++++++++++++--------------
 scrapy/spidermiddlewares/referer.py   |  3 +--
 scrapy/spidermiddlewares/urllength.py |  3 +--
 4 files changed, 19 insertions(+), 23 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 73079bca9..776a6879a 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -3,10 +3,10 @@ Depth Spider Middleware
 
 See documentation in docs/topics/spider-middleware.rst
 """
+
 import logging
 
 from scrapy.http import Request
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -55,4 +55,4 @@ class DepthMiddleware:
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index e7f481269..6e4efda97 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -10,7 +10,6 @@ import warnings
 from scrapy import signals
 from scrapy.http import Request
 from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -27,22 +26,21 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        def in_predicate(x):
-            if not isinstance(x, Request):
-                return True
-            if x.dont_filter or self.should_follow(x, spider):
-                return True
-            domain = urlparse_cached(x).hostname
-            if domain and domain not in self.domains_seen:
-                self.domains_seen.add(domain)
-                logger.debug(
-                    "Filtered offsite request to %(domain)r: %(request)s",
-                    {'domain': domain, 'request': x}, extra={'spider': spider})
-                self.stats.inc_value('offsite/domains', spider=spider)
-            self.stats.inc_value('offsite/filtered', spider=spider)
-            return False
-
-        return process_iterable_helper(result or (), in_predicate=in_predicate)
+        for x in result:
+            if isinstance(x, Request):
+                if x.dont_filter or self.should_follow(x, spider):
+                    yield x
+                else:
+                    domain = urlparse_cached(x).hostname
+                    if domain and domain not in self.domains_seen:
+                        self.domains_seen.add(domain)
+                        logger.debug(
+                            "Filtered offsite request to %(domain)r: %(request)s",
+                            {'domain': domain, 'request': x}, extra={'spider': spider})
+                        self.stats.inc_value('offsite/domains', spider=spider)
+                    self.stats.inc_value('offsite/filtered', spider=spider)
+            else:
+                yield x
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index 91c8727e1..f81041376 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,7 +10,6 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
-from scrapy.utils.middlewares import process_iterable_helper
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -338,7 +337,7 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return process_iterable_helper(result or (), processor=_set_referer)
+        return (_set_referer(r) for r in result or ())
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index c7359fecd..5be1f80cb 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,7 +8,6 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.middlewares import process_iterable_helper
 
 logger = logging.getLogger(__name__)
 
@@ -35,4 +34,4 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return process_iterable_helper(result or (), in_predicate=_filter)
+        return (r for r in result or () if _filter(r))

From c51ec1ae1cef3fd68fc512f0636b75d0f3a2da13 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Feb 2021 19:32:45 +0500
Subject: [PATCH 14/41] Drop process_iterable_helper, add
 _process_iterable_universal.

---
 scrapy/core/spidermw.py               | 28 +++------
 scrapy/spidermiddlewares/depth.py     | 18 ++++--
 scrapy/spidermiddlewares/offsite.py   | 32 +++++-----
 scrapy/spidermiddlewares/referer.py   |  9 ++-
 scrapy/spidermiddlewares/urllength.py |  9 ++-
 scrapy/utils/asyncgen.py              | 45 ++++++++++++++
 scrapy/utils/middlewares.py           | 35 -----------
 tests/test_utils_asyncgen.py          | 22 ++++++-
 tests/test_utils_middlewares.py       | 87 ---------------------------
 9 files changed, 120 insertions(+), 165 deletions(-)
 delete mode 100644 scrapy/utils/middlewares.py
 delete mode 100644 tests/test_utils_middlewares.py

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index d3d7e5f8c..33e215971 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -10,6 +10,7 @@ from twisted.python.failure import Failure
 
 from scrapy.exceptions import _InvalidOutput
 from scrapy.middleware import MiddlewareManager
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.conf import build_component_list
 from scrapy.utils.defer import mustbe_deferred
 from scrapy.utils.python import MutableAsyncChain, MutableChain
@@ -57,31 +58,18 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return scrape_func(response, request, spider)
 
     def _evaluate_iterable(self, response, spider, iterable, exception_processor_index, recover_to):
-        def _process_exception(ex):
-            exception_result = self._process_spider_exception(response, spider, Failure(ex),
-                                                              exception_processor_index)
-            if isinstance(exception_result, Failure):
-                raise  # pylint: disable=E0704
-            recover_to.extend(exception_result)
-
-        def _evaluate_normal_iterable(iterable):
-            try:
-                for r in iterable:
-                    yield r
-            except Exception as ex:
-                _process_exception(ex)
-
+        @_process_iterable_universal
         async def _evaluate_async_iterable(iterable):
             try:
                 async for r in iterable:
                     yield r
             except Exception as ex:
-                _process_exception(ex)
-
-        if inspect.isasyncgen(iterable):
-            return _evaluate_async_iterable(iterable)
-        else:
-            return _evaluate_normal_iterable(iterable)
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+        return _evaluate_async_iterable(iterable)
 
     def _process_spider_exception(self, response, spider, _failure, start_index=0):
         exception = _failure.value
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 776a6879a..973404b2b 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,6 +7,7 @@ See documentation in docs/topics/spider-middleware.rst
 import logging
 
 from scrapy.http import Request
+from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -49,10 +50,15 @@ class DepthMiddleware:
                                          spider=spider)
             return True
 
-        # base case (depth=0)
-        if 'depth' not in response.meta:
-            response.meta['depth'] = 0
-            if self.verbose_stats:
-                self.stats.inc_value('request_depth_count/0', spider=spider)
+        @_process_iterable_universal
+        async def process(result):
+            # base case (depth=0)
+            if 'depth' not in response.meta:
+                response.meta['depth'] = 0
+                if self.verbose_stats:
+                    self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        return (r for r in result or () if _filter(r))
+            async for r in result or ():
+                if _filter(r):
+                    yield r
+        return process(result)
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 6e4efda97..074ec7a4e 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -9,6 +9,7 @@ import warnings
 
 from scrapy import signals
 from scrapy.http import Request
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
@@ -26,21 +27,24 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        for x in result:
-            if isinstance(x, Request):
-                if x.dont_filter or self.should_follow(x, spider):
-                    yield x
+        @_process_iterable_universal
+        async def process(result):
+            async for x in result:
+                if isinstance(x, Request):
+                    if x.dont_filter or self.should_follow(x, spider):
+                        yield x
+                    else:
+                        domain = urlparse_cached(x).hostname
+                        if domain and domain not in self.domains_seen:
+                            self.domains_seen.add(domain)
+                            logger.debug(
+                                "Filtered offsite request to %(domain)r: %(request)s",
+                                {'domain': domain, 'request': x}, extra={'spider': spider})
+                            self.stats.inc_value('offsite/domains', spider=spider)
+                        self.stats.inc_value('offsite/filtered', spider=spider)
                 else:
-                    domain = urlparse_cached(x).hostname
-                    if domain and domain not in self.domains_seen:
-                        self.domains_seen.add(domain)
-                        logger.debug(
-                            "Filtered offsite request to %(domain)r: %(request)s",
-                            {'domain': domain, 'request': x}, extra={'spider': spider})
-                        self.stats.inc_value('offsite/domains', spider=spider)
-                    self.stats.inc_value('offsite/filtered', spider=spider)
-            else:
-                yield x
+                    yield x
+        return process(result)
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f81041376..8d862d1d0 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -10,6 +10,7 @@ from w3lib.url import safe_url_string
 from scrapy.http import Request, Response
 from scrapy.exceptions import NotConfigured
 from scrapy import signals
+from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.python import to_unicode
 from scrapy.utils.misc import load_object
 from scrapy.utils.url import strip_url
@@ -337,7 +338,13 @@ class RefererMiddleware:
                 if referrer is not None:
                     r.headers.setdefault('Referer', referrer)
             return r
-        return (_set_referer(r) for r in result or ())
+
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result or ():
+                yield _set_referer(r)
+
+        return process(result)
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..d40d43ff1 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,6 +8,7 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
+from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -34,4 +35,10 @@ class UrlLengthMiddleware:
             else:
                 return True
 
-        return (r for r in result or () if _filter(r))
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result or ():
+                if _filter(r):
+                    yield r
+
+        return process(result)
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index db2173f85..39c94ad8a 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,4 +1,6 @@
 import collections
+import functools
+import inspect
 
 
 async def collect_asyncgen(result):
@@ -15,3 +17,46 @@ async def as_async_generator(it):
     else:
         for r in it:
             yield r
+
+
+# https://stackoverflow.com/a/66170760/113586
+def _process_iterable_universal(process_async):
+    """ Takes a function that takes an async iterable, args and kwargs. Returns
+    a function that takes any iterable, args and kwargs.
+
+    Requires that process_async only awaits on the iterable and synchronous functions,
+    so it's better to use this only in the Scrapy code itself.
+    """
+
+    # If this stops working, all internal uses can be just replaced with manually-written
+    # process_sync functions.
+
+    def process_sync(iterable, *args, **kwargs):
+        agen = process_async(as_async_generator(iterable), *args, **kwargs)
+        if not inspect.isasyncgen(agen):
+            raise ValueError(f"process_async returned wrong type {type(agen)}")
+        sent = None
+        while True:
+            try:
+                gen = agen.asend(sent)
+                gen.send(None)
+            except StopIteration as e:
+                sent = yield e.value
+            except StopAsyncIteration:
+                return
+            else:
+                gen.throw(RuntimeError,
+                          f"Synchronously-called function '{process_async.__name__}' has blocked, "
+                          f"you can't use {_process_iterable_universal.__name__} with it.")
+
+    @functools.wraps(process_async)
+    def process(iterable, *args, **kwargs):
+        if inspect.isasyncgen(iterable):
+            # call process_async directly
+            return process_async(iterable, *args, **kwargs)
+        if hasattr(iterable, '__iter__'):
+            # convert process_async to process_sync
+            return process_sync(iterable, *args, **kwargs)
+        raise ValueError(f"Wrong iterable type {type(iterable)}")
+
+    return process
diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py
deleted file mode 100644
index da28e0ddf..000000000
--- a/scrapy/utils/middlewares.py
+++ /dev/null
@@ -1,35 +0,0 @@
-# coding: utf-8
-import inspect
-
-
-def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    for o in it:
-        if in_predicate and not in_predicate(o):
-            continue
-        if processor is not None:
-            o = processor(o)
-        if out_predicate and not out_predicate(o):
-            continue
-        yield o
-
-
-async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    async for o in it:
-        if in_predicate and not in_predicate(o):
-            continue
-        if processor is not None:
-            o = processor(o)
-        if out_predicate and not out_predicate(o):
-            continue
-        yield o
-
-
-def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
-    """
-    For each item in the iterable: skips it if in_predicate is False, applies processor,
-    skips the result if out_predicate is False, else yields it.
-    """
-    if inspect.isasyncgen(it):
-        return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
-    else:
-        return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 9ae66c57c..2f4181d3d 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,6 +1,6 @@
 from twisted.trial import unittest
 
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
 
 
@@ -18,3 +18,23 @@ class AsyncgenUtilsTest(unittest.TestCase):
         ag = as_async_generator(range(42))
         results = await collect_asyncgen(ag)
         self.assertEqual(results, list(range(42)))
+
+
+@_process_iterable_universal
+async def process_iterable(iterable):
+    async for i in iterable:
+        yield i * 2
+
+
+class ProcessIterableUniversalTest(unittest.TestCase):
+
+    def test_normal(self):
+        iterable = iter([1, 2, 3])
+        results = list(process_iterable(iterable))
+        self.assertEqual(results, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_async(self):
+        iterable = as_async_generator([1, 2, 3])
+        results = await collect_asyncgen(process_iterable(iterable))
+        self.assertEqual(results, [2, 4, 6])
diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py
deleted file mode 100644
index d395ba1a9..000000000
--- a/tests/test_utils_middlewares.py
+++ /dev/null
@@ -1,87 +0,0 @@
-import collections
-
-from twisted.trial import unittest
-
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_f_from_coro_f
-from scrapy.utils.middlewares import process_iterable_helper
-
-
-def predicate1(o):
-    return bool(o % 2)
-
-
-def predicate2(o):
-    return o < 10
-
-
-def processor(o):
-    return o * 2
-
-
-class ProcessIterableHelperNormalTest(unittest.TestCase):
-
-    def test_normal_in_predicate(self):
-        iterable1 = iter([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [1, 3])
-
-    def test_normal_out_predicate(self):
-        iterable1 = iter([1, 2, 10, 3, 15])
-        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [1, 2, 3])
-
-    def test_normal_processor(self):
-        iterable1 = iter([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [2, 4, 6])
-
-    def test_normal_combined(self):
-        iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
-                                            out_predicate=predicate2, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.Iterable)
-        list2 = list(iterable2)
-        self.assertEqual(list2, [2, 6])
-
-
-class ProcessIterableHelperAsyncTest(unittest.TestCase):
-
-    @deferred_f_from_coro_f
-    async def test_async_in_predicate(self):
-        iterable1 = as_async_generator([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [1, 3])
-
-    @deferred_f_from_coro_f
-    async def test_async_out_predicate(self):
-        iterable1 = as_async_generator([1, 2, 10, 3, 15])
-        iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [1, 2, 3])
-
-    @deferred_f_from_coro_f
-    async def test_async_processor(self):
-        iterable1 = as_async_generator([1, 2, 3])
-        iterable2 = process_iterable_helper(iterable1, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_async_combined(self):
-        iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])
-        iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
-                                            out_predicate=predicate2, processor=processor)
-        self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
-        list2 = await collect_asyncgen(iterable2)
-        self.assertEqual(list2, [2, 6])

From a6034f942b034946538855cc53644b5a89ba081a Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 22 Mar 2021 22:53:52 +0500
Subject: [PATCH 15/41] Add tests for _AsyncCooperatorAdapter.

---
 tests/test_utils_defer.py | 68 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 67 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py
index 62f6ff194..543bbee09 100644
--- a/tests/test_utils_defer.py
+++ b/tests/test_utils_defer.py
@@ -1,14 +1,18 @@
+import random
+
 from pytest import mark
 from twisted.trial import unittest
 from twisted.internet import reactor, defer
 from twisted.python.failure import Failure
 
-from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.asyncgen import collect_asyncgen, as_async_generator
 from scrapy.utils.defer import (
     aiter_errback,
     deferred_f_from_coro_f,
     iter_errback,
+    maybe_deferred_to_future,
     mustbe_deferred,
+    parallel_async,
     process_chain,
     process_chain_both,
     process_parallel,
@@ -164,3 +168,65 @@ class AsyncDefTestsuiteTest(unittest.TestCase):
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f_xfail(self):
         raise Exception("This is expected to be raised")
+
+
+class AsyncCooperatorTest(unittest.TestCase):
+    """ This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage.
+
+    parallel_async is called with the results of a callback (so an iterable of items, requests and None,
+    with arbitrary delays between values), and it uses Scraper._process_spidermw_output as the callable
+    (so a callable that returns a Deferred for an item, which will fire after pipelines process it, and
+    None for everything else). The concurrent task count is the CONCURRENT_ITEMS setting.
+
+    We want to test different concurrency values compared to the iterable length.
+    We also want to simulate the real usage, with arbitrary delays between getting the values
+    from the iterable. We also want to simulate sync and async results from the callable.
+    """
+    CONCURRENT_ITEMS = 50
+
+    @staticmethod
+    def callable(o, results):
+        if random.random() < 0.4:
+            # simulate async processing
+            dfd = defer.Deferred()
+            dfd.addCallback(lambda _: results.append(o))
+            delay = random.random() / 8
+            reactor.callLater(delay, dfd.callback, None)
+            return dfd
+        else:
+            # simulate trivial sync processing
+            results.append(o)
+
+    @staticmethod
+    def get_async_iterable(length):
+        # simulate a simple callback without delays between results
+        return as_async_generator(range(length))
+
+    @staticmethod
+    async def get_async_iterable_with_delays(length):
+        # simulate a callback with delays between some of the results
+        for i in range(length):
+            if random.random() < 0.1:
+                dfd = defer.Deferred()
+                delay = random.random() / 20
+                reactor.callLater(delay, dfd.callback, None)
+                await maybe_deferred_to_future(dfd)
+            yield i
+
+    @defer.inlineCallbacks
+    def test_simple(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))
+
+    @defer.inlineCallbacks
+    def test_delays(self):
+        for length in [20, 50, 100]:
+            results = []
+            ait = self.get_async_iterable_with_delays(length)
+            dl = parallel_async(ait, self.CONCURRENT_ITEMS, self.callable, results)
+            yield dl
+            self.assertEqual(list(range(length)), sorted(results))

From 0596f2bf6e7fc404333345d1cece01e31fabed8e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 23 Mar 2021 22:47:48 +0500
Subject: [PATCH 16/41] Remove not needed deferred_from_coro call.

---
 scrapy/core/scraper.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 3eae71af7..7eedbf33e 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -16,7 +16,6 @@ from scrapy.utils.defer import (
     aiter_errback,
     defer_fail,
     defer_succeed,
-    deferred_from_coro,
     iter_errback,
     parallel,
     parallel_async,
@@ -191,8 +190,8 @@ class Scraper:
             return defer_succeed(None)
         if isinstance(result, collections.abc.AsyncIterable):
             it = aiter_errback(result, self.handle_spider_error, request, response, spider)
-            dfd = deferred_from_coro(parallel_async(it, self.concurrent_items, self._process_spidermw_output,
-                                     request, response, spider))
+            dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
+                                 request, response, spider)
         else:
             it = iter_errback(result, self.handle_spider_error, request, response, spider)
             dfd = parallel(it, self.concurrent_items, self._process_spidermw_output,

From a97dc55c714d90378dbc8a819cae1a3a40056d6e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 17:37:32 +0500
Subject: [PATCH 17/41] Add/improve docs.

---
 docs/topics/asyncio.rst           |  1 -
 docs/topics/coroutines.rst        | 13 ++++++++++++-
 docs/topics/spider-middleware.rst | 17 +++++++++++------
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 4addaa178..18712c928 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -38,7 +38,6 @@ You can also use custom asyncio event loops with the asyncio reactor. Set the
 :setting:`ASYNCIO_EVENT_LOOP` setting to the import path of the desired event loop class to
 use it instead of the default asyncio event loop.
 
-
 .. _asyncio-await-dfd:
 
 Awaiting on Deferreds
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 279632653..9b50d9312 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,6 +17,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.http.Request` callbacks.
 
+    .. versionchanged:: VERSION
+       Output of async callbacks is now processed asynchronously instead of collecting
+       all of it first.
+
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
 
@@ -30,6 +34,13 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
+-   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <custom-spider-middleware>`.
+
+    .. versionadded:: VERSION
+    .. note:: This method needs to be an async generator, not just a coroutine that
+              returns an iterable.
+
 Usage
 =====
 
@@ -76,7 +87,7 @@ This means you can use many useful Python libraries providing such code::
         async def parse_with_asyncio(self, response):
             async with aiohttp.ClientSession() as session:
                 async with session.get('https://additional.url') as additional_response:
-                    additional_data = await r.text()
+                    additional_data = await additional_response.text()
             # ... use response and additional_data to yield items and requests
 
 .. note:: Many libraries that use coroutines, such as `aio-libs`_, require the
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index fc114a63f..d09693c16 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -98,20 +98,25 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
     .. method:: process_spider_output(response, result, spider)
 
+        .. versionchanged:: VERSION
+           Since VERSION this can take and return an :term:`python:asynchronous
+           iterable`.
+
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
-        :meth:`process_spider_output` must return an iterable of
-        :class:`~scrapy.http.Request` objects and :ref:`item object
-        <topics-items>`.
+        :meth:`process_spider_output` must return an iterable (normal or
+        asynchronous) of :class:`~scrapy.http.Request` objects and 
+        :ref:`item objects <topics-items>`.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable of :class:`~scrapy.http.Request` objects and
-          :ref:`item object <topics-items>`
+        :type result: an iterable (normal or asynchronous) of
+          :class:`~scrapy.http.Request` objects and :ref:`item objects
+          <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
@@ -122,7 +127,7 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         method (from a previous spider middleware) raises an exception.
 
         :meth:`process_spider_exception` should return either ``None`` or an
-        iterable of :class:`~scrapy.http.Request` objects and :ref:`item object
+        iterable of :class:`~scrapy.http.Request` objects and :ref:`item objects
         <topics-items>`.
 
         If it returns ``None``, Scrapy will continue processing this exception,

From f422861ef49e8d0e0c2aa4de937fb77b95e063ca Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 20:47:40 +0500
Subject: [PATCH 18/41] Add more tests for spider middlewares.

---
 tests/test_spidermiddleware.py | 188 +++++++++++++++++++++++++++++++++
 1 file changed, 188 insertions(+)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 78e926adc..2584dec21 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,11 +1,15 @@
+import collections.abc
 from unittest import mock
 
+from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
 
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
+from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
@@ -101,3 +105,187 @@ class ProcessSpiderExceptionReRaise(SpiderMiddlewareTestCase):
         result = self._scrape_response()
         self.assertIsInstance(result, Failure)
         self.assertIsInstance(result.value, ZeroDivisionError)
+
+
+class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
+    """ Helpers for testing sync, async and mixed middlewares.
+
+    Should work for process_spider_output and, when it's supported, process_start_requests.
+    """
+
+    RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes):
+        for mw_cls in mw_classes:
+            self.mwman._add_middleware(mw_cls())
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    def assertAsyncGeneratorNotIterable(self, o):
+        with self.assertRaisesRegex(TypeError,
+                                    "'(async_generator|MutableAsyncChain)' object is not iterable"):
+            list(o)
+
+    @defer.inlineCallbacks
+    def _test_simple_base(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.Iterable)
+        result_list = list(result)
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_base(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.AsyncIterator)
+        result_list = yield deferred_from_coro(collect_asyncgen(result))
+        self.assertEqual(len(result_list), self.RESULT_COUNT)
+        self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_fail(self, *mw_classes):
+        result = yield self._get_middleware_result(*mw_classes)
+        self.assertIsInstance(result, collections.abc.Iterable)
+        self.assertAsyncGeneratorNotIterable(result)
+
+
+class ProcessSpiderOutputSimpleMiddleware:
+    def process_spider_output(self, response, result, spider):
+        for r in result:
+            yield r
+
+
+class ProcessSpiderOutputAsyncGenMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        async for r in as_async_generator(result):
+            yield r
+
+
+class ProcessSpiderOutputUniversalMiddleware:
+    def process_spider_output(self, response, result, spider):
+        @_process_iterable_universal
+        async def process(result):
+            async for r in result:
+                yield r
+        return process(result)
+
+
+class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_spider_output tests for simple callbacks"""
+
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+    def test_simple(self):
+        """ Simple mw """
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_asyncgen(self):
+        """ Asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE)
+
+    def test_asyncgen_simple(self):
+        """ Asyncgen mw -> simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE,
+                                        self.MW_ASYNCGEN)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE)
+
+    def test_universal_asyncgen(self):
+        """ Universal mw -> asyncgen mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_UNIVERSAL)
+
+    def test_asyncgen_universal(self):
+        """ Asyncgen mw -> universal mw """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL,
+                                        self.MW_ASYNCGEN)
+
+
+class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
+    """ process_spider_output tests for async generator callbacks """
+
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_simple(self):
+        """ Simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE)
+
+    @defer.inlineCallbacks
+    def test_simple_asyncgen(self):
+        """ Simple mw -> asyncgen mw; cannot work """
+        result = yield self._get_middleware_result(
+            self.MW_ASYNCGEN,
+            self.MW_SIMPLE)
+        self.assertIsInstance(result, collections.abc.AsyncIterable)
+        self.assertAsyncGeneratorNotIterable(result)
+
+    def test_universal(self):
+        """ Universal mw """
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+    def test_universal_simple(self):
+        """ Universal mw -> simple mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_SIMPLE,
+                                        self.MW_UNIVERSAL)
+
+    def test_simple_universal(self):
+        """ Simple mw -> universal mw; cannot work """
+        return self._test_asyncgen_fail(self.MW_UNIVERSAL,
+                                        self.MW_SIMPLE)
+
+
+class ProcessStartRequestsSimpleMiddleware:
+    def process_start_requests(self, start_requests, spider):
+        for r in start_requests:
+            yield r
+
+
+class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
+    """ process_start_requests tests for simple start_requests"""
+
+    ITEM_TYPE = Request
+    MW_SIMPLE = ProcessStartRequestsSimpleMiddleware
+
+    def _start_requests(self):
+        for i in range(3):
+            yield Request(f'https://example.com/{i}', dont_filter=True)
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes):
+        for mw_cls in mw_classes:
+            self.mwman._add_middleware(mw_cls())
+        start_requests = iter(self._start_requests())
+        results = yield self.mwman.process_start_requests(start_requests, self.spider)
+        return results
+
+    def test_simple(self):
+        """ Simple mw """
+        self._test_simple_base(self.MW_SIMPLE)

From 0638d6f01c41f12311b21f06fee5c71f5d08569f Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 25 Mar 2021 21:58:29 +0500
Subject: [PATCH 19/41] Fix handling middlewares that change sync iterables
 into async.

---
 scrapy/core/spidermw.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 33e215971..230332673 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,7 +3,7 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-import inspect
+import collections.abc
 from itertools import islice
 
 from twisted.python.failure import Failure
@@ -96,11 +96,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        if inspect.isasyncgen(result):
-            iter_class = MutableAsyncChain
+        if isinstance(result, collections.abc.AsyncIterator):
+            recovered = MutableAsyncChain()
         else:
-            iter_class = MutableChain
-        recovered = iter_class()
+            recovered = MutableChain()
 
         method_list = islice(self.methods['process_spider_output'], start_index, None)
         for method_index, method in enumerate(method_list, start=start_index):
@@ -121,16 +120,23 @@ class SpiderMiddlewareManager(MiddlewareManager):
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
 
-        return iter_class(result, recovered)
+        # check this again as the middlewares could change "result" from sync to async
+        if isinstance(result, collections.abc.AsyncIterator):
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)
 
     def _process_callback_output(self, response, spider, result):
-        if inspect.isasyncgen(result):
-            iter_class = MutableAsyncChain
+        if isinstance(result, collections.abc.AsyncIterator):
+            recovered = MutableAsyncChain()
         else:
-            iter_class = MutableChain
-        recovered = iter_class()
+            recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        return iter_class(self._process_spider_output(response, spider, result), recovered)
+        result = self._process_spider_output(response, spider, result)
+        if isinstance(result, collections.abc.AsyncIterator):
+            return MutableAsyncChain(result, recovered)
+        else:
+            return MutableChain(result, recovered)
 
     def scrape_response(self, scrape_func, response, request, spider):
         def process_callback_output(result):

From b5f501df7bc3917af3d144bb6556a763f381cd7e Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Mar 2021 20:17:41 +0500
Subject: [PATCH 20/41] Remove some unneeded code from _AsyncCooperatorAdapter.

---
 scrapy/utils/defer.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index 554edc38c..ca3d79fa6 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -136,8 +136,6 @@ class _AsyncCooperatorAdapter:
         self.anext_deferred = None
         result = self.callable(result, *self.callable_args, **self.callable_kwargs)
         d = self.waiting_deferreds.pop(0)
-        if d.called:
-            raise ValueError('Deferred in waiting_deferreds already called')
         if isinstance(result, defer.Deferred):
             result.chainDeferred(d)
         else:
@@ -152,8 +150,6 @@ class _AsyncCooperatorAdapter:
         failure.trap(StopAsyncIteration)
         self.finished = True
         for d in self.waiting_deferreds:
-            if d.called:
-                raise ValueError('Deferred in waiting_deferreds already called')
             d.callback(None)
 
     def _call_anext(self):
@@ -162,9 +158,6 @@ class _AsyncCooperatorAdapter:
         self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
-    def __iter__(self):
-        return self
-
     def __next__(self):
         # This puts a new Deferred into self.waiting_deferreds and returns it.
         # It also calls __anext__() if needed.

From 6803779eb7e408b275da62f670aba9deaf3eade9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 26 Mar 2021 22:29:07 +0500
Subject: [PATCH 21/41] Add more tests for _process_iterable_universal.

---
 scrapy/utils/asyncgen.py     |  2 +-
 tests/test_utils_asyncgen.py | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 39c94ad8a..a79552f76 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -57,6 +57,6 @@ def _process_iterable_universal(process_async):
         if hasattr(iterable, '__iter__'):
             # convert process_async to process_sync
             return process_sync(iterable, *args, **kwargs)
-        raise ValueError(f"Wrong iterable type {type(iterable)}")
+        raise TypeError(f"Wrong iterable type {type(iterable)}")
 
     return process
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 2f4181d3d..41993a934 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -2,6 +2,7 @@ from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
+from scrapy.utils.test import get_web_client_agent_req
 
 
 class AsyncgenUtilsTest(unittest.TestCase):
@@ -26,6 +27,13 @@ async def process_iterable(iterable):
         yield i * 2
 
 
+@_process_iterable_universal
+async def process_iterable_awaiting(iterable):
+    async for i in iterable:
+        yield i * 2
+        await get_web_client_agent_req('http://example.com')
+
+
 class ProcessIterableUniversalTest(unittest.TestCase):
 
     def test_normal(self):
@@ -38,3 +46,22 @@ class ProcessIterableUniversalTest(unittest.TestCase):
         iterable = as_async_generator([1, 2, 3])
         results = await collect_asyncgen(process_iterable(iterable))
         self.assertEqual(results, [2, 4, 6])
+
+    @deferred_f_from_coro_f
+    async def test_blocking(self):
+        iterable = [1, 2, 3]
+        with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"):
+            list(process_iterable_awaiting(iterable))
+
+    def test_invalid_iterable(self):
+        with self.assertRaisesRegex(TypeError, "Wrong iterable type"):
+            process_iterable(None)
+
+    @deferred_f_from_coro_f
+    async def test_invalid_process(self):
+        @_process_iterable_universal
+        def process_iterable_invalid(iterable):
+            pass
+
+        with self.assertRaisesRegex(ValueError, "process_async returned wrong type"):
+            list(process_iterable_invalid([]))

From 849472535ef9490e8cc2a62cc7f1835b2bc3eaba Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 20:20:35 +0500
Subject: [PATCH 22/41] Update docs.

---
 docs/topics/coroutines.rst        | 65 +++++++++++++++++++++++++++++++
 docs/topics/spider-middleware.rst | 14 ++++---
 scrapy/utils/asyncgen.py          |  1 +
 3 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 9b50d9312..6a39dcb5e 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -108,3 +108,68 @@ Common use cases for asynchronous code include:
   :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
+
+.. _async-spider-middlewares:
+
+Asynchronous spider middlewares
+===============================
+
+.. versionadded:: VERSION
+.. note:: This currently applies to
+          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
+
+Middleware methods discussed here can take and return async iterables. They can
+return the same type of iterable or they can take a normal one and return an
+async one. If such method needs to return an async iterable it must be an async
+generator, not just a coroutine that returns an iterable.
+
+.. autofunction:: scrapy.utils.asyncgen.as_async_generator
+
+In the simplest form that supports both sync and async input it can be written
+like this::
+
+    from scrapy.utils.asyncgen import as_async_generator
+
+    class ProcessSpiderOutputAsyncGenMiddleware:
+        async def process_spider_output(self, response, result, spider):
+            async for r in as_async_generator(result):
+                # ... do something with r
+                yield r
+
+If the middleware input (the callback result for ``process_spider_output``) is
+an async iterable, all middlewares that process it must support it. The
+built-in ones do, but the ones in your project and 3rd-party ones will need to
+be updated to support it, as the code that expects a normal iterable will break
+on an async one. If these middlewares receive an async iterable, they must
+return one as well. On the other hand, if they receive a normal iterable, they
+shouldn't break and ideally should return a normal iterable too. There can be
+several possible implementations of this.
+
+The simplest one, always converting normal iterables to async ones, is provided
+above. Because a result of a middleware method is passed to the same method of
+the next middleware, it's only possible to mix middlewares with synchronous and
+asynchronous implementations of the same method if all synchronous ones are
+called first (which isn't always possible).
+
+Another option is to make separate methods for normal and async iterables and
+choose one at run time::
+
+    from inspect import isasyncgen
+
+    class ProcessSpiderOutputAsyncGenMiddleware:
+        def _normal_process_spider_output(self, response, result, spider):
+            # ... do something with normal result
+
+        async def _async_process_spider_output(self, response, result, spider):
+            # ... do the same with async result
+
+        def process_spider_output(self, response, result, spider):
+            if isasyncgen(result):
+                return self._async_process_spider_output(self, response, result, spider)
+            else:
+                return self._normal_process_spider_output(self, response, result, spider)
+
+If you are writing a middleware that you intend to publish or to use in many
+projects, this is likely the best way to implement it. It may be possible to
+extract common code from both methods to reduce code duplication, as in the
+simplest case the only difference between them will be ``for`` vs ``async for``.
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index d09693c16..d87b89292 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -105,18 +105,20 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
-        :meth:`process_spider_output` must return an iterable (normal or
-        asynchronous) of :class:`~scrapy.http.Request` objects and 
-        :ref:`item objects <topics-items>`.
+        :meth:`process_spider_output` must return an iterable of
+        :class:`~scrapy.http.Request` objects and :ref:`item objects
+        <topics-items>`.
+
+        .. note:: When defined as a :ref:`coroutine <async>`, this method needs
+                  to be an async generator, not just return an iterable.
 
         :param response: the response which generated this output from the
           spider
         :type response: :class:`~scrapy.http.Response` object
 
         :param result: the result returned by the spider
-        :type result: an iterable (normal or asynchronous) of
-          :class:`~scrapy.http.Request` objects and :ref:`item objects
-          <topics-items>`
+        :type result: an iterable of :class:`~scrapy.http.Request` objects and
+          :ref:`item objects <topics-items>`
 
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.spiders.Spider` object
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index a79552f76..6c0bb1d10 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -11,6 +11,7 @@ async def collect_asyncgen(result):
 
 
 async def as_async_generator(it):
+    """ Wraps an iterator (sync or async) into an async generator. """
     if isinstance(it, collections.abc.AsyncIterator):
         async for r in it:
             yield r

From 30ed7fa349214ad11b4d13c981cbd2fc63ddaf42 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 22:20:56 +0500
Subject: [PATCH 23/41] Some cleanup, make sync middlewares fail earlier.

---
 scrapy/core/spidermw.py        | 15 +++++++--------
 tests/test_spidermiddleware.py | 18 ++++--------------
 2 files changed, 11 insertions(+), 22 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 230332673..b24ccf6ae 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -20,10 +20,6 @@ def _isiterable(possible_iterator):
     return hasattr(possible_iterator, '__iter__') or hasattr(possible_iterator, '__aiter__')
 
 
-def _fname(f):
-    return f"{f.__self__.__class__.__name__}.{f.__func__.__name__}"
-
-
 class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
@@ -48,7 +44,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {_fname(method)} must return None "
+                    msg = (f"Middleware {method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -88,7 +84,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {_fname(method)} must return None "
+                msg = (f"Middleware {method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
@@ -96,7 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        if isinstance(result, collections.abc.AsyncIterator):
+        result_async = isinstance(result, collections.abc.AsyncIterator)
+        if result_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -116,9 +113,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {_fname(method)} must return an "
+                msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
+            if result_async and isinstance(result, collections.abc.Iterator):
+                raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
 
         # check this again as the middlewares could change "result" from sync to async
         if isinstance(result, collections.abc.AsyncIterator):
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 2584dec21..0a6b96c0c 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -122,11 +122,6 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
         result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
         return result
 
-    def assertAsyncGeneratorNotIterable(self, o):
-        with self.assertRaisesRegex(TypeError,
-                                    "'(async_generator|MutableAsyncChain)' object is not iterable"):
-            list(o)
-
     @defer.inlineCallbacks
     def _test_simple_base(self, *mw_classes):
         result = yield self._get_middleware_result(*mw_classes)
@@ -145,9 +140,8 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
     @defer.inlineCallbacks
     def _test_asyncgen_fail(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
-        self.assertIsInstance(result, collections.abc.Iterable)
-        self.assertAsyncGeneratorNotIterable(result)
+        with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
+            yield self._get_middleware_result(*mw_classes)
 
 
 class ProcessSpiderOutputSimpleMiddleware:
@@ -238,14 +232,10 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
         """ Simple mw; cannot work """
         return self._test_asyncgen_fail(self.MW_SIMPLE)
 
-    @defer.inlineCallbacks
     def test_simple_asyncgen(self):
         """ Simple mw -> asyncgen mw; cannot work """
-        result = yield self._get_middleware_result(
-            self.MW_ASYNCGEN,
-            self.MW_SIMPLE)
-        self.assertIsInstance(result, collections.abc.AsyncIterable)
-        self.assertAsyncGeneratorNotIterable(result)
+        return self._test_asyncgen_fail(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE)
 
     def test_universal(self):
         """ Universal mw """

From 7bd1d888d49d238622007e659e54af76e82bf1c1 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 2 Apr 2021 23:06:29 +0500
Subject: [PATCH 24/41] More robust sync/async middleware mix checking.

---
 scrapy/core/spidermw.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index b24ccf6ae..d0d292007 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -92,8 +92,8 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _process_spider_output(self, response, spider, result, start_index=0):
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        result_async = isinstance(result, collections.abc.AsyncIterator)
-        if result_async:
+        last_result_async = isinstance(result, collections.abc.AsyncIterator)
+        if last_result_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -116,11 +116,11 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if result_async and isinstance(result, collections.abc.Iterator):
+            if last_result_async and isinstance(result, collections.abc.Iterator):
                 raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
+            last_result_async = isinstance(result, collections.abc.AsyncIterator)
 
-        # check this again as the middlewares could change "result" from sync to async
-        if isinstance(result, collections.abc.AsyncIterator):
+        if last_result_async:
             return MutableAsyncChain(result, recovered)
         else:
             return MutableChain(result, recovered)

From 61197d3dba53cd67c41e5d23e1f0c11a864f539b Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 18:21:58 +0500
Subject: [PATCH 25/41] Add/update typing, cleanup iterator/iterable
 inconsistencies.

---
 scrapy/core/scraper.py   |  7 +++---
 scrapy/core/spidermw.py  | 36 +++++++++++++++-------------
 scrapy/utils/asyncgen.py | 19 +++++++--------
 scrapy/utils/defer.py    | 52 ++++++++++++++++++++++++----------------
 scrapy/utils/python.py   | 14 +++++------
 5 files changed, 71 insertions(+), 57 deletions(-)

diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 8ce57af2f..0630ce625 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -1,10 +1,8 @@
 """This module implements the Scraper component which parses responses and
 extracts information from them"""
-import collections
 import logging
 from collections import deque
-from collections.abc import Iterable
-from typing import Union
+from typing import AsyncGenerator, AsyncIterable, Generator, Iterable, Union
 
 from itemadapter import is_item
 from twisted.internet import defer
@@ -188,7 +186,8 @@ class Scraper:
     def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider):
         if not result:
             return defer_succeed(None)
-        if isinstance(result, collections.abc.AsyncIterable):
+        it: Union[Generator, AsyncGenerator]
+        if isinstance(result, AsyncIterable):
             it = aiter_errback(result, self.handle_spider_error, request, response, spider)
             dfd = parallel_async(it, self.concurrent_items, self._process_spidermw_output,
                                  request, response, spider)
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 6fcbc7492..9dd6c462d 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,9 +3,8 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
-import collections.abc
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable
+from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure
@@ -61,8 +60,9 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 return scrape_func(Failure(), request, spider)
         return scrape_func(response, request, spider)
 
-    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Iterable,
-                           exception_processor_index: int, recover_to: MutableChain) -> Generator:
+    def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
+                           exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
+                           ) -> Union[Generator, AsyncGenerator]:
         @_process_iterable_universal
         async def _evaluate_async_iterable(iterable):
             try:
@@ -100,11 +100,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
         return _failure
 
     def _process_spider_output(self, response: Response, spider: Spider,
-                               result: Iterable, start_index: int = 0) -> MutableChain:
+                               result: Union[Iterable, AsyncIterable], start_index: int = 0
+                               ) -> Union[MutableChain, MutableAsyncChain]:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
-        last_result_async = isinstance(result, collections.abc.AsyncIterator)
-        if last_result_async:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        last_result_is_async = isinstance(result, AsyncIterable)
+        if last_result_is_async:
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
@@ -127,30 +129,32 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if last_result_async and isinstance(result, collections.abc.Iterator):
+            if last_result_is_async and isinstance(result, Iterable):
                 raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
-            last_result_async = isinstance(result, collections.abc.AsyncIterator)
+            last_result_is_async = isinstance(result, AsyncIterable)
 
-        if last_result_async:
+        if last_result_is_async:
             return MutableAsyncChain(result, recovered)
         else:
-            return MutableChain(result, recovered)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Iterable) -> MutableChain:
-        if isinstance(result, collections.abc.AsyncIterator):
+    def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                 ) -> Union[MutableChain, MutableAsyncChain]:
+        recovered: Union[MutableChain, MutableAsyncChain]
+        if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
         result = self._process_spider_output(response, spider, result)
-        if isinstance(result, collections.abc.AsyncIterator):
+        if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
         else:
-            return MutableChain(result, recovered)
+            return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                         spider: Spider) -> Deferred:
-        def process_callback_output(result: Iterable) -> MutableChain:
+        def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]:
             return self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index 92118ddb9..ae9a79989 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,7 +1,6 @@
-import collections
 import functools
 import inspect
-from collections.abc import AsyncIterable
+from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
 
 
 async def collect_asyncgen(result: AsyncIterable):
@@ -11,9 +10,9 @@ async def collect_asyncgen(result: AsyncIterable):
     return results
 
 
-async def as_async_generator(it):
-    """ Wraps an iterator (sync or async) into an async generator. """
-    if isinstance(it, collections.abc.AsyncIterator):
+async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
+    """ Wraps an iterable (sync or async) into an async generator. """
+    if isinstance(it, AsyncIterable):
         async for r in it:
             yield r
     else:
@@ -22,7 +21,7 @@ async def as_async_generator(it):
 
 
 # https://stackoverflow.com/a/66170760/113586
-def _process_iterable_universal(process_async):
+def _process_iterable_universal(process_async: Callable):
     """ Takes a function that takes an async iterable, args and kwargs. Returns
     a function that takes any iterable, args and kwargs.
 
@@ -33,7 +32,7 @@ def _process_iterable_universal(process_async):
     # If this stops working, all internal uses can be just replaced with manually-written
     # process_sync functions.
 
-    def process_sync(iterable, *args, **kwargs):
+    def process_sync(iterable: Iterable, *args, **kwargs) -> Generator:
         agen = process_async(as_async_generator(iterable), *args, **kwargs)
         if not inspect.isasyncgen(agen):
             raise ValueError(f"process_async returned wrong type {type(agen)}")
@@ -52,11 +51,11 @@ def _process_iterable_universal(process_async):
                           f"you can't use {_process_iterable_universal.__name__} with it.")
 
     @functools.wraps(process_async)
-    def process(iterable, *args, **kwargs):
-        if inspect.isasyncgen(iterable):
+    def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]:
+        if isinstance(iterable, AsyncIterable):
             # call process_async directly
             return process_async(iterable, *args, **kwargs)
-        if hasattr(iterable, '__iter__'):
+        if isinstance(iterable, Iterable):
             # convert process_async to process_sync
             return process_sync(iterable, *args, **kwargs)
         raise TypeError(f"Wrong iterable type {type(iterable)}")
diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py
index f81faf6dd..39c8a85e9 100644
--- a/scrapy/utils/defer.py
+++ b/scrapy/utils/defer.py
@@ -3,9 +3,21 @@ Helper functions for dealing with Twisted deferreds
 """
 import asyncio
 import inspect
-from collections.abc import Coroutine
+from asyncio import Future
 from functools import wraps
-from typing import Any, Callable, Generator, Iterable
+from typing import (
+    Any,
+    AsyncGenerator,
+    AsyncIterable,
+    Callable,
+    Coroutine,
+    Generator,
+    Iterable,
+    Iterator,
+    List,
+    Optional,
+    Union
+)
 
 from twisted.internet import defer
 from twisted.internet.defer import Deferred, DeferredList, ensureDeferred
@@ -80,8 +92,8 @@ def parallel(iterable: Iterable, count: int, callable: Callable, *args, **named)
     return DeferredList([coop.coiterate(work) for _ in range(count)])
 
 
-class _AsyncCooperatorAdapter:
-    """ A class that wraps an async iterator into a normal iterator suitable
+class _AsyncCooperatorAdapter(Iterator):
+    """ A class that wraps an async iterable into a normal iterator suitable
     for using in Cooperator.coiterate(). As it's only needed for parallel_async(),
     it calls the callable directly in the callback, instead of providing a more
     generic interface.
@@ -125,30 +137,30 @@ class _AsyncCooperatorAdapter:
     Cooperator/CooperativeTask and use it instead of this adapter to achieve the same
     goal.
     """
-    def __init__(self, aiterator, callable, *callable_args, **callable_kwargs):
-        self.aiterator = aiterator
+    def __init__(self, aiterable: AsyncIterable, callable: Callable, *callable_args, **callable_kwargs):
+        self.aiterator = aiterable.__aiter__()
         self.callable = callable
         self.callable_args = callable_args
         self.callable_kwargs = callable_kwargs
         self.finished = False
-        self.waiting_deferreds = []
-        self.anext_deferred = None
+        self.waiting_deferreds: List[Deferred] = []
+        self.anext_deferred: Optional[Deferred] = None
 
-    def _callback(self, result):
+    def _callback(self, result: Any) -> None:
         # This gets called when the result from aiterator.__anext__() is available.
         # It calls the callable on it and sends the result to the oldest waiting Deferred
         # (by chaining if the result is a Deferred too or by firing if not).
         self.anext_deferred = None
         result = self.callable(result, *self.callable_args, **self.callable_kwargs)
         d = self.waiting_deferreds.pop(0)
-        if isinstance(result, defer.Deferred):
+        if isinstance(result, Deferred):
             result.chainDeferred(d)
         else:
             d.callback(None)
         if self.waiting_deferreds:
             self._call_anext()
 
-    def _errback(self, failure):
+    def _errback(self, failure: Failure) -> None:
         # This gets called on any exceptions in aiterator.__anext__().
         # It handles StopAsyncIteration by stopping the iteration and reraises all others.
         self.anext_deferred = None
@@ -157,29 +169,29 @@ class _AsyncCooperatorAdapter:
         for d in self.waiting_deferreds:
             d.callback(None)
 
-    def _call_anext(self):
+    def _call_anext(self) -> None:
         # This starts waiting for the next result from aiterator.
         # If aiterator is exhausted, _errback will be called.
         self.anext_deferred = deferred_from_coro(self.aiterator.__anext__())
         self.anext_deferred.addCallbacks(self._callback, self._errback)
 
-    def __next__(self):
+    def __next__(self) -> Deferred:
         # This puts a new Deferred into self.waiting_deferreds and returns it.
         # It also calls __anext__() if needed.
         if self.finished:
             raise StopIteration
-        d = defer.Deferred()
+        d = Deferred()
         self.waiting_deferreds.append(d)
         if not self.anext_deferred:
             self._call_anext()
         return d
 
 
-def parallel_async(async_iterable, count, callable, *args, **named):
-    """ Like parallel but for async iterables """
+def parallel_async(async_iterable: AsyncIterable, count: int, callable: Callable, *args, **named) -> DeferredList:
+    """ Like parallel but for async iterators """
     coop = Cooperator()
     work = _AsyncCooperatorAdapter(async_iterable, callable, *args, **named)
-    dl = defer.DeferredList([coop.coiterate(work) for _ in range(count)])
+    dl = DeferredList([coop.coiterate(work) for _ in range(count)])
     return dl
 
 
@@ -232,7 +244,7 @@ def iter_errback(iterable: Iterable, errback: Callable, *a, **kw) -> Generator:
             errback(failure.Failure(), *a, **kw)
 
 
-async def aiter_errback(aiterable, errback, *a, **kw):
+async def aiter_errback(aiterable: AsyncIterable, errback: Callable, *a, **kw) -> AsyncGenerator:
     """Wraps an async iterable calling an errback if an error is caught while
     iterating it. Similar to scrapy.utils.defer.iter_errback()
     """
@@ -290,13 +302,13 @@ def maybeDeferred_coro(f: Callable, *args, **kw) -> Deferred:
         return defer.succeed(result)
 
 
-def deferred_to_future(d):
+def deferred_to_future(d: Deferred) -> Future:
     """ Wraps a Deferred into a Future. Requires the asyncio reactor.
     """
     return d.asFuture(asyncio.get_event_loop())
 
 
-def maybe_deferred_to_future(d):
+def maybe_deferred_to_future(d: Deferred) -> Union[Deferred, Future]:
     """ Converts a Deferred to something that can be awaited in a callback or other user coroutine.
 
     If the asyncio reactor is installed, coroutines are wrapped into Futures, and only Futures can be
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 8b823d174..d086347bc 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -8,9 +8,9 @@ import re
 import sys
 import warnings
 import weakref
-from collections.abc import Iterable
 from functools import partial, wraps
 from itertools import chain
+from typing import AsyncIterable, Iterable, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
@@ -345,7 +345,7 @@ class MutableChain(Iterable):
     def __init__(self, *args: Iterable):
         self.data = chain.from_iterable(args)
 
-    def extend(self, *iterables: Iterable):
+    def extend(self, *iterables: Iterable) -> None:
         self.data = chain(self.data, chain.from_iterable(iterables))
 
     def __iter__(self):
@@ -359,22 +359,22 @@ class MutableChain(Iterable):
         return self.__next__()
 
 
-async def _async_chain(*iterables):
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]):
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
 
 
-class MutableAsyncChain:
+class MutableAsyncChain(AsyncIterable):
     """
     Similar to MutableChain but for async iterables
     """
 
-    def __init__(self, *args):
+    def __init__(self, *args: Union[Iterable, AsyncIterable]):
         self.data = _async_chain(*args)
 
-    def extend(self, *aiterables):
-        self.data = _async_chain(self.data, _async_chain(*aiterables))
+    def extend(self, *iterables: Union[Iterable, AsyncIterable]) -> None:
+        self.data = _async_chain(self.data, _async_chain(*iterables))
 
     def __aiter__(self):
         return self

From 9db01a483c729369b272235bfb6e1c66ff62d1f7 Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 14 Apr 2021 19:02:38 +0500
Subject: [PATCH 26/41] Update scrapy/core/spidermw.py
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 scrapy/core/spidermw.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 9dd6c462d..d1fedae07 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 from itertools import islice
-from typing import Any, Callable, Generator, Iterable, Union, AsyncIterable, AsyncGenerator
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
 
 from twisted.internet.defer import Deferred
 from twisted.python.failure import Failure

From de69d967f90bc70bc07f734b93a4e7b73f2a4aa9 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 11 Jun 2021 16:12:25 +0500
Subject: [PATCH 27/41] Fix async spider examples.

---
 docs/topics/coroutines.rst | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 6a39dcb5e..67f1a4098 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -77,14 +77,16 @@ coroutines, functions that return Deferreds and functions that return
 :term:`awaitable objects <awaitable>` such as :class:`~asyncio.Future`.
 This means you can use many useful Python libraries providing such code::
 
-    class MySpider(Spider):
+    class MySpiderDeferred(Spider):
         # ...
-        async def parse_with_deferred(self, response):
+        async def parse(self, response):
             additional_response = await treq.get('https://additional.url')
             additional_data = await treq.content(additional_response)
             # ... use response and additional_data to yield items and requests
 
-        async def parse_with_asyncio(self, response):
+    class MySpiderAsyncio(Spider):
+        # ...
+        async def parse(self, response):
             async with aiohttp.ClientSession() as session:
                 async with session.get('https://additional.url') as additional_response:
                     additional_data = await additional_response.text()

From 7306a81188f81964ad85f4936ce29e3aa0084447 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Mon, 19 Jul 2021 20:09:11 +0500
Subject: [PATCH 28/41] Disable builtin middlewares in spider middleware tests.

---
 tests/test_spidermiddleware.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index 0a6b96c0c..b0ca2f62e 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -19,7 +19,7 @@ class SpiderMiddlewareTestCase(TestCase):
     def setUp(self):
         self.request = Request('http://example.com/index.html')
         self.response = Response(self.request.url, request=self.request)
-        self.crawler = get_crawler(Spider)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}})
         self.spider = self.crawler._create_spider('foo')
         self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
 

From e5b057cfd4472e970b9b51757a1b823b2f585b09 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 13 Oct 2021 19:06:51 +0500
Subject: [PATCH 29/41] Don't use a HTTP request in a case that will not be
 awaited.

---
 tests/test_utils_asyncgen.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 41993a934..d9e6bc2eb 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,3 +1,4 @@
+from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
@@ -31,7 +32,10 @@ async def process_iterable(iterable):
 async def process_iterable_awaiting(iterable):
     async for i in iterable:
         yield i * 2
-        await get_web_client_agent_req('http://example.com')
+        d = Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, 42)
+        await d
 
 
 class ProcessIterableUniversalTest(unittest.TestCase):

From a642b73e1a1ba355bae9d5b9d1e87a9da0696293 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Wed, 13 Oct 2021 19:21:49 +0500
Subject: [PATCH 30/41] Remove an unused import.

---
 tests/test_utils_asyncgen.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index d9e6bc2eb..7abe17c22 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -3,7 +3,6 @@ from twisted.trial import unittest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
 from scrapy.utils.defer import deferred_f_from_coro_f
-from scrapy.utils.test import get_web_client_agent_req
 
 
 class AsyncgenUtilsTest(unittest.TestCase):

From f789547551ae0eb79f41c9de44525bf597a0ffa5 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Jan 2022 18:28:32 +0500
Subject: [PATCH 31/41] Implement spider middleware iterable upgrade/downgrade.

---
 docs/topics/coroutines.rst                  | 104 ++++---
 docs/topics/spider-middleware.rst           |   9 +
 scrapy/core/scraper.py                      |   3 +-
 scrapy/core/spidermw.py                     | 135 +++++++--
 scrapy/middleware.py                        |   7 +-
 scrapy/spidermiddlewares/depth.py           |  71 ++---
 scrapy/spidermiddlewares/offsite.py         |  40 +--
 scrapy/spidermiddlewares/referer.py         |  22 +-
 scrapy/spidermiddlewares/urllength.py       |  33 +--
 scrapy/utils/asyncgen.py                    |  47 +--
 scrapy/utils/python.py                      |   4 +-
 tests/test_spidermiddleware.py              | 302 +++++++++++++++++---
 tests/test_spidermiddleware_output_chain.py |  17 ++
 tests/test_utils_asyncgen.py                |  52 +---
 14 files changed, 553 insertions(+), 293 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index c514fc00a..073b6bd9a 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -35,11 +35,10 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
 -   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-    method of :ref:`spider middlewares <custom-spider-middleware>`.
+    method of :ref:`spider middlewares <custom-spider-middleware>`. See
+    :ref:`async-spider-middlewares`.
 
     .. versionadded:: VERSION
-    .. note:: This method needs to be an async generator, not just a coroutine that
-              returns an iterable.
 
 Usage
 =====
@@ -106,8 +105,8 @@ Common use cases for asynchronous code include:
 * storing data in databases (in pipelines and middlewares);
 * delaying the spider initialization until some external event (in the
   :signal:`spider_opened` handler);
-* calling asynchronous Scrapy methods like ``ExecutionEngine.download`` (see
-  :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
+* calling asynchronous Scrapy methods like :meth:`ExecutionEngine.download`
+  (see :ref:`the screenshot pipeline example<ScreenshotPipeline>`).
 
 .. _aio-libs: https://github.com/aio-libs
 
@@ -119,59 +118,72 @@ Asynchronous spider middlewares
 .. versionadded:: VERSION
 .. note:: This currently applies to
           :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
+          In the future it will also apply to
+          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`.
 
 Middleware methods discussed here can take and return async iterables. They can
 return the same type of iterable or they can take a normal one and return an
 async one. If such method needs to return an async iterable it must be an async
 generator, not just a coroutine that returns an iterable.
 
-.. autofunction:: scrapy.utils.asyncgen.as_async_generator
+As the result of a middleware method is passed to the same method of the next
+middleware, it needs to be adapted if the second method expects a different
+type. Scrapy will do this transparently:
 
-In the simplest form that supports both sync and async input it can be written
-like this::
+* A normal iterable is wrapped into an async one which shouldn't cause any side
+  effects.
+* An async iterable is downgraded to a normal one by waiting until all results
+  are available and wrapping them in a normal iterable. This is problematic
+  because it pauses the normal middleware processing for this iterable and
+  because all results can be skipped if exceptions are raised during
+  processing. This case emits a warning and will be deprecated and then removed
+  in a later Scrapy version.
+* Async iterables returned from
+  :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
+  won't be downgraded, an exception will be raised if that is needed.
 
-    from scrapy.utils.asyncgen import as_async_generator
+As downgrading is undesirable, here is the proposed way to avoid it. If all
+middlewares, including 3rd-party ones, support async iterables as input, no
+downgrading will happen. But removing normal iterable support (making the
+method a coroutine) from a middleware published as a separate project or used
+internally in projects for older Scrapy versions breaks backwards
+compatibility. So, as an interim measure (it will be deprecated and then
+removed in a later Scrapy version), a middleware can provide both sync and
+async methods in the following form::
 
-    class ProcessSpiderOutputAsyncGenMiddleware:
-        async def process_spider_output(self, response, result, spider):
-            async for r in as_async_generator(result):
+    class UniversalSpiderMiddleware:
+        def process_spider_output(self, response, result, spider):
+            for r in result:
                 # ... do something with r
                 yield r
 
-If the middleware input (the callback result for ``process_spider_output``) is
-an async iterable, all middlewares that process it must support it. The
-built-in ones do, but the ones in your project and 3rd-party ones will need to
-be updated to support it, as the code that expects a normal iterable will break
-on an async one. If these middlewares receive an async iterable, they must
-return one as well. On the other hand, if they receive a normal iterable, they
-shouldn't break and ideally should return a normal iterable too. There can be
-several possible implementations of this.
+        async def process_spider_output_async(self, response, result, spider):
+            async for r in result:
+                # ... do something with r
+                yield r
 
-The simplest one, always converting normal iterables to async ones, is provided
-above. Because a result of a middleware method is passed to the same method of
-the next middleware, it's only possible to mix middlewares with synchronous and
-asynchronous implementations of the same method if all synchronous ones are
-called first (which isn't always possible).
+In this case normal and async iterables will be passed to the respective
+methods without any wrapping or downgrading, and in older versions of Scrapy
+the coroutine method will just be ignored. When the backwards compatibility is
+no longer needed the non-coroutine method can be dropped and the coroutine one
+renamed to the normal name. It may be possible to extract common code from both
+methods to reduce code duplication, as in the simplest case the only difference
+between them will be ``for`` vs ``async for``.
 
-Another option is to make separate methods for normal and async iterables and
-choose one at run time::
+So, to recap:
 
-    from inspect import isasyncgen
-
-    class ProcessSpiderOutputAsyncGenMiddleware:
-        def _normal_process_spider_output(self, response, result, spider):
-            # ... do something with normal result
-
-        async def _async_process_spider_output(self, response, result, spider):
-            # ... do the same with async result
-
-        def process_spider_output(self, response, result, spider):
-            if isasyncgen(result):
-                return self._async_process_spider_output(self, response, result, spider)
-            else:
-                return self._normal_process_spider_output(self, response, result, spider)
-
-If you are writing a middleware that you intend to publish or to use in many
-projects, this is likely the best way to implement it. It may be possible to
-extract common code from both methods to reduce code duplication, as in the
-simplest case the only difference between them will be ``for`` vs ``async for``.
+* If you don't intend to use async callbacks or middlewares containing async
+  code in your project, nothing should change for you yet. At some point in the
+  future some of the 3rd-party middlewares you use may drop backwards
+  compatibility, which shouldn't lead to immediate problems but may be a sign
+  to start converting your code to ``async def`` too.
+* If you maintain a middleware that can be used with projects you can't control
+  (e.g. one you published for other people to use, or one that needs to support
+  some old project that can't be modernized), we recommend adding a
+  ``process_spider_output_async`` method so that the amount of unnecessary
+  iterable conversions is reduced but no compatibility is broken.
+* If you use async callbacks, try to make sure all middlewares support them.
+  Note that you can modernize 3rd-party middlewares by subclassing them.
+* If you want to write and publish a middleware that requires async code, you
+  should write in the docs that the minimum support Scrapy version is VERSION
+  (maybe even check this at the run time, using :attr:`scrapy.__version__`).
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index f8d4a356f..edfc2e4bb 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -123,6 +123,15 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :param spider: the spider whose result is being processed
         :type spider: :class:`~scrapy.Spider` object
 
+    .. method:: process_spider_output_async(response, result, spider)
+
+        .. versionadded:: VERSION
+
+        If exists, this methid will be called instead of
+        :meth:`process_spider_output` when ``result`` is an async iterable.
+        If this method exists, it must be a coroutine while
+        :meth:`process_spider_output` must not be a coroutine.
+
     .. method:: process_spider_exception(response, exception, spider)
 
         This method is called when a spider or :meth:`process_spider_output`
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index e64cfae0a..e1fdd8d13 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -192,7 +192,8 @@ class Scraper:
             spider=spider
         )
 
-    def handle_spider_output(self, result: Iterable, request: Request, response: Response, spider: Spider) -> Deferred:
+    def handle_spider_output(self, result: Union[Iterable, AsyncIterable], request: Request,
+                             response: Response, spider: Spider) -> Deferred:
         if not result:
             return defer_succeed(None)
         it: Union[Generator, AsyncGenerator]
diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 009ece06d..6075670b0 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -3,22 +3,27 @@ Spider Middleware manager
 
 See documentation in docs/topics/spider-middleware.rst
 """
+import logging
+from inspect import isasyncgenfunction
 from itertools import islice
-from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union, cast
+from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
 
-from twisted.internet.defer import Deferred
+from twisted.internet.defer import Deferred, inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy import Request, Spider
 from scrapy.exceptions import _InvalidOutput
 from scrapy.http import Response
 from scrapy.middleware import MiddlewareManager
-from scrapy.utils.asyncgen import _process_iterable_universal
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import mustbe_deferred
+from scrapy.utils.defer import mustbe_deferred, deferred_from_coro, deferred_f_from_coro_f, maybe_deferred_to_future
 from scrapy.utils.python import MutableAsyncChain, MutableChain
 
 
+logger = logging.getLogger(__name__)
+
+
 ScrapeFunc = Callable[[Union[Response, Failure], Request, Spider], Any]
 
 
@@ -30,6 +35,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
 
     component_name = 'spider middleware'
 
+    def __init__(self, *middlewares):
+        super().__init__(*middlewares)
+        self.downgrade_warning_done = False
+
     @classmethod
     def _get_mwlist_from_settings(cls, settings):
         return build_component_list(settings.getwithbase('SPIDER_MIDDLEWARES'))
@@ -40,7 +49,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             self.methods['process_spider_input'].append(mw.process_spider_input)
         if hasattr(mw, 'process_start_requests'):
             self.methods['process_start_requests'].appendleft(mw.process_start_requests)
-        process_spider_output = getattr(mw, 'process_spider_output', None)
+        process_spider_output = self._get_async_method_pair(mw, 'process_spider_output')
         self.methods['process_spider_output'].appendleft(process_spider_output)
         process_spider_exception = getattr(mw, 'process_spider_exception', None)
         self.methods['process_spider_exception'].appendleft(process_spider_exception)
@@ -64,8 +73,19 @@ class SpiderMiddlewareManager(MiddlewareManager):
     def _evaluate_iterable(self, response: Response, spider: Spider, iterable: Union[Iterable, AsyncIterable],
                            exception_processor_index: int, recover_to: Union[MutableChain, MutableAsyncChain]
                            ) -> Union[Generator, AsyncGenerator]:
-        @_process_iterable_universal
-        async def _evaluate_async_iterable(iterable):
+
+        def process_sync(iterable: Iterable):
+            try:
+                for r in iterable:
+                    yield r
+            except Exception as ex:
+                exception_result = self._process_spider_exception(response, spider, Failure(ex),
+                                                                  exception_processor_index)
+                if isinstance(exception_result, Failure):
+                    raise
+                recover_to.extend(exception_result)
+
+        async def process_async(iterable: AsyncIterable):
             try:
                 async for r in iterable:
                     yield r
@@ -75,7 +95,10 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 if isinstance(exception_result, Failure):
                     raise
                 recover_to.extend(exception_result)
-        return _evaluate_async_iterable(iterable)
+
+        if isinstance(iterable, AsyncIterable):
+            return process_async(iterable)
+        return process_sync(iterable)
 
     def _process_spider_exception(self, response: Response, spider: Spider, _failure: Failure,
                                   start_index: int = 0) -> Union[Failure, MutableChain]:
@@ -87,11 +110,22 @@ class SpiderMiddlewareManager(MiddlewareManager):
         for method_index, method in enumerate(method_list, start=start_index):
             if method is None:
                 continue
+            method = cast(Callable, method)
             result = method(response=response, exception=exception, spider=spider)
             if _isiterable(result):
                 # stop exception handling by handing control over to the
                 # process_spider_output chain if an iterable has been returned
-                return self._process_spider_output(response, spider, result, method_index + 1)
+                dfd: Deferred = self._process_spider_output(response, spider, result, method_index + 1)
+                # _process_spider_output() returns a Deferred only because of downgrading so this can be
+                # simplified when downgrading is removed.
+                if dfd.called:
+                    # the result is available immediately if _process_spider_output didn't do downgrading
+                    return dfd.result
+                else:
+                    # we forbid waiting here because otherwise we would need to return a deferred from
+                    # _process_spider_exception too, which complicates the architecture
+                    msg = f"Async iterable returned from {method.__qualname__} cannot be downgraded"
+                    raise _InvalidOutput(msg)
             elif result is None:
                 continue
             else:
@@ -100,9 +134,13 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 raise _InvalidOutput(msg)
         return _failure
 
+    # This method cannot be made async def, as _process_spider_exception relies on the Deferred result
+    # being available immediately which doesn't work when it's a wrapped coroutine.
+    # It also needs @inlineCallbacks only because of downgrading so it can be removed when downgrading is removed.
+    @inlineCallbacks
     def _process_spider_output(self, response: Response, spider: Spider,
                                result: Union[Iterable, AsyncIterable], start_index: int = 0
-                               ) -> Union[MutableChain, MutableAsyncChain]:
+                               ) -> Deferred:
         # items in this iterable do not need to go through the process_spider_output
         # chain, they went through it already from the process_spider_exception method
         recovered: Union[MutableChain, MutableAsyncChain]
@@ -112,11 +150,43 @@ class SpiderMiddlewareManager(MiddlewareManager):
         else:
             recovered = MutableChain()
 
+        # There are three cases for the middleware: def foo, async def foo, def foo + async def foo_async.
+        # 1. def foo. Sync iterables are passed as is, async ones are downgraded.
+        # 2. async def foo. Sync iterables are upgraded, async ones are passed as is.
+        # 3. def foo + async def foo_async. Iterables are passed to the respective method.
+        # Storing methods and method tuples in the same list is weird but we should be able to roll this back
+        # when we drop this compatibility feature.
+
         method_list = islice(self.methods['process_spider_output'], start_index, None)
-        for method_index, method in enumerate(method_list, start=start_index):
-            if method is None:
+        for method_index, method_pair in enumerate(method_list, start=start_index):
+            if method_pair is None:
                 continue
+            need_upgrade = need_downgrade = False
+            if isinstance(method_pair, tuple):
+                # This tuple handling is only needed until _async compatibility methods are removed.
+                method_sync, method_async = method_pair
+                method = method_async if last_result_is_async else method_sync
+            else:
+                method = method_pair
+                if not last_result_is_async and isasyncgenfunction(method):
+                    need_upgrade = True
+                elif last_result_is_async and not isasyncgenfunction(method):
+                    need_downgrade = True
             try:
+                if need_upgrade:
+                    # Iterable -> AsyncIterable
+                    result = as_async_generator(result)
+                elif need_downgrade:
+                    if not self.downgrade_warning_done:
+                        logger.warning(f"Async iterable passed to {method.__qualname__} "
+                                       f"was downgraded to a non-async one")
+                        self.downgrade_warning_done = True
+                    assert isinstance(result, AsyncIterable)
+                    # AsyncIterable -> Iterable
+                    result = yield deferred_from_coro(collect_asyncgen(result))
+                    if isinstance(recovered, AsyncIterable):
+                        recovered_collected = yield deferred_from_coro(collect_asyncgen(recovered))
+                        recovered = MutableChain(recovered_collected)
                 # might fail directly if the output value is not a generator
                 result = method(response=response, result=result, spider=spider)
             except Exception as ex:
@@ -130,8 +200,6 @@ class SpiderMiddlewareManager(MiddlewareManager):
                 msg = (f"Middleware {method.__qualname__} must return an "
                        f"iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
-            if last_result_is_async and isinstance(result, Iterable):
-                raise TypeError(f"Synchronous {method.__qualname__} called with an async iterable")
             last_result_is_async = isinstance(result, AsyncIterable)
 
         if last_result_is_async:
@@ -139,31 +207,58 @@ class SpiderMiddlewareManager(MiddlewareManager):
         else:
             return MutableChain(result, recovered)  # type: ignore[arg-type]
 
-    def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
-                                 ) -> Union[MutableChain, MutableAsyncChain]:
+    async def _process_callback_output(self, response: Response, spider: Spider, result: Union[Iterable, AsyncIterable]
+                                       ) -> Union[MutableChain, MutableAsyncChain]:
         recovered: Union[MutableChain, MutableAsyncChain]
         if isinstance(result, AsyncIterable):
             recovered = MutableAsyncChain()
         else:
             recovered = MutableChain()
         result = self._evaluate_iterable(response, spider, result, 0, recovered)
-        result = self._process_spider_output(response, spider, result)
+        result = await maybe_deferred_to_future(self._process_spider_output(response, spider, result))
         if isinstance(result, AsyncIterable):
             return MutableAsyncChain(result, recovered)
         else:
+            if isinstance(recovered, AsyncIterable):
+                recovered_collected = await collect_asyncgen(recovered)
+                recovered = MutableChain(recovered_collected)
             return MutableChain(result, recovered)  # type: ignore[arg-type]
 
     def scrape_response(self, scrape_func: ScrapeFunc, response: Response, request: Request,
                         spider: Spider) -> Deferred:
-        def process_callback_output(result: Union[Iterable, AsyncIterable]) -> Union[MutableChain, MutableAsyncChain]:
-            return self._process_callback_output(response, spider, result)
+        async def process_callback_output(result: Union[Iterable, AsyncIterable]
+                                          ) -> Union[MutableChain, MutableAsyncChain]:
+            return await self._process_callback_output(response, spider, result)
 
         def process_spider_exception(_failure: Failure) -> Union[Failure, MutableChain]:
             return self._process_spider_exception(response, spider, _failure)
 
         dfd = mustbe_deferred(self._process_spider_input, scrape_func, response, request, spider)
-        dfd.addCallbacks(callback=process_callback_output, errback=process_spider_exception)
+        dfd.addCallbacks(callback=deferred_f_from_coro_f(process_callback_output), errback=process_spider_exception)
         return dfd
 
     def process_start_requests(self, start_requests, spider: Spider) -> Deferred:
         return self._process_chain('process_start_requests', start_requests, spider)
+
+    # This method is only needed until _async compatibility methods are removed.
+    @staticmethod
+    def _get_async_method_pair(mw: Any, methodname: str) -> Union[None, Callable, Tuple[Callable, Callable]]:
+        normal_method = getattr(mw, methodname, None)
+        methodname_async = methodname + "_async"
+        async_method = getattr(mw, methodname_async, None)
+        if not async_method:
+            return normal_method
+        if not normal_method:
+            logger.error(f"Middleware {mw.__qualname__} has {methodname_async} "
+                         f"without {methodname}, skipping this method.")
+            return None
+        if not isasyncgenfunction(async_method):
+            logger.error(f"{async_method.__qualname__} is not "
+                         f"an async generator function, skipping this method.")
+            return normal_method
+        if isasyncgenfunction(normal_method):
+            logger.error(f"{normal_method.__qualname__} is an async "
+                         f"generator function while {methodname_async} exists, "
+                         f"skipping both methods.")
+            return None
+        return normal_method, async_method
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 2eb1d8609..8d7e5a602 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -1,7 +1,7 @@
 import logging
 import pprint
 from collections import defaultdict, deque
-from typing import Callable, Deque, Dict, Optional, cast, Iterable
+from typing import Callable, Deque, Dict, Iterable, Tuple, Union, cast
 
 from twisted.internet.defer import Deferred
 
@@ -21,8 +21,9 @@ class MiddlewareManager:
 
     def __init__(self, *middlewares):
         self.middlewares = middlewares
-        # Optional because process_spider_output and process_spider_exception can be None
-        self.methods: Dict[str, Deque[Optional[Callable]]] = defaultdict(deque)
+        # Only process_spider_output and process_spider_exception can be None.
+        # Only process_spider_output can be a tuple, and only until _async compatibility methods are removed.
+        self.methods: Dict[str, Deque[Union[None, Callable, Tuple[Callable, Callable]]]] = defaultdict(deque)
         for mw in middlewares:
             self._add_middleware(mw)
 
diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 973404b2b..29634c3ad 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -7,7 +7,6 @@ See documentation in docs/topics/spider-middleware.rst
 import logging
 
 from scrapy.http import Request
-from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -29,36 +28,42 @@ class DepthMiddleware:
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request):
-                depth = response.meta['depth'] + 1
-                request.meta['depth'] = depth
-                if self.prio:
-                    request.priority -= depth * self.prio
-                if self.maxdepth and depth > self.maxdepth:
-                    logger.debug(
-                        "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
-                        {'maxdepth': self.maxdepth, 'requrl': request.url},
-                        extra={'spider': spider}
-                    )
-                    return False
-                else:
-                    if self.verbose_stats:
-                        self.stats.inc_value(f'request_depth_count/{depth}',
-                                             spider=spider)
-                    self.stats.max_value('request_depth_max', depth,
-                                         spider=spider)
+        # base case (depth=0)
+        if 'depth' not in response.meta:
+            response.meta['depth'] = 0
+            if self.verbose_stats:
+                self.stats.inc_value('request_depth_count/0', spider=spider)
+
+        return (r for r in result or () if self._filter(r, response, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        # base case (depth=0)
+        if 'depth' not in response.meta:
+            response.meta['depth'] = 0
+            if self.verbose_stats:
+                self.stats.inc_value('request_depth_count/0', spider=spider)
+
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _filter(self, request, response, spider):
+        if not isinstance(request, Request):
             return True
-
-        @_process_iterable_universal
-        async def process(result):
-            # base case (depth=0)
-            if 'depth' not in response.meta:
-                response.meta['depth'] = 0
-                if self.verbose_stats:
-                    self.stats.inc_value('request_depth_count/0', spider=spider)
-
-            async for r in result or ():
-                if _filter(r):
-                    yield r
-        return process(result)
+        depth = response.meta['depth'] + 1
+        request.meta['depth'] = depth
+        if self.prio:
+            request.priority -= depth * self.prio
+        if self.maxdepth and depth > self.maxdepth:
+            logger.debug(
+                "Ignoring link (depth > %(maxdepth)d): %(requrl)s ",
+                {'maxdepth': self.maxdepth, 'requrl': request.url},
+                extra={'spider': spider}
+            )
+            return False
+        if self.verbose_stats:
+            self.stats.inc_value(f'request_depth_count/{depth}',
+                                 spider=spider)
+        self.stats.max_value('request_depth_max', depth,
+                             spider=spider)
+        return True
diff --git a/scrapy/spidermiddlewares/offsite.py b/scrapy/spidermiddlewares/offsite.py
index 074ec7a4e..448bc1367 100644
--- a/scrapy/spidermiddlewares/offsite.py
+++ b/scrapy/spidermiddlewares/offsite.py
@@ -9,7 +9,6 @@ import warnings
 
 from scrapy import signals
 from scrapy.http import Request
-from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.httpobj import urlparse_cached
 
 logger = logging.getLogger(__name__)
@@ -27,24 +26,27 @@ class OffsiteMiddleware:
         return o
 
     def process_spider_output(self, response, result, spider):
-        @_process_iterable_universal
-        async def process(result):
-            async for x in result:
-                if isinstance(x, Request):
-                    if x.dont_filter or self.should_follow(x, spider):
-                        yield x
-                    else:
-                        domain = urlparse_cached(x).hostname
-                        if domain and domain not in self.domains_seen:
-                            self.domains_seen.add(domain)
-                            logger.debug(
-                                "Filtered offsite request to %(domain)r: %(request)s",
-                                {'domain': domain, 'request': x}, extra={'spider': spider})
-                            self.stats.inc_value('offsite/domains', spider=spider)
-                        self.stats.inc_value('offsite/filtered', spider=spider)
-                else:
-                    yield x
-        return process(result)
+        return (r for r in result or () if self._filter(r, spider))
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
+
+    def _filter(self, request, spider) -> bool:
+        if not isinstance(request, Request):
+            return True
+        if request.dont_filter or self.should_follow(request, spider):
+            return True
+        domain = urlparse_cached(request).hostname
+        if domain and domain not in self.domains_seen:
+            self.domains_seen.add(domain)
+            logger.debug(
+                "Filtered offsite request to %(domain)r: %(request)s",
+                {'domain': domain, 'request': request}, extra={'spider': spider})
+            self.stats.inc_value('offsite/domains', spider=spider)
+        self.stats.inc_value('offsite/filtered', spider=spider)
+        return False
 
     def should_follow(self, request, spider):
         regex = self.host_regex
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index e0a22592f..8027beb92 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -11,7 +11,6 @@ from w3lib.url import safe_url_string
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
-from scrapy.utils.asyncgen import _process_iterable_universal
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_unicode
 from scrapy.utils.url import strip_url
@@ -334,19 +333,18 @@ class RefererMiddleware:
         return cls() if cls else self.default_policy()
 
     def process_spider_output(self, response, result, spider):
-        def _set_referer(r):
-            if isinstance(r, Request):
-                referrer = self.policy(response, r).referrer(response.url, r.url)
-                if referrer is not None:
-                    r.headers.setdefault('Referer', referrer)
-            return r
+        return (self._set_referer(r, response) for r in result or ())
 
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result or ():
-                yield _set_referer(r)
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            yield self._set_referer(r, response)
 
-        return process(result)
+    def _set_referer(self, r, response):
+        if isinstance(r, Request):
+            referrer = self.policy(response, r).referrer(response.url, r.url)
+            if referrer is not None:
+                r.headers.setdefault('Referer', referrer)
+        return r
 
     def request_scheduled(self, request, spider):
         # check redirected request to patch "Referer" header if necessary
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 63b1d36bd..7ad64d2af 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -8,7 +8,6 @@ import logging
 
 from scrapy.http import Request
 from scrapy.exceptions import NotConfigured
-from scrapy.utils.asyncgen import _process_iterable_universal
 
 logger = logging.getLogger(__name__)
 
@@ -26,22 +25,20 @@ class UrlLengthMiddleware:
         return cls(maxlength)
 
     def process_spider_output(self, response, result, spider):
-        def _filter(request):
-            if isinstance(request, Request) and len(request.url) > self.maxlength:
-                logger.info(
-                    "Ignoring link (url length > %(maxlength)d): %(url)s ",
-                    {'maxlength': self.maxlength, 'url': request.url},
-                    extra={'spider': spider}
-                )
-                spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
-                return False
-            else:
-                return True
+        return (r for r in result or () if self._filter(r, spider))
 
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result or ():
-                if _filter(r):
-                    yield r
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result or ():
+            if self._filter(r, spider):
+                yield r
 
-        return process(result)
+    def _filter(self, request, spider):
+        if isinstance(request, Request) and len(request.url) > self.maxlength:
+            logger.info(
+                "Ignoring link (url length > %(maxlength)d): %(url)s ",
+                {'maxlength': self.maxlength, 'url': request.url},
+                extra={'spider': spider}
+            )
+            spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
+            return False
+        return True
diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py
index ae9a79989..9f794de92 100644
--- a/scrapy/utils/asyncgen.py
+++ b/scrapy/utils/asyncgen.py
@@ -1,6 +1,4 @@
-import functools
-import inspect
-from typing import AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Union
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 
 async def collect_asyncgen(result: AsyncIterable):
@@ -18,46 +16,3 @@ async def as_async_generator(it: Union[Iterable, AsyncIterable]) -> AsyncGenerat
     else:
         for r in it:
             yield r
-
-
-# https://stackoverflow.com/a/66170760/113586
-def _process_iterable_universal(process_async: Callable):
-    """ Takes a function that takes an async iterable, args and kwargs. Returns
-    a function that takes any iterable, args and kwargs.
-
-    Requires that process_async only awaits on the iterable and synchronous functions,
-    so it's better to use this only in the Scrapy code itself.
-    """
-
-    # If this stops working, all internal uses can be just replaced with manually-written
-    # process_sync functions.
-
-    def process_sync(iterable: Iterable, *args, **kwargs) -> Generator:
-        agen = process_async(as_async_generator(iterable), *args, **kwargs)
-        if not inspect.isasyncgen(agen):
-            raise ValueError(f"process_async returned wrong type {type(agen)}")
-        sent = None
-        while True:
-            try:
-                gen = agen.asend(sent)
-                gen.send(None)
-            except StopIteration as e:
-                sent = yield e.value
-            except StopAsyncIteration:
-                return
-            else:
-                gen.throw(RuntimeError,
-                          f"Synchronously-called function '{process_async.__name__}' has blocked, "
-                          f"you can't use {_process_iterable_universal.__name__} with it.")
-
-    @functools.wraps(process_async)
-    def process(iterable: Union[Iterable, AsyncIterable], *args, **kwargs) -> Union[Generator, AsyncGenerator]:
-        if isinstance(iterable, AsyncIterable):
-            # call process_async directly
-            return process_async(iterable, *args, **kwargs)
-        if isinstance(iterable, Iterable):
-            # convert process_async to process_sync
-            return process_sync(iterable, *args, **kwargs)
-        raise TypeError(f"Wrong iterable type {type(iterable)}")
-
-    return process
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index d086347bc..11c089ac2 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -10,7 +10,7 @@ import warnings
 import weakref
 from functools import partial, wraps
 from itertools import chain
-from typing import AsyncIterable, Iterable, Union
+from typing import AsyncGenerator, AsyncIterable, Iterable, Union
 
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.asyncgen import as_async_generator
@@ -359,7 +359,7 @@ class MutableChain(Iterable):
         return self.__next__()
 
 
-async def _async_chain(*iterables: Union[Iterable, AsyncIterable]):
+async def _async_chain(*iterables: Union[Iterable, AsyncIterable]) -> AsyncGenerator:
     for it in iterables:
         async for o in as_async_generator(it):
             yield o
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index b0ca2f62e..f9f2b6642 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -1,6 +1,8 @@
 import collections.abc
+from typing import Optional
 from unittest import mock
 
+from testfixtures import LogCapture
 from twisted.internet import defer
 from twisted.trial.unittest import TestCase
 from twisted.python.failure import Failure
@@ -8,8 +10,8 @@ from twisted.python.failure import Failure
 from scrapy.spiders import Spider
 from scrapy.http import Request, Response
 from scrapy.exceptions import _InvalidOutput
-from scrapy.utils.asyncgen import _process_iterable_universal, as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_from_coro
+from scrapy.utils.asyncgen import collect_asyncgen
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from scrapy.core.spidermw import SpiderMiddlewareManager
 
@@ -115,33 +117,40 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
 
     RESULT_COUNT = 3  # to simplify checks, let everything return 3 objects
 
+    @staticmethod
+    def _construct_mw_setting(*mw_classes, start_index: Optional[int] = None):
+        if start_index is None:
+            start_index = 10
+        return {i: c for c, i in enumerate(mw_classes, start=start_index)}
+
     @defer.inlineCallbacks
-    def _get_middleware_result(self, *mw_classes):
-        for mw_cls in mw_classes:
-            self.mwman._add_middleware(mw_cls())
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
         result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
         return result
 
     @defer.inlineCallbacks
-    def _test_simple_base(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
+    def _test_simple_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
         self.assertIsInstance(result, collections.abc.Iterable)
         result_list = list(result)
         self.assertEqual(len(result_list), self.RESULT_COUNT)
         self.assertIsInstance(result_list[0], self.ITEM_TYPE)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
 
     @defer.inlineCallbacks
-    def _test_asyncgen_base(self, *mw_classes):
-        result = yield self._get_middleware_result(*mw_classes)
+    def _test_asyncgen_base(self, *mw_classes, downgrade: bool = False, start_index: Optional[int] = None):
+        with LogCapture() as log:
+            result = yield self._get_middleware_result(*mw_classes, start_index=start_index)
         self.assertIsInstance(result, collections.abc.AsyncIterator)
         result_list = yield deferred_from_coro(collect_asyncgen(result))
         self.assertEqual(len(result_list), self.RESULT_COUNT)
         self.assertIsInstance(result_list[0], self.ITEM_TYPE)
-
-    @defer.inlineCallbacks
-    def _test_asyncgen_fail(self, *mw_classes):
-        with self.assertRaisesRegex(TypeError, "Synchronous .+ called with an async iterable"):
-            yield self._get_middleware_result(*mw_classes)
+        self.assertEqual("downgraded to a non-async" in str(log), downgrade)
 
 
 class ProcessSpiderOutputSimpleMiddleware:
@@ -152,17 +161,36 @@ class ProcessSpiderOutputSimpleMiddleware:
 
 class ProcessSpiderOutputAsyncGenMiddleware:
     async def process_spider_output(self, response, result, spider):
-        async for r in as_async_generator(result):
+        async for r in result:
             yield r
 
 
 class ProcessSpiderOutputUniversalMiddleware:
     def process_spider_output(self, response, result, spider):
-        @_process_iterable_universal
-        async def process(result):
-            async for r in result:
-                yield r
-        return process(result)
+        for r in result:
+            yield r
+
+    async def process_spider_output_async(self, response, result, spider):
+        async for r in result:
+            yield r
+
+
+class ProcessSpiderExceptionSimpleIterableMiddleware:
+    def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+
+class ProcessSpiderExceptionAsyncIterableMiddleware:
+    async def process_spider_exception(self, response, exception, spider):
+        yield {'foo': 1}
+        d = defer.Deferred()
+        from twisted.internet import reactor
+        reactor.callLater(0, d.callback, None)
+        await maybe_deferred_to_future(d)
+        yield {'foo': 2}
+        yield {'foo': 3}
 
 
 class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
@@ -183,18 +211,19 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
         return self._test_simple_base(self.MW_SIMPLE)
 
     def test_asyncgen(self):
-        """ Asyncgen mw """
+        """ Asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw """
+        """ Simple mw -> asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN,
                                         self.MW_SIMPLE)
 
     def test_asyncgen_simple(self):
-        """ Asyncgen mw -> simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE,
-                                        self.MW_ASYNCGEN)
+        """ Asyncgen mw -> simple mw; upgrade then downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_ASYNCGEN,
+                                      downgrade=True)
 
     def test_universal(self):
         """ Universal mw """
@@ -211,12 +240,12 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
                                       self.MW_SIMPLE)
 
     def test_universal_asyncgen(self):
-        """ Universal mw -> asyncgen mw """
+        """ Universal mw -> asyncgen mw; upgrade """
         return self._test_asyncgen_base(self.MW_ASYNCGEN,
                                         self.MW_UNIVERSAL)
 
     def test_asyncgen_universal(self):
-        """ Asyncgen mw -> universal mw """
+        """ Asyncgen mw -> universal mw; upgrade """
         return self._test_asyncgen_base(self.MW_UNIVERSAL,
                                         self.MW_ASYNCGEN)
 
@@ -229,27 +258,31 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
             yield item
 
     def test_simple(self):
-        """ Simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE)
+        """ Simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      downgrade=True)
 
     def test_simple_asyncgen(self):
-        """ Simple mw -> asyncgen mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_ASYNCGEN,
-                                        self.MW_SIMPLE)
+        """ Simple mw -> asyncgen mw; downgrade then upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_SIMPLE,
+                                        downgrade=True)
 
     def test_universal(self):
         """ Universal mw """
         return self._test_asyncgen_base(self.MW_UNIVERSAL)
 
     def test_universal_simple(self):
-        """ Universal mw -> simple mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_SIMPLE,
-                                        self.MW_UNIVERSAL)
+        """ Universal mw -> simple mw; downgrade """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_UNIVERSAL,
+                                      downgrade=True)
 
     def test_simple_universal(self):
-        """ Simple mw -> universal mw; cannot work """
-        return self._test_asyncgen_fail(self.MW_UNIVERSAL,
-                                        self.MW_SIMPLE)
+        """ Simple mw -> universal mw; downgrade """
+        return self._test_simple_base(self.MW_UNIVERSAL,
+                                      self.MW_SIMPLE,
+                                      downgrade=True)
 
 
 class ProcessStartRequestsSimpleMiddleware:
@@ -269,13 +302,198 @@ class ProcessStartRequestsSimple(BaseAsyncSpiderMiddlewareTestCase):
             yield Request(f'https://example.com/{i}', dont_filter=True)
 
     @defer.inlineCallbacks
-    def _get_middleware_result(self, *mw_classes):
-        for mw_cls in mw_classes:
-            self.mwman._add_middleware(mw_cls())
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES_BASE': {}, 'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
         start_requests = iter(self._start_requests())
         results = yield self.mwman.process_start_requests(start_requests, self.spider)
         return results
 
     def test_simple(self):
         """ Simple mw """
-        self._test_simple_base(self.MW_SIMPLE)
+        return self._test_simple_base(self.MW_SIMPLE)
+
+
+class UniversalMiddlewareNoSync:
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothSync:
+    def process_spider_output(self, response, result, spider):
+        yield
+
+    def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareBothAsync:
+    async def process_spider_output(self, response, result, spider):
+        yield
+
+    async def process_spider_output_async(self, response, result, spider):
+        yield
+
+
+class UniversalMiddlewareManagerTest(TestCase):
+    def setUp(self):
+        self.mwman = SpiderMiddlewareManager()
+
+    def test_simple_mw(self):
+        mw = ProcessSpiderOutputSimpleMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_async_mw(self):
+        mw = ProcessSpiderOutputAsyncGenMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw(self):
+        mw = ProcessSpiderOutputUniversalMiddleware
+        self.mwman._add_middleware(mw)
+        self.assertEqual(self.mwman.methods['process_spider_output'][0],
+                         (mw.process_spider_output, mw.process_spider_output_async))
+
+    def test_universal_mw_no_sync(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareNoSync)
+        self.assertIn("UniversalMiddlewareNoSync has process_spider_output_async"
+                      " without process_spider_output", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+    def test_universal_mw_both_sync(self):
+        mw = UniversalMiddlewareBothSync
+        with LogCapture() as log:
+            self.mwman._add_middleware(mw)
+        self.assertIn("UniversalMiddlewareBothSync.process_spider_output_async "
+                      "is not an async generator function", str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], mw.process_spider_output)
+
+    def test_universal_mw_both_async(self):
+        with LogCapture() as log:
+            self.mwman._add_middleware(UniversalMiddlewareBothAsync)
+        self.assertIn("UniversalMiddlewareBothAsync.process_spider_output "
+                      "is an async generator function while process_spider_output_async exists",
+                      str(log))
+        self.assertEqual(self.mwman.methods['process_spider_output'][0], None)
+
+
+class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
+    @defer.inlineCallbacks
+    def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
+        setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
+        self.crawler = get_crawler(Spider, {'SPIDER_MIDDLEWARES': setting})
+        self.spider = self.crawler._create_spider('foo')
+        self.mwman = SpiderMiddlewareManager.from_crawler(self.crawler)
+        result = yield self.mwman.scrape_response(self._scrape_func, self.response, self.request, self.spider)
+        return result
+
+    def test_just_builtin(self):
+        return self._test_simple_base()
+
+    def test_builtin_simple(self):
+        return self._test_simple_base(self.MW_SIMPLE, start_index=1000)
+
+    def test_builtin_async(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_simple_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        return self._test_simple_base(self.MW_SIMPLE)
+
+    def test_async_builtin(self):
+        """ Upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_simple_base(self.MW_UNIVERSAL)
+
+
+class BuiltinMiddlewareAsyncGenTest(BuiltinMiddlewareSimpleTest):
+    async def _scrape_func(self, *args, **kwargs):
+        for item in super()._scrape_func():
+            yield item
+
+    def test_just_builtin(self):
+        return self._test_asyncgen_base()
+
+    def test_builtin_simple(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True, start_index=1000)
+
+    def test_builtin_async(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN, start_index=1000)
+
+    def test_builtin_universal(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL, start_index=1000)
+
+    def test_simple_builtin(self):
+        """ Downgrade """
+        return self._test_simple_base(self.MW_SIMPLE, downgrade=True)
+
+    def test_async_builtin(self):
+        return self._test_asyncgen_base(self.MW_ASYNCGEN)
+
+    def test_universal_builtin(self):
+        return self._test_asyncgen_base(self.MW_UNIVERSAL)
+
+
+class ProcessSpiderExceptionTest(BaseAsyncSpiderMiddlewareTestCase):
+    ITEM_TYPE = dict
+    MW_SIMPLE = ProcessSpiderOutputSimpleMiddleware
+    MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
+    MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
+    MW_EXC_SIMPLE = ProcessSpiderExceptionSimpleIterableMiddleware
+    MW_EXC_ASYNCGEN = ProcessSpiderExceptionAsyncIterableMiddleware
+
+    def _scrape_func(self, *args, **kwargs):
+        1 / 0
+
+    @defer.inlineCallbacks
+    def _test_asyncgen_nodowngrade(self, *mw_classes):
+        with self.assertRaisesRegex(_InvalidOutput, "Async iterable returned from .+ cannot be downgraded"):
+            yield self._get_middleware_result(*mw_classes)
+
+    def test_exc_simple(self):
+        """ Simple exc mw """
+        return self._test_simple_base(self.MW_EXC_SIMPLE)
+
+    def test_exc_async(self):
+        """ Async exc mw """
+        return self._test_asyncgen_base(self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_simple(self):
+        """ Simple exc mw -> simple output mw """
+        return self._test_simple_base(self.MW_SIMPLE,
+                                      self.MW_EXC_SIMPLE)
+
+    def test_exc_async_async(self):
+        """ Async exc mw -> async output mw """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_ASYNCGEN)
+
+    def test_exc_simple_async(self):
+        """ Simple exc mw -> async output mw; upgrade """
+        return self._test_asyncgen_base(self.MW_ASYNCGEN,
+                                        self.MW_EXC_SIMPLE)
+
+    def test_exc_async_simple(self):
+        """ Async exc mw -> simple output mw; cannot work as downgrading is not supported """
+        return self._test_asyncgen_nodowngrade(self.MW_SIMPLE,
+                                               self.MW_EXC_ASYNCGEN)
diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py
index 088c14ca8..dac246fb6 100644
--- a/tests/test_spidermiddleware_output_chain.py
+++ b/tests/test_spidermiddleware_output_chain.py
@@ -28,6 +28,7 @@ class RecoveryMiddleware:
 class RecoverySpider(Spider):
     name = 'RecoverySpider'
     custom_settings = {
+        'SPIDER_MIDDLEWARES_BASE': {},
         'SPIDER_MIDDLEWARES': {
             RecoveryMiddleware: 10,
         },
@@ -107,6 +108,13 @@ class GeneratorCallbackSpider(Spider):
         raise ImportError()
 
 
+class AsyncGeneratorCallbackSpider(GeneratorCallbackSpider):
+    async def parse(self, response):
+        yield {'test': 1}
+        yield {'test': 2}
+        raise ImportError()
+
+
 # ================================================================================
 # (2.1) exceptions from a spider callback (generator, middleware right after callback)
 class GeneratorCallbackSpiderMiddlewareRightAfterSpider(GeneratorCallbackSpider):
@@ -360,6 +368,15 @@ class TestSpiderMiddleware(TestCase):
         self.assertIn("Middleware: ImportError exception caught", str(log2))
         self.assertIn("'item_scraped_count': 2", str(log2))
 
+    @defer.inlineCallbacks
+    def test_async_generator_callback(self):
+        """
+        Same as test_generator_callback but with an async callback.
+        """
+        log2 = yield self.crawl_log(AsyncGeneratorCallbackSpider)
+        self.assertIn("Middleware: ImportError exception caught", str(log2))
+        self.assertIn("'item_scraped_count': 2", str(log2))
+
     @defer.inlineCallbacks
     def test_generator_callback_right_after_callback(self):
         """
diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py
index 7abe17c22..9ae66c57c 100644
--- a/tests/test_utils_asyncgen.py
+++ b/tests/test_utils_asyncgen.py
@@ -1,7 +1,6 @@
-from twisted.internet.defer import Deferred
 from twisted.trial import unittest
 
-from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen, _process_iterable_universal
+from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import deferred_f_from_coro_f
 
 
@@ -19,52 +18,3 @@ class AsyncgenUtilsTest(unittest.TestCase):
         ag = as_async_generator(range(42))
         results = await collect_asyncgen(ag)
         self.assertEqual(results, list(range(42)))
-
-
-@_process_iterable_universal
-async def process_iterable(iterable):
-    async for i in iterable:
-        yield i * 2
-
-
-@_process_iterable_universal
-async def process_iterable_awaiting(iterable):
-    async for i in iterable:
-        yield i * 2
-        d = Deferred()
-        from twisted.internet import reactor
-        reactor.callLater(0, d.callback, 42)
-        await d
-
-
-class ProcessIterableUniversalTest(unittest.TestCase):
-
-    def test_normal(self):
-        iterable = iter([1, 2, 3])
-        results = list(process_iterable(iterable))
-        self.assertEqual(results, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_async(self):
-        iterable = as_async_generator([1, 2, 3])
-        results = await collect_asyncgen(process_iterable(iterable))
-        self.assertEqual(results, [2, 4, 6])
-
-    @deferred_f_from_coro_f
-    async def test_blocking(self):
-        iterable = [1, 2, 3]
-        with self.assertRaisesRegex(RuntimeError, "Synchronously-called function"):
-            list(process_iterable_awaiting(iterable))
-
-    def test_invalid_iterable(self):
-        with self.assertRaisesRegex(TypeError, "Wrong iterable type"):
-            process_iterable(None)
-
-    @deferred_f_from_coro_f
-    async def test_invalid_process(self):
-        @_process_iterable_universal
-        def process_iterable_invalid(iterable):
-            pass
-
-        with self.assertRaisesRegex(ValueError, "process_async returned wrong type"):
-            list(process_iterable_invalid([]))

From e079bffdab11402d1936103ba453e43e97925079 Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Tue, 11 Jan 2022 19:21:07 +0500
Subject: [PATCH 32/41] Disable logging-fstring-interpolation in pylint.

---
 pylintrc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/pylintrc b/pylintrc
index 2cdd6321e..0d29dc709 100644
--- a/pylintrc
+++ b/pylintrc
@@ -49,6 +49,7 @@ disable=abstract-method,
         keyword-arg-before-vararg,
         line-too-long,
         logging-format-interpolation,
+        logging-fstring-interpolation,
         logging-not-lazy,
         lost-exception,
         method-hidden,

From fd08bb6cd99f16c9ce433583d4698801dc7e0ac3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 14:34:57 +0100
Subject: [PATCH 33/41] Refactor the asynchronous process_spider_output
 documentation

---
 docs/topics/coroutines.rst        | 157 +++++++++++++++++-------------
 docs/topics/spider-middleware.rst |  22 +++--
 2 files changed, 102 insertions(+), 77 deletions(-)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 073b6bd9a..361cd5e60 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -17,9 +17,12 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :class:`~scrapy.Request` callbacks.
 
+    If you are using any custom or third-party :ref:`spider middleware
+    <topics-spider-middleware>`, see :ref:`sync-async-spider-middleware`.
+
     .. versionchanged:: VERSION
-       Output of async callbacks is now processed asynchronously instead of collecting
-       all of it first.
+       Output of async callbacks is now processed asynchronously instead of
+       collecting all of it first.
 
 -   The :meth:`process_item` method of
     :ref:`item pipelines <topics-item-pipeline>`.
@@ -34,18 +37,26 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
--   The :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
-    method of :ref:`spider middlewares <custom-spider-middleware>`. See
-    :ref:`async-spider-middlewares`.
+-   The
+    :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`
+    method of :ref:`spider middlewares <topics-spider-middleware>`.
+
+    It must be defined as an :term:`asynchronous generator`. The input
+    ``result`` parameter is an :term:`asynchronous iterable`.
+
+    See also :ref:`sync-async-spider-middleware` and
+    :ref:`universal-spider-middleware`.
 
     .. versionadded:: VERSION
 
-Usage
-=====
+General usage
+=============
 
-There are several use cases for coroutines in Scrapy. Code that would
-return Deferreds when written for previous Scrapy versions, such as downloader
-middlewares and signal handlers, can be rewritten to be shorter and cleaner::
+There are several use cases for coroutines in Scrapy.
+
+Code that would return Deferreds when written for previous Scrapy versions,
+such as downloader middlewares and signal handlers, can be rewritten to be
+shorter and cleaner::
 
     from itemadapter import ItemAdapter
 
@@ -110,46 +121,73 @@ Common use cases for asynchronous code include:
 
 .. _aio-libs: https://github.com/aio-libs
 
-.. _async-spider-middlewares:
 
-Asynchronous spider middlewares
-===============================
+.. _sync-async-spider-middleware:
+
+Mixing synchronous and asynchronous spider middlewares
+======================================================
 
 .. versionadded:: VERSION
-.. note:: This currently applies to
-          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output`.
-          In the future it will also apply to
-          :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_start_requests`.
 
-Middleware methods discussed here can take and return async iterables. They can
-return the same type of iterable or they can take a normal one and return an
-async one. If such method needs to return an async iterable it must be an async
-generator, not just a coroutine that returns an iterable.
+The output of a :class:`~scrapy.Request` callback is passed as the ``result``
+parameter to the
+:meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_output` method
+of the first :ref:`spider middleware <topics-spider-middleware>` from the
+:ref:`list of active spider middlewares <topics-spider-middleware-setting>`.
+Then the output of that ``process_spider_output`` method is passed to the
+``process_spider_output`` method of the next spider middleware, and so on for
+every active spider middleware.
 
-As the result of a middleware method is passed to the same method of the next
-middleware, it needs to be adapted if the second method expects a different
-type. Scrapy will do this transparently:
+Scrapy supports mixing :ref:`coroutine methods <async>` and synchronous methods
+in this chain of calls.
 
-* A normal iterable is wrapped into an async one which shouldn't cause any side
-  effects.
-* An async iterable is downgraded to a normal one by waiting until all results
-  are available and wrapping them in a normal iterable. This is problematic
-  because it pauses the normal middleware processing for this iterable and
-  because all results can be skipped if exceptions are raised during
-  processing. This case emits a warning and will be deprecated and then removed
-  in a later Scrapy version.
-* Async iterables returned from
-  :meth:`~scrapy.spidermiddlewares.SpiderMiddleware.process_spider_exception`
-  won't be downgraded, an exception will be raised if that is needed.
+However, if any of the ``process_spider_output`` methods is defined as a
+synchronous method, and the previous ``Request`` callback or
+``process_spider_output`` method is a coroutine, there are some drawbacks to
+the asynchronous-to-synchronous conversion that Scrapy does so that the
+synchronous ``process_spider_output`` method gets a synchronous iterable as its
+``result`` parameter:
 
-As downgrading is undesirable, here is the proposed way to avoid it. If all
-middlewares, including 3rd-party ones, support async iterables as input, no
-downgrading will happen. But removing normal iterable support (making the
-method a coroutine) from a middleware published as a separate project or used
-internally in projects for older Scrapy versions breaks backwards
-compatibility. So, as an interim measure (it will be deprecated and then
-removed in a later Scrapy version), a middleware can provide both sync and
-async methods in the following form::
+-   The whole output of the previous ``Request`` callback or
+    ``process_spider_output`` method is awaited at this point.
+
+-   If an exception raises while awaiting the output of the previous
+    ``Request`` callback or ``process_spider_output`` method, none of that
+    output will be processed.
+
+Asynchronous-to-synchronous conversions are supported for backward
+compatibility, but they are deprecated and will stop working in a future
+version of Scrapy.
+
+To avoid asynchronous-to-synchronous conversion, when defining ``Request``
+callbacks as coroutine methods or when using spider middlewares whose
+``process_spider_output`` method is an :term:`asynchronous generator`, all
+active spider middlewares must either have their ``process_spider_output``
+method defined as an asynchronous generator or :ref:`define a
+process_spider_output_async method <universal-spider-middleware>`.
+
+.. note:: When using third-party spider middlewares that only define a
+          synchronous ``process_spider_output`` method, consider
+          :ref:`making them universal <universal-spider-middleware>` through
+          :ref:`subclassing <tut-inheritance>`.
+
+
+.. _universal-spider-middleware:
+
+Universal spider middleware
+===========================
+
+.. versionadded:: VERSION
+
+To allow writing a spider middleware that supports asynchronous execution of
+its ``process_spider_output`` method in Scrapy VERSION and later (avoiding
+:ref:`asynchronous-to-synchronous conversions <sync-async-spider-middleware>`)
+while maintaining support for older Scrapy versions, you may define
+``process_spider_output`` as a synchronous method and define an
+:term:`asynchronous generator` version of that method with an alternative name:
+``process_spider_output_async``.
+
+For example::
 
     class UniversalSpiderMiddleware:
         def process_spider_output(self, response, result, spider):
@@ -162,28 +200,13 @@ async methods in the following form::
                 # ... do something with r
                 yield r
 
-In this case normal and async iterables will be passed to the respective
-methods without any wrapping or downgrading, and in older versions of Scrapy
-the coroutine method will just be ignored. When the backwards compatibility is
-no longer needed the non-coroutine method can be dropped and the coroutine one
-renamed to the normal name. It may be possible to extract common code from both
-methods to reduce code duplication, as in the simplest case the only difference
-between them will be ``for`` vs ``async for``.
+.. note:: This is an interim measure to allow, for a time, to write code that
+          works in Scrapy VERSION and later without requiring
+          asynchronous-to-synchronous conversions, and works in earlier Scrapy
+          versions as well.
 
-So, to recap:
-
-* If you don't intend to use async callbacks or middlewares containing async
-  code in your project, nothing should change for you yet. At some point in the
-  future some of the 3rd-party middlewares you use may drop backwards
-  compatibility, which shouldn't lead to immediate problems but may be a sign
-  to start converting your code to ``async def`` too.
-* If you maintain a middleware that can be used with projects you can't control
-  (e.g. one you published for other people to use, or one that needs to support
-  some old project that can't be modernized), we recommend adding a
-  ``process_spider_output_async`` method so that the amount of unnecessary
-  iterable conversions is reduced but no compatibility is broken.
-* If you use async callbacks, try to make sure all middlewares support them.
-  Note that you can modernize 3rd-party middlewares by subclassing them.
-* If you want to write and publish a middleware that requires async code, you
-  should write in the docs that the minimum support Scrapy version is VERSION
-  (maybe even check this at the run time, using :attr:`scrapy.__version__`).
+          In some future version of Scrapy, however, this feature will be
+          deprecated and, eventually, in a later version of Scrapy, this
+          feature will be removed, and all spider middlewares will be expected
+          to define their ``process_spider_output`` method as an asynchronous
+          generator.
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index edfc2e4bb..787545ed2 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -98,10 +98,6 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
     .. method:: process_spider_output(response, result, spider)
 
-        .. versionchanged:: VERSION
-           Since VERSION this can take and return an :term:`python:asynchronous
-           iterable`.
-
         This method is called with the results returned from the Spider, after
         it has processed the response.
 
@@ -109,8 +105,15 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
         :class:`~scrapy.Request` objects and :ref:`item objects
         <topics-items>`.
 
-        .. note:: When defined as a :ref:`coroutine <async>`, this method needs
-                  to be an async generator, not just return an iterable.
+        .. versionchanged:: VERSION
+           This method may be defined as an :term:`asynchronous generator`, in
+           which case ``result`` is an :term:`asynchronous iterable`.
+
+        Consider defining this method as an :term:`asynchronous generator`,
+        which will be a requirement in a future version of Scrapy. However, if
+        you wish your spider middleware to work with Scrapy versions earlier
+        than Scrapy VERSION, :ref:`make your spider middleware universal
+        <universal-spider-middleware>` instead.
 
         :param response: the response which generated this output from the
           spider
@@ -127,10 +130,9 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
         .. versionadded:: VERSION
 
-        If exists, this methid will be called instead of
-        :meth:`process_spider_output` when ``result`` is an async iterable.
-        If this method exists, it must be a coroutine while
-        :meth:`process_spider_output` must not be a coroutine.
+        If defined, this method must be an :term:`asynchronous generator`,
+        which will be called instead of :meth:`process_spider_output` if
+        ``result`` is an :term:`asynchronous iterable`.
 
     .. method:: process_spider_exception(response, exception, spider)
 

From c961438d5d9998344460d930ea502fae40553043 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 18:45:56 +0100
Subject: [PATCH 34/41] tests: cover scenarios of bad results from
 process_spider_output

---
 scrapy/core/spidermw.py        | 19 ++++++++----
 tests/test_spidermiddleware.py | 54 +++++++++++++++++++++++++++-------
 2 files changed, 58 insertions(+), 15 deletions(-)

diff --git a/scrapy/core/spidermw.py b/scrapy/core/spidermw.py
index 6075670b0..1aa02f29f 100644
--- a/scrapy/core/spidermw.py
+++ b/scrapy/core/spidermw.py
@@ -4,7 +4,7 @@ Spider Middleware manager
 See documentation in docs/topics/spider-middleware.rst
 """
 import logging
-from inspect import isasyncgenfunction
+from inspect import isasyncgenfunction, iscoroutine
 from itertools import islice
 from typing import Any, AsyncGenerator, AsyncIterable, Callable, Generator, Iterable, Tuple, Union, cast
 
@@ -61,7 +61,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             try:
                 result = method(response=response, spider=spider)
                 if result is not None:
-                    msg = (f"Middleware {method.__qualname__} must return None "
+                    msg = (f"{method.__qualname__} must return None "
                            f"or raise an exception, got {type(result)}")
                     raise _InvalidOutput(msg)
             except _InvalidOutput:
@@ -129,7 +129,7 @@ class SpiderMiddlewareManager(MiddlewareManager):
             elif result is None:
                 continue
             else:
-                msg = (f"Middleware {method.__qualname__} must return None "
+                msg = (f"{method.__qualname__} must return None "
                        f"or an iterable, got {type(result)}")
                 raise _InvalidOutput(msg)
         return _failure
@@ -197,8 +197,17 @@ class SpiderMiddlewareManager(MiddlewareManager):
             if _isiterable(result):
                 result = self._evaluate_iterable(response, spider, result, method_index + 1, recovered)
             else:
-                msg = (f"Middleware {method.__qualname__} must return an "
-                       f"iterable, got {type(result)}")
+                if iscoroutine(result):
+                    result.close()  # Silence warning about not awaiting
+                    msg = (
+                        f"{method.__qualname__} must be an asynchronous "
+                        f"generator (i.e. use yield)"
+                    )
+                else:
+                    msg = (
+                        f"{method.__qualname__} must return an iterable, got "
+                        f"{type(result)}"
+                    )
                 raise _InvalidOutput(msg)
             last_result_is_async = isinstance(result, AsyncIterable)
 
diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index f9f2b6642..ed0912b82 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -123,6 +123,11 @@ class BaseAsyncSpiderMiddlewareTestCase(SpiderMiddlewareTestCase):
             start_index = 10
         return {i: c for c, i in enumerate(mw_classes, start=start_index)}
 
+    def _scrape_func(self, *args, **kwargs):
+        yield {'foo': 1}
+        yield {'foo': 2}
+        yield {'foo': 3}
+
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)
@@ -201,11 +206,6 @@ class ProcessSpiderOutputSimple(BaseAsyncSpiderMiddlewareTestCase):
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
-    def _scrape_func(self, *args, **kwargs):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
-
     def test_simple(self):
         """ Simple mw """
         return self._test_simple_base(self.MW_SIMPLE)
@@ -285,6 +285,45 @@ class ProcessSpiderOutputAsyncGen(ProcessSpiderOutputSimple):
                                       downgrade=True)
 
 
+class ProcessSpiderOutputNonIterableMiddleware:
+    def process_spider_output(self, response, result, spider):
+        return
+
+
+class ProcessSpiderOutputCoroutineMiddleware:
+    async def process_spider_output(self, response, result, spider):
+        results = []
+        for r in result:
+            results.append(r)
+        return results
+
+
+class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
+
+    @defer.inlineCallbacks
+    def test_non_iterable(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            (
+                "\.process_spider_output must return an iterable, got <class "
+                "'NoneType'>"
+            ),
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputNonIterableMiddleware,
+            )
+
+    @defer.inlineCallbacks
+    def test_coroutine(self):
+        with self.assertRaisesRegex(
+            _InvalidOutput,
+            "\.process_spider_output must be an asynchronous generator",
+        ):
+            yield self._get_middleware_result(
+                ProcessSpiderOutputCoroutineMiddleware,
+            )
+
+
 class ProcessStartRequestsSimpleMiddleware:
     def process_start_requests(self, start_requests, spider):
         for r in start_requests:
@@ -387,11 +426,6 @@ class BuiltinMiddlewareSimpleTest(BaseAsyncSpiderMiddlewareTestCase):
     MW_ASYNCGEN = ProcessSpiderOutputAsyncGenMiddleware
     MW_UNIVERSAL = ProcessSpiderOutputUniversalMiddleware
 
-    def _scrape_func(self, *args, **kwargs):
-        yield {'foo': 1}
-        yield {'foo': 2}
-        yield {'foo': 3}
-
     @defer.inlineCallbacks
     def _get_middleware_result(self, *mw_classes, start_index: Optional[int] = None):
         setting = self._construct_mw_setting(*mw_classes, start_index=start_index)

From b78e6915c6b259b31d3c37cae1529e85e797c964 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Wed, 16 Mar 2022 20:17:25 +0100
Subject: [PATCH 35/41] Clarify that without async-to-sync conversions items
 yielded before an exception are processed

---
 docs/topics/coroutines.rst | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index 361cd5e60..efc4566a0 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -155,6 +155,9 @@ synchronous ``process_spider_output`` method gets a synchronous iterable as its
     ``Request`` callback or ``process_spider_output`` method, none of that
     output will be processed.
 
+    This contrasts with the regular behavior, where all items yielded before
+    an exception raises are processed.
+
 Asynchronous-to-synchronous conversions are supported for backward
 compatibility, but they are deprecated and will stop working in a future
 version of Scrapy.

From b95c634b861bacc2b2ee3beeb5fcf64ab8607eea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A1n=20Chaves?= <adrian@chaves.io>
Date: Thu, 17 Mar 2022 22:23:25 +0100
Subject: [PATCH 36/41] Document how to enforce Scrapy versions on Scrapy
 components

---
 docs/index.rst                    | 13 +++--
 docs/topics/asyncio.rst           | 24 +++++++++
 docs/topics/components.rst        | 84 +++++++++++++++++++++++++++++++
 docs/topics/coroutines.rst        |  6 +--
 docs/topics/spider-middleware.rst |  8 +--
 5 files changed, 125 insertions(+), 10 deletions(-)
 create mode 100644 docs/topics/components.rst

diff --git a/docs/index.rst b/docs/index.rst
index 75e08f537..6e22db884 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -229,10 +229,11 @@ Extending Scrapy
    topics/downloader-middleware
    topics/spider-middleware
    topics/extensions
-   topics/api
    topics/signals
    topics/scheduler
    topics/exporters
+   topics/components
+   topics/api
 
 
 :doc:`topics/architecture`
@@ -247,9 +248,6 @@ Extending Scrapy
 :doc:`topics/extensions`
     Extend Scrapy with your custom functionality
 
-:doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
-
 :doc:`topics/signals`
     See all available signals and how to work with them.
 
@@ -259,6 +257,13 @@ Extending Scrapy
 :doc:`topics/exporters`
     Quickly export your scraped items to a file (XML, CSV, etc).
 
+:doc:`topics/components`
+    Learn the common API and some good practices when building custom Scrapy
+    components.
+
+:doc:`topics/api`
+    Use it on extensions and middlewares to extend Scrapy functionality
+
 
 All the rest
 ============
diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst
index 3a6941a2c..dbee7146d 100644
--- a/docs/topics/asyncio.rst
+++ b/docs/topics/asyncio.rst
@@ -96,3 +96,27 @@ Futures. Scrapy provides two helpers for this:
          down to Scrapy 2.0 (earlier versions do not support
          :mod:`asyncio`), you can copy the implementation of these functions
          into your own code.
+
+
+.. _enforce-asyncio-requirement:
+
+Enforcing asyncio as a requirement
+==================================
+
+If you are writing a :ref:`component <topics-components>` that requires asyncio
+to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to
+:ref:`enforce it as a requirement <enforce-component-requirements>`. For
+example::
+
+    from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+    class MyComponent:
+
+        def __init__(self):
+            if not is_asyncio_reactor_installed():
+                raise ValueError(
+                    f"{MyComponent.__qualname__} requires the asyncio Twisted "
+                    f"reactor. Make sure you have it configured in the "
+                    f"TWISTED_REACTOR setting. See the asyncio documentation "
+                    f"of Scrapy for more information."
+                )
diff --git a/docs/topics/components.rst b/docs/topics/components.rst
new file mode 100644
index 000000000..1fff2d61a
--- /dev/null
+++ b/docs/topics/components.rst
@@ -0,0 +1,84 @@
+.. _topics-components:
+
+==========
+Components
+==========
+
+A Scrapy component is any class whose objects are created using
+:func:`scrapy.utils.misc.create_instance`.
+
+That includes the classes that you may assign to the following settings:
+
+-   :setting:`DNS_RESOLVER`
+
+-   :setting:`DOWNLOAD_HANDLERS`
+
+-   :setting:`DOWNLOADER_CLIENTCONTEXTFACTORY`
+
+-   :setting:`DOWNLOADER_MIDDLEWARES`
+
+-   :setting:`DUPEFILTER_CLASS`
+
+-   :setting:`EXTENSIONS`
+
+-   :setting:`FEED_EXPORTERS`
+
+-   :setting:`FEED_STORAGES`
+
+-   :setting:`ITEM_PIPELINES`
+
+-   :setting:`SCHEDULER`
+
+-   :setting:`SCHEDULER_DISK_QUEUE`
+
+-   :setting:`SCHEDULER_MEMORY_QUEUE`
+
+-   :setting:`SCHEDULER_PRIORITY_QUEUE`
+
+-   :setting:`SPIDER_MIDDLEWARES`
+
+Third-party Scrapy components may also let you define additional Scrapy
+components, usually configurable through :ref:`settings <topics-settings>`, to
+modify their behavior.
+
+.. _enforce-component-requirements:
+
+Enforcing component requirements
+================================
+
+Sometimes, your components may only be intended to work under certain
+conditions. For example, the may require a minimum version of Scrapy to work as
+intended, or they may require certain settings to have specific values.
+
+In addition to describing those conditions in the documentation of your
+component, it is a good practice to raise an exception from the ``__init__``
+method of your component if those conditions are not met at run time.
+
+In the case of :ref:`downloader middlewares <topics-downloader-middleware>`,
+:ref:`extensions <topics-extensions>`, :ref:`item pipelines
+<topics-item-pipeline>`, and :ref:`spider middlewares
+<topics-spider-middleware>`, you should raise
+:exc:`scrapy.exceptions.NotConfigured`, passing a description of the issue as a
+parameter to the exception so that it is printed in the logs, for the user to
+see. For other components, feel free to raise whatever other exception feels
+right to you; for example, :exc:`RuntimeError` would make sense for a Scrapy
+version mismatch, while :exc:`ValueError` may be better if the issue is the
+value of a setting.
+
+If your requirement is a minimum Scrapy version, you may use
+:attr:`scrapy.__version__` to enforce your requirement. For example::
+
+    from pkg_resources import parse_version
+
+    import scrapy
+
+    class MyComponent:
+
+        def __init__(self):
+            if parse_version(scrapy.__version__) < parse_version('VERSION'):
+                raise RuntimeError(
+                    f"{MyComponent.__qualname__} requires Scrapy VERSION or "
+                    f"later, which allow defining the process_spider_output "
+                    f"method of spider middlewares as an asynchronous "
+                    f"generator."
+                )
diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst
index efc4566a0..55d013c06 100644
--- a/docs/topics/coroutines.rst
+++ b/docs/topics/coroutines.rst
@@ -162,7 +162,7 @@ Asynchronous-to-synchronous conversions are supported for backward
 compatibility, but they are deprecated and will stop working in a future
 version of Scrapy.
 
-To avoid asynchronous-to-synchronous conversion, when defining ``Request``
+To avoid asynchronous-to-synchronous conversions, when defining ``Request``
 callbacks as coroutine methods or when using spider middlewares whose
 ``process_spider_output`` method is an :term:`asynchronous generator`, all
 active spider middlewares must either have their ``process_spider_output``
@@ -177,8 +177,8 @@ process_spider_output_async method <universal-spider-middleware>`.
 
 .. _universal-spider-middleware:
 
-Universal spider middleware
-===========================
+Universal spider middlewares
+============================
 
 .. versionadded:: VERSION
 
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 787545ed2..816cb5e03 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -111,9 +111,11 @@ object gives you access, for example, to the :ref:`settings <topics-settings>`.
 
         Consider defining this method as an :term:`asynchronous generator`,
         which will be a requirement in a future version of Scrapy. However, if
-        you wish your spider middleware to work with Scrapy versions earlier
-        than Scrapy VERSION, :ref:`make your spider middleware universal
-        <universal-spider-middleware>` instead.
+        you plan on sharing your spider middleware with other people, consider
+        either :ref:`enforcing Scrapy VERSION <enforce-component-requirements>`
+        as a minimum requirement of your spider middleware, or :ref:`making
+        your spider middleware universal <universal-spider-middleware>` so that
+        it works with Scrapy versions earlier than Scrapy VERSION.
 
         :param response: the response which generated this output from the
           spider

From b21c16099ed0acb0589677afd98c0ae3b78cd17d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Fri, 22 Jul 2022 19:18:33 +0500
Subject: [PATCH 37/41] Fix flake8 issues.

---
 tests/test_spidermiddleware.py | 6 +++---
 tests/test_utils_python.py     | 2 --
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py
index ed0912b82..edde6f682 100644
--- a/tests/test_spidermiddleware.py
+++ b/tests/test_spidermiddleware.py
@@ -305,8 +305,8 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
         with self.assertRaisesRegex(
             _InvalidOutput,
             (
-                "\.process_spider_output must return an iterable, got <class "
-                "'NoneType'>"
+                r"\.process_spider_output must return an iterable, got <class "
+                r"'NoneType'>"
             ),
         ):
             yield self._get_middleware_result(
@@ -317,7 +317,7 @@ class ProcessSpiderOutputInvalidResult(BaseAsyncSpiderMiddlewareTestCase):
     def test_coroutine(self):
         with self.assertRaisesRegex(
             _InvalidOutput,
-            "\.process_spider_output must be an asynchronous generator",
+            r"\.process_spider_output must be an asynchronous generator",
         ):
             yield self._get_middleware_result(
                 ProcessSpiderOutputCoroutineMiddleware,
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 00b06b839..b1a8fdc04 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -2,8 +2,6 @@ import functools
 import gc
 import operator
 import platform
-import unittest
-from datetime import datetime
 from itertools import count
 from warnings import catch_warnings, filterwarnings
 

From 56e2eeac1066cd2d3c710b76a3d6210b3ac257f5 Mon Sep 17 00:00:00 2001
From: Mikhail Korobov <kmike84@gmail.com>
Date: Wed, 27 Jul 2022 09:41:12 +0500
Subject: [PATCH 38/41] fix typing issues by upgrading mypy

---
 tests/test_utils_request.py | 2 +-
 tox.ini                     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 5ee772c0b..8bc7922b6 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -52,7 +52,7 @@ class UtilsRequestTest(unittest.TestCase):
 class FingerprintTest(unittest.TestCase):
     maxDiff = None
 
-    function = staticmethod(fingerprint)
+    function: staticmethod = staticmethod(fingerprint)
     cache: Union[
         "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], bytes]]",
         "WeakKeyDictionary[Request, Dict[Tuple[Optional[Tuple[bytes, ...]], bool], str]]",
diff --git a/tox.ini b/tox.ini
index 4d1bb574d..2110e1020 100644
--- a/tox.ini
+++ b/tox.ini
@@ -38,7 +38,7 @@ install_command =
 basepython = python3
 deps =
     lxml-stubs==0.2.0
-    mypy==0.910
+    mypy==0.971
     types-pyOpenSSL==20.0.3
     types-setuptools==57.0.0
 commands =

From 83ecdf1bcab310be5f2c59e5c005b8968d72a72c Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 27 Jul 2022 23:12:31 +0500
Subject: [PATCH 39/41] Update docs/topics/components.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/topics/components.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/topics/components.rst b/docs/topics/components.rst
index 1fff2d61a..c44f3def2 100644
--- a/docs/topics/components.rst
+++ b/docs/topics/components.rst
@@ -47,7 +47,7 @@ Enforcing component requirements
 ================================
 
 Sometimes, your components may only be intended to work under certain
-conditions. For example, the may require a minimum version of Scrapy to work as
+conditions. For example, they may require a minimum version of Scrapy to work as
 intended, or they may require certain settings to have specific values.
 
 In addition to describing those conditions in the documentation of your

From 0f1112f3e22d91e505f736fc78fae94eda07c72d Mon Sep 17 00:00:00 2001
From: Andrey Rahmatullin <wrar@wrar.name>
Date: Wed, 27 Jul 2022 23:12:43 +0500
Subject: [PATCH 40/41] Update docs/index.rst
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adrián Chaves <adrian@chaves.io>
---
 docs/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/index.rst b/docs/index.rst
index ea4950e4c..5404969e0 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -258,7 +258,7 @@ Extending Scrapy
     components.
 
 :doc:`topics/api`
-    Use it on extensions and middlewares to extend Scrapy functionality
+    Use it on extensions and middlewares to extend Scrapy functionality.
 
 
 All the rest

From c7b90c6e1e3de20256d8bf6fc5d18da44d562b0d Mon Sep 17 00:00:00 2001
From: Andrey Rakhmatullin <wrar@wrar.name>
Date: Thu, 28 Jul 2022 13:44:36 +0500
Subject: [PATCH 41/41] Extract more common code.

---
 scrapy/spidermiddlewares/depth.py | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/scrapy/spidermiddlewares/depth.py b/scrapy/spidermiddlewares/depth.py
index 29634c3ad..4c923b1b3 100644
--- a/scrapy/spidermiddlewares/depth.py
+++ b/scrapy/spidermiddlewares/depth.py
@@ -28,25 +28,22 @@ class DepthMiddleware:
         return cls(maxdepth, crawler.stats, verbose, prio)
 
     def process_spider_output(self, response, result, spider):
-        # base case (depth=0)
-        if 'depth' not in response.meta:
-            response.meta['depth'] = 0
-            if self.verbose_stats:
-                self.stats.inc_value('request_depth_count/0', spider=spider)
-
+        self._init_depth(response, spider)
         return (r for r in result or () if self._filter(r, response, spider))
 
     async def process_spider_output_async(self, response, result, spider):
+        self._init_depth(response, spider)
+        async for r in result or ():
+            if self._filter(r, response, spider):
+                yield r
+
+    def _init_depth(self, response, spider):
         # base case (depth=0)
         if 'depth' not in response.meta:
             response.meta['depth'] = 0
             if self.verbose_stats:
                 self.stats.inc_value('request_depth_count/0', spider=spider)
 
-        async for r in result or ():
-            if self._filter(r, response, spider):
-                yield r
-
     def _filter(self, request, response, spider):
         if not isinstance(request, Request):
             return True