From 76abcedaf4f31a7c76de9e19680dd7499d9eccf4 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 2 Feb 2021 13:36:25 +0500 Subject: [PATCH] Add as_async_generator. --- scrapy/utils/asyncgen.py | 12 ++++++++++++ tests/test_utils_asyncgen.py | 20 ++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 tests/test_utils_asyncgen.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 7f697af5f..db2173f85 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,5 +1,17 @@ +import collections + + async def collect_asyncgen(result): results = [] async for x in result: results.append(x) return results + + +async def as_async_generator(it): + if isinstance(it, collections.abc.AsyncIterator): + async for r in it: + yield r + else: + for r in it: + yield r diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py new file mode 100644 index 000000000..9ae66c57c --- /dev/null +++ b/tests/test_utils_asyncgen.py @@ -0,0 +1,20 @@ +from twisted.trial import unittest + +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen +from scrapy.utils.defer import deferred_f_from_coro_f + + +class AsyncgenUtilsTest(unittest.TestCase): + @deferred_f_from_coro_f + async def test_as_async_generator(self): + ag = as_async_generator(range(42)) + results = [] + async for i in ag: + results.append(i) + self.assertEqual(results, list(range(42))) + + @deferred_f_from_coro_f + async def test_collect_asyncgen(self): + ag = as_async_generator(range(42)) + results = await collect_asyncgen(ag) + self.assertEqual(results, list(range(42)))