From c0b2488213ca853fc00cc4d1b085a1ddad2c9f65 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 26 Aug 2020 19:23:15 +0500 Subject: [PATCH] Drop python 3.5 support from this PR. --- conftest.py | 8 ---- docs/topics/coroutines.rst | 4 +- docs/topics/spider-middleware.rst | 13 +++--- scrapy/core/engine.py | 7 ++-- scrapy/crawler.py | 5 +-- scrapy/utils/asyncgen.py | 3 -- scrapy/utils/defer.py | 7 ---- scrapy/utils/spider.py | 9 ++-- tests/middlewares.py | 10 ++--- tests/py36/_test_crawl.py | 63 ---------------------------- tests/py36/_test_engine.py | 19 --------- tests/py36/_test_spidermiddleware.py | 8 ---- tests/py36/_test_utils_asyncgen.py | 3 -- tests/py36/middlewares.py | 3 -- tests/spiders.py | 53 +++++++++++++++++++++++ tests/test_crawl.py | 13 +++--- tests/test_engine.py | 19 ++++++--- tests/test_spidermiddleware.py | 21 ++++------ tests/test_utils_asyncgen.py | 16 +++---- 19 files changed, 110 insertions(+), 174 deletions(-) delete mode 100644 tests/py36/_test_crawl.py delete mode 100644 tests/py36/_test_engine.py delete mode 100644 tests/py36/_test_spidermiddleware.py delete mode 100644 tests/py36/_test_utils_asyncgen.py delete mode 100644 tests/py36/middlewares.py diff --git a/conftest.py b/conftest.py index 6ecf7304b..be97b7714 100644 --- a/conftest.py +++ b/conftest.py @@ -1,4 +1,3 @@ -import sys from pathlib import Path import pytest @@ -15,15 +14,8 @@ collect_ignore = [ *_py_files("tests/CrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::CrawlerRunnerSubprocess *_py_files("tests/CrawlerRunner"), - # Py36-only parts of respective tests - *_py_files("tests/py36"), ] -if sys.version_info < (3, 6): - # not importable on 3.5 - collect_ignore.append("scrapy/utils/asyncgen.py") - collect_ignore.append("scrapy/utils/py36.py") - for line in open('tests/ignores.txt'): file_path = line.strip() if file_path and file_path[0] != '#': diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 0cd607bd8..43fee1a7b 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -137,9 +137,7 @@ A better behavior, which should treat requests from requests, is enabled when :meth:`~scrapy.spiders.Spider.start_requests` is an async function (declared using ``async def``). It doesn't need to contain ``await`` for this to work, so if you want the new queue behavior, you can just -change ``def`` to ``async def``. Note though, that using ``yield`` statements -inside an async function makes it an async generator which are only supported -since Python 3.6. +change ``def`` to ``async def``. Usage ===== diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst index 262d2266d..671bffa8a 100644 --- a/docs/topics/spider-middleware.rst +++ b/docs/topics/spider-middleware.rst @@ -159,13 +159,12 @@ object gives you access, for example, to the :ref:`settings `. It receives an iterable (in the ``start_requests`` parameter) and must return another iterable of :class:`~scrapy.http.Request` objects. - When using Python 3.6+ ``start_requests`` may be an - :term:`python:asynchronous iterable`. To support such spiders, this - middleware method should be an :term:`python:asynchronous generator`, - which should support both synchronous and asynchronous - ``start_requests``. :func:`scrapy.utils.asyncgen.as_async_generator` - can be used to convert both kinds of ``start_requests`` into an - asynchronous iterable. + ``start_requests`` may be an :term:`python:asynchronous iterable`. To + support such spiders, this middleware method should be an + :term:`python:asynchronous generator`, which should support both + synchronous and asynchronous ``start_requests``. + :func:`scrapy.utils.asyncgen.as_async_generator` can be used to convert + both kinds of ``start_requests`` into an asynchronous iterable. .. note:: When implementing this method in your spider middleware, you should always return an iterable (that follows the input one) and diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py index ccec3a123..afdd31ff3 100644 --- a/scrapy/core/engine.py +++ b/scrapy/core/engine.py @@ -4,6 +4,7 @@ This is the Scrapy engine which controls the Scheduler, Downloader and Spiders. For more information see docs/topics/architecture.rst """ +import inspect import logging from time import time @@ -14,7 +15,7 @@ from scrapy import signals from scrapy.core.scraper import Scraper from scrapy.exceptions import DontCloseSpider from scrapy.http import Response, Request -from scrapy.utils.defer import deferred_from_coro, _isasyncgen +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import load_object from scrapy.utils.reactor import CallLaterOnce from scrapy.utils.log import logformatter_adapter, failure_to_exc_info @@ -27,7 +28,7 @@ class Slot: def __init__(self, start_requests, close_if_idle, nextcall, scheduler, new_queue_behavior=False): self.closing = False self.inprogress = set() # requests in progress - if _isasyncgen(start_requests): + if inspect.isasyncgen(start_requests): self.start_requests = start_requests else: self.start_requests = iter(start_requests) @@ -126,7 +127,7 @@ class ExecutionEngine: @defer.inlineCallbacks def _schedule_next_req(self, spider, slot): try: - if _isasyncgen(slot.start_requests): + if inspect.isasyncgen(slot.start_requests): request = yield deferred_from_coro(slot.start_requests.__anext__()) else: request = next(slot.start_requests) diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9303e0ec6..64514fc3e 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -98,8 +98,7 @@ class Crawler: raise def call_start_requests(self): - if hasattr(inspect, 'isasyncgenfunction') and inspect.isasyncgenfunction(self.spider.start_requests): - # requires Python 3.6+ + if inspect.isasyncgenfunction(self.spider.start_requests): return self.spider.start_requests().__aiter__() elif inspect.iscoroutinefunction(self.spider.start_requests): return deferred_from_coro(self.spider.start_requests()) @@ -108,7 +107,7 @@ class Crawler: @staticmethod def is_start_requests_async(start_requests_function): - if hasattr(inspect, 'isasyncgenfunction') and inspect.isasyncgenfunction(start_requests_function): + if inspect.isasyncgenfunction(start_requests_function): return True if inspect.iscoroutinefunction(start_requests_function): return True diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 6130e81be..db2173f85 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -1,6 +1,3 @@ -""" -Helpers using Python 3.6+ async generator syntax (ignore SyntaxError on import). -""" import collections diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 7daee3ebb..a3950db75 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -173,10 +173,3 @@ def maybeDeferred_coro(f, *args, **kw): return defer.fail(result) else: return defer.succeed(result) - - -def _isasyncgen(o): - """ Returns inspect.isasyncgen() result if it's available (requires Python 3.6), - otherwise returns False. - """ - return hasattr(inspect, 'isasyncgen') and inspect.isasyncgen(o) diff --git a/scrapy/utils/spider.py b/scrapy/utils/spider.py index 80a008d80..987615d17 100644 --- a/scrapy/utils/spider.py +++ b/scrapy/utils/spider.py @@ -2,19 +2,16 @@ import inspect import logging from scrapy.spiders import Spider -from scrapy.utils.defer import deferred_from_coro, _isasyncgen +from scrapy.utils.asyncgen import collect_asyncgen +from scrapy.utils.defer import deferred_from_coro from scrapy.utils.misc import arg_to_iter -try: - from scrapy.utils.asyncgen import collect_asyncgen -except SyntaxError: - collect_asyncgen = None logger = logging.getLogger(__name__) def iterate_spider_output(result): - if collect_asyncgen and _isasyncgen(result): + if inspect.isasyncgen(result): d = deferred_from_coro(collect_asyncgen(result)) d.addCallback(iterate_spider_output) return d diff --git a/tests/middlewares.py b/tests/middlewares.py index 2f8f7dc09..6ccb9f828 100644 --- a/tests/middlewares.py +++ b/tests/middlewares.py @@ -1,15 +1,15 @@ +import inspect + from scrapy.http import Request -from scrapy.utils.defer import _isasyncgen class RequestInOrderMiddleware: def process_spider_output(self, response, result, spider): return (self._preserve_in_order(r, spider) for r in result or ()) - def process_start_requests(self, start_requests, spider): - if _isasyncgen(start_requests): - from tests.py36.middlewares import RequestInOrderMiddleware_process_start_requests - return RequestInOrderMiddleware_process_start_requests(self._preserve_in_order, start_requests, spider) + async def process_start_requests(self, start_requests, spider): + if inspect.isasyncgen(start_requests): + return (self._preserve_in_order(r, spider) async for r in start_requests or ()) else: return (self._preserve_in_order(r, spider) for r in start_requests or ()) diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py deleted file mode 100644 index 678b19647..000000000 --- a/tests/py36/_test_crawl.py +++ /dev/null @@ -1,63 +0,0 @@ -import asyncio - -from scrapy import Request -from tests.spiders import SimpleSpider, YieldingRequestsSpider - - -class AsyncDefAsyncioGenSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen' - - async def parse(self, response): - await asyncio.sleep(0.2) - yield {'foo': 42} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenLoopSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_loop' - - async def parse(self, response): - for i in range(10): - await asyncio.sleep(0.1) - yield {'foo': i} - self.logger.info("Got response %d" % response.status) - - -class AsyncDefAsyncioGenComplexSpider(SimpleSpider): - - name = 'asyncdef_asyncio_gen_complex' - initial_reqs = 4 - following_reqs = 3 - depth = 2 - - def _get_req(self, index, cb=None): - return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}, - dont_filter=True, - callback=cb) - - def start_requests(self): - for i in range(1, self.initial_reqs + 1): - yield self._get_req(i) - - async def parse(self, response): - index = response.meta['index'] - yield {'index': index} - if index < 10 ** self.depth: - for new_index in range(10 * index, 10 * index + self.following_reqs): - yield self._get_req(new_index) - yield self._get_req(index, cb=self.parse2) - await asyncio.sleep(0.1) - yield {'index': index + 5} - - async def parse2(self, response): - await asyncio.sleep(0.1) - yield {'index2': response.meta['index']} - - -class EagerAsyncGenSpider(YieldingRequestsSpider): - async def start_requests(self): - for r in super().start_requests(): - yield r diff --git a/tests/py36/_test_engine.py b/tests/py36/_test_engine.py deleted file mode 100644 index 0d02ff9f0..000000000 --- a/tests/py36/_test_engine.py +++ /dev/null @@ -1,19 +0,0 @@ -#coding: utf-8 -import asyncio - -from scrapy import Request - -from tests.test_engine import TestSpider - - -class StartRequestsAsyncGenSpider(TestSpider): - async def start_requests(self): - for url in self.start_urls: - yield Request(url, dont_filter=True) - - -class StartRequestsAsyncGenAsyncioSpider(TestSpider): - async def start_requests(self): - for url in self.start_urls: - yield Request(url, dont_filter=True) - await asyncio.sleep(1) diff --git a/tests/py36/_test_spidermiddleware.py b/tests/py36/_test_spidermiddleware.py deleted file mode 100644 index be49aadfb..000000000 --- a/tests/py36/_test_spidermiddleware.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding: utf-8 -from scrapy.utils.asyncgen import as_async_generator - - -class ProcessStartRequestsAsyncGenMiddleware: - async def process_start_requests(self, start_requests, spider): - async for r in as_async_generator(start_requests): - yield r diff --git a/tests/py36/_test_utils_asyncgen.py b/tests/py36/_test_utils_asyncgen.py deleted file mode 100644 index d66e28487..000000000 --- a/tests/py36/_test_utils_asyncgen.py +++ /dev/null @@ -1,3 +0,0 @@ -async def async_gen(): - for i in range(3): - yield i diff --git a/tests/py36/middlewares.py b/tests/py36/middlewares.py deleted file mode 100644 index 182610f30..000000000 --- a/tests/py36/middlewares.py +++ /dev/null @@ -1,3 +0,0 @@ -async def RequestInOrderMiddleware_process_start_requests(f, start_requests, spider): - async for r in start_requests or (): - yield f(r, spider) diff --git a/tests/spiders.py b/tests/spiders.py index 3855835c1..af12ead82 100644 --- a/tests/spiders.py +++ b/tests/spiders.py @@ -148,6 +148,59 @@ class AsyncDefAsyncioReqsReturnSpider(SimpleSpider): return reqs +class AsyncDefAsyncioGenSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen' + + async def parse(self, response): + await asyncio.sleep(0.2) + yield {'foo': 42} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenLoopSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_loop' + + async def parse(self, response): + for i in range(10): + await asyncio.sleep(0.1) + yield {'foo': i} + self.logger.info("Got response %d" % response.status) + + +class AsyncDefAsyncioGenComplexSpider(SimpleSpider): + + name = 'asyncdef_asyncio_gen_complex' + initial_reqs = 4 + following_reqs = 3 + depth = 2 + + def _get_req(self, index, cb=None): + return Request(self.mockserver.url("/status?n=200&request=%d" % index), + meta={'index': index}, + dont_filter=True, + callback=cb) + + def start_requests(self): + for i in range(1, self.initial_reqs + 1): + yield self._get_req(i) + + async def parse(self, response): + index = response.meta['index'] + yield {'index': index} + if index < 10 ** self.depth: + for new_index in range(10 * index, 10 * index + self.following_reqs): + yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) + await asyncio.sleep(0.1) + yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} + + class ItemSpider(FollowAllSpider): name = 'item' diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 56b286e03..d8c0ceb63 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -20,6 +20,9 @@ from scrapy.http.response import Response from scrapy.utils.python import to_unicode from tests.mockserver import MockServer from tests.spiders import ( + AsyncDefAsyncioGenComplexSpider, + AsyncDefAsyncioGenLoopSpider, + AsyncDefAsyncioGenSpider, AsyncDefAsyncioReqsReturnSpider, AsyncDefAsyncioReturnSingleElementSpider, AsyncDefAsyncioReturnSpider, @@ -210,10 +213,13 @@ class CrawlTestCase(TestCase): yield self._test_start_requests_eagerness(EagerAsyncDefSpider) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @defer.inlineCallbacks def test_start_requests_eagerness_asyncgen(self): - from tests.py36._test_crawl import EagerAsyncGenSpider + class EagerAsyncGenSpider(YieldingRequestsSpider): + async def start_requests(self): + for r in super().start_requests(): + yield r + yield self._test_start_requests_eagerness(EagerAsyncGenSpider) @defer.inlineCallbacks @@ -475,7 +481,6 @@ class CrawlSpiderTestCase(TestCase): @mark.only_asyncio() @defer.inlineCallbacks def test_async_def_asyncgen_parse(self): - from tests.py36._test_crawl import AsyncDefAsyncioGenSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenSpider) with LogCapture() as log: yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver) @@ -492,7 +497,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenLoopSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenLoopSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) with LogCapture() as log: @@ -512,7 +516,6 @@ class CrawlSpiderTestCase(TestCase): def _on_item_scraped(item): items.append(item) - from tests.py36._test_crawl import AsyncDefAsyncioGenComplexSpider crawler = self.runner.create_crawler(AsyncDefAsyncioGenComplexSpider) crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) diff --git a/tests/test_engine.py b/tests/test_engine.py index 4dc7288fc..4634187ce 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -9,7 +9,7 @@ module with the ``runserver`` argument:: python test_engine.py runserver """ - +import asyncio import os import re import sys @@ -123,6 +123,19 @@ class StartRequestsAsyncDefSpider(TestSpider): return [Request(url, dont_filter=True) for url in self.start_urls] +class StartRequestsAsyncGenSpider(TestSpider): + async def start_requests(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + + +class StartRequestsAsyncGenAsyncioSpider(TestSpider): + async def start_requests(self): + for url in self.start_urls: + yield Request(url, dont_filter=True) + await asyncio.sleep(1) + + def start_test_site(debug=False): root_dir = os.path.join(tests_datadir, "test_site") r = static.File(root_dir) @@ -272,19 +285,15 @@ class EngineTest(unittest.TestCase): yield self.run.run() self._assert_visited_urls() - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @defer.inlineCallbacks def test_crawler_startrequests_asyncgen(self): - from tests.py36._test_engine import StartRequestsAsyncGenSpider self.run = CrawlerRun(StartRequestsAsyncGenSpider) yield self.run.run() self._assert_visited_urls() - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @mark.only_asyncio() @defer.inlineCallbacks def test_crawler_startrequests_asyncgen_asyncio(self): - from tests.py36._test_engine import StartRequestsAsyncGenAsyncioSpider self.run = CrawlerRun(StartRequestsAsyncGenAsyncioSpider) yield self.run.run() self._assert_visited_urls() diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 6f931e8f8..939423e3b 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -1,9 +1,7 @@ import collections import inspect -import sys from unittest import mock -from pytest import mark from twisted.internet import defer from twisted.trial.unittest import TestCase from twisted.python.failure import Failure @@ -11,10 +9,11 @@ from twisted.python.failure import Failure from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_from_coro from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager -from tests.test_engine import StartRequestsAsyncDefSpider +from tests.test_engine import StartRequestsAsyncDefSpider, StartRequestsAsyncGenSpider class SpiderMiddlewareTestCase(TestCase): @@ -121,6 +120,12 @@ class ProcessStartRequestsAsyncDefMiddleware: return start_requests +class ProcessStartRequestsAsyncGenMiddleware: + async def process_start_requests(self, start_requests, spider): + async for r in as_async_generator(start_requests): + yield r + + class ProcessStartRequestsSimple(TestCase): """ process_start_requests tests for simple start_requests""" @@ -152,7 +157,6 @@ class ProcessStartRequestsSimple(TestCase): @defer.inlineCallbacks def _test_asyncgen_base(self, *mw_classes): - from scrapy.utils.asyncgen import collect_asyncgen processed_start_requests = yield self._get_processed_start_requests(*mw_classes) self.assertTrue(inspect.isasyncgen(processed_start_requests)) start_requests_list = yield deferred_from_coro(collect_asyncgen(processed_start_requests)) @@ -169,26 +173,20 @@ class ProcessStartRequestsSimple(TestCase): """ Async def mw """ yield self._test_simple_base(ProcessStartRequestsAsyncDefMiddleware) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @defer.inlineCallbacks def test_asyncgen(self): """ Asyncgen mw """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @defer.inlineCallbacks def test_simple_asyncgen(self): """ Simple mw -> asyncgen mw """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware, ProcessStartRequestsSimpleMiddleware) - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") @defer.inlineCallbacks def test_asyncgen_simple(self): """ Asyncgen mw -> simple mw; cannot work """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware processed_start_requests = yield self._get_processed_start_requests( ProcessStartRequestsSimpleMiddleware, ProcessStartRequestsAsyncGenMiddleware) @@ -202,13 +200,11 @@ class ProcessStartRequestsAsyncDef(ProcessStartRequestsSimple): spider_cls = StartRequestsAsyncDefSpider -@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple): """ process_start_requests tests for async generator start_requests """ def __init__(self, methodName='runTest'): super().__init__(methodName) - from tests.py36._test_engine import StartRequestsAsyncGenSpider self.spider_cls = StartRequestsAsyncGenSpider @defer.inlineCallbacks @@ -227,7 +223,6 @@ class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple): @defer.inlineCallbacks def test_simple_asyncgen(self): """ Simple mw -> asyncgen mw; cannot work """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware processed_start_requests = yield self._get_processed_start_requests( ProcessStartRequestsAsyncGenMiddleware, ProcessStartRequestsSimpleMiddleware) diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index 1e4dfbaf9..ce7e66fc1 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,17 +1,18 @@ -import sys - -from pytest import mark from twisted.trial import unittest +from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import deferred_f_from_coro_f -@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") +async def async_gen(): + for i in range(3): + yield i + + class AsyncGeneratorTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator_simple(self): - from scrapy.utils.asyncgen import as_async_generator gen = (i for i in range(3)) results = [] async for i in as_async_generator(gen): @@ -20,7 +21,6 @@ class AsyncGeneratorTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator_list(self): - from scrapy.utils.asyncgen import as_async_generator L = [i for i in range(3)] results = [] async for i in as_async_generator(L): @@ -29,8 +29,6 @@ class AsyncGeneratorTest(unittest.TestCase): @deferred_f_from_coro_f async def test_as_async_generator_async(self): - from scrapy.utils.asyncgen import as_async_generator - from tests.py36._test_utils_asyncgen import async_gen results = [] async for i in as_async_generator(async_gen()): results.append(i) @@ -38,7 +36,5 @@ class AsyncGeneratorTest(unittest.TestCase): @deferred_f_from_coro_f async def test_collect_asyncgen(self): - from scrapy.utils.asyncgen import collect_asyncgen - from tests.py36._test_utils_asyncgen import async_gen results = await collect_asyncgen(async_gen()) self.assertEqual(results, [0, 1, 2])