From 548281960b9c4e46025490149184d32cfde790cd Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 22 May 2020 20:02:06 +0500 Subject: [PATCH] Remove process_iterable_helper from this PR. --- scrapy/utils/asyncgen.py | 11 ---- scrapy/utils/middlewares.py | 25 --------- tests/test_spidermiddleware.py | 62 ---------------------- tests/test_utils_middlewares.py | 93 --------------------------------- 4 files changed, 191 deletions(-) delete mode 100644 scrapy/utils/middlewares.py delete mode 100644 tests/test_utils_middlewares.py diff --git a/scrapy/utils/asyncgen.py b/scrapy/utils/asyncgen.py index 9c3ac83d5..6130e81be 100644 --- a/scrapy/utils/asyncgen.py +++ b/scrapy/utils/asyncgen.py @@ -18,14 +18,3 @@ async def as_async_generator(it): else: for r in it: yield r - - -async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - async for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o diff --git a/scrapy/utils/middlewares.py b/scrapy/utils/middlewares.py deleted file mode 100644 index fdde3c9a2..000000000 --- a/scrapy/utils/middlewares.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding: utf-8 -from scrapy.utils.defer import isasyncgen - - -def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - for o in it: - if in_predicate and not in_predicate(o): - continue - if processor is not None: - o = processor(o) - if out_predicate and not out_predicate(o): - continue - yield o - - -def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None): - """ - For each item in the iterable: skips it if in_predicate is False, applies processor, - skips the result if out_predicate is False, else yields it. - """ - if isasyncgen(it): - from scrapy.utils.asyncgen import process_async_iterable_helper # Python 3.5 limitation - return process_async_iterable_helper(it, in_predicate, out_predicate, processor) - else: - return process_normal_iterable_helper(it, in_predicate, out_predicate, processor) diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 44c35b506..7adf9a84b 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -11,7 +11,6 @@ from scrapy.spiders import Spider from scrapy.http import Request, Response from scrapy.exceptions import _InvalidOutput from scrapy.utils.defer import deferred_from_coro -from scrapy.utils.middlewares import process_iterable_helper from scrapy.utils.test import get_crawler from scrapy.core.spidermw import SpiderMiddlewareManager from tests.test_engine import StartRequestsAsyncDefSpider @@ -116,11 +115,6 @@ class ProcessStartRequestsSimpleMiddleware: yield r -class ProcessStartRequestsUniversalMiddleware: - def process_start_requests(self, start_requests, spider): - return process_iterable_helper(start_requests) - - class ProcessStartRequestsSimple(TestCase): """ process_start_requests tests for simple start_requests""" @@ -190,39 +184,6 @@ class ProcessStartRequestsSimple(TestCase): self.assertTrue(inspect.isgenerator(processed_start_requests)) self.assertAsyncGeneratorNotIterable(processed_start_requests) - @defer.inlineCallbacks - def test_universal(self): - """ Universal mw """ - yield self._test_simple_base(ProcessStartRequestsUniversalMiddleware) - - @defer.inlineCallbacks - def test_universal_simple(self): - """ Universal mw -> simple mw """ - yield self._test_simple_base(ProcessStartRequestsSimpleMiddleware, - ProcessStartRequestsUniversalMiddleware) - - @defer.inlineCallbacks - def test_simple_universal(self): - """ Simple mw -> universal mw """ - yield self._test_simple_base(ProcessStartRequestsUniversalMiddleware, - ProcessStartRequestsSimpleMiddleware) - - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") - @defer.inlineCallbacks - def test_universal_asyncgen(self): - """ Universal mw -> asyncgen mw """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware - yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware, - ProcessStartRequestsUniversalMiddleware) - - @mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") - @defer.inlineCallbacks - def test_asyncgen_universal(self): - """ Asyncgen mw -> universal mw """ - from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware - yield self._test_asyncgen_base(ProcessStartRequestsUniversalMiddleware, - ProcessStartRequestsAsyncGenMiddleware) - class ProcessStartRequestsAsyncDef(ProcessStartRequestsSimple): """ process_start_requests tests for async def start_requests """ @@ -256,26 +217,3 @@ class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple): ProcessStartRequestsSimpleMiddleware) self.assertTrue(inspect.isasyncgen(processed_start_requests)) self.assertAsyncGeneratorNotIterable(processed_start_requests) - - @defer.inlineCallbacks - def test_universal(self): - """ Universal mw """ - yield self._test_asyncgen_base(ProcessStartRequestsUniversalMiddleware) - - @defer.inlineCallbacks - def test_universal_simple(self): - """ Universal mw -> simple mw; cannot work """ - processed_start_requests = yield self._get_processed_start_requests( - ProcessStartRequestsSimpleMiddleware, - ProcessStartRequestsUniversalMiddleware) - self.assertTrue(inspect.isgenerator(processed_start_requests)) - self.assertAsyncGeneratorNotIterable(processed_start_requests) - - @defer.inlineCallbacks - def test_simple_universal(self): - """ Simple mw -> universal mw; cannot work """ - processed_start_requests = yield self._get_processed_start_requests( - ProcessStartRequestsUniversalMiddleware, - ProcessStartRequestsSimpleMiddleware) - self.assertTrue(inspect.isgenerator(processed_start_requests)) - self.assertAsyncGeneratorNotIterable(processed_start_requests) diff --git a/tests/test_utils_middlewares.py b/tests/test_utils_middlewares.py deleted file mode 100644 index fb4d54fb5..000000000 --- a/tests/test_utils_middlewares.py +++ /dev/null @@ -1,93 +0,0 @@ -import collections -import sys - -from pytest import mark -from twisted.trial import unittest - -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.middlewares import process_iterable_helper - - -def predicate1(o): - return bool(o % 2) - - -def predicate2(o): - return o < 10 - - -def processor(o): - return o * 2 - - -class ProcessIterableHelperNormalTest(unittest.TestCase): - - def test_normal_in_predicate(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 3]) - - def test_normal_out_predicate(self): - iterable1 = iter([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - def test_normal_processor(self): - iterable1 = iter([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - def test_normal_combined(self): - iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.Iterable) - list2 = list(iterable2) - self.assertEqual(list2, [2, 6]) - - -@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher") -class ProcessIterableHelperAsyncTest(unittest.TestCase): - - @deferred_f_from_coro_f - async def test_async_in_predicate(self): - from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 3]) - - @deferred_f_from_coro_f - async def test_async_out_predicate(self): - from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen - iterable1 = as_async_generator([1, 2, 10, 3, 15]) - iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [1, 2, 3]) - - @deferred_f_from_coro_f - async def test_async_processor(self): - from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen - iterable1 = as_async_generator([1, 2, 3]) - iterable2 = process_iterable_helper(iterable1, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 4, 6]) - - @deferred_f_from_coro_f - async def test_async_combined(self): - from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen - iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15]) - iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1, - out_predicate=predicate2, processor=processor) - self.assertIsInstance(iterable2, collections.abc.AsyncIterable) - list2 = await collect_asyncgen(iterable2) - self.assertEqual(list2, [2, 6])