mirror of https://github.com/scrapy/scrapy.git
Remove process_iterable_helper from this PR.
This commit is contained in:
Andrey Rakhmatullin
parent
3bacf8e421
commit
548281960b
|
|
@ -18,14 +18,3 @@ async def as_async_generator(it):
|
|||
else:
|
||||
for r in it:
|
||||
yield r
|
||||
|
||||
|
||||
async def process_async_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
|
||||
async for o in it:
|
||||
if in_predicate and not in_predicate(o):
|
||||
continue
|
||||
if processor is not None:
|
||||
o = processor(o)
|
||||
if out_predicate and not out_predicate(o):
|
||||
continue
|
||||
yield o
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
# coding: utf-8
|
||||
from scrapy.utils.defer import isasyncgen
|
||||
|
||||
|
||||
def process_normal_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
|
||||
for o in it:
|
||||
if in_predicate and not in_predicate(o):
|
||||
continue
|
||||
if processor is not None:
|
||||
o = processor(o)
|
||||
if out_predicate and not out_predicate(o):
|
||||
continue
|
||||
yield o
|
||||
|
||||
|
||||
def process_iterable_helper(it, in_predicate=None, out_predicate=None, processor=None):
|
||||
"""
|
||||
For each item in the iterable: skips it if in_predicate is False, applies processor,
|
||||
skips the result if out_predicate is False, else yields it.
|
||||
"""
|
||||
if isasyncgen(it):
|
||||
from scrapy.utils.asyncgen import process_async_iterable_helper # Python 3.5 limitation
|
||||
return process_async_iterable_helper(it, in_predicate, out_predicate, processor)
|
||||
else:
|
||||
return process_normal_iterable_helper(it, in_predicate, out_predicate, processor)
|
||||
|
|
@ -11,7 +11,6 @@ from scrapy.spiders import Spider
|
|||
from scrapy.http import Request, Response
|
||||
from scrapy.exceptions import _InvalidOutput
|
||||
from scrapy.utils.defer import deferred_from_coro
|
||||
from scrapy.utils.middlewares import process_iterable_helper
|
||||
from scrapy.utils.test import get_crawler
|
||||
from scrapy.core.spidermw import SpiderMiddlewareManager
|
||||
from tests.test_engine import StartRequestsAsyncDefSpider
|
||||
|
|
@ -116,11 +115,6 @@ class ProcessStartRequestsSimpleMiddleware:
|
|||
yield r
|
||||
|
||||
|
||||
class ProcessStartRequestsUniversalMiddleware:
|
||||
def process_start_requests(self, start_requests, spider):
|
||||
return process_iterable_helper(start_requests)
|
||||
|
||||
|
||||
class ProcessStartRequestsSimple(TestCase):
|
||||
""" process_start_requests tests for simple start_requests"""
|
||||
|
||||
|
|
@ -190,39 +184,6 @@ class ProcessStartRequestsSimple(TestCase):
|
|||
self.assertTrue(inspect.isgenerator(processed_start_requests))
|
||||
self.assertAsyncGeneratorNotIterable(processed_start_requests)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_universal(self):
|
||||
""" Universal mw """
|
||||
yield self._test_simple_base(ProcessStartRequestsUniversalMiddleware)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_universal_simple(self):
|
||||
""" Universal mw -> simple mw """
|
||||
yield self._test_simple_base(ProcessStartRequestsSimpleMiddleware,
|
||||
ProcessStartRequestsUniversalMiddleware)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_simple_universal(self):
|
||||
""" Simple mw -> universal mw """
|
||||
yield self._test_simple_base(ProcessStartRequestsUniversalMiddleware,
|
||||
ProcessStartRequestsSimpleMiddleware)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@defer.inlineCallbacks
|
||||
def test_universal_asyncgen(self):
|
||||
""" Universal mw -> asyncgen mw """
|
||||
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
|
||||
yield self._test_asyncgen_base(ProcessStartRequestsAsyncGenMiddleware,
|
||||
ProcessStartRequestsUniversalMiddleware)
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
@defer.inlineCallbacks
|
||||
def test_asyncgen_universal(self):
|
||||
""" Asyncgen mw -> universal mw """
|
||||
from tests.py36._test_spidermiddleware import ProcessStartRequestsAsyncGenMiddleware
|
||||
yield self._test_asyncgen_base(ProcessStartRequestsUniversalMiddleware,
|
||||
ProcessStartRequestsAsyncGenMiddleware)
|
||||
|
||||
|
||||
class ProcessStartRequestsAsyncDef(ProcessStartRequestsSimple):
|
||||
""" process_start_requests tests for async def start_requests """
|
||||
|
|
@ -256,26 +217,3 @@ class ProcessStartRequestsAsyncGen(ProcessStartRequestsSimple):
|
|||
ProcessStartRequestsSimpleMiddleware)
|
||||
self.assertTrue(inspect.isasyncgen(processed_start_requests))
|
||||
self.assertAsyncGeneratorNotIterable(processed_start_requests)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_universal(self):
|
||||
""" Universal mw """
|
||||
yield self._test_asyncgen_base(ProcessStartRequestsUniversalMiddleware)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_universal_simple(self):
|
||||
""" Universal mw -> simple mw; cannot work """
|
||||
processed_start_requests = yield self._get_processed_start_requests(
|
||||
ProcessStartRequestsSimpleMiddleware,
|
||||
ProcessStartRequestsUniversalMiddleware)
|
||||
self.assertTrue(inspect.isgenerator(processed_start_requests))
|
||||
self.assertAsyncGeneratorNotIterable(processed_start_requests)
|
||||
|
||||
@defer.inlineCallbacks
|
||||
def test_simple_universal(self):
|
||||
""" Simple mw -> universal mw; cannot work """
|
||||
processed_start_requests = yield self._get_processed_start_requests(
|
||||
ProcessStartRequestsUniversalMiddleware,
|
||||
ProcessStartRequestsSimpleMiddleware)
|
||||
self.assertTrue(inspect.isgenerator(processed_start_requests))
|
||||
self.assertAsyncGeneratorNotIterable(processed_start_requests)
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
import collections
|
||||
import sys
|
||||
|
||||
from pytest import mark
|
||||
from twisted.trial import unittest
|
||||
|
||||
from scrapy.utils.defer import deferred_f_from_coro_f
|
||||
from scrapy.utils.middlewares import process_iterable_helper
|
||||
|
||||
|
||||
def predicate1(o):
|
||||
return bool(o % 2)
|
||||
|
||||
|
||||
def predicate2(o):
|
||||
return o < 10
|
||||
|
||||
|
||||
def processor(o):
|
||||
return o * 2
|
||||
|
||||
|
||||
class ProcessIterableHelperNormalTest(unittest.TestCase):
|
||||
|
||||
def test_normal_in_predicate(self):
|
||||
iterable1 = iter([1, 2, 3])
|
||||
iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
|
||||
self.assertIsInstance(iterable2, collections.abc.Iterable)
|
||||
list2 = list(iterable2)
|
||||
self.assertEqual(list2, [1, 3])
|
||||
|
||||
def test_normal_out_predicate(self):
|
||||
iterable1 = iter([1, 2, 10, 3, 15])
|
||||
iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
|
||||
self.assertIsInstance(iterable2, collections.abc.Iterable)
|
||||
list2 = list(iterable2)
|
||||
self.assertEqual(list2, [1, 2, 3])
|
||||
|
||||
def test_normal_processor(self):
|
||||
iterable1 = iter([1, 2, 3])
|
||||
iterable2 = process_iterable_helper(iterable1, processor=processor)
|
||||
self.assertIsInstance(iterable2, collections.abc.Iterable)
|
||||
list2 = list(iterable2)
|
||||
self.assertEqual(list2, [2, 4, 6])
|
||||
|
||||
def test_normal_combined(self):
|
||||
iterable1 = iter([1, 2, 10, 3, 6, 18, 5, 15])
|
||||
iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
|
||||
out_predicate=predicate2, processor=processor)
|
||||
self.assertIsInstance(iterable2, collections.abc.Iterable)
|
||||
list2 = list(iterable2)
|
||||
self.assertEqual(list2, [2, 6])
|
||||
|
||||
|
||||
@mark.skipif(sys.version_info < (3, 6), reason="Async generators require Python 3.6 or higher")
|
||||
class ProcessIterableHelperAsyncTest(unittest.TestCase):
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_in_predicate(self):
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
iterable1 = as_async_generator([1, 2, 3])
|
||||
iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1)
|
||||
self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
|
||||
list2 = await collect_asyncgen(iterable2)
|
||||
self.assertEqual(list2, [1, 3])
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_out_predicate(self):
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
iterable1 = as_async_generator([1, 2, 10, 3, 15])
|
||||
iterable2 = process_iterable_helper(iterable1, out_predicate=predicate2)
|
||||
self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
|
||||
list2 = await collect_asyncgen(iterable2)
|
||||
self.assertEqual(list2, [1, 2, 3])
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_processor(self):
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
iterable1 = as_async_generator([1, 2, 3])
|
||||
iterable2 = process_iterable_helper(iterable1, processor=processor)
|
||||
self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
|
||||
list2 = await collect_asyncgen(iterable2)
|
||||
self.assertEqual(list2, [2, 4, 6])
|
||||
|
||||
@deferred_f_from_coro_f
|
||||
async def test_async_combined(self):
|
||||
from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
|
||||
iterable1 = as_async_generator([1, 2, 10, 3, 6, 18, 5, 15])
|
||||
iterable2 = process_iterable_helper(iterable1, in_predicate=predicate1,
|
||||
out_predicate=predicate2, processor=processor)
|
||||
self.assertIsInstance(iterable2, collections.abc.AsyncIterable)
|
||||
list2 = await collect_asyncgen(iterable2)
|
||||
self.assertEqual(list2, [2, 6])
|
||||
Loading…
Reference in New Issue