From 36dcf901849014d7db00a0294ed86c6cc79b5cc6 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Tue, 11 Feb 2020 00:57:58 +0500 Subject: [PATCH] Also test non-default async callbacks. --- tests/py36/_test_crawl.py | 13 ++++++++++--- tests/test_crawl.py | 7 +++++-- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/py36/_test_crawl.py b/tests/py36/_test_crawl.py index 74c7daf53..162a53760 100644 --- a/tests/py36/_test_crawl.py +++ b/tests/py36/_test_crawl.py @@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): following_reqs = 3 depth = 2 - def _get_req(self, index): + def _get_req(self, index, cb=None): return Request(self.mockserver.url("/status?n=200&request=%d" % index), - meta={'index': index}) + meta={'index': index}, + dont_filter=True, + callback=cb) def start_requests(self): - for i in range(self.initial_reqs): + for i in range(1, self.initial_reqs + 1): yield self._get_req(i) async def parse(self, response): @@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider): if index < 10 ** self.depth: for new_index in range(10 * index, 10 * index + self.following_reqs): yield self._get_req(new_index) + yield self._get_req(index, cb=self.parse2) await asyncio.sleep(0.1) yield {'index': index + 5} + + async def parse2(self, response): + await asyncio.sleep(0.1) + yield {'index2': response.meta['index']} diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 626000147..64819acb6 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -392,9 +392,12 @@ with multiples lines crawler.signals.connect(_on_item_scraped, signals.item_scraped) yield crawler.crawl(mockserver=self.mockserver) itemcount = crawler.stats.get_value('item_scraped_count') - self.assertEqual(itemcount, 80) - for i in [0, 3, 21, 22, 207, 311]: # some random items + self.assertEqual(itemcount, 156) + # some random items + for i in [1, 4, 21, 22, 207, 311]: self.assertIn({'index': i}, items) + for i in [10, 30, 122]: + self.assertIn({'index2': i}, items) @mark.only_asyncio() @defer.inlineCallbacks