RomanNum3ral
/
scrapy
mirror of https://github.com/scrapy/scrapy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Also test non-default async callbacks.

This commit is contained in:
Andrey Rakhmatullin 2020-02-11 00:57:58 +05:00
parent 4c6993f264
commit 36dcf90184
2 changed files with 15 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
tests/py36/_test_crawl.py
View File

@ -32,12 +32,14 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
following_reqs = 3 following_reqs = 3
depth = 2 depth = 2
def _get_req(self, index): def _get_req(self, index, cb=None):
return Request(self.mockserver.url("/status?n=200&request=%d" % index), return Request(self.mockserver.url("/status?n=200&request=%d" % index),
meta={'index': index}) meta={'index': index},
dont_filter=True,
callback=cb)
def start_requests(self): def start_requests(self):
for i in range(self.initial_reqs): for i in range(1, self.initial_reqs + 1):
yield self._get_req(i) yield self._get_req(i)
async def parse(self, response): async def parse(self, response):
@ -46,5 +48,10 @@ class AsyncDefAsyncioGenComplexSpider(SimpleSpider):
if index < 10 ** self.depth: if index < 10 ** self.depth:
for new_index in range(10 * index, 10 * index + self.following_reqs): for new_index in range(10 * index, 10 * index + self.following_reqs):
yield self._get_req(new_index) yield self._get_req(new_index)
yield self._get_req(index, cb=self.parse2)
await asyncio.sleep(0.1) await asyncio.sleep(0.1)
yield {'index': index + 5} yield {'index': index + 5}
async def parse2(self, response):
await asyncio.sleep(0.1)
yield {'index2': response.meta['index']}

7
tests/test_crawl.py
View File

@ -392,9 +392,12 @@ with multiples lines
crawler.signals.connect(_on_item_scraped, signals.item_scraped) crawler.signals.connect(_on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver) yield crawler.crawl(mockserver=self.mockserver)
itemcount = crawler.stats.get_value('item_scraped_count') itemcount = crawler.stats.get_value('item_scraped_count')
self.assertEqual(itemcount, 80) self.assertEqual(itemcount, 156)
for i in [0, 3, 21, 22, 207, 311]: # some random items # some random items
for i in [1, 4, 21, 22, 207, 311]:
self.assertIn({'index': i}, items) self.assertIn({'index': i}, items)
for i in [10, 30, 122]:
self.assertIn({'index2': i}, items)
@mark.only_asyncio() @mark.only_asyncio()
@defer.inlineCallbacks @defer.inlineCallbacks