Add a test for an async callbacks that returns requests.

This commit is contained in:
Andrey Rakhmatullin 2020-02-07 17:14:52 +05:00
parent 4f31c3ce01
commit 31f6c7112f
2 changed files with 30 additions and 2 deletions

View File

@ -117,6 +117,24 @@ class AsyncDefAsyncioReturnSpider(SimpleSpider):
return [{'id': 1}, {'id': 2}]
class AsyncDefAsyncioReqsReturnSpider(SimpleSpider):
name = 'asyncdef_asyncio_reqs_return'
async def parse(self, response):
await asyncio.sleep(0.2)
req_id = response.meta.get('req_id', 0)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d, req_id %d" % (status, req_id))
if req_id > 0:
return
reqs = []
for i in range(1, 3):
req = Request(self.start_urls[0], dont_filter=True, meta={'req_id': i})
reqs.append(req)
return reqs
class ItemSpider(FollowAllSpider):
name = 'item'

View File

@ -13,7 +13,8 @@ from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider,
AsyncDefAsyncioReqsReturnSpider)
class CrawlTestCase(TestCase):
@ -330,7 +331,7 @@ with multiples lines
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_list(self):
def test_async_def_asyncio_parse_items_list(self):
items = []
def _on_item_scraped(item):
@ -343,3 +344,12 @@ with multiples lines
self.assertIn("Got response 200", str(log))
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_reqs_list(self):
crawler = self.runner.create_crawler(AsyncDefAsyncioReqsReturnSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
for req_id in range(3):
self.assertIn("Got response 200, req_id %d" % req_id, str(log))