Test returning items from an async def callback.

This commit is contained in:
Andrey Rakhmatullin 2020-01-30 16:17:06 +05:00
parent a91a13b443
commit cc825c21de
2 changed files with 29 additions and 1 deletions

View File

@ -106,6 +106,17 @@ class AsyncDefAsyncioSpider(SimpleSpider):
self.logger.info("Got response %d" % status)
class AsyncDefAsyncioReturnSpider(SimpleSpider):
name = 'asyncdef_asyncio_return'
async def parse(self, response):
await asyncio.sleep(0.2)
status = await get_from_asyncio_queue(response.status)
self.logger.info("Got response %d" % status)
return [{'id': 1}, {'id': 2}]
class ItemSpider(FollowAllSpider):
name = 'item'

View File

@ -6,13 +6,14 @@ from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial.unittest import TestCase
from scrapy import signals
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from scrapy.utils.python import to_unicode
from tests.mockserver import MockServer
from tests.spiders import (FollowAllSpider, DelaySpider, SimpleSpider, BrokenStartRequestsSpider,
SingleRequestSpider, DuplicateStartRequestsSpider, CrawlSpiderWithErrback,
AsyncDefSpider, AsyncDefAsyncioSpider)
AsyncDefSpider, AsyncDefAsyncioSpider, AsyncDefAsyncioReturnSpider)
class CrawlTestCase(TestCase):
@ -326,3 +327,19 @@ with multiples lines
with LogCapture() as log:
yield runner.join()
self.assertIn("Got response 200", str(log))
@mark.only_asyncio()
@defer.inlineCallbacks
def test_async_def_asyncio_parse_list(self):
items = []
def _on_item_scraped(item):
items.append(item)
crawler = self.runner.create_crawler(AsyncDefAsyncioReturnSpider)
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/status?n=200"), mockserver=self.mockserver)
self.assertIn("Got response 200", str(log))
self.assertIn({'id': 1}, items)
self.assertIn({'id': 2}, items)