Improve async signal tests.

This commit is contained in:
Andrey Rakhmatullin 2020-02-11 01:05:45 +05:00
parent 4a7c7340a0
commit 1f0f52cbf7
1 changed files with 10 additions and 5 deletions

View File

@ -1,8 +1,9 @@
from pytest import mark
from twisted.internet import defer
from twisted.trial import unittest
from scrapy import signals, Request, Spider
from scrapy.utils.test import get_crawler
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.mockserver import MockServer
@ -11,12 +12,12 @@ class ItemSpider(Spider):
name = 'itemspider'
def start_requests(self):
for _ in range(10):
yield Request(self.mockserver.url('/status?n=200'),
dont_filter=True)
for index in range(10):
yield Request(self.mockserver.url('/status?n=200&id=%d' % index),
meta={'index': index})
def parse(self, response):
return {'field': 42}
return {'index': response.meta['index']}
class AsyncSignalTestCase(unittest.TestCase):
@ -29,11 +30,15 @@ class AsyncSignalTestCase(unittest.TestCase):
self.mockserver.__exit__(None, None, None)
async def _on_item_scraped(self, item):
item = await get_from_asyncio_queue(item)
self.items.append(item)
@mark.only_asyncio()
@defer.inlineCallbacks
def test_simple_pipeline(self):
crawler = get_crawler(ItemSpider)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
yield crawler.crawl(mockserver=self.mockserver)
self.assertEqual(len(self.items), 10)
for index in range(10):
self.assertIn({'index': index}, self.items)