mirror of https://github.com/scrapy/scrapy.git
62 lines
1.7 KiB
Python
62 lines
1.7 KiB
Python
import pytest
|
|
|
|
from scrapy import Request, Spider, signals
|
|
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
|
|
from tests.mockserver.http import MockServer
|
|
from tests.utils.decorators import coroutine_test, inline_callbacks_test
|
|
|
|
|
|
class ItemSpider(Spider):
|
|
name = "itemspider"
|
|
|
|
async def start(self):
|
|
for index in range(10):
|
|
yield Request(
|
|
self.mockserver.url(f"/status?n=200&id={index}"), meta={"index": index}
|
|
)
|
|
|
|
def parse(self, response):
|
|
return {"index": response.meta["index"]}
|
|
|
|
|
|
class TestMain:
|
|
@coroutine_test
|
|
async def test_scheduler_empty(self):
|
|
crawler = get_crawler()
|
|
calls = []
|
|
|
|
def track_call():
|
|
calls.append(object())
|
|
|
|
crawler.signals.connect(track_call, signals.scheduler_empty)
|
|
await crawler.crawl_async()
|
|
assert len(calls) >= 1
|
|
|
|
|
|
class TestMockServer:
|
|
@classmethod
|
|
def setup_class(cls):
|
|
cls.mockserver = MockServer()
|
|
cls.mockserver.__enter__()
|
|
|
|
@classmethod
|
|
def teardown_class(cls):
|
|
cls.mockserver.__exit__(None, None, None)
|
|
|
|
def setup_method(self):
|
|
self.items = []
|
|
|
|
async def _on_item_scraped(self, item):
|
|
item = await get_from_asyncio_queue(item)
|
|
self.items.append(item)
|
|
|
|
@pytest.mark.only_asyncio
|
|
@inline_callbacks_test
|
|
def test_simple_pipeline(self):
|
|
crawler = get_crawler(ItemSpider)
|
|
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
|
|
yield crawler.crawl(mockserver=self.mockserver)
|
|
assert len(self.items) == 10
|
|
for index in range(10):
|
|
assert {"index": index} in self.items
|