diff --git a/conftest.py b/conftest.py index 106f4c9e0..f33ffb1a4 100644 --- a/conftest.py +++ b/conftest.py @@ -48,6 +48,14 @@ def chdir(tmpdir): tmpdir.chdir() +def pytest_addoption(parser): + parser.addoption( + "--reactor", + default="default", + choices=["default", "asyncio"], + ) + + @pytest.fixture(scope="class") def reactor_pytest(request): if not request.cls: @@ -58,11 +66,8 @@ def reactor_pytest(request): @pytest.fixture(autouse=True) -def only_asyncio(request): - if ( - request.node.get_closest_marker("only_asyncio") - and request.config.getoption("--reactor") != "asyncio" - ): +def only_asyncio(request, reactor_pytest): + if request.node.get_closest_marker("only_asyncio") and reactor_pytest != "asyncio": pytest.skip("This test is only run with --reactor=asyncio") diff --git a/tests/test_spider_yield_seeds.py b/tests/test_spider_yield_seeds.py index dcda67f29..a65a0fb51 100644 --- a/tests/test_spider_yield_seeds.py +++ b/tests/test_spider_yield_seeds.py @@ -1,51 +1,41 @@ from asyncio import sleep import pytest -from pytest_twisted import ensureDeferred +from twisted.internet.defer import inlineCallbacks +from twisted.trial.unittest import TestCase from scrapy import Spider, signals from scrapy.core.engine import ExecutionEngine from scrapy.utils.test import get_crawler -class Scenario: - pass +class MainTestCase(TestCase): + @inlineCallbacks + def _test_scenario(self, scenario): + class TestSpider(Spider): + name = "test" + yield_seeds = scenario.yield_seeds + actual_items = [] -class AsyncioScenario(Scenario): - expected_items = [{"a": "b"}] - only_asyncio = True + def track_item(item, response, spider): + actual_items.append(item) - async def yield_seeds(self): - await sleep(ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01) - yield {"a": "b"} + crawler = get_crawler(TestSpider) + crawler.signals.connect(track_item, signals.item_scraped) + yield crawler.crawl() + assert crawler.stats.get_value("finish_reason") == "finished" + assert actual_items == scenario.expected_items + @pytest.mark.only_asyncio + @inlineCallbacks + def test_asyncio_delayed(self): + class Scenario: + expected_items = [{"a": "b"}] + only_asyncio = True -@pytest.mark.parametrize( - "scenario", - [ - pytest.param( - scenario, - marks=pytest.mark.only_asyncio - if getattr(scenario, "only_asyncio", False) - else [], - ) - for scenario in Scenario.__subclasses__() - ], -) -@ensureDeferred -async def test_main(scenario): - class TestSpider(Spider): - name = "test" - yield_seeds = scenario.yield_seeds + async def yield_seeds(self): + await sleep(ExecutionEngine._SLOT_HEARTBEAT_INTERVAL + 0.01) + yield {"a": "b"} - actual_items = [] - - def track_item(item, response, spider): - actual_items.append(item) - - crawler = get_crawler(TestSpider) - crawler.signals.connect(track_item, signals.item_scraped) - await crawler.crawl() - assert crawler.stats.get_value("finish_reason") == "finished" - assert actual_items == scenario.expected_items + yield self._test_scenario(Scenario) diff --git a/tox.ini b/tox.ini index 71a58d52f..041fcffca 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,6 @@ deps = pygments pytest != 8.2.* # https://github.com/pytest-dev/pytest/issues/12275 pytest-cov >= 4.0.0 - pytest-twisted pytest-xdist sybil >= 1.3.0 # https://github.com/cjw296/sybil/issues/20#issuecomment-605433422 testfixtures