From c8aa429c9beccd271b12c0b385fdf8e9352838b9 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Mon, 2 Feb 2026 23:36:23 +0500 Subject: [PATCH] Reactorless tests 2: enable pytest-asyncio. (#7233) * Move no-reactor to pytest-asyncio. * Unify the style. * Skip a test that installs a reactor in no-reactor. * Fix pinned env deps. * Strict pytest-asyncio mode. --- conftest.py | 5 ++- pyproject.toml | 3 +- tests/test_addons.py | 7 ++- tests/test_closespider.py | 3 ++ tests/test_contracts.py | 1 + tests/test_core_downloader.py | 1 + tests/test_core_scraper.py | 5 ++- tests/test_crawl.py | 2 + tests/test_crawler.py | 11 +++-- .../test_downloader_handler_twisted_http2.py | 4 +- tests/test_downloaderslotssettings.py | 2 + tests/test_engine.py | 4 ++ tests/test_engine_loop.py | 5 ++- tests/test_engine_stop_download_bytes.py | 5 ++- tests/test_engine_stop_download_headers.py | 5 ++- tests/test_extension_telnet.py | 2 + tests/test_feedexport.py | 6 +++ tests/test_logformatter.py | 1 + tests/test_pipeline_crawl.py | 1 + tests/test_pipeline_files.py | 1 + tests/test_pipelines.py | 4 ++ tests/test_proxy_connect.py | 1 + tests/test_request_attribute_binding.py | 2 + tests/test_request_cb_kwargs.py | 2 + tests/test_scheduler.py | 1 + tests/test_scheduler_base.py | 1 + tests/test_signals.py | 1 + tests/test_spider_start.py | 1 + tests/test_spidermiddleware_httperror.py | 1 + tests/test_spidermiddleware_output_chain.py | 2 + tests/test_spidermiddleware_process_start.py | 2 + tests/test_utils_defer.py | 2 + tests/test_utils_reactor.py | 1 + tests/test_zz_resources.py | 12 ++++++ tests/utils/decorators.py | 43 ++++++++++++++----- tox.ini | 7 ++- 36 files changed, 129 insertions(+), 28 deletions(-) diff --git a/conftest.py b/conftest.py index b17789b3a..50925a311 100644 --- a/conftest.py +++ b/conftest.py @@ -88,7 +88,10 @@ def pytest_runtest_setup(item): # Skip tests based on reactor markers reactor = item.config.getoption("--reactor") - if item.get_closest_marker("requires_reactor") and reactor == "none": + if ( + item.get_closest_marker("requires_reactor") + or item.get_closest_marker("requires_http_handler") + ) and reactor == "none": pytest.skip('This test is only run when the --reactor value is not "none"') if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: diff --git a/pyproject.toml b/pyproject.toml index a2ace0ec7..e2484b02a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,13 +230,14 @@ markers = [ "only_asyncio: marks tests that require the asyncio loop to be used", "only_not_asyncio: marks tests that require the asyncio loop to not be used", "requires_reactor: marks tests that require a reactor", + "requires_http_handler: marks tests that require a HTTP handler", "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", "requires_mitmproxy: marks tests that need mitmproxy", ] filterwarnings = [ - "ignore::DeprecationWarning:twisted.web.static" + "ignore::DeprecationWarning:twisted.web.static", ] [tool.ruff.lint] diff --git a/tests/test_addons.py b/tests/test_addons.py index fd673b2a5..675574d48 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -199,7 +199,12 @@ class TestAddonManager: settings = Settings() settings.setdict(get_reactor_settings()) settings.set("KEY", "default", priority="default") - runner = CrawlerRunner(settings) + runner_cls = ( + CrawlerRunner + if settings.getbool("TWISTED_ENABLED", True) + else AsyncCrawlerRunner + ) + runner = runner_cls(settings) crawler = runner.create_crawler(MySpider) assert crawler.settings.get("KEY") == "default" yield crawler.crawl() diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 623ed89c8..20c311c5a 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,3 +1,5 @@ +import pytest + from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ( @@ -10,6 +12,7 @@ from tests.spiders import ( from tests.utils.decorators import inlineCallbacks +@pytest.mark.requires_http_handler class TestCloseSpider: @classmethod def setup_class(cls): diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 505960685..9dface8aa 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -501,6 +501,7 @@ class TestContractsManager: assert not self.results.failures assert self.results.errors + @pytest.mark.requires_http_handler @inlineCallbacks def test_same_url(self): class TestSameUrlSpider(Spider): diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index d7940842a..c479e66c9 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -38,6 +38,7 @@ class TestSlot: assert repr(slot) == "Slot(concurrency=8, delay=0.10, randomize_delay=True)" +@pytest.mark.requires_reactor class TestContextFactoryBase: context_factory = None diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py index 3986656f0..249daf943 100644 --- a/tests/test_core_scraper.py +++ b/tests/test_core_scraper.py @@ -2,16 +2,17 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: - import pytest - from tests.mockserver.http import MockServer +@pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_scraper_exception( mockserver: MockServer, diff --git a/tests/test_crawl.py b/tests/test_crawl.py index c88526eea..61a4d387f 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -61,6 +61,7 @@ if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector +@pytest.mark.requires_http_handler # easier than marking many individual tests class TestCrawl: mockserver: MockServer @@ -428,6 +429,7 @@ with multiples lines assert "NotSupported: Unsupported URL scheme 'foo'" in caplog.text +@pytest.mark.requires_http_handler class TestCrawlSpider: mockserver: MockServer diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 9235ef885..8de38114c 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -646,12 +646,16 @@ class TestCrawlerProcess(TestBaseCrawler): @pytest.mark.only_asyncio class TestAsyncCrawlerProcess(TestBaseCrawler): - def test_crawler_process_accepts_dict(self): - runner = AsyncCrawlerProcess({"foo": "bar"}, install_root_handler=False) + def test_crawler_process_accepts_dict(self, reactor_pytest: str) -> None: + runner = AsyncCrawlerProcess( + {"foo": "bar", "TWISTED_ENABLED": reactor_pytest != "none"}, + install_root_handler=False, + ) assert runner.settings["foo"] == "bar" self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") - def test_crawler_process_accepts_None(self): + @pytest.mark.requires_reactor # can't pass TWISTED_ENABLED=False + def test_crawler_process_accepts_None(self) -> None: runner = AsyncCrawlerProcess(install_root_handler=False) self.assertOptionIsDefault(runner.settings, "RETRY_ENABLED") @@ -672,6 +676,7 @@ class NoRequestsSpider(scrapy.Spider): yield +@pytest.mark.requires_reactor class TestCrawlerRunnerHasSpider: @staticmethod def _runner(): diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 22eb56a94..df19646ff 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -200,7 +200,7 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): ) -> None: with pytest.raises(NotImplementedError): await maybe_deferred_to_future( - super().test_download_with_proxy_https_timeout(proxy_mockserver) + super().test_download_with_proxy_https_timeout(proxy_mockserver) # type: ignore[arg-type] ) @deferred_f_from_coro_f @@ -209,5 +209,5 @@ class TestHttps2Proxy(H2DownloadHandlerMixin, TestHttpProxyBase): ) -> None: with pytest.raises(UnsupportedURLSchemeError): await maybe_deferred_to_future( - super().test_download_with_proxy_without_http_scheme(proxy_mockserver) + super().test_download_with_proxy_without_http_scheme(proxy_mockserver) # type: ignore[arg-type] ) diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index 66dafd9b6..7d2001e47 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -67,6 +67,7 @@ class TestCrawl: def setup_method(self): self.runner = CrawlerRunner() + @pytest.mark.requires_http_handler @inlineCallbacks def test_delay(self): crawler = get_crawler(DownloaderSlotsSettingsTestSpider) @@ -128,6 +129,7 @@ def test_get_slot_deprecated_spider_arg(): assert slot1 == slot2 +@pytest.mark.requires_http_handler @pytest.mark.parametrize( "priority_queue_class", [ diff --git a/tests/test_engine.py b/tests/test_engine.py index d4f9fd794..939ea0f6f 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -373,6 +373,7 @@ class TestEngineBase: class TestEngine(TestEngineBase): + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler(self, mockserver: MockServer) -> None: for spider in ( @@ -390,6 +391,7 @@ class TestEngine(TestEngineBase): self._assert_signals_caught(run) self._assert_bytes_received(run) + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler_dupefilter(self, mockserver: MockServer) -> None: run = CrawlerRun(DupeFilterSpider) @@ -397,12 +399,14 @@ class TestEngine(TestEngineBase): self._assert_scheduled_requests(run, count=8) self._assert_dropped_requests(run) + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler_itemerror(self, mockserver: MockServer) -> None: run = CrawlerRun(ItemZeroDivisionErrorSpider) await run.run(mockserver) self._assert_items_error(run) + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler_change_close_reason_on_idle( self, mockserver: MockServer diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index f9cd437f7..f12409029 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -4,6 +4,7 @@ from collections import deque from logging import ERROR from typing import TYPE_CHECKING +import pytest from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals @@ -14,8 +15,6 @@ from tests.test_scheduler import MemoryScheduler from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: - import pytest - from scrapy.http import Response @@ -28,6 +27,7 @@ async def sleep(seconds: float = 0.001) -> None: class TestMain: + @pytest.mark.requires_reactor # TODO @deferred_f_from_coro_f async def test_sleep(self): """Neither asynchronous sleeps on Spider.start() nor the equivalent on @@ -332,6 +332,7 @@ class TestRequestSendOrder: # Examples from the “Start requests” section of the documentation about # spiders. + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_lazy(self): start_nums = [1, 2, 4] diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index e330adb3f..bd29e9ef6 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -2,6 +2,8 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + from scrapy.exceptions import StopDownload from tests.test_engine import ( AttrsItemsSpider, @@ -14,8 +16,6 @@ from tests.test_engine import ( from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: - import pytest - from tests.mockserver.http import MockServer @@ -26,6 +26,7 @@ class BytesReceivedCrawlerRun(CrawlerRun): class TestBytesReceivedEngine(TestEngineBase): + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index f46f46bd7..7dbaa41d2 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -2,6 +2,8 @@ from __future__ import annotations from typing import TYPE_CHECKING +import pytest + from scrapy.exceptions import StopDownload from tests.test_engine import ( AttrsItemsSpider, @@ -14,8 +16,6 @@ from tests.test_engine import ( from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: - import pytest - from tests.mockserver.http import MockServer @@ -26,6 +26,7 @@ class HeadersReceivedCrawlerRun(CrawlerRun): class TestHeadersReceivedEngine(TestEngineBase): + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_crawler( self, mockserver: MockServer, caplog: pytest.LogCaptureFixture diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 15e5aacce..fcb027832 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -6,6 +6,8 @@ from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler from tests.utils.decorators import inlineCallbacks +pytestmark = pytest.mark.requires_reactor + class TestTelnetExtension: def _get_console_and_portal(self, settings=None): diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index 914d9a950..c65bd9df9 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -163,6 +163,7 @@ class TestFileFeedStorage: assert storage.path == path +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage class TestFTPFeedStorage: def get_test_spider(self, settings=None): class TestSpider(scrapy.Spider): @@ -277,6 +278,7 @@ class TestBlockingFeedStorage: @pytest.mark.requires_boto3 +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage class TestS3FeedStorage: def test_parse_credentials(self): aws_credentials = { @@ -506,6 +508,7 @@ class TestS3FeedStorage: assert "S3 does not support appending to files" in str(log) +@pytest.mark.requires_reactor # needs a reactor for BlockingFeedStorage class TestGCSFeedStorage: def test_parse_settings(self): try: @@ -846,6 +849,7 @@ class ExceptionJsonItemExporter(JsonItemExporter): raise RuntimeError("foo") +@pytest.mark.requires_http_handler class TestFeedExport(TestFeedExportBase): async def run_and_export( self, spider_cls: type[Spider], settings: dict[str, Any] @@ -1830,6 +1834,7 @@ class TestFeedExport(TestFeedExportBase): assert not Storage.file_was_closed +@pytest.mark.requires_http_handler class TestFeedPostProcessedExports(TestFeedExportBase): items = [{"foo": "bar"}] expected = b"foo\r\nbar\r\n" @@ -2348,6 +2353,7 @@ class TestFeedPostProcessedExports(TestFeedExportBase): assert result == expected +@pytest.mark.requires_http_handler class TestBatchDeliveries(TestFeedExportBase): _file_mark = "_%(batch_time)s_#%(batch_id)02d_" diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 5cd816d76..0602be0c8 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -253,6 +253,7 @@ class DropSomeItemsPipeline: self.drop = True +@pytest.mark.requires_http_handler class TestShowOrSkipMessages: @classmethod def setup_class(cls): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index f29bc90e8..dddb6d335 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -57,6 +57,7 @@ class RedirectedMediaDownloadSpider(MediaDownloadSpider): ) +@pytest.mark.requires_http_handler class TestFileDownloadCrawl: pipeline_class = "scrapy.pipelines.files.FilesPipeline" store_setting_key = "FILES_STORE" diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 18fa6cdf2..7a6e9a942 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -691,6 +691,7 @@ class TestGCSFilesStore: store.bucket.get_blob.assert_called_with(expected_blob_path) +@pytest.mark.requires_reactor # needs a reactor for FTPFilesStore class TestFTPFileStore: @inlineCallbacks def test_persist(self): diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index 5bdca3035..69a6c890d 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -127,6 +127,7 @@ class ItemSpider(Spider): return {"field": 42} +@pytest.mark.requires_http_handler class TestPipeline: def _on_item_scraped(self, item): assert isinstance(item, dict) @@ -265,6 +266,7 @@ class TestCustomPipelineManager: ): itemproc.process_item({}, crawler.spider) + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_integration_recommended(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): @@ -291,6 +293,7 @@ class TestCustomPipelineManager: assert len(items) == 1 + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None: class CustomPipelineManager(ItemPipelineManager): @@ -349,6 +352,7 @@ class TestCustomPipelineManager: assert len(items) == 1 + @pytest.mark.requires_http_handler @deferred_f_from_coro_f async def test_integration_no_async_not_subclass( self, mockserver: MockServer diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 4071224d6..daa635b54 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -61,6 +61,7 @@ def _wrong_credentials(proxy_url): return urlunsplit(bad_auth_proxy) +@pytest.mark.requires_http_handler @pytest.mark.requires_mitmproxy class TestProxyConnect: @classmethod diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index 4065a5a2c..48234adb3 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,3 +1,4 @@ +import pytest from testfixtures import LogCapture from scrapy import Request, signals @@ -62,6 +63,7 @@ class AlternativeCallbacksMiddleware: return response.replace(request=new_request) +@pytest.mark.requires_http_handler class TestCrawl: @classmethod def setup_class(cls): diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 56d7264da..c91a670c1 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,3 +1,4 @@ +import pytest from testfixtures import LogCapture from scrapy.http import Request @@ -148,6 +149,7 @@ class KeywordArgumentsSpider(MockServerSpider): self.crawler.stats.inc_value("boolean_checks", 1) +@pytest.mark.requires_http_handler class TestCallbackKeywordArguments: @classmethod def setup_class(cls): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index 8730489f7..f363b1cd5 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -369,6 +369,7 @@ class TestIntegrationWithDownloaderAwareInMemory: }, ) + @pytest.mark.requires_http_handler @inlineCallbacks def test_integration_downloader_aware_priority_queue(self): with MockServer() as mockserver: diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 538464788..277c00623 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -144,6 +144,7 @@ class TestSimpleScheduler(InterfaceCheckMixin): assert close_result == "close" +@pytest.mark.requires_http_handler class TestMinimalSchedulerCrawl: scheduler_cls = MinimalScheduler diff --git a/tests/test_signals.py b/tests/test_signals.py index 048af00ab..5769f6588 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -50,6 +50,7 @@ class TestMockServer: item = await get_from_asyncio_queue(item) self.items.append(item) + @pytest.mark.requires_http_handler @pytest.mark.only_asyncio @inlineCallbacks def test_simple_pipeline(self): diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index d6767d74c..b84b0d8ac 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -162,6 +162,7 @@ class TestMain: await self._test_start(start, [ITEM_A]) + @pytest.mark.requires_reactor # needs a reactor for twisted_sleep() @deferred_f_from_coro_f async def test_twisted_delayed(self): async def start(spider): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index c0284c5ed..d4d0a4fb1 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -191,6 +191,7 @@ class TestHttpErrorMiddlewareHandleAll: mw.process_spider_input(res402) +@pytest.mark.requires_http_handler class TestHttpErrorMiddlewareIntegrational: @classmethod def setup_class(cls): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index b35a4c704..07df2ecc4 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,3 +1,4 @@ +import pytest from testfixtures import LogCapture from scrapy import Request, Spider @@ -318,6 +319,7 @@ class NotGeneratorOutputChainSpider(Spider): # ================================================================================ +@pytest.mark.requires_http_handler class TestSpiderMiddleware: mockserver: MockServer diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 250b58544..e7c9640b8 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -341,10 +341,12 @@ class TestMain: [NoOpSpiderMiddleware, AsyncioSleepSpiderMiddleware, NoOpSpiderMiddleware] ) + @pytest.mark.requires_reactor @deferred_f_from_coro_f async def test_twisted_sleep_single(self): await self._test_sleep([TwistedSleepSpiderMiddleware]) + @pytest.mark.requires_reactor @deferred_f_from_coro_f async def test_twisted_sleep_multiple(self): await self._test_sleep( diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index ad317f3ae..ad4cc466d 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -26,6 +26,7 @@ if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator +@pytest.mark.requires_reactor @pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning") class TestMustbeDeferred: @inlineCallbacks @@ -153,6 +154,7 @@ class TestAsyncDefTestsuite: raise RuntimeError("This is expected to be raised") +@pytest.mark.requires_reactor class TestParallelAsync: """This tests _AsyncCooperatorAdapter by testing parallel_async which is its only usage. diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 658e68016..a045c05a4 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -29,6 +29,7 @@ class TestAsyncio: assert original_reactor == reactor + @pytest.mark.requires_reactor @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_set_asyncio_event_loop(self): diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py index 7b11aac9e..e6ff46f5c 100644 --- a/tests/test_zz_resources.py +++ b/tests/test_zz_resources.py @@ -8,6 +8,7 @@ import logging import pytest from scrapy.utils.log import LogCounterHandler +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed def test_counter_handler() -> None: @@ -35,3 +36,14 @@ def test_stderr_log_handler() -> None: def test_pending_asyncio_tasks() -> None: """Test that there are no pending asyncio tasks.""" assert not asyncio.all_tasks() + + +def test_installed_reactor(reactor_pytest: str) -> None: + """Test that the correct reactor is installed.""" + match reactor_pytest: + case "asyncio": + assert is_asyncio_reactor_installed() + case "default": + assert not is_asyncio_reactor_installed() + case "none": + assert not is_reactor_installed() diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py index dc9d2e113..633d8d97d 100644 --- a/tests/utils/decorators.py +++ b/tests/utils/decorators.py @@ -7,7 +7,8 @@ import pytest from twisted.internet.defer import Deferred from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig -from scrapy.utils.defer import deferred_from_coro +from scrapy.utils.defer import deferred_from_coro, deferred_to_future +from scrapy.utils.reactor import is_reactor_installed if TYPE_CHECKING: from collections.abc import Awaitable, Callable, Generator @@ -18,28 +19,48 @@ _P = ParamSpec("_P") def inlineCallbacks( f: Callable[_P, Generator[Deferred[Any], Any, None]], -) -> Callable[_P, Deferred[None]]: - """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the - decorated function with ``@pytest.mark.requires_reactor``.""" +) -> Callable[_P, Awaitable[None]]: + """Mark a test function written in a :func:`twisted.internet.defer.inlineCallbacks` style. + + This calls :func:`twisted.internet.defer.inlineCallbacks` and then: + + * with ``pytest-twisted`` this returns the resulting Deferred + * with ``pytest-asyncio`` this converts the resulting Deferred into a + coroutine + """ + + if not is_reactor_installed(): + + @pytest.mark.asyncio + @wraps(f) + async def wrapper_coro(*args: _P.args, **kwargs: _P.kwargs) -> None: + await deferred_to_future(inlineCallbacks_orig(f)(*args, **kwargs)) + + return wrapper_coro - @pytest.mark.requires_reactor @wraps(f) @inlineCallbacks_orig - def wrapper( + def wrapper_dfd( *args: _P.args, **kwargs: _P.kwargs ) -> Generator[Deferred[Any], Any, None]: return f(*args, **kwargs) - return wrapper + return wrapper_dfd def deferred_f_from_coro_f( coro_f: Callable[_P, Awaitable[None]], -) -> Callable[_P, Deferred[None]]: - """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the - decorated function with ``@pytest.mark.requires_reactor``.""" +) -> Callable[_P, Awaitable[None]]: + """Mark a test function that returns a coroutine. + + * with ``pytest-twisted`` this converts a coroutine into a + :class:`twisted.internet.defer.Deferred` + * with ``pytest-asyncio`` this is a no-op + """ + + if not is_reactor_installed(): + return pytest.mark.asyncio(coro_f) - @pytest.mark.requires_reactor @wraps(coro_f) def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]: return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) diff --git a/tox.ini b/tox.ini index fc4d1b824..83cd06561 100644 --- a/tox.ini +++ b/tox.ini @@ -162,6 +162,9 @@ commands = {[testenv]commands} --reactor=default [testenv:no-reactor] +deps = + {[testenv]deps} + pytest-asyncio commands = {[testenv]commands} -p no:twisted --reactor=none @@ -174,7 +177,9 @@ setenv = [testenv:no-reactor-pinned] basepython = {[pinned]basepython} -deps = {[testenv:pinned]deps} +deps = + {[testenv:pinned]deps} + pytest-asyncio commands = {[pinned]commands} -p no:twisted --reactor=none setenv = {[pinned]setenv}