diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml index 31ca6b2e9..585cdd307 100644 --- a/.github/workflows/tests-ubuntu.yml +++ b/.github/workflows/tests-ubuntu.yml @@ -32,6 +32,9 @@ jobs: - python-version: "3.13" env: TOXENV: default-reactor + - python-version: "3.13" + env: + TOXENV: no-reactor - python-version: pypy3.11 env: TOXENV: pypy3 @@ -43,6 +46,9 @@ jobs: - python-version: "3.10.19" env: TOXENV: default-reactor-pinned + - python-version: "3.10.19" + env: + TOXENV: no-reactor-pinned - python-version: pypy3.11 env: TOXENV: pypy3-pinned diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml index 5f4b383fd..9b03b594d 100644 --- a/.github/workflows/tests-windows.yml +++ b/.github/workflows/tests-windows.yml @@ -32,6 +32,9 @@ jobs: - python-version: "3.13" env: TOXENV: default-reactor + - python-version: "3.13" + env: + TOXENV: no-reactor # pinned deps - python-version: "3.10.11" diff --git a/conftest.py b/conftest.py index 55b5a28c1..b17789b3a 100644 --- a/conftest.py +++ b/conftest.py @@ -8,6 +8,7 @@ import pytest from twisted.web.http import H2_ENABLED from scrapy.utils.reactor import set_asyncio_event_loop_policy +from scrapy.utils.reactorless import install_reactor_import_hook from tests.keys import generate_keys from tests.mockserver.http import MockServer @@ -25,10 +26,6 @@ collect_ignore = [ # not a test, but looks like a test "scrapy/utils/testproc.py", "scrapy/utils/testsite.py", - "tests/ftpserver.py", - "tests/mockserver.py", - "tests/pipelines.py", - "tests/spiders.py", # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess *_py_files("tests/AsyncCrawlerProcess"), # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess @@ -56,6 +53,17 @@ if not H2_ENABLED: ) +def pytest_addoption(parser, pluginmanager): + if pluginmanager.hasplugin("twisted"): + return + # add the full choice set so that pytest doesn't complain about invalid choices in some cases + parser.addoption( + "--reactor", + default="none", + choices=["asyncio", "default", "none"], + ) + + @pytest.fixture(scope="session") def mockserver() -> Generator[MockServer]: with MockServer() as mockserver: @@ -72,17 +80,26 @@ def pytest_configure(config): # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility. # If we decide to run tests with both, we will need to add a new option and check it here. set_asyncio_event_loop_policy() + elif config.getoption("--reactor") == "none": + install_reactor_import_hook() def pytest_runtest_setup(item): # Skip tests based on reactor markers reactor = item.config.getoption("--reactor") - if item.get_closest_marker("only_asyncio") and reactor != "asyncio": - pytest.skip("This test is only run with --reactor=asyncio") + if item.get_closest_marker("requires_reactor") and reactor == "none": + pytest.skip('This test is only run when the --reactor value is not "none"') - if item.get_closest_marker("only_not_asyncio") and reactor == "asyncio": - pytest.skip("This test is only run without --reactor=asyncio") + if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is "asyncio" (default) or "none"' + ) + + if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}: + pytest.skip( + 'This test is only run when the --reactor value is not "asyncio" (default) or "none"' + ) # Skip tests requiring optional dependencies optional_deps = [ diff --git a/pyproject.toml b/pyproject.toml index 7982b43b8..a2ace0ec7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -227,8 +227,9 @@ addopts = [ xfail_strict = true python_files = ["test_*.py", "test_*/__init__.py"] markers = [ - "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed", - "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed", + "only_asyncio: marks tests that require the asyncio loop to be used", + "only_not_asyncio: marks tests that require the asyncio loop to not be used", + "requires_reactor: marks tests that require a reactor", "requires_uvloop: marks tests as only enabled when uvloop is known to be working", "requires_botocore: marks tests that need botocore (but not boto3)", "requires_boto3: marks tests that need botocore and boto3", diff --git a/scrapy/utils/test.py b/scrapy/utils/test.py index 4e0356350..7d81868a9 100644 --- a/scrapy/utils/test.py +++ b/scrapy/utils/test.py @@ -17,7 +17,7 @@ from unittest import mock from twisted.trial.unittest import SkipTest from twisted.web.client import Agent -from scrapy.crawler import CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.utils.boto import is_botocore_available from scrapy.utils.deprecate import create_deprecated_class @@ -116,16 +116,24 @@ def get_reactor_settings() -> dict[str, Any]: ``Crawler._apply_settings()`` checks that the installed reactor matches the settings, so tests that run the crawler in the current process may need to - pass a correct ``"TWISTED_REACTOR"`` setting value when creating it. + pass a correct :setting:`TWISTED_REACTOR` setting value when creating it. """ - if not is_reactor_installed(): - raise RuntimeError( - "get_reactor_settings() called without an installed reactor," - " you may need to install a reactor explicitly when running your tests." - ) settings: dict[str, Any] = {} - if not is_asyncio_reactor_installed(): - settings["TWISTED_REACTOR"] = None + if is_reactor_installed(): + if not is_asyncio_reactor_installed(): + settings["TWISTED_REACTOR"] = None + else: + # We are either running Scrapy tests for the reactorless mode, or + # running some 3rd-party library tests for the reactorless mode, or + # running some 3rd-party library tests without initializing a reactor + # properly. The first two cases are fine, but we cannot distinguish the + # last one from them. + settings["TWISTED_ENABLED"] = False + settings["DOWNLOAD_HANDLERS"] = { + "ftp": None, + "http": None, + "https": None, + } return settings @@ -144,7 +152,11 @@ def get_crawler( **get_reactor_settings(), **(settings_dict or {}), } - runner = CrawlerRunner(settings) + runner: CrawlerRunnerBase + if is_reactor_installed(): + runner = CrawlerRunner(settings) + else: + runner = AsyncCrawlerRunner(settings) crawler = runner.create_crawler(spidercls or DefaultSpider) crawler._apply_settings() return crawler diff --git a/tests/test_addons.py b/tests/test_addons.py index 457945ea5..fd673b2a5 100644 --- a/tests/test_addons.py +++ b/tests/test_addons.py @@ -2,13 +2,12 @@ import itertools from typing import Any from unittest.mock import patch -from twisted.internet.defer import inlineCallbacks - from scrapy import Spider -from scrapy.crawler import Crawler, CrawlerRunner +from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner from scrapy.exceptions import NotConfigured from scrapy.settings import BaseSettings, Settings from scrapy.utils.test import get_crawler, get_reactor_settings +from tests.utils.decorators import inlineCallbacks class SimpleAddon: @@ -109,9 +108,15 @@ class TestAddonManager: crawler = get_crawler(settings_dict=settings_dict) assert crawler.settings.getint("KEY") == 15 + runner_cls = ( + CrawlerRunner + if settings_dict.get("TWISTED_ENABLED", True) + else AsyncCrawlerRunner + ) + settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") - runner = CrawlerRunner(settings) + runner = runner_cls(settings) crawler = runner.create_crawler(Spider) crawler._apply_settings() assert crawler.settings.getint("KEY") == 15 @@ -123,44 +128,39 @@ class TestAddonManager: } settings = Settings(settings_dict) settings.set("KEY", 0, priority="default") - runner = CrawlerRunner(settings) + runner = runner_cls(settings) crawler = runner.create_crawler(Spider) assert crawler.settings.getint("KEY") == 20 def test_fallback_workflow(self): - FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER" + FALLBACK_SETTING = "MY_FALLBACK_SCHEDULER" class AddonWithFallback: def update_settings(self, settings): if not settings.get(FALLBACK_SETTING): settings.set( FALLBACK_SETTING, - settings.getwithbase("DOWNLOAD_HANDLERS")["https"], + settings.get("SCHEDULER"), "addon", ) - settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler" + settings["SCHEDULER"] = "AddonScheduler" settings_dict = { "ADDONS": {AddonWithFallback: 1}, } crawler = get_crawler(settings_dict=settings_dict) + assert crawler.settings.get("SCHEDULER") == "AddonScheduler" assert ( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" - ) - assert ( - crawler.settings.get(FALLBACK_SETTING) - == "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler" + crawler.settings.get(FALLBACK_SETTING) == "scrapy.core.scheduler.Scheduler" ) settings_dict = { "ADDONS": {AddonWithFallback: 1}, - "DOWNLOAD_HANDLERS": {"https": "UserHandler"}, + "SCHEDULER": "UserScheduler", } crawler = get_crawler(settings_dict=settings_dict) - assert ( - crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler" - ) - assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler" + assert crawler.settings.get("SCHEDULER") == "AddonScheduler" + assert crawler.settings.get(FALLBACK_SETTING) == "UserScheduler" def test_logging_message(self): class LoggedAddon: diff --git a/tests/test_closespider.py b/tests/test_closespider.py index 2243cd371..623ed89c8 100644 --- a/tests/test_closespider.py +++ b/tests/test_closespider.py @@ -1,5 +1,3 @@ -from twisted.internet.defer import inlineCallbacks - from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ( @@ -9,6 +7,7 @@ from tests.spiders import ( MaxItemsAndRequestsSpider, SlowSpider, ) +from tests.utils.decorators import inlineCallbacks class TestCloseSpider: diff --git a/tests/test_contracts.py b/tests/test_contracts.py index 326be570e..505960685 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -1,7 +1,6 @@ from unittest import TextTestResult import pytest -from twisted.internet.defer import inlineCallbacks from twisted.python import failure from scrapy import FormRequest @@ -19,6 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.utils.decorators import inlineCallbacks class DemoItem(Item): diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py index a35bfd590..d7940842a 100644 --- a/tests/test_core_downloader.py +++ b/tests/test_core_downloader.py @@ -18,13 +18,14 @@ from scrapy.core.downloader.contextfactory import ( from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.settings import Settings -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.python import to_bytes from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http_resources import PayloadResource from tests.mockserver.utils import ssl_context_factory +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from twisted.internet.defer import Deferred diff --git a/tests/test_core_scraper.py b/tests/test_core_scraper.py index 47dee1271..3986656f0 100644 --- a/tests/test_core_scraper.py +++ b/tests/test_core_scraper.py @@ -2,9 +2,9 @@ from __future__ import annotations from typing import TYPE_CHECKING -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: import pytest diff --git a/tests/test_crawl.py b/tests/test_crawl.py index 1d64c2f39..c88526eea 100644 --- a/tests/test_crawl.py +++ b/tests/test_crawl.py @@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlparse import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks, succeed +from twisted.internet.defer import succeed from twisted.internet.ssl import Certificate from twisted.python.failure import Failure @@ -18,7 +18,7 @@ from scrapy.crawler import CrawlerRunner from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload from scrapy.http import Request from scrapy.http.response import Response -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.engine import format_engine_status, get_engine_status from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler, get_reactor_settings @@ -55,6 +55,7 @@ from tests.spiders import ( StartGoodAndBadOutput, StartItemSpider, ) +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks if TYPE_CHECKING: from scrapy.statscollectors import StatsCollector diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 6d0c82825..9235ef885 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -14,7 +14,7 @@ from typing import Any import pytest from packaging.version import parse as parse_version from pexpect.popen_spawn import PopenSpawn -from twisted.internet.defer import Deferred, inlineCallbacks +from twisted.internet.defer import Deferred from w3lib import __version__ as w3lib_version from zope.interface.exceptions import MultipleInvalid @@ -31,11 +31,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.throttle import AutoThrottle from scrapy.settings import Settings, default_settings from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - deferred_from_coro, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.log import ( _uninstall_scrapy_root_handler, configure_logging, @@ -45,6 +41,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_reactor_settings from tests.mockserver.http import MockServer from tests.utils import get_script_run_env +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks BASE_SETTINGS: dict[str, Any] = {} diff --git a/tests/test_downloader_handler_twisted_ftp.py b/tests/test_downloader_handler_twisted_ftp.py index 6cda607cc..60d2dfd12 100644 --- a/tests/test_downloader_handler_twisted_ftp.py +++ b/tests/test_downloader_handler_twisted_ftp.py @@ -25,6 +25,9 @@ if TYPE_CHECKING: from twisted.protocols.ftp import FTPFactory +pytestmark = pytest.mark.requires_reactor + + class TestFTPBase(ABC): username = "scrapy" password = "passwd" diff --git a/tests/test_downloader_handler_twisted_http10.py b/tests/test_downloader_handler_twisted_http10.py index 939e2c8ad..99f3697f8 100644 --- a/tests/test_downloader_handler_twisted_http10.py +++ b/tests/test_downloader_handler_twisted_http10.py @@ -16,6 +16,9 @@ if TYPE_CHECKING: from tests.mockserver.http import MockServer +pytestmark = pytest.mark.requires_reactor + + class HTTP10DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: diff --git a/tests/test_downloader_handler_twisted_http11.py b/tests/test_downloader_handler_twisted_http11.py index eb2735c4d..ffaa2c569 100644 --- a/tests/test_downloader_handler_twisted_http11.py +++ b/tests/test_downloader_handler_twisted_http11.py @@ -4,6 +4,8 @@ from __future__ import annotations from typing import TYPE_CHECKING, Any +import pytest + from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler from tests.test_downloader_handlers_http_base import ( TestHttp11Base, @@ -21,6 +23,9 @@ if TYPE_CHECKING: from scrapy.core.downloader.handlers import DownloadHandlerProtocol +pytestmark = pytest.mark.requires_reactor + + class HTTP11DownloadHandlerMixin: @property def download_handler_cls(self) -> type[DownloadHandlerProtocol]: diff --git a/tests/test_downloader_handler_twisted_http2.py b/tests/test_downloader_handler_twisted_http2.py index 5ae35a461..22eb56a94 100644 --- a/tests/test_downloader_handler_twisted_http2.py +++ b/tests/test_downloader_handler_twisted_http2.py @@ -27,9 +27,13 @@ if TYPE_CHECKING: from tests.mockserver.http import MockServer from tests.mockserver.proxy_echo import ProxyEchoMockServer -pytestmark = pytest.mark.skipif( - not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" -) + +pytestmark = [ + pytest.mark.requires_reactor, + pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" + ), +] class H2DownloadHandlerMixin: diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py index 602b2186f..7844415f5 100644 --- a/tests/test_downloader_handlers.py +++ b/tests/test_downloader_handlers.py @@ -19,9 +19,9 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning from scrapy.http import Request from scrapy.responsetypes import responsetypes from scrapy.utils.boto import is_botocore_available -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler +from tests.utils.decorators import deferred_f_from_coro_f class DummyDH: diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py index 7ad787346..4d4b4ecaa 100644 --- a/tests/test_downloader_handlers_http_base.py +++ b/tests/test_downloader_handlers_http_base.py @@ -26,11 +26,7 @@ from scrapy.exceptions import ( UnsupportedURLSchemeError, ) from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - deferred_from_coro, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from scrapy.utils.misc import build_from_crawler from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler @@ -38,6 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE from tests.mockserver.proxy_echo import ProxyEchoMockServer from tests.mockserver.simple_https import SimpleMockServer from tests.spiders import SingleRequestSpider +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from collections.abc import AsyncGenerator, Generator diff --git a/tests/test_downloadermiddleware.py b/tests/test_downloadermiddleware.py index bc1748dfb..10ac77aa0 100644 --- a/tests/test_downloadermiddleware.py +++ b/tests/test_downloadermiddleware.py @@ -13,9 +13,10 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput from scrapy.http import Request, Response from scrapy.spiders import Spider -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_bytes from scrapy.utils.test import get_crawler, get_from_asyncio_queue +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from collections.abc import AsyncGenerator diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py index 1e99d5162..9b4b2750b 100644 --- a/tests/test_downloadermiddleware_robotstxt.py +++ b/tests/test_downloadermiddleware_robotstxt.py @@ -14,12 +14,9 @@ from scrapy.http import Request, Response, TextResponse from scrapy.http.request import NO_CALLBACK from scrapy.settings import Settings from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - deferred_from_coro, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future from tests.test_robotstxt_interface import rerp_available +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py index dffe95406..66dafd9b6 100644 --- a/tests/test_downloaderslotssettings.py +++ b/tests/test_downloaderslotssettings.py @@ -2,17 +2,16 @@ import time from typing import Any import pytest -from twisted.internet.defer import inlineCallbacks from scrapy import Request from scrapy.core.downloader import Downloader, Slot from scrapy.crawler import CrawlerRunner from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MetaSpider +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks class DownloaderSlotsSettingsTestSpider(MetaSpider): @@ -85,6 +84,7 @@ class TestCrawl: assert max(list(error_delta.values())) < tolerance +@pytest.mark.requires_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop def test_params(): params = { "concurrency": 1, @@ -109,6 +109,7 @@ def test_params(): ) +@pytest.mark.requires_reactor # needs a reactor or an event loop for Downloader._slot_gc_loop def test_get_slot_deprecated_spider_arg(): crawler = get_crawler(DefaultSpider) crawler.spider = crawler._create_spider() diff --git a/tests/test_engine.py b/tests/test_engine.py index d3485e344..d4f9fd794 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -17,7 +17,6 @@ from itemadapter import ItemAdapter from pydispatch import dispatcher from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from scrapy import signals from scrapy.core.engine import ExecutionEngine, _Slot @@ -29,7 +28,6 @@ from scrapy.linkextractors import LinkExtractor from scrapy.spiders import Spider from scrapy.utils.defer import ( _schedule_coro, - deferred_f_from_coro_f, deferred_from_coro, maybe_deferred_to_future, ) @@ -37,6 +35,7 @@ from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests import get_testdata +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks if TYPE_CHECKING: from scrapy.core.scheduler import Scheduler @@ -599,6 +598,7 @@ class TestEngineDownload(TestEngineDownloadAsync): return await maybe_deferred_to_future(engine.download(request)) +@pytest.mark.requires_reactor # needs a reactor or an event loop for _Slot.heartbeat def test_request_scheduled_signal(caplog): class TestScheduler(BaseScheduler): def __init__(self): diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py index c219a0f6f..f9cd437f7 100644 --- a/tests/test_engine_loop.py +++ b/tests/test_engine_loop.py @@ -7,10 +7,11 @@ from typing import TYPE_CHECKING from twisted.internet.defer import Deferred from scrapy import Request, Spider, signals -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.test_scheduler import MemoryScheduler +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: import pytest diff --git a/tests/test_engine_stop_download_bytes.py b/tests/test_engine_stop_download_bytes.py index b46ebe2d9..e330adb3f 100644 --- a/tests/test_engine_stop_download_bytes.py +++ b/tests/test_engine_stop_download_bytes.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from scrapy.utils.defer import deferred_f_from_coro_f from tests.test_engine import ( AttrsItemsSpider, CrawlerRun, @@ -12,6 +11,7 @@ from tests.test_engine import ( MySpider, TestEngineBase, ) +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: import pytest diff --git a/tests/test_engine_stop_download_headers.py b/tests/test_engine_stop_download_headers.py index 76cfbe8ed..f46f46bd7 100644 --- a/tests/test_engine_stop_download_headers.py +++ b/tests/test_engine_stop_download_headers.py @@ -3,7 +3,6 @@ from __future__ import annotations from typing import TYPE_CHECKING from scrapy.exceptions import StopDownload -from scrapy.utils.defer import deferred_f_from_coro_f from tests.test_engine import ( AttrsItemsSpider, CrawlerRun, @@ -12,6 +11,7 @@ from tests.test_engine import ( MySpider, TestEngineBase, ) +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: import pytest diff --git a/tests/test_extension_periodic_log.py b/tests/test_extension_periodic_log.py index 8d6c22ad7..4f368d5ec 100644 --- a/tests/test_extension_periodic_log.py +++ b/tests/test_extension_periodic_log.py @@ -3,6 +3,8 @@ from __future__ import annotations import datetime from typing import TYPE_CHECKING, Any +import pytest + from scrapy.extensions.periodic_log import PeriodicLog from scrapy.utils.test import get_crawler @@ -86,6 +88,7 @@ class TestPeriodicLog: assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60}) assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60}) + @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task def test_log_delta(self): def emulate(settings=None): spider = MetaSpider() @@ -143,6 +146,7 @@ class TestPeriodicLog: and ("downloader/" in k and "bytes" not in k), ) + @pytest.mark.requires_reactor # needs a reactor or an event loop for PeriodicLog.task def test_log_stats(self): def emulate(settings=None): spider = MetaSpider() diff --git a/tests/test_extension_telnet.py b/tests/test_extension_telnet.py index 6b4ad450f..15e5aacce 100644 --- a/tests/test_extension_telnet.py +++ b/tests/test_extension_telnet.py @@ -1,10 +1,10 @@ import pytest from twisted.conch.telnet import ITelnetProtocol from twisted.cred import credentials -from twisted.internet.defer import inlineCallbacks from scrapy.extensions.telnet import TelnetConsole from scrapy.utils.test import get_crawler +from tests.utils.decorators import inlineCallbacks class TestTelnetExtension: diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py index a9d608e52..914d9a950 100644 --- a/tests/test_feedexport.py +++ b/tests/test_feedexport.py @@ -29,7 +29,6 @@ import lxml.etree import pytest from packaging.version import Version from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from w3lib.url import file_uri_to_path, path_to_file_uri from zope.interface import implementer from zope.interface.verify import verifyObject @@ -50,12 +49,13 @@ from scrapy.extensions.feedexport import ( StdoutFeedStorage, ) from scrapy.settings import Settings -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.python import to_unicode from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks if TYPE_CHECKING: from collections.abc import Callable, Iterable diff --git a/tests/test_http2_client_protocol.py b/tests/test_http2_client_protocol.py index 46ef75064..944a67920 100644 --- a/tests/test_http2_client_protocol.py +++ b/tests/test_http2_client_protocol.py @@ -39,9 +39,12 @@ if TYPE_CHECKING: from scrapy.core.http2.protocol import H2ClientProtocol -pytestmark = pytest.mark.skipif( - not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" -) +pytestmark = [ + pytest.mark.requires_reactor, + pytest.mark.skipif( + not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled" + ), +] def generate_random_string(size: int) -> str: diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index 426be13a7..5cd816d76 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -2,7 +2,6 @@ import logging import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from scrapy.exceptions import DropItem @@ -13,6 +12,7 @@ from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import ItemSpider +from tests.utils.decorators import inlineCallbacks class CustomItem(Item): diff --git a/tests/test_logstats.py b/tests/test_logstats.py index d25bc0c93..c2e6d3456 100644 --- a/tests/test_logstats.py +++ b/tests/test_logstats.py @@ -16,6 +16,7 @@ class TestLogStats: self.stats.set_value("response_received_count", 4802) self.stats.set_value("item_scraped_count", 3201) + @pytest.mark.requires_reactor # needs a reactor or an event loop for LogStats.task def test_stats_calculations(self): logstats = LogStats.from_crawler(self.crawler) diff --git a/tests/test_mail.py b/tests/test_mail.py index 4e3523208..c845ba320 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -1,12 +1,14 @@ from email.charset import Charset from io import BytesIO +import pytest from twisted.internet import defer from twisted.internet._sslverify import ClientTLSOptions from scrapy.mail import MailSender +@pytest.mark.requires_reactor class TestMailSender: def test_send(self): mailsender = MailSender(debug=True) diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 494833fe1..f29bc90e8 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from w3lib.url import add_or_replace_parameter from scrapy import Spider, signals @@ -15,6 +14,7 @@ from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SimpleSpider +from tests.utils.decorators import inlineCallbacks if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py index 6ab5d9a8e..18fa6cdf2 100644 --- a/tests/test_pipeline_files.py +++ b/tests/test_pipeline_files.py @@ -19,7 +19,6 @@ from urllib.parse import urlparse import attr import pytest from itemadapter import ItemAdapter -from twisted.internet.defer import inlineCallbacks from scrapy.exceptions import NotConfigured from scrapy.http import Request, Response @@ -32,10 +31,10 @@ from scrapy.pipelines.files import ( S3FilesStore, ) from scrapy.settings import Settings -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.ftp import MockFTPServer +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks from .test_pipeline_media import _mocked_download_func diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 05ad43d16..37e87f1a9 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -13,11 +13,12 @@ from scrapy.http import Request, Response from scrapy.http.request import NO_CALLBACK from scrapy.pipelines.files import FileException from scrapy.pipelines.media import MediaPipeline -from scrapy.utils.defer import _defer_sleep_async, deferred_f_from_coro_f +from scrapy.utils.defer import _defer_sleep_async from scrapy.utils.log import failure_to_exc_info from scrapy.utils.signal import disconnect_all from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler +from tests.utils.decorators import deferred_f_from_coro_f async def _mocked_download_func(request): diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index acf8dd412..5bdca3035 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -10,14 +10,11 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.pipelines import ItemPipelineManager from scrapy.utils.asyncio import call_later from scrapy.utils.conf import build_component_list -from scrapy.utils.defer import ( - deferred_f_from_coro_f, - deferred_to_future, - maybe_deferred_to_future, -) +from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver.http import MockServer +from tests.utils.decorators import deferred_f_from_coro_f class SimplePipeline: @@ -251,6 +248,9 @@ class TestPipeline: class TestCustomPipelineManager: + # needs a reactor or an event loop for is_asyncio_available() + # (for ItemPipelineManager.process_item()) + @pytest.mark.requires_reactor def test_deprecated_process_item_spider_arg(self) -> None: class CustomPipelineManager(ItemPipelineManager): def process_item(self, item, spider): # pylint: disable=useless-parent-delegation diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.py index 27a78f802..4071224d6 100644 --- a/tests/test_proxy_connect.py +++ b/tests/test_proxy_connect.py @@ -8,12 +8,12 @@ from urllib.parse import urlsplit, urlunsplit import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from scrapy.http import Request from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SimpleSpider, SingleRequestSpider +from tests.utils.decorators import inlineCallbacks class MitmProxy: diff --git a/tests/test_request_attribute_binding.py b/tests/test_request_attribute_binding.py index e2c1668d8..4065a5a2c 100644 --- a/tests/test_request_attribute_binding.py +++ b/tests/test_request_attribute_binding.py @@ -1,11 +1,11 @@ from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from scrapy import Request, signals from scrapy.http.response import Response from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import SingleRequestSpider +from tests.utils.decorators import inlineCallbacks OVERRIDDEN_URL = "https://example.org" diff --git a/tests/test_request_cb_kwargs.py b/tests/test_request_cb_kwargs.py index 67ad4ba1e..56d7264da 100644 --- a/tests/test_request_cb_kwargs.py +++ b/tests/test_request_cb_kwargs.py @@ -1,10 +1,10 @@ from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from scrapy.http import Request from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider +from tests.utils.decorators import inlineCallbacks class InjectArgumentsDownloaderMiddleware: diff --git a/tests/test_request_left.py b/tests/test_request_left.py index 721bfc76e..a5d5efffa 100644 --- a/tests/test_request_left.py +++ b/tests/test_request_left.py @@ -1,9 +1,8 @@ -from twisted.internet.defer import inlineCallbacks - from scrapy.signals import request_left_downloader from scrapy.spiders import Spider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.utils.decorators import inlineCallbacks class SignalCatcherSpider(Spider): diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py index ef1da9b3a..8730489f7 100644 --- a/tests/test_scheduler.py +++ b/tests/test_scheduler.py @@ -8,7 +8,6 @@ from collections import deque from typing import Any, NamedTuple import pytest -from twisted.internet.defer import inlineCallbacks from scrapy.core.downloader import Downloader from scrapy.core.scheduler import BaseScheduler, Scheduler @@ -20,6 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.misc import load_object from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.utils.decorators import inlineCallbacks class MemoryScheduler(BaseScheduler): @@ -99,6 +99,9 @@ class MockCrawler(Crawler): self.stats = load_object(self.settings["STATS_CLASS"])(self) +# needs a reactor or an event loop for is_asyncio_available() +# (for _schedule_coro()) +@pytest.mark.requires_reactor class SchedulerHandler(ABC): jobdir = None @@ -243,6 +246,9 @@ _URLS_WITH_SLOTS = [ class TestMigration: + # needs a reactor or an event loop for is_asyncio_available() + # (for _schedule_coro()) + @pytest.mark.requires_reactor def test_migration(self, tmpdir): class PrevSchedulerHandler(SchedulerHandler): jobdir = tmpdir diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.py index 5dd2de8d8..538464788 100644 --- a/tests/test_scheduler_base.py +++ b/tests/test_scheduler_base.py @@ -5,7 +5,6 @@ from urllib.parse import urljoin import pytest from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from scrapy.core.scheduler import BaseScheduler from scrapy.http import Request @@ -14,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached from scrapy.utils.request import fingerprint from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.utils.decorators import inlineCallbacks PATHS = ["/a", "/b", "/c"] URLS = [urljoin("https://example.org", p) for p in PATHS] diff --git a/tests/test_signals.py b/tests/test_signals.py index 89b7a74dc..048af00ab 100644 --- a/tests/test_signals.py +++ b/tests/test_signals.py @@ -1,10 +1,9 @@ import pytest -from twisted.internet.defer import inlineCallbacks from scrapy import Request, Spider, signals -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler, get_from_asyncio_queue from tests.mockserver.http import MockServer +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks class ItemSpider(Spider): diff --git a/tests/test_spider.py b/tests/test_spider.py index a1649aa6f..dd1cc15b0 100644 --- a/tests/test_spider.py +++ b/tests/test_spider.py @@ -12,7 +12,6 @@ from unittest import mock import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from w3lib.url import safe_url_string from scrapy import signals @@ -29,9 +28,9 @@ from scrapy.spiders import ( XMLFeedSpider, ) from scrapy.spiders.init import InitSpider -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler, get_reactor_settings from tests import get_testdata, tests_datadir +from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks class TestSpider: diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py index e608d8975..d6767d74c 100644 --- a/tests/test_spider_start.py +++ b/tests/test_spider_start.py @@ -9,10 +9,11 @@ from testfixtures import LogCapture from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from .utils import twisted_sleep +from .utils.decorators import deferred_f_from_coro_f SLEEP_SECONDS = 0.1 diff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.py index 3ac6d9af7..17d897526 100644 --- a/tests/test_spidermiddleware.py +++ b/tests/test_spidermiddleware.py @@ -15,9 +15,10 @@ from scrapy.http import Request, Response from scrapy.spiders import Spider from scrapy.utils.asyncgen import collect_asyncgen from scrapy.utils.asyncio import call_later -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from twisted.python.failure import Failure diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 9899a0cd7..c0284c5ed 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -4,7 +4,6 @@ import logging import pytest from testfixtures import LogCapture -from twisted.internet.defer import inlineCallbacks from scrapy.http import Request, Response from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware @@ -12,6 +11,7 @@ from scrapy.utils.spider import DefaultSpider from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer from tests.spiders import MockServerSpider +from tests.utils.decorators import inlineCallbacks class _HttpErrorSpider(MockServerSpider): diff --git a/tests/test_spidermiddleware_output_chain.py b/tests/test_spidermiddleware_output_chain.py index 97d9d6470..b35a4c704 100644 --- a/tests/test_spidermiddleware_output_chain.py +++ b/tests/test_spidermiddleware_output_chain.py @@ -1,9 +1,9 @@ from testfixtures import LogCapture from scrapy import Request, Spider -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler from tests.mockserver.http import MockServer +from tests.utils.decorators import deferred_f_from_coro_f class _BaseSpiderMiddleware: diff --git a/tests/test_spidermiddleware_process_start.py b/tests/test_spidermiddleware_process_start.py index 18868bf8c..250b58544 100644 --- a/tests/test_spidermiddleware_process_start.py +++ b/tests/test_spidermiddleware_process_start.py @@ -5,11 +5,12 @@ import pytest from scrapy import Spider, signals from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future +from scrapy.utils.defer import maybe_deferred_to_future from scrapy.utils.test import get_crawler from tests.test_spider_start import SLEEP_SECONDS from .utils import twisted_sleep +from .utils.decorators import deferred_f_from_coro_f ITEM_A = {"id": "a"} ITEM_B = {"id": "b"} diff --git a/tests/test_spidermiddleware_start.py b/tests/test_spidermiddleware_start.py index c2efe47c9..1f2209f60 100644 --- a/tests/test_spidermiddleware_start.py +++ b/tests/test_spidermiddleware_start.py @@ -1,9 +1,9 @@ from scrapy.http import Request from scrapy.spidermiddlewares.start import StartSpiderMiddleware from scrapy.spiders import Spider -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.misc import build_from_crawler from scrapy.utils.test import get_crawler +from tests.utils.decorators import deferred_f_from_coro_f class TestMiddleware: diff --git a/tests/test_stats.py b/tests/test_stats.py index b5ab06901..187d240a8 100644 --- a/tests/test_stats.py +++ b/tests/test_stats.py @@ -10,9 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning from scrapy.extensions.corestats import CoreStats from scrapy.spiders import Spider from scrapy.statscollectors import DummyStatsCollector, StatsCollector -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.test import get_crawler from tests.spiders import SimpleSpider +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from scrapy.crawler import Crawler diff --git a/tests/test_utils_asyncgen.py b/tests/test_utils_asyncgen.py index b2d2b04c7..dba1ac5da 100644 --- a/tests/test_utils_asyncgen.py +++ b/tests/test_utils_asyncgen.py @@ -1,5 +1,5 @@ from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import deferred_f_from_coro_f +from tests.utils.decorators import deferred_f_from_coro_f class TestAsyncgenUtils: diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index b6b77c9e3..6706c1c80 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -14,13 +14,14 @@ from scrapy.utils.asyncio import ( _parallel_asyncio, is_asyncio_available, ) -from scrapy.utils.defer import deferred_f_from_coro_f +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from collections.abc import AsyncGenerator class TestAsyncio: + @pytest.mark.requires_reactor # needs a reactor or an event loop for is_asyncio_available() def test_is_asyncio_available(self, reactor_pytest: str) -> None: # the result should depend only on the pytest --reactor argument assert is_asyncio_available() == (reactor_pytest == "asyncio") @@ -103,6 +104,7 @@ class TestParallelAsyncio: assert max_parallel_count[0] <= self.CONCURRENT_ITEMS +@pytest.mark.requires_reactor # needs a running event loop for AsyncioLoopingCall.start() @pytest.mark.only_asyncio class TestAsyncioLoopingCall: def test_looping_call(self): diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index be623c3fb..ad317f3ae 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -6,7 +6,8 @@ from asyncio import Future from typing import TYPE_CHECKING, Any import pytest -from twisted.internet.defer import Deferred, inlineCallbacks, succeed +from twisted.internet.defer import Deferred, succeed +from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen from scrapy.utils.defer import ( @@ -19,6 +20,7 @@ from scrapy.utils.defer import ( mustbe_deferred, parallel_async, ) +from tests.utils.decorators import inlineCallbacks if TYPE_CHECKING: from collections.abc import AsyncGenerator, Awaitable, Callable, Generator @@ -107,6 +109,7 @@ class TestIterErrback: assert isinstance(errors[0].value, ZeroDivisionError) +@pytest.mark.requires_reactor class TestAiterErrback: @deferred_f_from_coro_f async def test_aiter_errback_good(self): @@ -134,6 +137,7 @@ class TestAiterErrback: assert isinstance(errors[0].value, ZeroDivisionError) +@pytest.mark.requires_reactor class TestAsyncDefTestsuite: @deferred_f_from_coro_f async def test_deferred_f_from_coro_f(self): @@ -304,7 +308,7 @@ class TestDeferredFromCoro: class TestDeferredFFromCoroF: - @inlineCallbacks + @inlineCallbacks_orig def _assert_result( self, c_f: Callable[[], Awaitable[int]] ) -> Generator[Deferred[Any], Any, None]: @@ -342,6 +346,7 @@ class TestDeferredFFromCoroF: @pytest.mark.only_asyncio +@pytest.mark.requires_reactor class TestDeferredToFuture: @deferred_f_from_coro_f async def test_deferred(self): @@ -377,6 +382,9 @@ class TestDeferredToFuture: @pytest.mark.only_asyncio +# needs a reactor or an event loop for is_asyncio_available() +# (for maybe_deferred_to_future()) +@pytest.mark.requires_reactor class TestMaybeDeferredToFutureAsyncio: @deferred_f_from_coro_f async def test_deferred(self): @@ -412,6 +420,9 @@ class TestMaybeDeferredToFutureAsyncio: @pytest.mark.only_not_asyncio +# needs a reactor or an event loop for is_asyncio_available() +# (for maybe_deferred_to_future()) +@pytest.mark.requires_reactor class TestMaybeDeferredToFutureNotAsyncio: def test_deferred(self): d = Deferred() diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py index 34fca06c1..4a9e375fb 100644 --- a/tests/test_utils_python.py +++ b/tests/test_utils_python.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, TypeVar import pytest from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen -from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f +from scrapy.utils.defer import aiter_errback from scrapy.utils.python import ( MutableAsyncChain, MutableChain, @@ -20,6 +20,7 @@ from scrapy.utils.python import ( to_unicode, without_none_values, ) +from tests.utils.decorators import deferred_f_from_coro_f if TYPE_CHECKING: from collections.abc import Iterable, Mapping diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py index 6cbdcfccc..658e68016 100644 --- a/tests/test_utils_reactor.py +++ b/tests/test_utils_reactor.py @@ -3,20 +3,22 @@ import warnings import pytest -from scrapy.utils.defer import deferred_f_from_coro_f from scrapy.utils.reactor import ( _asyncio_reactor_path, install_reactor, is_asyncio_reactor_installed, set_asyncio_event_loop, ) +from tests.utils.decorators import deferred_f_from_coro_f class TestAsyncio: + @pytest.mark.requires_reactor def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None: # the result should depend only on the pytest --reactor argument assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio") + @pytest.mark.requires_reactor def test_install_asyncio_reactor(self): from twisted.internet import reactor as original_reactor diff --git a/tests/test_utils_signal.py b/tests/test_utils_signal.py index 51e496575..e47e9013a 100644 --- a/tests/test_utils_signal.py +++ b/tests/test_utils_signal.py @@ -4,7 +4,6 @@ import pytest from pydispatch import dispatcher from testfixtures import LogCapture from twisted.internet import defer -from twisted.internet.defer import inlineCallbacks from twisted.python.failure import Failure from scrapy.utils.asyncio import call_later @@ -15,6 +14,7 @@ from scrapy.utils.signal import ( send_catch_log_deferred, ) from scrapy.utils.test import get_from_asyncio_queue +from tests.utils.decorators import inlineCallbacks class TestSendCatchLog: diff --git a/tests/test_webclient.py b/tests/test_webclient.py index 50cab9a60..0ad6bd9d1 100644 --- a/tests/test_webclient.py +++ b/tests/test_webclient.py @@ -30,6 +30,8 @@ from tests.mockserver.http_resources import ( from tests.mockserver.utils import ssl_context_factory from tests.test_core_downloader import TestContextFactoryBase +pytestmark = pytest.mark.requires_reactor + def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs): """Adapted version of twisted.web.client.getPage""" diff --git a/tests/test_zz_resources.py b/tests/test_zz_resources.py index e2c8ed205..7b11aac9e 100644 --- a/tests/test_zz_resources.py +++ b/tests/test_zz_resources.py @@ -30,6 +30,7 @@ def test_stderr_log_handler() -> None: assert c == 0 +@pytest.mark.requires_reactor # needs a running event loop for asyncio.all_tasks() @pytest.mark.only_asyncio def test_pending_asyncio_tasks() -> None: """Test that there are no pending asyncio tasks.""" diff --git a/tests/utils/decorators.py b/tests/utils/decorators.py new file mode 100644 index 000000000..dc9d2e113 --- /dev/null +++ b/tests/utils/decorators.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from functools import wraps +from typing import TYPE_CHECKING, Any, ParamSpec + +import pytest +from twisted.internet.defer import Deferred +from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig + +from scrapy.utils.defer import deferred_from_coro + +if TYPE_CHECKING: + from collections.abc import Awaitable, Callable, Generator + + +_P = ParamSpec("_P") + + +def inlineCallbacks( + f: Callable[_P, Generator[Deferred[Any], Any, None]], +) -> Callable[_P, Deferred[None]]: + """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the + decorated function with ``@pytest.mark.requires_reactor``.""" + + @pytest.mark.requires_reactor + @wraps(f) + @inlineCallbacks_orig + def wrapper( + *args: _P.args, **kwargs: _P.kwargs + ) -> Generator[Deferred[Any], Any, None]: + return f(*args, **kwargs) + + return wrapper + + +def deferred_f_from_coro_f( + coro_f: Callable[_P, Awaitable[None]], +) -> Callable[_P, Deferred[None]]: + """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the + decorated function with ``@pytest.mark.requires_reactor``.""" + + @pytest.mark.requires_reactor + @wraps(coro_f) + def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]: + return deferred_from_coro(coro_f(*coro_args, **coro_kwargs)) + + return f diff --git a/tox.ini b/tox.ini index 5200b26e0..fc4d1b824 100644 --- a/tox.ini +++ b/tox.ini @@ -161,6 +161,10 @@ commands = {[pinned]commands} commands = {[testenv]commands} --reactor=default +[testenv:no-reactor] +commands = + {[testenv]commands} -p no:twisted --reactor=none + [testenv:default-reactor-pinned] basepython = {[pinned]basepython} deps = {[testenv:pinned]deps} @@ -168,6 +172,13 @@ commands = {[pinned]commands} --reactor=default setenv = {[pinned]setenv} +[testenv:no-reactor-pinned] +basepython = {[pinned]basepython} +deps = {[testenv:pinned]deps} +commands = {[pinned]commands} -p no:twisted --reactor=none +setenv = + {[pinned]setenv} + [testenv:pypy3] basepython = pypy3 commands =