Reactorless tests 1: the test env (#6952)

* Reactorless tests.

* Updates after merging master.

* Update a new import.

* Updates after merging master.

* Update testenv:no-reactor-pinned.

* Foundations for the reactorless mode.

* Fixes after merging master.

* Updates after merging reactorless-base.

* Add simple subprocess tests for reactorless AsyncCrawler*.

* More reactorless tests.

* Refactor AsyncCrawlerProcess.start().

* More checks.

* Fix test_reactorless_import_hook.

* More tests.

* Call install_reactor() before asyncio.run().

* Cleanup.

* Rephrase.

* Fix a test.

* Fixes.

* Rephrasing.

* Set TELNETCONSOLE_ENABLED=False in the reactorless mode.

* Fix a new import.

* Clarify --reactor marks.

* Cleanup.

* More docs/comments.

* Update test_fallback_workflow().

* Update TestDeferredFFromCoroF.

* Typing improvements.

.github/workflows/tests-ubuntu.yml

@@ -32,6 +32,9 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: default-reactor
+        - python-version: "3.13"
+          env:
+            TOXENV: no-reactor
         - python-version: pypy3.11
           env:
             TOXENV: pypy3
@@ -43,6 +46,9 @@ jobs:
         - python-version: "3.10.19"
           env:
             TOXENV: default-reactor-pinned
+        - python-version: "3.10.19"
+          env:
+            TOXENV: no-reactor-pinned
         - python-version: pypy3.11
           env:
             TOXENV: pypy3-pinned

.github/workflows/tests-windows.yml

@@ -32,6 +32,9 @@ jobs:
         - python-version: "3.13"
           env:
             TOXENV: default-reactor
+        - python-version: "3.13"
+          env:
+            TOXENV: no-reactor
 
         # pinned deps
         - python-version: "3.10.11"

conftest.py

@@ -8,6 +8,7 @@ import pytest
 from twisted.web.http import H2_ENABLED
 
 from scrapy.utils.reactor import set_asyncio_event_loop_policy
+from scrapy.utils.reactorless import install_reactor_import_hook
 from tests.keys import generate_keys
 from tests.mockserver.http import MockServer
 
@@ -25,10 +26,6 @@ collect_ignore = [
     # not a test, but looks like a test
     "scrapy/utils/testproc.py",
     "scrapy/utils/testsite.py",
-    "tests/ftpserver.py",
-    "tests/mockserver.py",
-    "tests/pipelines.py",
-    "tests/spiders.py",
     # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerProcessSubprocess
     *_py_files("tests/AsyncCrawlerProcess"),
     # contains scripts to be run by tests/test_crawler.py::AsyncCrawlerRunnerSubprocess
@@ -56,6 +53,17 @@ if not H2_ENABLED:
     )
 
 
+def pytest_addoption(parser, pluginmanager):
+    if pluginmanager.hasplugin("twisted"):
+        return
+    # add the full choice set so that pytest doesn't complain about invalid choices in some cases
+    parser.addoption(
+        "--reactor",
+        default="none",
+        choices=["asyncio", "default", "none"],
+    )
+
+
 @pytest.fixture(scope="session")
 def mockserver() -> Generator[MockServer]:
     with MockServer() as mockserver:
@@ -72,17 +80,26 @@ def pytest_configure(config):
         # Needed on Windows to switch from proactor to selector for Twisted reactor compatibility.
         # If we decide to run tests with both, we will need to add a new option and check it here.
         set_asyncio_event_loop_policy()
+    elif config.getoption("--reactor") == "none":
+        install_reactor_import_hook()
 
 
 def pytest_runtest_setup(item):
     # Skip tests based on reactor markers
     reactor = item.config.getoption("--reactor")
 
-    if item.get_closest_marker("only_asyncio") and reactor != "asyncio":
-        pytest.skip("This test is only run with --reactor=asyncio")
+    if item.get_closest_marker("requires_reactor") and reactor == "none":
+        pytest.skip('This test is only run when the --reactor value is not "none"')
 
-    if item.get_closest_marker("only_not_asyncio") and reactor == "asyncio":
-        pytest.skip("This test is only run without --reactor=asyncio")
+    if item.get_closest_marker("only_asyncio") and reactor not in {"asyncio", "none"}:
+        pytest.skip(
+            'This test is only run when the --reactor value is "asyncio" (default) or "none"'
+        )
+
+    if item.get_closest_marker("only_not_asyncio") and reactor in {"asyncio", "none"}:
+        pytest.skip(
+            'This test is only run when the --reactor value is not "asyncio" (default) or "none"'
+        )
 
     # Skip tests requiring optional dependencies
     optional_deps = [

pyproject.toml

@@ -227,8 +227,9 @@ addopts = [
 xfail_strict = true
 python_files = ["test_*.py", "test_*/__init__.py"]
 markers = [
-    "only_asyncio: marks tests as only enabled when --reactor=asyncio is passed",
-    "only_not_asyncio: marks tests as only enabled when --reactor=asyncio is not passed",
+    "only_asyncio: marks tests that require the asyncio loop to be used",
+    "only_not_asyncio: marks tests that require the asyncio loop to not be used",
+    "requires_reactor: marks tests that require a reactor",
     "requires_uvloop: marks tests as only enabled when uvloop is known to be working",
     "requires_botocore: marks tests that need botocore (but not boto3)",
     "requires_boto3: marks tests that need botocore and boto3",

scrapy/utils/test.py

@@ -17,7 +17,7 @@ from unittest import mock
 from twisted.trial.unittest import SkipTest
 from twisted.web.client import Agent
 
-from scrapy.crawler import CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, CrawlerRunner, CrawlerRunnerBase
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.utils.boto import is_botocore_available
 from scrapy.utils.deprecate import create_deprecated_class
@@ -116,16 +116,24 @@ def get_reactor_settings() -> dict[str, Any]:
 
     ``Crawler._apply_settings()`` checks that the installed reactor matches the
     settings, so tests that run the crawler in the current process may need to
-    pass a correct ``"TWISTED_REACTOR"`` setting value when creating it.
+    pass a correct :setting:`TWISTED_REACTOR` setting value when creating it.
     """
-    if not is_reactor_installed():
-        raise RuntimeError(
-            "get_reactor_settings() called without an installed reactor,"
-            " you may need to install a reactor explicitly when running your tests."
-        )
     settings: dict[str, Any] = {}
-    if not is_asyncio_reactor_installed():
-        settings["TWISTED_REACTOR"] = None
+    if is_reactor_installed():
+        if not is_asyncio_reactor_installed():
+            settings["TWISTED_REACTOR"] = None
+    else:
+        # We are either running Scrapy tests for the reactorless mode, or
+        # running some 3rd-party library tests for the reactorless mode, or
+        # running some 3rd-party library tests without initializing a reactor
+        # properly. The first two cases are fine, but we cannot distinguish the
+        # last one from them.
+        settings["TWISTED_ENABLED"] = False
+        settings["DOWNLOAD_HANDLERS"] = {
+            "ftp": None,
+            "http": None,
+            "https": None,
+        }
     return settings
 
 
@@ -144,7 +152,11 @@ def get_crawler(
         **get_reactor_settings(),
         **(settings_dict or {}),
     }
-    runner = CrawlerRunner(settings)
+    runner: CrawlerRunnerBase
+    if is_reactor_installed():
+        runner = CrawlerRunner(settings)
+    else:
+        runner = AsyncCrawlerRunner(settings)
     crawler = runner.create_crawler(spidercls or DefaultSpider)
     crawler._apply_settings()
     return crawler

tests/test_addons.py

@@ -2,13 +2,12 @@ import itertools
 from typing import Any
 from unittest.mock import patch
 
-from twisted.internet.defer import inlineCallbacks
-
 from scrapy import Spider
-from scrapy.crawler import Crawler, CrawlerRunner
+from scrapy.crawler import AsyncCrawlerRunner, Crawler, CrawlerRunner
 from scrapy.exceptions import NotConfigured
 from scrapy.settings import BaseSettings, Settings
 from scrapy.utils.test import get_crawler, get_reactor_settings
+from tests.utils.decorators import inlineCallbacks
 
 
 class SimpleAddon:
@@ -109,9 +108,15 @@ class TestAddonManager:
         crawler = get_crawler(settings_dict=settings_dict)
         assert crawler.settings.getint("KEY") == 15
 
+        runner_cls = (
+            CrawlerRunner
+            if settings_dict.get("TWISTED_ENABLED", True)
+            else AsyncCrawlerRunner
+        )
+
         settings = Settings(settings_dict)
         settings.set("KEY", 0, priority="default")
-        runner = CrawlerRunner(settings)
+        runner = runner_cls(settings)
         crawler = runner.create_crawler(Spider)
         crawler._apply_settings()
         assert crawler.settings.getint("KEY") == 15
@@ -123,44 +128,39 @@ class TestAddonManager:
         }
         settings = Settings(settings_dict)
         settings.set("KEY", 0, priority="default")
-        runner = CrawlerRunner(settings)
+        runner = runner_cls(settings)
         crawler = runner.create_crawler(Spider)
         assert crawler.settings.getint("KEY") == 20
 
     def test_fallback_workflow(self):
-        FALLBACK_SETTING = "MY_FALLBACK_DOWNLOAD_HANDLER"
+        FALLBACK_SETTING = "MY_FALLBACK_SCHEDULER"
 
         class AddonWithFallback:
             def update_settings(self, settings):
                 if not settings.get(FALLBACK_SETTING):
                     settings.set(
                         FALLBACK_SETTING,
-                        settings.getwithbase("DOWNLOAD_HANDLERS")["https"],
+                        settings.get("SCHEDULER"),
                         "addon",
                     )
-                settings["DOWNLOAD_HANDLERS"]["https"] = "AddonHandler"
+                settings["SCHEDULER"] = "AddonScheduler"
 
         settings_dict = {
             "ADDONS": {AddonWithFallback: 1},
         }
         crawler = get_crawler(settings_dict=settings_dict)
+        assert crawler.settings.get("SCHEDULER") == "AddonScheduler"
         assert (
-            crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"
-        )
-        assert (
-            crawler.settings.get(FALLBACK_SETTING)
-            == "scrapy.core.downloader.handlers.http11.HTTP11DownloadHandler"
+            crawler.settings.get(FALLBACK_SETTING) == "scrapy.core.scheduler.Scheduler"
         )
 
         settings_dict = {
             "ADDONS": {AddonWithFallback: 1},
-            "DOWNLOAD_HANDLERS": {"https": "UserHandler"},
+            "SCHEDULER": "UserScheduler",
         }
         crawler = get_crawler(settings_dict=settings_dict)
-        assert (
-            crawler.settings.getwithbase("DOWNLOAD_HANDLERS")["https"] == "AddonHandler"
-        )
-        assert crawler.settings.get(FALLBACK_SETTING) == "UserHandler"
+        assert crawler.settings.get("SCHEDULER") == "AddonScheduler"
+        assert crawler.settings.get(FALLBACK_SETTING) == "UserScheduler"
 
     def test_logging_message(self):
         class LoggedAddon:

tests/test_closespider.py

@@ -1,5 +1,3 @@
-from twisted.internet.defer import inlineCallbacks
-
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import (
@@ -9,6 +7,7 @@ from tests.spiders import (
     MaxItemsAndRequestsSpider,
     SlowSpider,
 )
+from tests.utils.decorators import inlineCallbacks
 
 
 class TestCloseSpider:

tests/test_contracts.py

@@ -1,7 +1,6 @@
 from unittest import TextTestResult
 
 import pytest
-from twisted.internet.defer import inlineCallbacks
 from twisted.python import failure
 
 from scrapy import FormRequest
@@ -19,6 +18,7 @@ from scrapy.spidermiddlewares.httperror import HttpError
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
 
 
 class DemoItem(Item):

tests/test_core_downloader.py

@@ -18,13 +18,14 @@ from scrapy.core.downloader.contextfactory import (
 from scrapy.core.downloader.handlers.http11 import _RequestBodyProducer
 from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.python import to_bytes
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http_resources import PayloadResource
 from tests.mockserver.utils import ssl_context_factory
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from twisted.internet.defer import Deferred

tests/test_core_scraper.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     import pytest

tests/test_crawl.py

@@ -9,7 +9,7 @@ from urllib.parse import urlencode, urlparse
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks, succeed
+from twisted.internet.defer import succeed
 from twisted.internet.ssl import Certificate
 from twisted.python.failure import Failure
 
@@ -18,7 +18,7 @@ from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning, StopDownload
 from scrapy.http import Request
 from scrapy.http.response import Response
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.engine import format_engine_status, get_engine_status
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler, get_reactor_settings
@@ -55,6 +55,7 @@ from tests.spiders import (
     StartGoodAndBadOutput,
     StartItemSpider,
 )
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 if TYPE_CHECKING:
     from scrapy.statscollectors import StatsCollector

tests/test_crawler.py

@@ -14,7 +14,7 @@ from typing import Any
 import pytest
 from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
-from twisted.internet.defer import Deferred, inlineCallbacks
+from twisted.internet.defer import Deferred
 from w3lib import __version__ as w3lib_version
 from zope.interface.exceptions import MultipleInvalid
 
@@ -31,11 +31,7 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.throttle import AutoThrottle
 from scrapy.settings import Settings, default_settings
 from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import (
-    deferred_f_from_coro_f,
-    deferred_from_coro,
-    maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.log import (
     _uninstall_scrapy_root_handler,
     configure_logging,
@@ -45,6 +41,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests.mockserver.http import MockServer
 from tests.utils import get_script_run_env
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 BASE_SETTINGS: dict[str, Any] = {}
 

tests/test_downloader_handler_twisted_ftp.py

@@ -25,6 +25,9 @@ if TYPE_CHECKING:
     from twisted.protocols.ftp import FTPFactory
 
 
+pytestmark = pytest.mark.requires_reactor
+
+
 class TestFTPBase(ABC):
     username = "scrapy"
     password = "passwd"

tests/test_downloader_handler_twisted_http10.py

@@ -16,6 +16,9 @@ if TYPE_CHECKING:
     from tests.mockserver.http import MockServer
 
 
+pytestmark = pytest.mark.requires_reactor
+
+
 class HTTP10DownloadHandlerMixin:
     @property
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:

tests/test_downloader_handler_twisted_http11.py

@@ -4,6 +4,8 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+import pytest
+
 from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
 from tests.test_downloader_handlers_http_base import (
     TestHttp11Base,
@@ -21,6 +23,9 @@ if TYPE_CHECKING:
     from scrapy.core.downloader.handlers import DownloadHandlerProtocol
 
 
+pytestmark = pytest.mark.requires_reactor
+
+
 class HTTP11DownloadHandlerMixin:
     @property
     def download_handler_cls(self) -> type[DownloadHandlerProtocol]:

tests/test_downloader_handler_twisted_http2.py

@@ -27,9 +27,13 @@ if TYPE_CHECKING:
     from tests.mockserver.http import MockServer
     from tests.mockserver.proxy_echo import ProxyEchoMockServer
 
-pytestmark = pytest.mark.skipif(
-    not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
-)
+
+pytestmark = [
+    pytest.mark.requires_reactor,
+    pytest.mark.skipif(
+        not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
+    ),
+]
 
 
 class H2DownloadHandlerMixin:

tests/test_downloader_handlers.py

@@ -19,9 +19,9 @@ from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
 from scrapy.http import Request
 from scrapy.responsetypes import responsetypes
 from scrapy.utils.boto import is_botocore_available
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class DummyDH:

tests/test_downloader_handlers_http_base.py

@@ -26,11 +26,7 @@ from scrapy.exceptions import (
     UnsupportedURLSchemeError,
 )
 from scrapy.http import Headers, HtmlResponse, Request, Response, TextResponse
-from scrapy.utils.defer import (
-    deferred_f_from_coro_f,
-    deferred_from_coro,
-    maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
@@ -38,6 +34,7 @@ from tests import NON_EXISTING_RESOLVABLE
 from tests.mockserver.proxy_echo import ProxyEchoMockServer
 from tests.mockserver.simple_https import SimpleMockServer
 from tests.spiders import SingleRequestSpider
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Generator

tests/test_downloadermiddleware.py

@@ -13,9 +13,10 @@ from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
 from scrapy.exceptions import ScrapyDeprecationWarning, _InvalidOutput
 from scrapy.http import Request, Response
 from scrapy.spiders import Spider
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.python import to_bytes
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator

tests/test_downloadermiddleware_robotstxt.py

@@ -14,12 +14,9 @@ from scrapy.http import Request, Response, TextResponse
 from scrapy.http.request import NO_CALLBACK
 from scrapy.settings import Settings
 from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import (
-    deferred_f_from_coro_f,
-    deferred_from_coro,
-    maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
 from tests.test_robotstxt_interface import rerp_available
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler

tests/test_downloaderslotssettings.py

@@ -2,17 +2,16 @@ import time
 from typing import Any
 
 import pytest
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy import Request
 from scrapy.core.downloader import Downloader, Slot
 from scrapy.crawler import CrawlerRunner
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import MetaSpider
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 
 class DownloaderSlotsSettingsTestSpider(MetaSpider):
@@ -85,6 +84,7 @@ class TestCrawl:
         assert max(list(error_delta.values())) < tolerance
 
 
+@pytest.mark.requires_reactor  # needs a reactor or an event loop for Downloader._slot_gc_loop
 def test_params():
     params = {
         "concurrency": 1,
@@ -109,6 +109,7 @@ def test_params():
         )
 
 
+@pytest.mark.requires_reactor  # needs a reactor or an event loop for Downloader._slot_gc_loop
 def test_get_slot_deprecated_spider_arg():
     crawler = get_crawler(DefaultSpider)
     crawler.spider = crawler._create_spider()

tests/test_engine.py

@@ -17,7 +17,6 @@ from itemadapter import ItemAdapter
 from pydispatch import dispatcher
 from testfixtures import LogCapture
 from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy import signals
 from scrapy.core.engine import ExecutionEngine, _Slot
@@ -29,7 +28,6 @@ from scrapy.linkextractors import LinkExtractor
 from scrapy.spiders import Spider
 from scrapy.utils.defer import (
     _schedule_coro,
-    deferred_f_from_coro_f,
     deferred_from_coro,
     maybe_deferred_to_future,
 )
@@ -37,6 +35,7 @@ from scrapy.utils.signal import disconnect_all
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests import get_testdata
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 if TYPE_CHECKING:
     from scrapy.core.scheduler import Scheduler
@@ -599,6 +598,7 @@ class TestEngineDownload(TestEngineDownloadAsync):
         return await maybe_deferred_to_future(engine.download(request))
 
 
+@pytest.mark.requires_reactor  # needs a reactor or an event loop for _Slot.heartbeat
 def test_request_scheduled_signal(caplog):
     class TestScheduler(BaseScheduler):
         def __init__(self):

tests/test_engine_loop.py

@@ -7,10 +7,11 @@ from typing import TYPE_CHECKING
 from twisted.internet.defer import Deferred
 
 from scrapy import Request, Spider, signals
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.test_scheduler import MemoryScheduler
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     import pytest

tests/test_engine_stop_download_bytes.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from scrapy.utils.defer import deferred_f_from_coro_f
 from tests.test_engine import (
     AttrsItemsSpider,
     CrawlerRun,
@@ -12,6 +11,7 @@ from tests.test_engine import (
     MySpider,
     TestEngineBase,
 )
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     import pytest

tests/test_engine_stop_download_headers.py

@@ -3,7 +3,6 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 
 from scrapy.exceptions import StopDownload
-from scrapy.utils.defer import deferred_f_from_coro_f
 from tests.test_engine import (
     AttrsItemsSpider,
     CrawlerRun,
@@ -12,6 +11,7 @@ from tests.test_engine import (
     MySpider,
     TestEngineBase,
 )
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     import pytest

tests/test_extension_periodic_log.py

@@ -3,6 +3,8 @@ from __future__ import annotations
 import datetime
 from typing import TYPE_CHECKING, Any
 
+import pytest
+
 from scrapy.extensions.periodic_log import PeriodicLog
 from scrapy.utils.test import get_crawler
 
@@ -86,6 +88,7 @@ class TestPeriodicLog:
         assert extension({"PERIODIC_LOG_DELTA": True, "LOGSTATS_INTERVAL": 60})
         assert extension({"PERIODIC_LOG_DELTA": "True", "LOGSTATS_INTERVAL": 60})
 
+    @pytest.mark.requires_reactor  # needs a reactor or an event loop for PeriodicLog.task
     def test_log_delta(self):
         def emulate(settings=None):
             spider = MetaSpider()
@@ -143,6 +146,7 @@ class TestPeriodicLog:
             and ("downloader/" in k and "bytes" not in k),
         )
 
+    @pytest.mark.requires_reactor  # needs a reactor or an event loop for PeriodicLog.task
     def test_log_stats(self):
         def emulate(settings=None):
             spider = MetaSpider()

tests/test_extension_telnet.py

@@ -1,10 +1,10 @@
 import pytest
 from twisted.conch.telnet import ITelnetProtocol
 from twisted.cred import credentials
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.extensions.telnet import TelnetConsole
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import inlineCallbacks
 
 
 class TestTelnetExtension:

tests/test_feedexport.py

@@ -29,7 +29,6 @@ import lxml.etree
 import pytest
 from packaging.version import Version
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 from w3lib.url import file_uri_to_path, path_to_file_uri
 from zope.interface import implementer
 from zope.interface.verify import verifyObject
@@ -50,12 +49,13 @@ from scrapy.extensions.feedexport import (
     StdoutFeedStorage,
 )
 from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
 from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 if TYPE_CHECKING:
     from collections.abc import Callable, Iterable

tests/test_http2_client_protocol.py

@@ -39,9 +39,12 @@ if TYPE_CHECKING:
     from scrapy.core.http2.protocol import H2ClientProtocol
 
 
-pytestmark = pytest.mark.skipif(
-    not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
-)
+pytestmark = [
+    pytest.mark.requires_reactor,
+    pytest.mark.skipif(
+        not H2_ENABLED, reason="HTTP/2 support in Twisted is not enabled"
+    ),
+]
 
 
 def generate_random_string(size: int) -> str:

tests/test_logformatter.py

@@ -2,7 +2,6 @@ import logging
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy.exceptions import DropItem
@@ -13,6 +12,7 @@ from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import ItemSpider
+from tests.utils.decorators import inlineCallbacks
 
 
 class CustomItem(Item):

tests/test_logstats.py

@@ -16,6 +16,7 @@ class TestLogStats:
         self.stats.set_value("response_received_count", 4802)
         self.stats.set_value("item_scraped_count", 3201)
 
+    @pytest.mark.requires_reactor  # needs a reactor or an event loop for LogStats.task
     def test_stats_calculations(self):
         logstats = LogStats.from_crawler(self.crawler)
 

tests/test_mail.py

@@ -1,12 +1,14 @@
 from email.charset import Charset
 from io import BytesIO
 
+import pytest
 from twisted.internet import defer
 from twisted.internet._sslverify import ClientTLSOptions
 
 from scrapy.mail import MailSender
 
 
+@pytest.mark.requires_reactor
 class TestMailSender:
     def test_send(self):
         mailsender = MailSender(debug=True)

tests/test_pipeline_crawl.py

@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 from w3lib.url import add_or_replace_parameter
 
 from scrapy import Spider, signals
@@ -15,6 +14,7 @@ from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SimpleSpider
+from tests.utils.decorators import inlineCallbacks
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler

tests/test_pipeline_files.py

@@ -19,7 +19,6 @@ from urllib.parse import urlparse
 import attr
 import pytest
 from itemadapter import ItemAdapter
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.exceptions import NotConfigured
 from scrapy.http import Request, Response
@@ -32,10 +31,10 @@ from scrapy.pipelines.files import (
     S3FilesStore,
 )
 from scrapy.settings import Settings
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.ftp import MockFTPServer
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 from .test_pipeline_media import _mocked_download_func
 

tests/test_pipeline_media.py

@@ -13,11 +13,12 @@ from scrapy.http import Request, Response
 from scrapy.http.request import NO_CALLBACK
 from scrapy.pipelines.files import FileException
 from scrapy.pipelines.media import MediaPipeline
-from scrapy.utils.defer import _defer_sleep_async, deferred_f_from_coro_f
+from scrapy.utils.defer import _defer_sleep_async
 from scrapy.utils.log import failure_to_exc_info
 from scrapy.utils.signal import disconnect_all
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 async def _mocked_download_func(request):

tests/test_pipelines.py

@@ -10,14 +10,11 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.pipelines import ItemPipelineManager
 from scrapy.utils.asyncio import call_later
 from scrapy.utils.conf import build_component_list
-from scrapy.utils.defer import (
-    deferred_f_from_coro_f,
-    deferred_to_future,
-    maybe_deferred_to_future,
-)
+from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class SimplePipeline:
@@ -251,6 +248,9 @@ class TestPipeline:
 
 
 class TestCustomPipelineManager:
+    # needs a reactor or an event loop for is_asyncio_available()
+    # (for ItemPipelineManager.process_item())
+    @pytest.mark.requires_reactor
     def test_deprecated_process_item_spider_arg(self) -> None:
         class CustomPipelineManager(ItemPipelineManager):
             def process_item(self, item, spider):  # pylint: disable=useless-parent-delegation

tests/test_proxy_connect.py

@@ -8,12 +8,12 @@ from urllib.parse import urlsplit, urlunsplit
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SimpleSpider, SingleRequestSpider
+from tests.utils.decorators import inlineCallbacks
 
 
 class MitmProxy:

tests/test_request_attribute_binding.py

@@ -1,11 +1,11 @@
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy import Request, signals
 from scrapy.http.response import Response
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import SingleRequestSpider
+from tests.utils.decorators import inlineCallbacks
 
 OVERRIDDEN_URL = "https://example.org"
 

tests/test_request_cb_kwargs.py

@@ -1,10 +1,10 @@
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.http import Request
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import MockServerSpider
+from tests.utils.decorators import inlineCallbacks
 
 
 class InjectArgumentsDownloaderMiddleware:

tests/test_request_left.py

@@ -1,9 +1,8 @@
-from twisted.internet.defer import inlineCallbacks
-
 from scrapy.signals import request_left_downloader
 from scrapy.spiders import Spider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
 
 
 class SignalCatcherSpider(Spider):

tests/test_scheduler.py

@@ -8,7 +8,6 @@ from collections import deque
 from typing import Any, NamedTuple
 
 import pytest
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import Downloader
 from scrapy.core.scheduler import BaseScheduler, Scheduler
@@ -20,6 +19,7 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
 
 
 class MemoryScheduler(BaseScheduler):
@@ -99,6 +99,9 @@ class MockCrawler(Crawler):
         self.stats = load_object(self.settings["STATS_CLASS"])(self)
 
 
+# needs a reactor or an event loop for is_asyncio_available()
+# (for _schedule_coro())
+@pytest.mark.requires_reactor
 class SchedulerHandler(ABC):
     jobdir = None
 
@@ -243,6 +246,9 @@ _URLS_WITH_SLOTS = [
 
 
 class TestMigration:
+    # needs a reactor or an event loop for is_asyncio_available()
+    # (for _schedule_coro())
+    @pytest.mark.requires_reactor
     def test_migration(self, tmpdir):
         class PrevSchedulerHandler(SchedulerHandler):
             jobdir = tmpdir

tests/test_scheduler_base.py

@@ -5,7 +5,6 @@ from urllib.parse import urljoin
 import pytest
 from testfixtures import LogCapture
 from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.scheduler import BaseScheduler
 from scrapy.http import Request
@@ -14,6 +13,7 @@ from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.request import fingerprint
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import inlineCallbacks
 
 PATHS = ["/a", "/b", "/c"]
 URLS = [urljoin("https://example.org", p) for p in PATHS]

tests/test_signals.py

@@ -1,10 +1,9 @@
 import pytest
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy import Request, Spider, signals
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler, get_from_asyncio_queue
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 
 class ItemSpider(Spider):

tests/test_spider.py

@@ -12,7 +12,6 @@ from unittest import mock
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 from w3lib.url import safe_url_string
 
 from scrapy import signals
@@ -29,9 +28,9 @@ from scrapy.spiders import (
     XMLFeedSpider,
 )
 from scrapy.spiders.init import InitSpider
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler, get_reactor_settings
 from tests import get_testdata, tests_datadir
+from tests.utils.decorators import deferred_f_from_coro_f, inlineCallbacks
 
 
 class TestSpider:

tests/test_spider_start.py

@@ -9,10 +9,11 @@ from testfixtures import LogCapture
 
 from scrapy import Spider, signals
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 
 from .utils import twisted_sleep
+from .utils.decorators import deferred_f_from_coro_f
 
 SLEEP_SECONDS = 0.1
 

tests/test_spidermiddleware.py

@@ -15,9 +15,10 @@ from scrapy.http import Request, Response
 from scrapy.spiders import Spider
 from scrapy.utils.asyncgen import collect_asyncgen
 from scrapy.utils.asyncio import call_later
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from twisted.python.failure import Failure

tests/test_spidermiddleware_httperror.py

@@ -4,7 +4,6 @@ import logging
 
 import pytest
 from testfixtures import LogCapture
-from twisted.internet.defer import inlineCallbacks
 
 from scrapy.http import Request, Response
 from scrapy.spidermiddlewares.httperror import HttpError, HttpErrorMiddleware
@@ -12,6 +11,7 @@ from scrapy.utils.spider import DefaultSpider
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
 from tests.spiders import MockServerSpider
+from tests.utils.decorators import inlineCallbacks
 
 
 class _HttpErrorSpider(MockServerSpider):

tests/test_spidermiddleware_output_chain.py

@@ -1,9 +1,9 @@
 from testfixtures import LogCapture
 
 from scrapy import Request, Spider
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class _BaseSpiderMiddleware:

tests/test_spidermiddleware_process_start.py

@@ -5,11 +5,12 @@ import pytest
 
 from scrapy import Spider, signals
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import maybe_deferred_to_future
 from scrapy.utils.test import get_crawler
 from tests.test_spider_start import SLEEP_SECONDS
 
 from .utils import twisted_sleep
+from .utils.decorators import deferred_f_from_coro_f
 
 ITEM_A = {"id": "a"}
 ITEM_B = {"id": "b"}

tests/test_spidermiddleware_start.py

@@ -1,9 +1,9 @@
 from scrapy.http import Request
 from scrapy.spidermiddlewares.start import StartSpiderMiddleware
 from scrapy.spiders import Spider
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.misc import build_from_crawler
 from scrapy.utils.test import get_crawler
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class TestMiddleware:

tests/test_stats.py

@@ -10,9 +10,9 @@ from scrapy.exceptions import ScrapyDeprecationWarning
 from scrapy.extensions.corestats import CoreStats
 from scrapy.spiders import Spider
 from scrapy.statscollectors import DummyStatsCollector, StatsCollector
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.test import get_crawler
 from tests.spiders import SimpleSpider
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from scrapy.crawler import Crawler

tests/test_utils_asyncgen.py

@@ -1,5 +1,5 @@
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import deferred_f_from_coro_f
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class TestAsyncgenUtils:

tests/test_utils_asyncio.py

@@ -14,13 +14,14 @@ from scrapy.utils.asyncio import (
     _parallel_asyncio,
     is_asyncio_available,
 )
-from scrapy.utils.defer import deferred_f_from_coro_f
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator
 
 
 class TestAsyncio:
+    @pytest.mark.requires_reactor  # needs a reactor or an event loop for is_asyncio_available()
     def test_is_asyncio_available(self, reactor_pytest: str) -> None:
         # the result should depend only on the pytest --reactor argument
         assert is_asyncio_available() == (reactor_pytest == "asyncio")
@@ -103,6 +104,7 @@ class TestParallelAsyncio:
             assert max_parallel_count[0] <= self.CONCURRENT_ITEMS
 
 
+@pytest.mark.requires_reactor  # needs a running event loop for AsyncioLoopingCall.start()
 @pytest.mark.only_asyncio
 class TestAsyncioLoopingCall:
     def test_looping_call(self):

tests/test_utils_defer.py

@@ -6,7 +6,8 @@ from asyncio import Future
 from typing import TYPE_CHECKING, Any
 
 import pytest
-from twisted.internet.defer import Deferred, inlineCallbacks, succeed
+from twisted.internet.defer import Deferred, succeed
+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
 from scrapy.utils.defer import (
@@ -19,6 +20,7 @@ from scrapy.utils.defer import (
     mustbe_deferred,
     parallel_async,
 )
+from tests.utils.decorators import inlineCallbacks
 
 if TYPE_CHECKING:
     from collections.abc import AsyncGenerator, Awaitable, Callable, Generator
@@ -107,6 +109,7 @@ class TestIterErrback:
         assert isinstance(errors[0].value, ZeroDivisionError)
 
 
+@pytest.mark.requires_reactor
 class TestAiterErrback:
     @deferred_f_from_coro_f
     async def test_aiter_errback_good(self):
@@ -134,6 +137,7 @@ class TestAiterErrback:
         assert isinstance(errors[0].value, ZeroDivisionError)
 
 
+@pytest.mark.requires_reactor
 class TestAsyncDefTestsuite:
     @deferred_f_from_coro_f
     async def test_deferred_f_from_coro_f(self):
@@ -304,7 +308,7 @@ class TestDeferredFromCoro:
 
 
 class TestDeferredFFromCoroF:
-    @inlineCallbacks
+    @inlineCallbacks_orig
     def _assert_result(
         self, c_f: Callable[[], Awaitable[int]]
     ) -> Generator[Deferred[Any], Any, None]:
@@ -342,6 +346,7 @@ class TestDeferredFFromCoroF:
 
 
 @pytest.mark.only_asyncio
+@pytest.mark.requires_reactor
 class TestDeferredToFuture:
     @deferred_f_from_coro_f
     async def test_deferred(self):
@@ -377,6 +382,9 @@ class TestDeferredToFuture:
 
 
 @pytest.mark.only_asyncio
+# needs a reactor or an event loop for is_asyncio_available()
+# (for maybe_deferred_to_future())
+@pytest.mark.requires_reactor
 class TestMaybeDeferredToFutureAsyncio:
     @deferred_f_from_coro_f
     async def test_deferred(self):
@@ -412,6 +420,9 @@ class TestMaybeDeferredToFutureAsyncio:
 
 
 @pytest.mark.only_not_asyncio
+# needs a reactor or an event loop for is_asyncio_available()
+# (for maybe_deferred_to_future())
+@pytest.mark.requires_reactor
 class TestMaybeDeferredToFutureNotAsyncio:
     def test_deferred(self):
         d = Deferred()

tests/test_utils_python.py

@@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, TypeVar
 import pytest
 
 from scrapy.utils.asyncgen import as_async_generator, collect_asyncgen
-from scrapy.utils.defer import aiter_errback, deferred_f_from_coro_f
+from scrapy.utils.defer import aiter_errback
 from scrapy.utils.python import (
     MutableAsyncChain,
     MutableChain,
@@ -20,6 +20,7 @@ from scrapy.utils.python import (
     to_unicode,
     without_none_values,
 )
+from tests.utils.decorators import deferred_f_from_coro_f
 
 if TYPE_CHECKING:
     from collections.abc import Iterable, Mapping

tests/test_utils_reactor.py

@@ -3,20 +3,22 @@ import warnings
 
 import pytest
 
-from scrapy.utils.defer import deferred_f_from_coro_f
 from scrapy.utils.reactor import (
     _asyncio_reactor_path,
     install_reactor,
     is_asyncio_reactor_installed,
     set_asyncio_event_loop,
 )
+from tests.utils.decorators import deferred_f_from_coro_f
 
 
 class TestAsyncio:
+    @pytest.mark.requires_reactor
     def test_is_asyncio_reactor_installed(self, reactor_pytest: str) -> None:
         # the result should depend only on the pytest --reactor argument
         assert is_asyncio_reactor_installed() == (reactor_pytest == "asyncio")
 
+    @pytest.mark.requires_reactor
     def test_install_asyncio_reactor(self):
         from twisted.internet import reactor as original_reactor
 

tests/test_utils_signal.py

@@ -4,7 +4,6 @@ import pytest
 from pydispatch import dispatcher
 from testfixtures import LogCapture
 from twisted.internet import defer
-from twisted.internet.defer import inlineCallbacks
 from twisted.python.failure import Failure
 
 from scrapy.utils.asyncio import call_later
@@ -15,6 +14,7 @@ from scrapy.utils.signal import (
     send_catch_log_deferred,
 )
 from scrapy.utils.test import get_from_asyncio_queue
+from tests.utils.decorators import inlineCallbacks
 
 
 class TestSendCatchLog:

tests/test_webclient.py

@@ -30,6 +30,8 @@ from tests.mockserver.http_resources import (
 from tests.mockserver.utils import ssl_context_factory
 from tests.test_core_downloader import TestContextFactoryBase
 
+pytestmark = pytest.mark.requires_reactor
+
 
 def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):
     """Adapted version of twisted.web.client.getPage"""

tests/test_zz_resources.py

@@ -30,6 +30,7 @@ def test_stderr_log_handler() -> None:
     assert c == 0
 
 
+@pytest.mark.requires_reactor  # needs a running event loop for asyncio.all_tasks()
 @pytest.mark.only_asyncio
 def test_pending_asyncio_tasks() -> None:
     """Test that there are no pending asyncio tasks."""

tests/utils/decorators.py

@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+from functools import wraps
+from typing import TYPE_CHECKING, Any, ParamSpec
+
+import pytest
+from twisted.internet.defer import Deferred
+from twisted.internet.defer import inlineCallbacks as inlineCallbacks_orig
+
+from scrapy.utils.defer import deferred_from_coro
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Generator
+
+
+_P = ParamSpec("_P")
+
+
+def inlineCallbacks(
+    f: Callable[_P, Generator[Deferred[Any], Any, None]],
+) -> Callable[_P, Deferred[None]]:
+    """Like :func:`twisted.internet.defer.inlineCallbacks`, but marks the
+    decorated function with ``@pytest.mark.requires_reactor``."""
+
+    @pytest.mark.requires_reactor
+    @wraps(f)
+    @inlineCallbacks_orig
+    def wrapper(
+        *args: _P.args, **kwargs: _P.kwargs
+    ) -> Generator[Deferred[Any], Any, None]:
+        return f(*args, **kwargs)
+
+    return wrapper
+
+
+def deferred_f_from_coro_f(
+    coro_f: Callable[_P, Awaitable[None]],
+) -> Callable[_P, Deferred[None]]:
+    """Like :func:`scrapy.utils.defer.deferred_f_from_coro_f`, but marks the
+    decorated function with ``@pytest.mark.requires_reactor``."""
+
+    @pytest.mark.requires_reactor
+    @wraps(coro_f)
+    def f(*coro_args: _P.args, **coro_kwargs: _P.kwargs) -> Deferred[None]:
+        return deferred_from_coro(coro_f(*coro_args, **coro_kwargs))
+
+    return f

tox.ini

@@ -161,6 +161,10 @@ commands = {[pinned]commands}
 commands =
     {[testenv]commands} --reactor=default
 
+[testenv:no-reactor]
+commands =
+    {[testenv]commands} -p no:twisted --reactor=none
+
 [testenv:default-reactor-pinned]
 basepython = {[pinned]basepython}
 deps = {[testenv:pinned]deps}
@@ -168,6 +172,13 @@ commands = {[pinned]commands} --reactor=default
 setenv =
     {[pinned]setenv}
 
+[testenv:no-reactor-pinned]
+basepython = {[pinned]basepython}
+deps = {[testenv:pinned]deps}
+commands = {[pinned]commands} -p no:twisted --reactor=none
+setenv =
+    {[pinned]setenv}
+
 [testenv:pypy3]
 basepython = pypy3
 commands =