From dceb85bf3e41d06dfac81c037c69fd0f1ab61156 Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Wed, 28 May 2025 14:46:39 +0500 Subject: [PATCH] Add is_asyncio_available(). (#6827) * Add is_asyncio_available(). * Print unexpected warnings in test_install_asyncio_reactor(). * Fix printing warnings. * Fix printing warnings - 2. * Skip TestDeferredToFuture on non-asyncio. * Test the is_asyncio_available() exception. --- docs/topics/asyncio.rst | 11 +++--- scrapy/utils/asyncio.py | 38 +++++++++++++++++++ scrapy/utils/defer.py | 7 ++-- scrapy/utils/reactor.py | 6 +++ .../CrawlerProcess/asyncio_enabled_reactor.py | 10 +++++ tests/test_utils_asyncio.py | 33 ++-------------- tests/test_utils_defer.py | 4 +- tests/test_utils_deprecate.py | 2 +- tests/test_utils_reactor.py | 35 +++++++++++++++++ 9 files changed, 106 insertions(+), 40 deletions(-) create mode 100644 scrapy/utils/asyncio.py create mode 100644 tests/test_utils_reactor.py diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 0490129b3..473ef7bfa 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -105,25 +105,26 @@ Enforcing asyncio as a requirement ================================== If you are writing a :ref:`component ` that requires asyncio -to work, use :func:`scrapy.utils.reactor.is_asyncio_reactor_installed` to +to work, use :func:`scrapy.utils.asyncio.is_asyncio_available` to :ref:`enforce it as a requirement `. For example: .. code-block:: python - from scrapy.utils.reactor import is_asyncio_reactor_installed + from scrapy.utils.asyncio import is_asyncio_available class MyComponent: def __init__(self): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): raise ValueError( - f"{MyComponent.__qualname__} requires the asyncio Twisted " - f"reactor. Make sure you have it configured in the " + f"{MyComponent.__qualname__} requires the asyncio support. " + f"Make sure you have configured the asyncio reactor in the " f"TWISTED_REACTOR setting. See the asyncio documentation " f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.asyncio.is_asyncio_available .. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py new file mode 100644 index 000000000..4469369fa --- /dev/null +++ b/scrapy/utils/asyncio.py @@ -0,0 +1,38 @@ +"""Utilities related to asyncio and its support in Scrapy.""" + +from scrapy.utils.reactor import is_asyncio_reactor_installed, is_reactor_installed + + +def is_asyncio_available() -> bool: + """Check if it's possible to call asyncio code that relies on the asyncio event loop. + + .. versionadded:: VERSION + + Currently this function is identical to + :func:`scrapy.utils.reactor.is_asyncio_reactor_installed`: it returns + ``True`` if the Twisted reactor that is installed is + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, returns + ``False`` if a different reactor is installed, and raises a + :exc:`RuntimeError` if no reactor is installed. In a future Scrapy version, + when Scrapy supports running without a Twisted reactor, this function will + also return ``True`` when running in that mode, so code that doesn't + directly require a Twisted reactor should use this function instead of + :func:`~scrapy.utils.reactor.is_asyncio_reactor_installed`. + + When this returns ``True``, an asyncio loop is installed and used by + Scrapy. It's possible to call functions that require it, such as + :func:`asyncio.sleep`, and await on :class:`asyncio.Future` objects in + Scrapy-related code. + + When this returns ``False``, a non-asyncio Twisted reactor is installed. + It's not possible to use asyncio features that require an asyncio event + loop or await on :class:`asyncio.Future` objects in Scrapy-related code, + but it's possible to await on :class:`~twisted.internet.defer.Deferred` + objects. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_available() called without an installed reactor." + ) + + return is_asyncio_reactor_installed() diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index d06397f50..4649c4daa 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -22,7 +22,8 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import IgnoreRequest, ScrapyDeprecationWarning -from scrapy.utils.reactor import _get_asyncio_event_loop, is_asyncio_reactor_installed +from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.reactor import _get_asyncio_event_loop if TYPE_CHECKING: from collections.abc import AsyncIterator, Callable @@ -379,7 +380,7 @@ def deferred_from_coro(o: Awaitable[_T] | _T2) -> Deferred[_T] | _T2: if isinstance(o, Deferred): return o if inspect.isawaitable(o): - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): # wrapping the coroutine directly into a Deferred, this doesn't work correctly with coroutines # that use asyncio, e.g. "await asyncio.sleep(1)" return Deferred.fromCoroutine(cast(Coroutine[Deferred[Any], Any, _T], o)) @@ -471,6 +472,6 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: deferred = self.crawler.engine.download(additional_request) additional_response = await maybe_deferred_to_future(deferred) """ - if not is_asyncio_reactor_installed(): + if not is_asyncio_available(): return d return deferred_to_future(d) diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 9c2754394..5e76da37b 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -202,6 +202,12 @@ def is_asyncio_reactor_installed() -> bool: """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. Raise a :exc:`RuntimeError` if no reactor is installed. + + In a future Scrapy version, when Scrapy supports running without a Twisted + reactor, this function won't be useful for checking if it's possible to use + asyncio features, so the code that that doesn't directly require a Twisted + reactor should use :func:`scrapy.utils.asyncio.is_asyncio_available` + instead of this function. """ if not is_reactor_installed(): raise RuntimeError( diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index f3dab12fe..4e8d3db12 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,5 +1,6 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.asyncio import is_asyncio_available from scrapy.utils.reactor import ( install_reactor, is_asyncio_reactor_installed, @@ -18,6 +19,13 @@ except RuntimeError: else: raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") +try: + is_asyncio_available() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_available() did not raise RuntimeError.") + if is_reactor_installed(): raise RuntimeError( "Reactor already installed after is_asyncio_reactor_installed()." @@ -33,6 +41,8 @@ class ReactorCheckExtension: def __init__(self): if not is_asyncio_reactor_installed(): raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + if not is_asyncio_available(): + raise RuntimeError("ReactorCheckExtension requires asyncio support.") class NoRequestsSpider(scrapy.Spider): diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py index 901e03d59..fe44748f9 100644 --- a/tests/test_utils_asyncio.py +++ b/tests/test_utils_asyncio.py @@ -1,35 +1,10 @@ -import asyncio -import warnings - import pytest -from twisted.trial.unittest import TestCase -from scrapy.utils.defer import deferred_f_from_coro_f -from scrapy.utils.reactor import ( - install_reactor, - is_asyncio_reactor_installed, - set_asyncio_event_loop, -) +from scrapy.utils.asyncio import is_asyncio_available @pytest.mark.usefixtures("reactor_pytest") -class TestAsyncio(TestCase): - def test_is_asyncio_reactor_installed(self): +class TestAsyncio: + def test_is_asyncio_available(self): # the result should depend only on the pytest --reactor argument - assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") - - def test_install_asyncio_reactor(self): - from twisted.internet import reactor as original_reactor - - with warnings.catch_warnings(record=True) as w: - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert len(w) == 0 - from twisted.internet import reactor # pylint: disable=reimported - - assert original_reactor == reactor - - @pytest.mark.only_asyncio - @deferred_f_from_coro_f - async def test_set_asyncio_event_loop(self): - install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") - assert set_asyncio_event_loop(None) is asyncio.get_running_loop() + assert is_asyncio_available() == (self.reactor_pytest != "default") diff --git a/tests/test_utils_defer.py b/tests/test_utils_defer.py index 29cd5fbf2..c565c1c4e 100644 --- a/tests/test_utils_defer.py +++ b/tests/test_utils_defer.py @@ -311,6 +311,7 @@ class TestDeferredFFromCoroF(unittest.TestCase): yield self._assert_result(c_f) +@pytest.mark.only_asyncio class TestDeferredToFuture(unittest.TestCase): @deferred_f_from_coro_f async def test_deferred(self): @@ -332,7 +333,6 @@ class TestDeferredToFuture(unittest.TestCase): future_result = await result assert future_result == 42 - @pytest.mark.only_asyncio @deferred_f_from_coro_f async def test_wrapped_coroutine_asyncio(self): async def c_f() -> int: @@ -340,7 +340,7 @@ class TestDeferredToFuture(unittest.TestCase): return 42 d = deferred_from_coro(c_f()) - result = maybe_deferred_to_future(d) + result = deferred_to_future(d) assert isinstance(result, Future) future_result = await result assert future_result == 42 diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.py index 52c165bb4..662de0dc3 100644 --- a/tests/test_utils_deprecate.py +++ b/tests/test_utils_deprecate.py @@ -243,7 +243,7 @@ class TestWarnWhenSubclassed: ) w = self._mywarnings(w) - assert len(w) == 0, str(map(str, w)) + assert len(w) == 0, [str(warning) for warning in w] with warnings.catch_warnings(record=True) as w: AlsoDeprecated() diff --git a/tests/test_utils_reactor.py b/tests/test_utils_reactor.py new file mode 100644 index 000000000..99f175c60 --- /dev/null +++ b/tests/test_utils_reactor.py @@ -0,0 +1,35 @@ +import asyncio +import warnings + +import pytest +from twisted.trial.unittest import TestCase + +from scrapy.utils.defer import deferred_f_from_coro_f +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + set_asyncio_event_loop, +) + + +@pytest.mark.usefixtures("reactor_pytest") +class TestAsyncio(TestCase): + def test_is_asyncio_reactor_installed(self): + # the result should depend only on the pytest --reactor argument + assert is_asyncio_reactor_installed() == (self.reactor_pytest != "default") + + def test_install_asyncio_reactor(self): + from twisted.internet import reactor as original_reactor + + with warnings.catch_warnings(record=True) as w: + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert len(w) == 0, [str(warning) for warning in w] + from twisted.internet import reactor # pylint: disable=reimported + + assert original_reactor == reactor + + @pytest.mark.only_asyncio + @deferred_f_from_coro_f + async def test_set_asyncio_event_loop(self): + install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + assert set_asyncio_event_loop(None) is asyncio.get_running_loop()