diff --git a/conftest.py b/conftest.py index 9999e41d2..8e0c429a0 100644 --- a/conftest.py +++ b/conftest.py @@ -119,6 +119,9 @@ def requires_boto3(request): def pytest_configure(config): if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + else: + # install the reactor explicitly + from twisted.internet import reactor # noqa: F401 # Generate localhost certificate files, needed by some tests diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 35afdc11b..0490129b3 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -72,24 +72,32 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + .. _enforce-asyncio-requirement: @@ -116,6 +124,8 @@ example: f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + .. _asyncio-windows: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index a7ba5443f..bed13f483 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -9,6 +9,7 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax ` (i.e. ``async def``). + .. _coroutine-support: Supported callables @@ -60,6 +61,29 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): - :ref:`Signal handlers that support deferreds `. + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +For example: + +- The :meth:`ExecutionEngine.download` method returns a + :class:`~twisted.internet.defer.Deferred` object. +- A custom download handler needs to define a ``download_request()`` method that + returns a :class:`~twisted.internet.defer.Deferred` object. + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + + General usage ============= diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8f52836c4..42ad28d8d 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -362,7 +362,8 @@ def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Deferred | _T: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" + """Converts a coroutine or other awaitable object into a Deferred, + or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): @@ -442,12 +443,12 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: What you can await in Scrapy callables defined as coroutines depends on the value of :setting:`TWISTED_REACTOR`: - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - When :ref:`using the asyncio reactor `, you can only await on :class:`asyncio.Future` objects. + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + If you want to write code that uses ``Deferred`` objects but works with any reactor, use this function on all ``Deferred`` objects:: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 679e38206..099c81f0e 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -175,7 +175,20 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: ) +def is_reactor_installed() -> bool: + return "twisted.internet.reactor" in sys.modules + + def is_asyncio_reactor_installed() -> bool: + """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + + Raise a :exc:`RuntimeError` if no reactor is installed. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_reactor_installed() called without an installed reactor." + ) + from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index ff2e47a70..950bdb006 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -1,5 +1,12 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") class NoRequestsSpider(scrapy.Spider): @@ -13,6 +20,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 9959f31fe..a80a1180a 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,9 +1,39 @@ import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + class NoRequestsSpider(scrapy.Spider): name = "no_request" @@ -16,6 +46,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index ad2db4128..fa1c90880 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -748,6 +748,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") @@ -756,6 +757,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"),