From fc566a7ff9913be5fce2b961a6599f861e1ef59a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Fri, 14 Mar 2025 23:46:17 +0400 Subject: [PATCH 1/3] Don't install the reactor in is_asyncio_reactor_installed(). (#6732) --- docs/topics/asyncio.rst | 2 ++ scrapy/utils/reactor.py | 13 ++++++++ .../asyncio_enabled_no_reactor.py | 8 +++++ .../CrawlerProcess/asyncio_enabled_reactor.py | 33 ++++++++++++++++++- tests/test_crawler.py | 2 ++ 5 files changed, 57 insertions(+), 1 deletion(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index 35afdc11b..b61a6e4a8 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -116,6 +116,8 @@ example: f"of Scrapy for more information." ) +.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed + .. _asyncio-windows: diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py index 679e38206..099c81f0e 100644 --- a/scrapy/utils/reactor.py +++ b/scrapy/utils/reactor.py @@ -175,7 +175,20 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None: ) +def is_reactor_installed() -> bool: + return "twisted.internet.reactor" in sys.modules + + def is_asyncio_reactor_installed() -> bool: + """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`. + + Raise a :exc:`RuntimeError` if no reactor is installed. + """ + if not is_reactor_installed(): + raise RuntimeError( + "is_asyncio_reactor_installed() called without an installed reactor." + ) + from twisted.internet import reactor return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor) diff --git a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py index 6df6d76fa..6f82cf589 100644 --- a/tests/CrawlerProcess/asyncio_enabled_no_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_no_reactor.py @@ -1,5 +1,12 @@ import scrapy from scrapy.crawler import CrawlerProcess +from scrapy.utils.reactor import is_asyncio_reactor_installed + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") class NoRequestsSpider(scrapy.Spider): @@ -12,6 +19,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/CrawlerProcess/asyncio_enabled_reactor.py b/tests/CrawlerProcess/asyncio_enabled_reactor.py index 0c380610d..a8bf1bc3c 100644 --- a/tests/CrawlerProcess/asyncio_enabled_reactor.py +++ b/tests/CrawlerProcess/asyncio_enabled_reactor.py @@ -1,9 +1,39 @@ import scrapy from scrapy.crawler import CrawlerProcess -from scrapy.utils.reactor import install_reactor +from scrapy.utils.reactor import ( + install_reactor, + is_asyncio_reactor_installed, + is_reactor_installed, +) + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed before is_asyncio_reactor_installed()." + ) + +try: + is_asyncio_reactor_installed() +except RuntimeError: + pass +else: + raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.") + +if is_reactor_installed(): + raise RuntimeError( + "Reactor already installed after is_asyncio_reactor_installed()." + ) install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") +if not is_asyncio_reactor_installed(): + raise RuntimeError("Wrong reactor installed after install_reactor().") + + +class ReactorCheckExtension: + def __init__(self): + if not is_asyncio_reactor_installed(): + raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.") + class NoRequestsSpider(scrapy.Spider): name = "no_request" @@ -15,6 +45,7 @@ class NoRequestsSpider(scrapy.Spider): process = CrawlerProcess( settings={ "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor", + "EXTENSIONS": {ReactorCheckExtension: 0}, } ) process.crawl(NoRequestsSpider) diff --git a/tests/test_crawler.py b/tests/test_crawler.py index 98352b66e..6c465f000 100644 --- a/tests/test_crawler.py +++ b/tests/test_crawler.py @@ -747,6 +747,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log def test_asyncio_enabled_reactor(self): log = self.run_script("asyncio_enabled_reactor.py") @@ -755,6 +756,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase): "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor" in log ) + assert "RuntimeError" not in log @pytest.mark.skipif( parse_version(w3lib_version) >= parse_version("2.0.0"), From 9057bf4e1e08dccac4fa6d9f0f191d1f4708a43a Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 14:47:16 +0400 Subject: [PATCH 2/3] More docs about Deferred<->Future interoperability. (#6734) --- docs/topics/asyncio.rst | 24 ++++++++++++++++-------- docs/topics/coroutines.rst | 24 ++++++++++++++++++++++++ scrapy/utils/defer.py | 9 +++++---- 3 files changed, 45 insertions(+), 12 deletions(-) diff --git a/docs/topics/asyncio.rst b/docs/topics/asyncio.rst index b61a6e4a8..0490129b3 100644 --- a/docs/topics/asyncio.rst +++ b/docs/topics/asyncio.rst @@ -72,24 +72,32 @@ those imports happen. .. _asyncio-await-dfd: -Awaiting on Deferreds -===================== +Integrating Deferred code and asyncio code +========================================== -When the asyncio reactor isn't installed, you can await on Deferreds in the -coroutines directly. When it is installed, this is not possible anymore, due to -specifics of the Scrapy coroutine integration (the coroutines are wrapped into -:class:`asyncio.Future` objects, not into -:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into -Futures. Scrapy provides two helpers for this: +Coroutine functions can await on Deferreds by wrapping them into +:class:`asyncio.Future` objects. Scrapy provides two helpers for this: .. autofunction:: scrapy.utils.defer.deferred_to_future .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future + +.. tip:: If you don't need to support reactors other than the default + :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you + can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you + should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`. + .. tip:: If you need to use these functions in code that aims to be compatible with lower versions of Scrapy that do not provide these functions, down to Scrapy 2.0 (earlier versions do not support :mod:`asyncio`), you can copy the implementation of these functions into your own code. +Coroutines and futures can be wrapped into Deferreds (for example, when a +Scrapy API requires passing a Deferred to it) using the following helpers: + +.. autofunction:: scrapy.utils.defer.deferred_from_coro +.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f + .. _enforce-asyncio-requirement: diff --git a/docs/topics/coroutines.rst b/docs/topics/coroutines.rst index 57aa3a62d..1c80857f6 100644 --- a/docs/topics/coroutines.rst +++ b/docs/topics/coroutines.rst @@ -9,6 +9,7 @@ Coroutines Scrapy has :ref:`partial support ` for the :ref:`coroutine syntax `. + .. _coroutine-support: Supported callables @@ -51,6 +52,29 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``): .. versionadded:: 2.7 + +.. _coroutine-deferred-apis: + +Using Deferred-based APIs +========================= + +In addition to native coroutine APIs Scrapy has some APIs that return a +:class:`~twisted.internet.defer.Deferred` object or take a user-supplied +function that returns a :class:`~twisted.internet.defer.Deferred` object. These +APIs are also asynchronous but don't yet support native ``async def`` syntax. +For example: + +- The :meth:`ExecutionEngine.download` method returns a + :class:`~twisted.internet.defer.Deferred` object. +- A custom download handler needs to define a ``download_request()`` method that + returns a :class:`~twisted.internet.defer.Deferred` object. + +In most cases you can use these APIs in code that otherwise uses coroutines, by +wrapping a :class:`~twisted.internet.defer.Deferred` object into a +:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for +more information about this. + + General usage ============= diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 8f52836c4..42ad28d8d 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -362,7 +362,8 @@ def deferred_from_coro(o: _T) -> _T: ... def deferred_from_coro(o: _T) -> Deferred | _T: - """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine""" + """Converts a coroutine or other awaitable object into a Deferred, + or returns the object as is if it isn't a coroutine.""" if isinstance(o, Deferred): return o if asyncio.isfuture(o) or inspect.isawaitable(o): @@ -442,12 +443,12 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: What you can await in Scrapy callables defined as coroutines depends on the value of :setting:`TWISTED_REACTOR`: - - When not using the asyncio reactor, you can only await on - :class:`~twisted.internet.defer.Deferred` objects. - - When :ref:`using the asyncio reactor `, you can only await on :class:`asyncio.Future` objects. + - When not using the asyncio reactor, you can only await on + :class:`~twisted.internet.defer.Deferred` objects. + If you want to write code that uses ``Deferred`` objects but works with any reactor, use this function on all ``Deferred`` objects:: From 872924721344502cca92a72ce68d5f78a9b9e5bc Mon Sep 17 00:00:00 2001 From: Andrey Rakhmatullin Date: Sat, 15 Mar 2025 23:33:10 +0400 Subject: [PATCH 3/3] Fix running simple tests with --reactor=default. (#6735) --- conftest.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conftest.py b/conftest.py index 9999e41d2..8e0c429a0 100644 --- a/conftest.py +++ b/conftest.py @@ -119,6 +119,9 @@ def requires_boto3(request): def pytest_configure(config): if config.getoption("--reactor") != "default": install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor") + else: + # install the reactor explicitly + from twisted.internet import reactor # noqa: F401 # Generate localhost certificate files, needed by some tests