Merge remote-tracking branch 'scrapy/master' into async-seeds

conftest.py

@@ -119,6 +119,9 @@ def requires_boto3(request):
 def pytest_configure(config):
     if config.getoption("--reactor") != "default":
         install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+    else:
+        # install the reactor explicitly
+        from twisted.internet import reactor  # noqa: F401
 
 
 # Generate localhost certificate files, needed by some tests

docs/topics/asyncio.rst

@@ -72,24 +72,32 @@ those imports happen.
 
 .. _asyncio-await-dfd:
 
-Awaiting on Deferreds
-=====================
+Integrating Deferred code and asyncio code
+==========================================
 
-When the asyncio reactor isn't installed, you can await on Deferreds in the
-coroutines directly. When it is installed, this is not possible anymore, due to
-specifics of the Scrapy coroutine integration (the coroutines are wrapped into
-:class:`asyncio.Future` objects, not into
-:class:`~twisted.internet.defer.Deferred` directly), and you need to wrap them into
-Futures. Scrapy provides two helpers for this:
+Coroutine functions can await on Deferreds by wrapping them into
+:class:`asyncio.Future` objects. Scrapy provides two helpers for this:
 
 .. autofunction:: scrapy.utils.defer.deferred_to_future
 .. autofunction:: scrapy.utils.defer.maybe_deferred_to_future
+
+.. tip:: If you don't need to support reactors other than the default
+         :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`, you
+         can use :func:`~scrapy.utils.defer.deferred_to_future`, otherwise you
+         should use :func:`~scrapy.utils.defer.maybe_deferred_to_future`.
+
 .. tip:: If you need to use these functions in code that aims to be compatible
          with lower versions of Scrapy that do not provide these functions,
          down to Scrapy 2.0 (earlier versions do not support
          :mod:`asyncio`), you can copy the implementation of these functions
          into your own code.
 
+Coroutines and futures can be wrapped into Deferreds (for example, when a
+Scrapy API requires passing a Deferred to it) using the following helpers:
+
+.. autofunction:: scrapy.utils.defer.deferred_from_coro
+.. autofunction:: scrapy.utils.defer.deferred_f_from_coro_f
+
 
 .. _enforce-asyncio-requirement:
 
@@ -116,6 +124,8 @@ example:
                     f"of Scrapy for more information."
                 )
 
+.. autofunction:: scrapy.utils.reactor.is_asyncio_reactor_installed
+
 
 .. _asyncio-windows:
 

docs/topics/coroutines.rst

@@ -9,6 +9,7 @@ Coroutines
 Scrapy has :ref:`partial support <coroutine-support>` for the :ref:`coroutine
 syntax <async>` (i.e. ``async def``).
 
+
 .. _coroutine-support:
 
 Supported callables
@@ -60,6 +61,29 @@ hence use coroutine syntax (e.g. ``await``, ``async for``, ``async with``):
 
 -   :ref:`Signal handlers that support deferreds <signal-deferred>`.
 
+
+.. _coroutine-deferred-apis:
+
+Using Deferred-based APIs
+=========================
+
+In addition to native coroutine APIs Scrapy has some APIs that return a
+:class:`~twisted.internet.defer.Deferred` object or take a user-supplied
+function that returns a :class:`~twisted.internet.defer.Deferred` object. These
+APIs are also asynchronous but don't yet support native ``async def`` syntax.
+For example:
+
+-   The :meth:`ExecutionEngine.download` method returns a
+    :class:`~twisted.internet.defer.Deferred` object.
+-   A custom download handler needs to define a ``download_request()`` method that
+    returns a :class:`~twisted.internet.defer.Deferred` object.
+
+In most cases you can use these APIs in code that otherwise uses coroutines, by
+wrapping a :class:`~twisted.internet.defer.Deferred` object into a
+:class:`~asyncio.Future` object or vice versa. See :ref:`asyncio-await-dfd` for
+more information about this.
+
+
 General usage
 =============
 

scrapy/utils/defer.py

@@ -362,7 +362,8 @@ def deferred_from_coro(o: _T) -> _T: ...
 
 
 def deferred_from_coro(o: _T) -> Deferred | _T:
-    """Converts a coroutine into a Deferred, or returns the object as is if it isn't a coroutine"""
+    """Converts a coroutine or other awaitable object into a Deferred,
+    or returns the object as is if it isn't a coroutine."""
     if isinstance(o, Deferred):
         return o
     if asyncio.isfuture(o) or inspect.isawaitable(o):
@@ -442,12 +443,12 @@ def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
     What you can await in Scrapy callables defined as coroutines depends on the
     value of :setting:`TWISTED_REACTOR`:
 
-    -   When not using the asyncio reactor, you can only await on
-        :class:`~twisted.internet.defer.Deferred` objects.
-
     -   When :ref:`using the asyncio reactor <install-asyncio>`, you can only
         await on :class:`asyncio.Future` objects.
 
+    -   When not using the asyncio reactor, you can only await on
+        :class:`~twisted.internet.defer.Deferred` objects.
+
     If you want to write code that uses ``Deferred`` objects but works with any
     reactor, use this function on all ``Deferred`` objects::
 

scrapy/utils/reactor.py

@@ -175,7 +175,20 @@ def verify_installed_asyncio_event_loop(loop_path: str) -> None:
     )
 
 
+def is_reactor_installed() -> bool:
+    return "twisted.internet.reactor" in sys.modules
+
+
 def is_asyncio_reactor_installed() -> bool:
+    """Check whether the installed reactor is :class:`~twisted.internet.asyncioreactor.AsyncioSelectorReactor`.
+
+    Raise a :exc:`RuntimeError` if no reactor is installed.
+    """
+    if not is_reactor_installed():
+        raise RuntimeError(
+            "is_asyncio_reactor_installed() called without an installed reactor."
+        )
+
     from twisted.internet import reactor
 
     return isinstance(reactor, asyncioreactor.AsyncioSelectorReactor)

tests/CrawlerProcess/asyncio_enabled_no_reactor.py

@@ -1,5 +1,12 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
+from scrapy.utils.reactor import is_asyncio_reactor_installed
+
+
+class ReactorCheckExtension:
+    def __init__(self):
+        if not is_asyncio_reactor_installed():
+            raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")
 
 
 class NoRequestsSpider(scrapy.Spider):
@@ -13,6 +20,7 @@ class NoRequestsSpider(scrapy.Spider):
 process = CrawlerProcess(
     settings={
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "EXTENSIONS": {ReactorCheckExtension: 0},
     }
 )
 process.crawl(NoRequestsSpider)

tests/CrawlerProcess/asyncio_enabled_reactor.py

@@ -1,9 +1,39 @@
 import scrapy
 from scrapy.crawler import CrawlerProcess
-from scrapy.utils.reactor import install_reactor
+from scrapy.utils.reactor import (
+    install_reactor,
+    is_asyncio_reactor_installed,
+    is_reactor_installed,
+)
+
+if is_reactor_installed():
+    raise RuntimeError(
+        "Reactor already installed before is_asyncio_reactor_installed()."
+    )
+
+try:
+    is_asyncio_reactor_installed()
+except RuntimeError:
+    pass
+else:
+    raise RuntimeError("is_asyncio_reactor_installed() did not raise RuntimeError.")
+
+if is_reactor_installed():
+    raise RuntimeError(
+        "Reactor already installed after is_asyncio_reactor_installed()."
+    )
 
 install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
 
+if not is_asyncio_reactor_installed():
+    raise RuntimeError("Wrong reactor installed after install_reactor().")
+
+
+class ReactorCheckExtension:
+    def __init__(self):
+        if not is_asyncio_reactor_installed():
+            raise RuntimeError("ReactorCheckExtension requires the asyncio reactor.")
+
 
 class NoRequestsSpider(scrapy.Spider):
     name = "no_request"
@@ -16,6 +46,7 @@ class NoRequestsSpider(scrapy.Spider):
 process = CrawlerProcess(
     settings={
         "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
+        "EXTENSIONS": {ReactorCheckExtension: 0},
     }
 )
 process.crawl(NoRequestsSpider)

tests/test_crawler.py

@@ -748,6 +748,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
             in log
         )
+        assert "RuntimeError" not in log
 
     def test_asyncio_enabled_reactor(self):
         log = self.run_script("asyncio_enabled_reactor.py")
@@ -756,6 +757,7 @@ class TestCrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
             "Using reactor: twisted.internet.asyncioreactor.AsyncioSelectorReactor"
             in log
         )
+        assert "RuntimeError" not in log
 
     @pytest.mark.skipif(
         parse_version(w3lib_version) >= parse_version("2.0.0"),