Handle get_event_loop() deprecation

scrapy/crawler.py

@@ -20,6 +20,7 @@ from scrapy.extension import ExtensionManager
 from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings
 from scrapy.signalmanager import SignalManager
 from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader
+from scrapy.utils.asyncio import _get_running_or_installed_loop
 from scrapy.utils.defer import deferred_from_coro
 from scrapy.utils.log import (
     configure_logging,
@@ -587,10 +588,15 @@ class AsyncCrawlerRunner(CrawlerRunnerBase):
         self.crawlers.discard(crawler)
         self.bootstrap_failed |= not getattr(crawler, "spider", None)
 
+    def _get_event_loop(self) -> asyncio.AbstractEventLoop:
+        # The asyncio loop has been installed either by the user or by
+        # AsyncCrawlerProcess. It may not be running yet (e.g. when crawl() is
+        # called before AsyncCrawlerProcess.start()), so we cannot rely on
+        # asyncio.create_task() / get_running_loop() alone.
+        return _get_running_or_installed_loop()
+
     def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]:
-        # At this point the asyncio loop has been installed either by the user
-        # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()).
-        loop = asyncio.get_event_loop()
+        loop = self._get_event_loop()
         self.crawlers.add(crawler)
 
         task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs))
@@ -865,6 +871,14 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
     def _stop_dfd(self) -> Deferred[Any]:
         return deferred_from_coro(self.stop())
 
+    def _get_event_loop(self) -> asyncio.AbstractEventLoop:
+        # In reactorless mode no reactor is installed, so the loop created in
+        # __init__() is the one to use (it isn't running yet when crawl() is
+        # called before start()).
+        if self._reactorless_loop is not None:
+            return self._reactorless_loop
+        return super()._get_event_loop()
+
     def start(
         self, stop_after_crawl: bool = True, install_signal_handlers: bool = True
     ) -> None:
@@ -1046,7 +1060,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner):
         from twisted.internet import reactor
 
         if stop_after_crawl:
-            loop = asyncio.get_event_loop()
+            loop = self._get_event_loop()
             join_task = loop.create_task(self.join())
             join_task.add_done_callback(self._stop_reactor)
 

scrapy/utils/asyncio.py

@@ -92,6 +92,31 @@ def is_asyncio_available() -> bool:
     return is_asyncio_reactor_installed()
 
 
+def _get_running_or_installed_loop() -> asyncio.AbstractEventLoop:
+    """Return the asyncio event loop that Scrapy uses, even if it isn't running.
+
+    If an asyncio event loop is running in the current thread, it's returned.
+    Otherwise the event loop of the installed asyncio reactor is returned (the
+    reactor may not be running yet, e.g. while scheduling work before
+    :meth:`AsyncCrawlerProcess.start()
+    <scrapy.crawler.AsyncCrawlerProcess.start>`).
+
+    This is meant to replace :func:`asyncio.get_event_loop`, which is deprecated
+    and raises a :exc:`RuntimeError` on Python 3.14 and later when there is no
+    current event loop. It must only be called when an asyncio event loop is
+    running or an asyncio reactor is installed (i.e. when
+    :func:`is_asyncio_available` returns ``True``).
+    """
+    try:
+        return asyncio.get_running_loop()
+    except RuntimeError:
+        # No running loop: the asyncio reactor's loop is installed but not
+        # running yet (or it runs in a different thread).
+        from twisted.internet import reactor
+
+        return reactor._asyncioEventloop  # type: ignore[no-any-return]
+
+
 async def _parallel_asyncio(
     iterable: Iterable[_T] | AsyncIterator[_T],
     count: int,
@@ -175,7 +200,7 @@ class AsyncioLoopingCall:
         self._start_time = time.monotonic()
         if now:
             self._call()
-        loop = asyncio.get_event_loop()
+        loop = _get_running_or_installed_loop()
         self._task = loop.create_task(self._loop())
 
     def _to_sleep(self) -> float:
@@ -243,7 +268,7 @@ def call_later(
     .. versionadded:: 2.14.0
     """
     if is_asyncio_available():
-        loop = asyncio.get_event_loop()
+        loop = _get_running_or_installed_loop()
         return CallLaterResult.from_asyncio(loop.call_later(delay, func, *args))
 
     from twisted.internet import reactor

scrapy/utils/defer.py

@@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
 from twisted.python import failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.asyncio import _get_running_or_installed_loop, is_asyncio_available
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
@@ -493,7 +493,7 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]:
     """
     if not is_asyncio_available():
         raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.")
-    return d.asFuture(asyncio.get_event_loop())
+    return d.asFuture(_get_running_or_installed_loop())
 
 
 def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]:
@@ -535,8 +535,8 @@ def _schedule_coro(coro: Coroutine[Any, Any, Any]) -> None:
     if not is_asyncio_available():
         Deferred.fromCoroutine(coro)
         return
-    loop = asyncio.get_event_loop()
-    loop.create_task(coro)  # noqa: RUF006
+    loop = _get_running_or_installed_loop()
+    loop.create_task(coro)
 
 
 @overload

tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py

@@ -8,7 +8,9 @@ from scrapy.crawler import AsyncCrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncioreactor.install(asyncio.get_event_loop())
+# Install the asyncio reactor with the default (non-uvloop) event loop, so it
+# doesn't match the uvloop.Loop requested via the ASYNCIO_EVENT_LOOP setting.
+asyncioreactor.install(asyncio.new_event_loop())
 
 
 class NoRequestsSpider(scrapy.Spider):

tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py

@@ -9,8 +9,9 @@ from scrapy.crawler import AsyncCrawlerProcess
 
 if sys.platform == "win32":
     asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())
-asyncio.set_event_loop(Loop())
-asyncioreactor.install(asyncio.get_event_loop())
+loop = Loop()
+asyncio.set_event_loop(loop)
+asyncioreactor.install(loop)
 
 
 class NoRequestsSpider(scrapy.Spider):

tests/test_pipelines.py

@@ -65,7 +65,7 @@ class AsyncDefPipeline:
 class AsyncDefAsyncioPipeline:
     async def process_item(self, item):
         d = Deferred()
-        loop = asyncio.get_event_loop()
+        loop = asyncio.get_running_loop()
         loop.call_later(0, d.callback, None)
         await deferred_to_future(d)
         await asyncio.sleep(0.2)