diff --git a/scrapy/crawler.py b/scrapy/crawler.py index 9828fe6f5..1ec9eae7f 100644 --- a/scrapy/crawler.py +++ b/scrapy/crawler.py @@ -20,6 +20,7 @@ from scrapy.extension import ExtensionManager from scrapy.settings import SETTINGS_PRIORITIES, Settings, overridden_settings from scrapy.signalmanager import SignalManager from scrapy.spiderloader import SpiderLoaderProtocol, get_spider_loader +from scrapy.utils.asyncio import _get_running_or_installed_loop from scrapy.utils.defer import deferred_from_coro from scrapy.utils.log import ( configure_logging, @@ -587,10 +588,15 @@ class AsyncCrawlerRunner(CrawlerRunnerBase): self.crawlers.discard(crawler) self.bootstrap_failed |= not getattr(crawler, "spider", None) + def _get_event_loop(self) -> asyncio.AbstractEventLoop: + # The asyncio loop has been installed either by the user or by + # AsyncCrawlerProcess. It may not be running yet (e.g. when crawl() is + # called before AsyncCrawlerProcess.start()), so we cannot rely on + # asyncio.create_task() / get_running_loop() alone. + return _get_running_or_installed_loop() + def _crawl(self, crawler: Crawler, *args: Any, **kwargs: Any) -> asyncio.Task[None]: - # At this point the asyncio loop has been installed either by the user - # or by AsyncCrawlerProcess (but it isn't running yet, so no asyncio.create_task()). - loop = asyncio.get_event_loop() + loop = self._get_event_loop() self.crawlers.add(crawler) task = loop.create_task(self._crawl_and_track(crawler, *args, **kwargs)) @@ -865,6 +871,14 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): def _stop_dfd(self) -> Deferred[Any]: return deferred_from_coro(self.stop()) + def _get_event_loop(self) -> asyncio.AbstractEventLoop: + # In reactorless mode no reactor is installed, so the loop created in + # __init__() is the one to use (it isn't running yet when crawl() is + # called before start()). + if self._reactorless_loop is not None: + return self._reactorless_loop + return super()._get_event_loop() + def start( self, stop_after_crawl: bool = True, install_signal_handlers: bool = True ) -> None: @@ -1046,7 +1060,7 @@ class AsyncCrawlerProcess(CrawlerProcessBase, AsyncCrawlerRunner): from twisted.internet import reactor if stop_after_crawl: - loop = asyncio.get_event_loop() + loop = self._get_event_loop() join_task = loop.create_task(self.join()) join_task.add_done_callback(self._stop_reactor) diff --git a/scrapy/utils/asyncio.py b/scrapy/utils/asyncio.py index 44604c0fe..27d1ad9ee 100644 --- a/scrapy/utils/asyncio.py +++ b/scrapy/utils/asyncio.py @@ -92,6 +92,31 @@ def is_asyncio_available() -> bool: return is_asyncio_reactor_installed() +def _get_running_or_installed_loop() -> asyncio.AbstractEventLoop: + """Return the asyncio event loop that Scrapy uses, even if it isn't running. + + If an asyncio event loop is running in the current thread, it's returned. + Otherwise the event loop of the installed asyncio reactor is returned (the + reactor may not be running yet, e.g. while scheduling work before + :meth:`AsyncCrawlerProcess.start() + `). + + This is meant to replace :func:`asyncio.get_event_loop`, which is deprecated + and raises a :exc:`RuntimeError` on Python 3.14 and later when there is no + current event loop. It must only be called when an asyncio event loop is + running or an asyncio reactor is installed (i.e. when + :func:`is_asyncio_available` returns ``True``). + """ + try: + return asyncio.get_running_loop() + except RuntimeError: + # No running loop: the asyncio reactor's loop is installed but not + # running yet (or it runs in a different thread). + from twisted.internet import reactor + + return reactor._asyncioEventloop # type: ignore[no-any-return] + + async def _parallel_asyncio( iterable: Iterable[_T] | AsyncIterator[_T], count: int, @@ -175,7 +200,7 @@ class AsyncioLoopingCall: self._start_time = time.monotonic() if now: self._call() - loop = asyncio.get_event_loop() + loop = _get_running_or_installed_loop() self._task = loop.create_task(self._loop()) def _to_sleep(self) -> float: @@ -243,7 +268,7 @@ def call_later( .. versionadded:: 2.14.0 """ if is_asyncio_available(): - loop = asyncio.get_event_loop() + loop = _get_running_or_installed_loop() return CallLaterResult.from_asyncio(loop.call_later(delay, func, *args)) from twisted.internet import reactor diff --git a/scrapy/utils/defer.py b/scrapy/utils/defer.py index 29a34d4ef..5492e4024 100644 --- a/scrapy/utils/defer.py +++ b/scrapy/utils/defer.py @@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator from twisted.python import failure from scrapy.exceptions import ScrapyDeprecationWarning -from scrapy.utils.asyncio import is_asyncio_available +from scrapy.utils.asyncio import _get_running_or_installed_loop, is_asyncio_available from scrapy.utils.python import global_object_name if TYPE_CHECKING: @@ -493,7 +493,7 @@ def deferred_to_future(d: Deferred[_T]) -> Future[_T]: """ if not is_asyncio_available(): raise RuntimeError("deferred_to_future() requires AsyncioSelectorReactor.") - return d.asFuture(asyncio.get_event_loop()) + return d.asFuture(_get_running_or_installed_loop()) def maybe_deferred_to_future(d: Deferred[_T]) -> Deferred[_T] | Future[_T]: @@ -535,8 +535,8 @@ def _schedule_coro(coro: Coroutine[Any, Any, Any]) -> None: if not is_asyncio_available(): Deferred.fromCoroutine(coro) return - loop = asyncio.get_event_loop() - loop.create_task(coro) # noqa: RUF006 + loop = _get_running_or_installed_loop() + loop.create_task(coro) @overload diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py index 4257bc0ac..bd4c790eb 100644 --- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_different_loop.py @@ -8,7 +8,9 @@ from scrapy.crawler import AsyncCrawlerProcess if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncioreactor.install(asyncio.get_event_loop()) +# Install the asyncio reactor with the default (non-uvloop) event loop, so it +# doesn't match the uvloop.Loop requested via the ASYNCIO_EVENT_LOOP setting. +asyncioreactor.install(asyncio.new_event_loop()) class NoRequestsSpider(scrapy.Spider): diff --git a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py index 9c6fd090b..bd9b25e3e 100644 --- a/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py +++ b/tests/AsyncCrawlerProcess/asyncio_enabled_reactor_same_loop.py @@ -9,8 +9,9 @@ from scrapy.crawler import AsyncCrawlerProcess if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) -asyncio.set_event_loop(Loop()) -asyncioreactor.install(asyncio.get_event_loop()) +loop = Loop() +asyncio.set_event_loop(loop) +asyncioreactor.install(loop) class NoRequestsSpider(scrapy.Spider): diff --git a/tests/test_pipelines.py b/tests/test_pipelines.py index e65b5bc32..760cc2f4c 100644 --- a/tests/test_pipelines.py +++ b/tests/test_pipelines.py @@ -65,7 +65,7 @@ class AsyncDefPipeline: class AsyncDefAsyncioPipeline: async def process_item(self, item): d = Deferred() - loop = asyncio.get_event_loop() + loop = asyncio.get_running_loop() loop.call_later(0, d.callback, None) await deferred_to_future(d) await asyncio.sleep(0.2)