Add scrapy.utils.asyncio.sleep() (#7843)

docs/topics/asyncio.rst

@@ -267,6 +267,7 @@ Here are some examples of APIs and patterns that need a replacement:
 
 Scrapy provides unified helpers for some of these examples:
 
+.. autofunction:: scrapy.utils.asyncio.sleep
 .. autofunction:: scrapy.utils.asyncio.call_later
 .. autofunction:: scrapy.utils.asyncio.create_looping_call
 .. autoclass:: scrapy.utils.asyncio.AsyncioLoopingCall

scrapy/utils/asyncio.py

@@ -9,7 +9,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable
 from typing import TYPE_CHECKING, Any, Concatenate, ParamSpec, TypeVar
 
 from twisted.internet.defer import Deferred
-from twisted.internet.task import LoopingCall
+from twisted.internet.task import LoopingCall, deferLater
 from twisted.internet.threads import deferToThread
 
 from scrapy.utils.asyncgen import as_async_generator
@@ -293,6 +293,24 @@ class CallLaterResult:
             self._delayed_call = None
 
 
+async def sleep(seconds: float) -> None:
+    """Sleep for *seconds*.
+
+    .. versionadded:: VERSION
+
+    This uses either :func:`asyncio.sleep` or
+    :func:`~twisted.internet.task.deferLater`, depending on whether asyncio
+    support is available.
+    """
+    if is_asyncio_available():
+        await asyncio.sleep(seconds)
+        return
+
+    from twisted.internet import reactor
+
+    await deferLater(reactor, seconds)
+
+
 async def run_in_thread(
     func: Callable[_P, _T], *args: _P.args, **kwargs: _P.kwargs
 ) -> _T:

scrapy/utils/defer.py

@@ -27,7 +27,7 @@ from twisted.internet.task import Cooperator
 from twisted.python import failure
 
 from scrapy.exceptions import ScrapyDeprecationWarning
-from scrapy.utils.asyncio import is_asyncio_available
+from scrapy.utils.asyncio import is_asyncio_available, sleep
 from scrapy.utils.python import global_object_name
 
 if TYPE_CHECKING:
@@ -90,14 +90,7 @@ async def _defer_sleep_async() -> None:
     """Delay by _DEFER_DELAY so reactor has a chance to go through readers and writers
     before attending pending delayed calls, so do not set delay to zero.
     """
-    if is_asyncio_available():
-        await asyncio.sleep(_DEFER_DELAY)
-    else:
-        from twisted.internet import reactor
-
-        d: Deferred[None] = Deferred()
-        reactor.callLater(_DEFER_DELAY, d.callback, None)
-        await d
+    await sleep(_DEFER_DELAY)
 
 
 def defer_result(result: Any) -> Deferred[Any]:  # pragma: no cover

tests/test_crawler_subprocess.py

@@ -14,7 +14,8 @@ from packaging.version import parse as parse_version
 from pexpect.popen_spawn import PopenSpawn
 from w3lib import __version__ as w3lib_version
 
-from tests.utils import async_sleep, get_script_run_env
+from scrapy.utils.asyncio import sleep
+from tests.utils import get_script_run_env
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -244,7 +245,7 @@ class TestCrawlerProcessSubprocessBase(ScriptRunnerMixin):
         p.kill(sig)
         p.expect_exact("shutting down gracefully")
         # sending the second signal too fast often causes problems
-        await async_sleep(0.01)
+        await sleep(0.01)
         p.kill(sig)
         p.expect_exact("forcing unclean shutdown")
         p.wait()  # type: ignore[no-untyped-call]

tests/test_engine_loop.py

@@ -6,10 +6,9 @@ from typing import TYPE_CHECKING, Any
 
 from scrapy import Request, Spider, signals
 from scrapy.core.scheduler import BaseScheduler
-from scrapy.utils.asyncio import call_later
+from scrapy.utils.asyncio import call_later, sleep
 from scrapy.utils.test import get_crawler
 from tests.mockserver.http import MockServer
-from tests.utils import async_sleep
 from tests.utils.decorators import coroutine_test
 
 if TYPE_CHECKING:
@@ -65,23 +64,23 @@ class TestMain:
             async def start(self):
                 yield Request("data:,a")
 
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.pause()
                 self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,b"))
 
                 # During this time, the scheduler reports having requests but
                 # returns None.
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.unpause()
 
                 # The scheduler request is processed.
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 yield Request("data:,c")
 
-                await async_sleep(seconds)
+                await sleep(seconds)
 
                 self.crawler.engine._slot.scheduler.pause()
                 self.crawler.engine._slot.scheduler.enqueue_request(Request("data:,d"))

tests/test_utils_asyncio.py

@@ -12,7 +12,9 @@ from scrapy.utils.asyncgen import as_async_generator
 from scrapy.utils.asyncio import (
     AsyncioLoopingCall,
     _parallel_asyncio,
+    call_later,
     is_asyncio_available,
+    sleep,
 )
 from tests.utils.decorators import coroutine_test
 
@@ -26,6 +28,15 @@ async def test_is_asyncio_available(reactor_pytest: str) -> None:
     assert is_asyncio_available() == (reactor_pytest != "default")
 
 
+@coroutine_test
+async def test_sleep() -> None:
+    events: list[str] = []
+    call_later(0.05, events.append, "call_later")
+    await sleep(0.1)
+    events.append("sleep")
+    assert events == ["call_later", "sleep"]
+
+
 @pytest.mark.only_asyncio
 class TestParallelAsyncio:
     """Test for scrapy.utils.asyncio.parallel_asyncio(), based on tests.test_utils_defer.TestParallelAsync."""

tests/utils/__init__.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import asyncio
 import os
 from pathlib import Path
 from typing import TYPE_CHECKING
@@ -8,8 +7,6 @@ from typing import TYPE_CHECKING
 from twisted.internet.defer import Deferred
 
 from scrapy.settings import Settings, default_settings
-from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.defer import maybe_deferred_to_future
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -23,13 +20,6 @@ def twisted_sleep(seconds: float):
     return d
 
 
-async def async_sleep(seconds: float) -> None:
-    if is_asyncio_available():
-        await asyncio.sleep(seconds)
-    else:
-        await maybe_deferred_to_future(twisted_sleep(seconds))
-
-
 def get_script_run_env() -> dict[str, str]:
     """Return a OS environment dict suitable to run scripts shipped with tests."""