diff --git a/gateway/run.py b/gateway/run.py index e5483319e831e..a274636e1b11f 100644 --- a/gateway/run.py +++ b/gateway/run.py @@ -7652,8 +7652,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew counted by _running_agent_count(), but suspending mid-flight loses them. Checks the runner's own tracked tasks + the process registry's running processes + any pending process-completion watchers. + + PERMANENT supervised watchers (tagged _hermes_supervised_watcher by + _spawn_supervised) are excluded: they live for the whole process — + including the scale-to-zero watcher itself — so counting them would + make this predicate True forever and the gateway could never go + dormant. Verified live on staging (2026-08-12): an armed, fully idle + instance never logged "going dormant" because ~9 supervised watchers + sat in _background_tasks. Fly's coarse autostop used to mask this; + with the gateway owning the suspend it became load-bearing. """ - if any(not t.done() for t in self._background_tasks): + if any( + not t.done() and not getattr(t, "_hermes_supervised_watcher", False) + for t in self._background_tasks + ): return True try: from tools.async_delegation import active_count @@ -12021,6 +12033,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew # Deliberately do NOT pass name= to create_task — some test doubles mock # create_task with a signature that rejects the name kwarg. task = asyncio.create_task(coro_factory()) + # Mark this as a PERMANENT supervised watcher, not transient background + # WORK. The scale-to-zero idle check must ignore these: supervised + # watchers (session-expiry, kanban, reconnect, the scale-to-zero watcher + # itself, ...) live for the whole process, so counting them as "live + # background work" would make the gateway consider itself busy forever + # and never go dormant/suspend. Transient tasks added to + # _background_tasks elsewhere (startup-resume events etc.) stay counted. + task._hermes_supervised_watcher = True # type: ignore[attr-defined] self._background_tasks.add(task) if on_spawn is not None: # Record the live handle NOW so an external tracker (e.g. diff --git a/tests/gateway/test_scale_to_zero_watcher.py b/tests/gateway/test_scale_to_zero_watcher.py index 6cb4ad5f15328..781db57c4b8c4 100644 --- a/tests/gateway/test_scale_to_zero_watcher.py +++ b/tests/gateway/test_scale_to_zero_watcher.py @@ -248,3 +248,68 @@ async def test_self_suspend_noop_off_fly(monkeypatch): ) await r._scale_to_zero_self_suspend() assert called == [] + +# ── supervised watchers must NOT count as live background work (staging bug) ── +# +# _spawn_supervised parks every permanent watcher task (session-expiry, kanban, +# reconnect, the scale-to-zero watcher ITSELF, ...) in _background_tasks. The +# bg-work check counted them, so an armed gateway considered itself busy +# forever and never went dormant — verified live on staging 2026-08-12 (armed +# at 05:25, fully idle 25+ min, zero "going dormant" lines). Fly's coarse +# autostop masked this until the gateway took ownership of the suspend. +# These tests exercise the REAL _spawn_supervised path — the earlier tests +# stubbed _background_tasks and missed the call site (same trap as F25). + + +@pytest.mark.asyncio +async def test_supervised_watchers_do_not_block_idle(): + r = GatewayRunner.__new__(GatewayRunner) + r._running = True + r._background_tasks = set() + + async def _forever(): + await asyncio.sleep(3600) + + # Spawn like production does — through _spawn_supervised. + for name in ("session_expiry", "kanban", "scale_to_zero_watcher"): + r._spawn_supervised(lambda: _forever(), name) + await asyncio.sleep(0) # let tasks start + try: + assert r._scale_to_zero_has_live_background_work() is False + finally: + for t in r._background_tasks: + t.cancel() + await asyncio.gather(*r._background_tasks, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_transient_background_task_still_blocks_idle(): + """A plain (untagged) task in _background_tasks — startup-resume events, + ad-hoc work — must still count as live background work.""" + r = GatewayRunner.__new__(GatewayRunner) + r._running = True + + async def _work(): + await asyncio.sleep(3600) + + t = asyncio.create_task(_work()) + r._background_tasks = {t} + try: + assert r._scale_to_zero_has_live_background_work() is True + finally: + t.cancel() + await asyncio.gather(t, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_done_supervised_watcher_is_ignored_either_way(): + r = GatewayRunner.__new__(GatewayRunner) + r._running = True + + async def _quick(): + return None + + t = asyncio.create_task(_quick()) + await t + r._background_tasks = {t} + assert r._scale_to_zero_has_live_background_work() is False