fix(gateway): exclude permanent supervised watchers from the scale-to-zero busy check (#84327)

_scale_to_zero_has_live_background_work() counted every task in
_background_tasks — but _spawn_supervised parks all permanent watchers
there (session-expiry, kanban, reconnect, the scale-to-zero watcher
itself, ...). An armed gateway therefore considered itself busy forever
and never went dormant or suspended. Verified live on staging
(hermes-agent-stg-test-6698, 2026-08-12): armed at 05:25, fully idle for
25+ minutes, zero 'going dormant' lines. Fly's coarse proxy autostop used
to mask the bug; once the gateway took ownership of the suspend (#84295)
it became load-bearing.

_spawn_supervised now tags its tasks and the busy check skips them.
Transient tasks (startup-resume events, delegation, tracked processes)
still block suspend. New tests exercise the REAL _spawn_supervised path
rather than a stubbed _background_tasks set — the stubbing is exactly why
the earlier tests missed this (same call-site trap as the F25 arm bug);
the key test fails on main and passes with the fix.

gateway/run.py

@@ -7652,8 +7652,20 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
         counted by _running_agent_count(), but suspending mid-flight loses them.
         Checks the runner's own tracked tasks + the process registry's running
         processes + any pending process-completion watchers.
+
+        PERMANENT supervised watchers (tagged _hermes_supervised_watcher by
+        _spawn_supervised) are excluded: they live for the whole process —
+        including the scale-to-zero watcher itself — so counting them would
+        make this predicate True forever and the gateway could never go
+        dormant. Verified live on staging (2026-08-12): an armed, fully idle
+        instance never logged "going dormant" because ~9 supervised watchers
+        sat in _background_tasks. Fly's coarse autostop used to mask this;
+        with the gateway owning the suspend it became load-bearing.
         """
-        if any(not t.done() for t in self._background_tasks):
+        if any(
+            not t.done() and not getattr(t, "_hermes_supervised_watcher", False)
+            for t in self._background_tasks
+        ):
             return True
         try:
             from tools.async_delegation import active_count
@@ -12021,6 +12033,14 @@ class GatewayRunner(GatewayAuthorizationMixin, GatewayKanbanWatchersMixin, Gatew
         # Deliberately do NOT pass name= to create_task — some test doubles mock
         # create_task with a signature that rejects the name kwarg.
         task = asyncio.create_task(coro_factory())
+        # Mark this as a PERMANENT supervised watcher, not transient background
+        # WORK. The scale-to-zero idle check must ignore these: supervised
+        # watchers (session-expiry, kanban, reconnect, the scale-to-zero watcher
+        # itself, ...) live for the whole process, so counting them as "live
+        # background work" would make the gateway consider itself busy forever
+        # and never go dormant/suspend. Transient tasks added to
+        # _background_tasks elsewhere (startup-resume events etc.) stay counted.
+        task._hermes_supervised_watcher = True  # type: ignore[attr-defined]
         self._background_tasks.add(task)
         if on_spawn is not None:
             # Record the live handle NOW so an external tracker (e.g.

tests/gateway/test_scale_to_zero_watcher.py

@@ -248,3 +248,68 @@ async def test_self_suspend_noop_off_fly(monkeypatch):
     )
     await r._scale_to_zero_self_suspend()
     assert called == []
+
+# ── supervised watchers must NOT count as live background work (staging bug) ──
+#
+# _spawn_supervised parks every permanent watcher task (session-expiry, kanban,
+# reconnect, the scale-to-zero watcher ITSELF, ...) in _background_tasks. The
+# bg-work check counted them, so an armed gateway considered itself busy
+# forever and never went dormant — verified live on staging 2026-08-12 (armed
+# at 05:25, fully idle 25+ min, zero "going dormant" lines). Fly's coarse
+# autostop masked this until the gateway took ownership of the suspend.
+# These tests exercise the REAL _spawn_supervised path — the earlier tests
+# stubbed _background_tasks and missed the call site (same trap as F25).
+
+
+@pytest.mark.asyncio
+async def test_supervised_watchers_do_not_block_idle():
+    r = GatewayRunner.__new__(GatewayRunner)
+    r._running = True
+    r._background_tasks = set()
+
+    async def _forever():
+        await asyncio.sleep(3600)
+
+    # Spawn like production does — through _spawn_supervised.
+    for name in ("session_expiry", "kanban", "scale_to_zero_watcher"):
+        r._spawn_supervised(lambda: _forever(), name)
+    await asyncio.sleep(0)  # let tasks start
+    try:
+        assert r._scale_to_zero_has_live_background_work() is False
+    finally:
+        for t in r._background_tasks:
+            t.cancel()
+        await asyncio.gather(*r._background_tasks, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+async def test_transient_background_task_still_blocks_idle():
+    """A plain (untagged) task in _background_tasks — startup-resume events,
+    ad-hoc work — must still count as live background work."""
+    r = GatewayRunner.__new__(GatewayRunner)
+    r._running = True
+
+    async def _work():
+        await asyncio.sleep(3600)
+
+    t = asyncio.create_task(_work())
+    r._background_tasks = {t}
+    try:
+        assert r._scale_to_zero_has_live_background_work() is True
+    finally:
+        t.cancel()
+        await asyncio.gather(t, return_exceptions=True)
+
+
+@pytest.mark.asyncio
+async def test_done_supervised_watcher_is_ignored_either_way():
+    r = GatewayRunner.__new__(GatewayRunner)
+    r._running = True
+
+    async def _quick():
+        return None
+
+    t = asyncio.create_task(_quick())
+    await t
+    r._background_tasks = {t}
+    assert r._scale_to_zero_has_live_background_work() is False