diff --git a/gateway/lifecycle_ledger.py b/gateway/lifecycle_ledger.py index 594da94e5ca45..0950035939500 100644 --- a/gateway/lifecycle_ledger.py +++ b/gateway/lifecycle_ledger.py @@ -254,15 +254,24 @@ def record_startup(home: Optional[Path] = None) -> Optional[Dict[str, Any]]: logger.debug("Unclean-exit detection failed", exc_info=True) try: - _write_sentinel( - { - "phase": "running", - "pid": os.getpid(), - "start_time": time.time(), - "started_at": datetime.now(timezone.utc).isoformat(), - }, - home, - ) + claim: Dict[str, Any] = { + "phase": "running", + "pid": os.getpid(), + "start_time": time.time(), + "started_at": datetime.now(timezone.utc).isoformat(), + } + # Carry the verdict on the PREVIOUS life forward on the new + # sentinel: it is the only place the finding survives in + # machine-readable form (the exit-diag log is append-only prose + # for humans), and /api/status reads it to tell the user "your + # agent restarted after (suspected) running out of memory" + # (NS-656). Scoped to this life only — the next clean exit or + # boot rewrites the sentinel and the flags age out with it. + if evidence is not None: + claim["prior_unclean_exit"] = True + if evidence.get("suspected_oom"): + claim["prior_suspected_oom"] = True + _write_sentinel(claim, home) except Exception: logger.debug("Failed to claim lifecycle sentinel", exc_info=True) return evidence diff --git a/gateway/memory_status.py b/gateway/memory_status.py new file mode 100644 index 0000000000000..f440d9ff7b4b2 --- /dev/null +++ b/gateway/memory_status.py @@ -0,0 +1,194 @@ +"""Memory status rollup for ``/api/status`` (NS-656). + +The gateway already *produces* every memory-pressure signal a user would +want to know about, but all of it dies in log files: + +* :func:`gateway.shutdown_watchdog.write_loop_heartbeat` embeds a + :func:`gateway.lifecycle_ledger.sample_memory` snapshot (gateway RSS + + system MemAvailable/MemTotal + swap) in ``state/gateway.heartbeat`` + every 30 seconds. +* :func:`gateway.lifecycle_ledger.record_startup` detects an unclean + previous death and flags ``suspected_oom`` — but only into + ``gateway-exit-diag.log`` and a WARNING line. +* ``gateway/agent_cache_pressure.py`` evicts transcripts under pressure, + again log-only. + +So a hosted agent can be OOM-killed hourly (the BlueAtlas incident, +NS-608) while its dashboard and the NAS agent card both look perfectly +healthy. This module is the read side that closes the gap: it distills +the *already-persisted* heartbeat + lifecycle sentinel into a compact, +public-safe block that ``/api/status`` can serve to the dashboard SPA +and the NAS availability sweep — no new sampling, no IPC with the +gateway process, just two small file reads. + +Public-safety note: ``/api/status`` is an unauthenticated liveness probe +(``PUBLIC_API_PATHS``), which is exactly why NAS can consume it. This +block therefore carries only coarse numbers (MB granularity), enums, and +booleans — the same disclosure class as the existing ``active_agents`` +count and ``nous_session_valid`` field (which was added for the same +NAS-sweep audience). + +Everything here is best-effort and read-only: a missing/corrupt file +degrades to ``pressure="unknown"`` rather than raising into the status +endpoint. +""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, Optional + +logger = logging.getLogger(__name__) + +# Pressure thresholds on system MemAvailable. ``critical`` deliberately +# mirrors the lifecycle ledger's OOM-suspicion heuristics +# (:data:`gateway.lifecycle_ledger._LOW_MEM_AVAILABLE_KIB` / +# ``_LOW_MEM_AVAILABLE_FRACTION``): if a memory level would make a +# subsequent unclean death "suspected OOM", the user should already have +# been warned at that level while the process was still alive. +_CRITICAL_AVAILABLE_KIB = 64 * 1024 # < 64 MiB available +_CRITICAL_AVAILABLE_FRACTION = 0.05 # < 5% of MemTotal +_ELEVATED_AVAILABLE_KIB = 128 * 1024 # < 128 MiB available +_ELEVATED_AVAILABLE_FRACTION = 0.15 # < 15% of MemTotal + +# A heartbeat older than this no longer describes the present. The writer +# cadence is 30s (DEFAULT_HEARTBEAT_INTERVAL_S); 150s of slack tolerates a +# briefly stalled loop without letting a long-dead gateway's last sample +# masquerade as current pressure. +_HEARTBEAT_FRESH_TTL_S = 150.0 + +_KIB_PER_MB = 1024 + + +def _mb(kib: Any) -> Optional[int]: + if isinstance(kib, bool) or not isinstance(kib, int) or kib < 0: + return None + return kib // _KIB_PER_MB + + +def _parse_iso(value: Any) -> Optional[datetime]: + if not isinstance(value, str) or not value: + return None + try: + parsed = datetime.fromisoformat(value) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed + + +def classify_pressure( + available_kib: Any, total_kib: Any +) -> str: + """Map a MemAvailable/MemTotal pair to ``ok``/``elevated``/``critical``. + + ``unknown`` when the sample is missing or malformed — the caller must + not treat "we could not read it" as "memory is fine". + """ + if ( + isinstance(available_kib, bool) + or not isinstance(available_kib, int) + or available_kib < 0 + ): + return "unknown" + fraction: Optional[float] = None + if ( + not isinstance(total_kib, bool) + and isinstance(total_kib, int) + and total_kib > 0 + ): + fraction = available_kib / total_kib + if available_kib < _CRITICAL_AVAILABLE_KIB or ( + fraction is not None and fraction < _CRITICAL_AVAILABLE_FRACTION + ): + return "critical" + if available_kib < _ELEVATED_AVAILABLE_KIB or ( + fraction is not None and fraction < _ELEVATED_AVAILABLE_FRACTION + ): + return "elevated" + return "ok" + + +def _read_heartbeat(home: Optional[Path]) -> Optional[Dict[str, Any]]: + try: + from gateway.lifecycle_ledger import _read_json + from gateway.shutdown_watchdog import get_loop_heartbeat_path + + return _read_json(get_loop_heartbeat_path(home)) + except Exception: + return None + + +def _read_sentinel(home: Optional[Path]) -> Optional[Dict[str, Any]]: + try: + from gateway.lifecycle_ledger import ( + _read_json, + get_lifecycle_sentinel_path, + ) + + return _read_json(get_lifecycle_sentinel_path(home)) + except Exception: + return None + + +def collect_memory_status( + home: Optional[Path] = None, + *, + now: Optional[datetime] = None, +) -> Dict[str, Any]: + """Build the ``memory`` block for ``/api/status``. + + ``home`` scopes the read to a profile's HERMES_HOME (the status + endpoint's ``?profile=`` handling passes it through); ``None`` means + the active profile. ``now`` is injectable for tests. + + Always returns a dict — a gateway that is down, has never written a + heartbeat, or whose files are corrupt yields + ``{"pressure": "unknown", ...}`` with whatever fields could still be + recovered. Never raises. + """ + moment = now or datetime.now(timezone.utc) + status: Dict[str, Any] = { + "pressure": "unknown", + "gateway_rss_mb": None, + "system_total_mb": None, + "system_available_mb": None, + "swap_used_mb": None, + "sampled_at": None, + "last_boot_unclean": False, + "last_boot_suspected_oom": False, + } + + heartbeat = _read_heartbeat(home) + if heartbeat: + sampled_at = _parse_iso(heartbeat.get("updated_at")) + mem = heartbeat.get("mem") + if isinstance(mem, dict): + status["gateway_rss_mb"] = _mb(mem.get("rss_kib")) + status["system_total_mb"] = _mb(mem.get("mem_total_kib")) + status["system_available_mb"] = _mb(mem.get("mem_available_kib")) + status["swap_used_mb"] = _mb(mem.get("swap_used_kib")) + if sampled_at is not None: + status["sampled_at"] = sampled_at.isoformat() + age_s = (moment - sampled_at).total_seconds() + if 0 <= age_s <= _HEARTBEAT_FRESH_TTL_S: + status["pressure"] = classify_pressure( + mem.get("mem_available_kib"), + mem.get("mem_total_kib"), + ) + # else: stale sample — numbers are still reported (they are + # honest about *when* via sampled_at) but pressure stays + # "unknown" so a dead gateway's final gasp cannot render a + # live "critical" banner forever. + + sentinel = _read_sentinel(home) + if sentinel: + status["last_boot_unclean"] = bool(sentinel.get("prior_unclean_exit")) + status["last_boot_suspected_oom"] = bool( + sentinel.get("prior_suspected_oom") + ) + + return status diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index a533fbc2a5964..543e1c0b9ac44 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -3332,6 +3332,28 @@ async def get_status(profile: Optional[str] = None): else "degraded" ) + # Memory-pressure rollup (NS-656). Distilled from the gateway's + # 30s loop heartbeat + lifecycle sentinel — two small file reads, + # no gateway IPC. Coarse MB numbers/enums/booleans only: this + # endpoint is public (PUBLIC_API_PATHS), same disclosure class as + # nous_session_valid above. Deliberately NOT folded into + # components/overall — memory pressure is advisory (toast/notice + # material), not a liveness verdict, and flipping `overall` to + # "degraded" on it would page NAS's availability sweep for a + # condition the valve is already handling. + try: + from gateway.memory_status import collect_memory_status + + status["memory"] = await asyncio.get_running_loop().run_in_executor( + None, + functools.partial( + collect_memory_status, + profile_dir if profile_dir else get_hermes_home(), + ), + ) + except Exception: + status["memory"] = {"pressure": "unknown"} + # Deferred FTS rebuild progress (schema v23): lets the desktop / # dashboard render a "search index rebuilding: N%" indicator instead # of users wondering why old-message search is slower after an diff --git a/tests/gateway/test_lifecycle_ledger.py b/tests/gateway/test_lifecycle_ledger.py index 6ec2beeeb8b8c..016a8b2bcd147 100644 --- a/tests/gateway/test_lifecycle_ledger.py +++ b/tests/gateway/test_lifecycle_ledger.py @@ -143,6 +143,52 @@ def test_record_startup_persists_unclean_report_and_reclaims(tmp_path: Path) -> assert sentinel["pid"] == os.getpid() +def test_record_startup_carries_unclean_flags_onto_new_sentinel( + tmp_path: Path, +) -> None: + """The unclean-death verdict must survive on the reclaimed sentinel so + /api/status can surface "restarted after (suspected) OOM" (NS-656).""" + _write_sentinel(tmp_path, { + "phase": "running", + "pid": _DEAD_PID, + "start_time": 1000.0, + "started_at": "2026-07-11T04:30:00+00:00", + }) + # Last heartbeat shows near-exhausted memory → suspected OOM. + from gateway.shutdown_watchdog import get_loop_heartbeat_path + + hb_path = get_loop_heartbeat_path(tmp_path) + hb_path.parent.mkdir(parents=True, exist_ok=True) + hb_path.write_text(json.dumps({ + "pid": _DEAD_PID, + "updated_at": "2026-07-11T05:00:00+00:00", + "mem": {"mem_total_kib": 1024 * 1024, "mem_available_kib": 20 * 1024}, + }), encoding="utf-8") + + evidence = record_startup(home=tmp_path) + assert evidence is not None + assert evidence.get("suspected_oom") is True + + sentinel = _read_sentinel(tmp_path) + assert sentinel["phase"] == "running" + assert sentinel["prior_unclean_exit"] is True + assert sentinel["prior_suspected_oom"] is True + + +def test_record_startup_clean_boot_has_no_prior_flags(tmp_path: Path) -> None: + _write_sentinel(tmp_path, { + "phase": "exited", + "pid": _DEAD_PID, + "exit_code": 0, + "exit_reason": "graceful_shutdown", + }) + assert record_startup(home=tmp_path) is None + sentinel = _read_sentinel(tmp_path) + assert sentinel["phase"] == "running" + assert "prior_unclean_exit" not in sentinel + assert "prior_suspected_oom" not in sentinel + + # --------------------------------------------------------------------------- # Takeover ownership guard on mark_exited # --------------------------------------------------------------------------- diff --git a/tests/gateway/test_memory_status.py b/tests/gateway/test_memory_status.py new file mode 100644 index 0000000000000..0b533bd6325ce --- /dev/null +++ b/tests/gateway/test_memory_status.py @@ -0,0 +1,169 @@ +"""Tests for gateway.memory_status — the /api/status memory rollup (NS-656).""" + +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path + +from gateway.memory_status import classify_pressure, collect_memory_status +from gateway.shutdown_watchdog import get_loop_heartbeat_path +from gateway.lifecycle_ledger import get_lifecycle_sentinel_path + +_NOW = datetime(2026, 8, 13, 12, 0, 0, tzinfo=timezone.utc) + + +def _write_heartbeat( + home: Path, + *, + updated_at: datetime = _NOW, + mem: dict | None = None, +) -> None: + path = get_loop_heartbeat_path(home) + path.parent.mkdir(parents=True, exist_ok=True) + payload = { + "pid": 12345, + "updated_at": updated_at.isoformat(), + "monotonic": 1.0, + } + if mem is not None: + payload["mem"] = mem + path.write_text(json.dumps(payload), encoding="utf-8") + + +def _write_sentinel(home: Path, payload: dict) -> None: + path = get_lifecycle_sentinel_path(home) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload), encoding="utf-8") + + +class TestClassifyPressure: + def test_plentiful_memory_is_ok(self) -> None: + # 1 GiB available of 2 GiB total. + assert classify_pressure(1024 * 1024, 2048 * 1024) == "ok" + + def test_low_absolute_available_is_critical(self) -> None: + # 32 MiB available — below the 64 MiB floor regardless of total. + assert classify_pressure(32 * 1024, 8 * 1024 * 1024) == "critical" + + def test_low_fraction_is_critical(self) -> None: + # 300 MiB available of 8 GiB ≈ 3.7% < 5%. + assert classify_pressure(300 * 1024, 8 * 1024 * 1024) == "critical" + + def test_elevated_band(self) -> None: + # 100 MiB available of 1 GiB ≈ 9.8% — above critical, below elevated + # thresholds (128 MiB / 15%). + assert classify_pressure(100 * 1024, 1024 * 1024) == "elevated" + + def test_missing_sample_is_unknown(self) -> None: + assert classify_pressure(None, None) == "unknown" + + def test_bool_is_not_an_int(self) -> None: + # True == 1 in Python — must not classify as "1 KiB available". + assert classify_pressure(True, 2048 * 1024) == "unknown" + + def test_absolute_floor_works_without_total(self) -> None: + assert classify_pressure(32 * 1024, None) == "critical" + # 1 GiB available, unknown total: passes both absolute floors → ok. + assert classify_pressure(1024 * 1024, None) == "ok" + + +class TestCollectMemoryStatus: + def test_no_files_yields_unknown(self, tmp_path: Path) -> None: + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "unknown" + assert status["gateway_rss_mb"] is None + assert status["last_boot_unclean"] is False + assert status["last_boot_suspected_oom"] is False + + def test_fresh_heartbeat_reports_pressure_and_numbers( + self, tmp_path: Path + ) -> None: + _write_heartbeat( + tmp_path, + updated_at=_NOW - timedelta(seconds=30), + mem={ + "rss_kib": 400 * 1024, + "mem_total_kib": 1024 * 1024, + "mem_available_kib": 50 * 1024, + "swap_used_kib": 200 * 1024, + }, + ) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "critical" + assert status["gateway_rss_mb"] == 400 + assert status["system_total_mb"] == 1024 + assert status["system_available_mb"] == 50 + assert status["swap_used_mb"] == 200 + assert status["sampled_at"] is not None + + def test_stale_heartbeat_keeps_numbers_but_unknown_pressure( + self, tmp_path: Path + ) -> None: + # A dead gateway's final gasp must not render a live "critical" + # banner forever. + _write_heartbeat( + tmp_path, + updated_at=_NOW - timedelta(hours=2), + mem={ + "rss_kib": 400 * 1024, + "mem_total_kib": 1024 * 1024, + "mem_available_kib": 10 * 1024, + }, + ) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "unknown" + assert status["system_available_mb"] == 10 + assert status["sampled_at"] is not None + + def test_future_heartbeat_is_treated_as_stale(self, tmp_path: Path) -> None: + # Clock skew / restored snapshots: a timestamp from the future is + # not evidence about the present either. + _write_heartbeat( + tmp_path, + updated_at=_NOW + timedelta(hours=1), + mem={"mem_total_kib": 1024 * 1024, "mem_available_kib": 10 * 1024}, + ) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "unknown" + + def test_sentinel_flags_surface(self, tmp_path: Path) -> None: + _write_sentinel( + tmp_path, + { + "phase": "running", + "pid": 999, + "prior_unclean_exit": True, + "prior_suspected_oom": True, + }, + ) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["last_boot_unclean"] is True + assert status["last_boot_suspected_oom"] is True + + def test_clean_sentinel_has_no_flags(self, tmp_path: Path) -> None: + _write_sentinel( + tmp_path, + {"phase": "exited", "pid": 999, "exit_reason": "graceful_shutdown"}, + ) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["last_boot_unclean"] is False + assert status["last_boot_suspected_oom"] is False + + def test_corrupt_files_never_raise(self, tmp_path: Path) -> None: + hb = get_loop_heartbeat_path(tmp_path) + hb.parent.mkdir(parents=True, exist_ok=True) + hb.write_text("{not json", encoding="utf-8") + sentinel = get_lifecycle_sentinel_path(tmp_path) + sentinel.parent.mkdir(parents=True, exist_ok=True) + sentinel.write_text("[]", encoding="utf-8") # valid JSON, wrong shape + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "unknown" + + def test_heartbeat_without_mem_block(self, tmp_path: Path) -> None: + # Non-Linux hosts: sample_memory() returns {} so the heartbeat has + # no mem key at all. + _write_heartbeat(tmp_path, updated_at=_NOW) + status = collect_memory_status(tmp_path, now=_NOW) + assert status["pressure"] == "unknown" + assert status["gateway_rss_mb"] is None diff --git a/tests/hermes_cli/test_web_server.py b/tests/hermes_cli/test_web_server.py index a3c809d445e9d..5a06173807b0e 100644 --- a/tests/hermes_cli/test_web_server.py +++ b/tests/hermes_cli/test_web_server.py @@ -2994,6 +2994,41 @@ class TestGatewayBusyReadout: assert data["gateway_busy"] is False +class TestStatusMemoryBlock: + """NS-656: /api/status must always carry a `memory` block.""" + + @pytest.fixture(autouse=True) + def _setup_test_client(self): + try: + from starlette.testclient import TestClient + except ImportError: + pytest.skip("fastapi/starlette not installed") + + from hermes_cli.web_server import app, _SESSION_HEADER_NAME, _SESSION_TOKEN + self.client = TestClient(app) + self.client.headers[_SESSION_HEADER_NAME] = _SESSION_TOKEN + + def test_memory_block_present_with_pressure_field(self): + data = self.client.get("/api/status").json() + assert "memory" in data + assert data["memory"]["pressure"] in { + "ok", "elevated", "critical", "unknown", + } + + def test_memory_block_degrades_when_collector_raises(self, monkeypatch): + """A broken collector must never take down the status endpoint — + the block degrades to pressure=unknown.""" + import gateway.memory_status as ms + + def _boom(*_a, **_k): + raise RuntimeError("collector exploded") + + monkeypatch.setattr(ms, "collect_memory_status", _boom) + resp = self.client.get("/api/status") + assert resp.status_code == 200 + assert resp.json()["memory"] == {"pressure": "unknown"} + + class TestGatewayUpdatedAtContract: """Contract tests for /api/status ``gateway_updated_at``. diff --git a/web/src/App.tsx b/web/src/App.tsx index 7fe8c7f8c206a..6f57d4349397a 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -72,6 +72,7 @@ import { ProfileProvider } from "@/contexts/ProfileProvider"; import { useProfileScope } from "@/contexts/useProfileScope"; import { ProfileSwitcher } from "@/components/ProfileSwitcher"; import { ProfileScopeBanner } from "@/components/ProfileScopeBanner"; +import { MemoryPressureBanner } from "@/components/MemoryPressureBanner"; import { useSystemActions } from "@/contexts/useSystemActions"; import type { SystemAction } from "@/contexts/system-actions-context"; // Route pages are lazy-loaded so the initial dashboard shell does not pay for @@ -567,6 +568,7 @@ export default function App() { +
diff --git a/web/src/components/MemoryPressureBanner.test.tsx b/web/src/components/MemoryPressureBanner.test.tsx new file mode 100644 index 0000000000000..3baa30b564c5c --- /dev/null +++ b/web/src/components/MemoryPressureBanner.test.tsx @@ -0,0 +1,124 @@ +// @vitest-environment jsdom +// Tests for the NS-656 memory-pressure banner: trigger selection, +// severity precedence, dismissal semantics, and escalation re-opening. + +import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import type { ReactNode } from "react"; + +import { I18nProvider } from "@/i18n"; +import { MemoryPressureBanner } from "./MemoryPressureBanner"; +import type { StatusResponse, MemoryStatus } from "@/lib/api"; + +let container: HTMLDivElement; +let root: Root; + +async function render(ui: ReactNode) { + container = document.createElement("div"); + document.body.append(container); + root = createRoot(container); + await act(async () => root.render({ui})); +} + +async function rerender(ui: ReactNode) { + await act(async () => root.render({ui})); +} + +beforeEach(() => { + sessionStorage.clear(); +}); + +afterEach(async () => { + await act(async () => root?.unmount()); + container?.remove(); +}); + +function statusWith(memory: MemoryStatus | undefined): StatusResponse { + return { memory } as StatusResponse; +} + +function banner(): HTMLElement | null { + return container.querySelector('[data-testid="memory-pressure-banner"]'); +} + +describe("MemoryPressureBanner", () => { + it("renders nothing for null status or healthy memory", async () => { + await render(); + expect(banner()).toBeNull(); + await rerender( + , + ); + expect(banner()).toBeNull(); + // Older gateways: no memory block at all. + await rerender(); + expect(banner()).toBeNull(); + }); + + it("renders nothing for unknown pressure (absence of evidence)", async () => { + await render( + , + ); + expect(banner()).toBeNull(); + }); + + it("shows the elevated warning", async () => { + await render( + , + ); + expect(banner()?.textContent).toContain("running low on memory"); + }); + + it("shows the OOM-restart notice even when current pressure is ok", async () => { + await render( + , + ); + expect(banner()?.textContent).toContain( + "restarted after running out of memory", + ); + }); + + it("critical pressure outranks the OOM-restart notice", async () => { + await render( + , + ); + expect(banner()?.textContent).toContain("almost out of memory"); + }); + + it("dismissal hides the banner and persists across re-renders", async () => { + await render( + , + ); + const dismiss = container.querySelector( + '[data-testid="memory-pressure-banner"] button', + ) as HTMLButtonElement; + await act(async () => dismiss.click()); + expect(banner()).toBeNull(); + await rerender( + , + ); + expect(banner()).toBeNull(); + }); + + it("escalation to critical re-opens a dismissed banner", async () => { + await render( + , + ); + const dismiss = container.querySelector( + '[data-testid="memory-pressure-banner"] button', + ) as HTMLButtonElement; + await act(async () => dismiss.click()); + expect(banner()).toBeNull(); + await rerender( + , + ); + expect(banner()?.textContent).toContain("almost out of memory"); + }); +}); diff --git a/web/src/components/MemoryPressureBanner.tsx b/web/src/components/MemoryPressureBanner.tsx new file mode 100644 index 0000000000000..9a7ef628ee417 --- /dev/null +++ b/web/src/components/MemoryPressureBanner.tsx @@ -0,0 +1,96 @@ +import { useState } from "react"; +import { AlertTriangle, X } from "lucide-react"; +import type { StatusResponse } from "@/lib/api"; +import { useI18n } from "@/i18n"; + +/** + * App-wide warning banner for memory trouble (NS-656). + * + * Two independent triggers, worst-first: + * 1. Live pressure — the gateway's heartbeat shows system memory in the + * `elevated`/`critical` band right now. + * 2. Post-mortem — the previous gateway life died uncleanly and its last + * heartbeat showed near-exhausted memory (`last_boot_suspected_oom`). + * + * Both previously died in server-side log files; a hosted agent could be + * OOM-killed hourly while the dashboard looked healthy. + * + * Dismissal is session-scoped per trigger kind (sessionStorage), so a user + * who acknowledged "restarted after OOM" is not re-nagged on every poll, + * but a NEW escalation (ok → critical) still surfaces. + */ +export function MemoryPressureBanner({ + status, +}: { + status: StatusResponse | null; +}) { + const { t } = useI18n(); + const memory = status?.memory; + + // Highest-severity active trigger, or null. + const trigger = !memory + ? null + : memory.pressure === "critical" + ? "critical" + : memory.last_boot_suspected_oom + ? "oom_restart" + : memory.pressure === "elevated" + ? "elevated" + : null; + + const [dismissed, setDismissed] = useState(() => { + try { + return sessionStorage.getItem("memoryBannerDismissed"); + } catch { + return null; + } + }); + + // Dismissal only masks the exact trigger that was dismissed — an + // escalation (elevated → critical) changes `trigger` and therefore + // re-opens the banner without any effect/state churn. + if (!trigger || dismissed === trigger) return null; + + const dismiss = () => { + setDismissed(trigger); + try { + sessionStorage.setItem("memoryBannerDismissed", trigger); + } catch { + /* ignore */ + } + }; + + const critical = trigger === "critical"; + const message = + trigger === "oom_restart" + ? (t.app.memoryOomRestartBanner ?? + "Your agent restarted after running out of memory. Long sessions and many concurrent tasks increase memory use.") + : critical + ? (t.app.memoryCriticalBanner ?? + "Your agent is almost out of memory and may restart. Consider closing idle sessions or upgrading its memory.") + : (t.app.memoryElevatedBanner ?? + "Your agent is running low on memory."); + + return ( +
+ + {message} + +
+ ); +} diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts index dda0dd0e937aa..4fa280106edee 100644 --- a/web/src/i18n/en.ts +++ b/web/src/i18n/en.ts @@ -97,6 +97,12 @@ export const en: Translations = { currentProfileOption: "this dashboard ({name})", managingProfileBanner: "Managing profile \u201c{name}\u201d \u2014 config, keys, skills, MCPs, model, and new chats apply to that profile.", + memoryOomRestartBanner: + "Your agent restarted after running out of memory. Long sessions and many concurrent tasks increase memory use.", + memoryCriticalBanner: + "Your agent is almost out of memory and may restart. Consider closing idle sessions or upgrading its memory.", + memoryElevatedBanner: "Your agent is running low on memory.", + dismiss: "Dismiss", }, status: { diff --git a/web/src/i18n/types.ts b/web/src/i18n/types.ts index 5c79a937e5fee..b669d8877c7c6 100644 --- a/web/src/i18n/types.ts +++ b/web/src/i18n/types.ts @@ -115,6 +115,11 @@ export interface Translations { managingProfile?: string; currentProfileOption?: string; managingProfileBanner?: string; + /** NS-656 memory-pressure banner — optional, English fallback. */ + memoryOomRestartBanner?: string; + memoryCriticalBanner?: string; + memoryElevatedBanner?: string; + dismiss?: string; }; // ── Status page ── diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 02ff011945a7c..79f7dbc991362 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1882,10 +1882,27 @@ export interface StatusResponse { gateway_updated_at: string | null; hermes_home: string; latest_config_version: number; + /** NS-656: memory-pressure rollup from the gateway heartbeat + + * lifecycle ledger. Absent on older gateways. */ + memory?: MemoryStatus; release_date: string; version: string; } +/** NS-656: coarse memory telemetry served by /api/status. */ +export interface MemoryStatus { + pressure: "ok" | "elevated" | "critical" | "unknown"; + gateway_rss_mb?: number | null; + system_total_mb?: number | null; + system_available_mb?: number | null; + swap_used_mb?: number | null; + sampled_at?: string | null; + /** Previous gateway life died without running any exit path. */ + last_boot_unclean?: boolean; + /** ...and its final heartbeat showed near-exhausted memory. */ + last_boot_suspected_oom?: boolean; +} + export interface SessionInfo { id: string; source: string | null;