From 6386c753063d40e9fd0f874f0c31e8f59d6e63cd Mon Sep 17 00:00:00 2001 From: Leon Phull Date: Tue, 4 Aug 2026 21:46:27 +0200 Subject: [PATCH] fix(desktop): reap orphaned local serve backends on desktop boot When Desktop exits uncleanly, leftover `hermes serve --host 127.0.0.1 --port 0` processes can be reparented to pid 1 and keep full MCP trees alive. The next boot then stacks another backend on top of the corpses until EMFILE kills sidebar/session APIs and tabs disappear. - Detect Desktop-local serve shape (loopback + ephemeral port 0) - Only reap processes whose ppid is 0/1 (true orphans) - Spare fixed-port remote serves (e.g. --port 9119) and HERMES_DESKTOP_CHILD_PID - Run at Desktop backend start (HERMES_DESKTOP=1) before parent-death watchdog Complements parent-death watchdog (prevents future orphans) and configurable nofile soft limit (capacity floor). Together these stop the multi-backend pile-up cascade observed on macOS Desktop SSH/local installs. --- hermes_cli/dashboard_procs.py | 185 ++++++++++++++++++ hermes_cli/web_server.py | 13 ++ .../test_orphan_desktop_serve_reap.py | 110 +++++++++++ 3 files changed, 308 insertions(+) create mode 100644 tests/hermes_cli/test_orphan_desktop_serve_reap.py diff --git a/hermes_cli/dashboard_procs.py b/hermes_cli/dashboard_procs.py index 7379ac972e2e1..d5b27a081b5da 100644 --- a/hermes_cli/dashboard_procs.py +++ b/hermes_cli/dashboard_procs.py @@ -456,3 +456,188 @@ def _detect_concurrent_hermes_instances( matches.append((int(pid), str(name))) return matches + + +def _is_desktop_local_serve_cmdline(command: str) -> bool: + """True for the Desktop-local serve spawn shape (loopback + ephemeral port). + + Desktop primary/pool backends launch as:: + + hermes serve --host 127.0.0.1 --port 0 + hermes serve --isolated --host 127.0.0.1 --port 0 ... + + Intentional long-lived headless serves (e.g. ``--host + --port 9119``) must never match — those are operator-managed remote + backends and may legitimately run with ppid 1 under launchd/nohup. + """ + cmd = command.lower() + if "serve" not in cmd: + return False + if "hermes" not in cmd and "hermes_cli" not in cmd: + return False + # Ephemeral desktop bind: host loopback + port 0 (exact tokens). + has_loopback = ( + "--host 127.0.0.1" in cmd + or "--host=127.0.0.1" in cmd + or "--host localhost" in cmd + or "--host=localhost" in cmd + ) + has_ephemeral = "--port 0" in cmd or "--port=0" in cmd + if not (has_loopback and has_ephemeral): + return False + # Spare anything with a concrete non-zero port flag first (defensive). + # (port 0 already required above.) + return True + + +def _process_ppid(pid: int) -> int | None: + """Best-effort parent pid lookup. None on failure.""" + try: + if sys.platform == "win32": + return None # Windows orphan reap is handled by desktop tree-kill. + result = subprocess.run( + ["ps", "-o", "ppid=", "-p", str(pid)], + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + timeout=5, + ) + if result.returncode != 0 or not result.stdout: + return None + return int(result.stdout.strip().split()[0]) + except (ValueError, FileNotFoundError, subprocess.TimeoutExpired, OSError): + return None + + +def _exclude_pids_from_env() -> set[int]: + """PIDs Desktop marks as live backends (HERMES_DESKTOP_CHILD_PID).""" + raw = os.environ.get("HERMES_DESKTOP_CHILD_PID", "") + out: set[int] = set() + for part in raw.split(","): + part = part.strip() + if not part: + continue + try: + out.add(int(part)) + except ValueError: + continue + return out + + +def _reap_orphaned_desktop_local_serves( + *, + reason: str = "orphaned desktop-local hermes serve", + signal_term=None, + signal_kill=None, + sleep_fn=None, +) -> dict[str, list]: + """Kill leftover Desktop-local ``hermes serve`` backends with no parent. + + When Electron dies uncleanly (crash / SIGKILL / update handoff), local + ``serve --host 127.0.0.1 --port 0`` children can be reparented to pid 1 and + keep their full MCP trees alive. The next Desktop boot then stacks a fresh + backend on top of the corpses until the machine hits EMFILE and the UI + loses tabs/sidebar. + + The parent-death watchdog prevents *future* orphans once a backend is + running under HERMES_PARENT_PID; this helper clears *already* orphaned + corpses at the start of a new Desktop backend. + + Safety: + - only the Desktop-local spawn shape (loopback + ``--port 0``) + - only processes whose current ppid is 1 (or 0 on some supervisors) + - never self / never HERMES_DESKTOP_CHILD_PID entries + - never fixed-port remote serves (e.g. ``--port 9119``) + - best-effort; failures never raise to the caller + """ + import signal as _signal + import time as _time + + if signal_term is None: + signal_term = _signal.SIGTERM + if signal_kill is None: + signal_kill = getattr(_signal, "SIGKILL", _signal.SIGTERM) + if sleep_fn is None: + sleep_fn = _time.sleep + + if sys.platform == "win32": + # Windows desktop uses taskkill tree teardown; orphan scan here is POSIX. + return {"matched": [], "killed": [], "failed": []} + + exclude = _exclude_pids_from_env() + exclude.add(os.getpid()) + # Also spare our direct parent (the desktop / sshd wrapper). + try: + exclude.add(os.getppid()) + except Exception: + pass + + try: + scanned = _scan_dashboard_processes(exclude_pids=exclude) + except Exception: + return {"matched": [], "killed": [], "failed": []} + + targets: list[tuple[int, str]] = [] + for pid, cmd in scanned: + if not _is_desktop_local_serve_cmdline(cmd): + continue + ppid = _process_ppid(pid) + if ppid is None: + continue + # Orphaned under init/launchd. + if ppid not in (0, 1): + continue + targets.append((pid, cmd)) + + if not targets: + return {"matched": [], "killed": [], "failed": []} + + matched = [pid for pid, _ in targets] + killed: list[int] = [] + failed: list[int] = [] + + for pid, _cmd in targets: + try: + os.kill(pid, signal_term) + except ProcessLookupError: + continue + except PermissionError: + failed.append(pid) + continue + except OSError: + failed.append(pid) + continue + + # Brief grace, then SIGKILL survivors. + sleep_fn(1.5) + for pid, _cmd in targets: + if pid in failed: + continue + try: + os.kill(pid, 0) + except ProcessLookupError: + killed.append(pid) + continue + except OSError: + killed.append(pid) + continue + try: + os.kill(pid, signal_kill) + killed.append(pid) + except ProcessLookupError: + killed.append(pid) + except OSError: + failed.append(pid) + + if matched: + try: + print( + f"⟲ Reaped {len(killed)} orphaned desktop-local serve " + f"backend(s) ({reason}): {killed or matched}" + ) + except Exception: + pass + + return {"matched": matched, "killed": killed, "failed": failed} + diff --git a/hermes_cli/web_server.py b/hermes_cli/web_server.py index 5bc2d6472adcf..5260a288bc18e 100644 --- a/hermes_cli/web_server.py +++ b/hermes_cli/web_server.py @@ -17893,6 +17893,19 @@ def start_server( # serving — leaking the whole backend + its MCP child subtree # (each MCP watchdog is parented to THIS process, so os._exit here # cascades their teardown). Same pattern as + # Clear corpses left by a previous unclean Desktop exit before we + # stack another backend + MCP tree (EMFILE / missing tabs). + # Parent-death watchdog only protects *this* process going forward. + if os.getenv("HERMES_DESKTOP") == "1": + try: + from hermes_cli.dashboard_procs import ( + _reap_orphaned_desktop_local_serves, + ) + + _reap_orphaned_desktop_local_serves() + except Exception as exc: + _log.debug("orphan desktop-local serve reap skipped: %s", exc) + # tui_gateway/slash_worker.py::_start_parent_death_watchdog. No-op # for standalone `hermes serve` (no HERMES_PARENT_PID env). _start_parent_death_watchdog() diff --git a/tests/hermes_cli/test_orphan_desktop_serve_reap.py b/tests/hermes_cli/test_orphan_desktop_serve_reap.py new file mode 100644 index 0000000000000..01c38790b0816 --- /dev/null +++ b/tests/hermes_cli/test_orphan_desktop_serve_reap.py @@ -0,0 +1,110 @@ +"""Orphan Desktop-local ``hermes serve`` reap at backend start. + +When Desktop dies uncleanly, local ``serve --host 127.0.0.1 --port 0`` +children can be reparented to pid 1 and keep full MCP trees alive. The next +boot must clear those corpses without touching intentional fixed-port serves +(e.g. ``--port 9119`` remote dashboards). +""" + +from __future__ import annotations + +import os +from unittest.mock import patch + +from hermes_cli.dashboard_procs import ( + _is_desktop_local_serve_cmdline, + _reap_orphaned_desktop_local_serves, +) + + +def test_desktop_local_serve_shape_matches_ephemeral_loopback(): + assert _is_desktop_local_serve_cmdline( + "python -m hermes_cli.main serve --host 127.0.0.1 --port 0" + ) + assert _is_desktop_local_serve_cmdline( + "hermes serve --isolated --host 127.0.0.1 --port 0 --ssh-owner-nonce abc" + ) + assert _is_desktop_local_serve_cmdline( + "/venv/bin/hermes serve --host=127.0.0.1 --port=0" + ) + + +def test_desktop_local_serve_shape_spares_fixed_port_and_non_serve(): + assert not _is_desktop_local_serve_cmdline( + "hermes serve --host 100.106.105.2 --port 9119 --skip-build" + ) + assert not _is_desktop_local_serve_cmdline( + "hermes serve --host 127.0.0.1 --port 9119" + ) + assert not _is_desktop_local_serve_cmdline("hermes gateway run --replace") + assert not _is_desktop_local_serve_cmdline( + "vim notes about hermes serve --port 0" + ) + + +def test_reap_only_kills_ppid1_local_serves(): + scanned = [ + (111, "hermes serve --host 127.0.0.1 --port 0"), # orphan local + (222, "hermes serve --host 127.0.0.1 --port 0"), # still has parent + (333, "hermes serve --host 100.1.2.3 --port 9119"), # fixed remote + (444, "hermes serve --isolated --host 127.0.0.1 --port 0"), # orphan isolated + ] + ppids = {111: 1, 222: 50, 333: 1, 444: 1} + terms: list[int] = [] + live = {111, 222, 333, 444} + + def fake_kill(pid, sig): + if sig == 0: + if pid in live: + return None + raise ProcessLookupError() + if sig == 15: + terms.append(pid) + live.discard(pid) + return None + if sig == 9: + live.discard(pid) + return None + return None + + with ( + patch( + "hermes_cli.dashboard_procs._scan_dashboard_processes", + return_value=scanned, + ), + patch( + "hermes_cli.dashboard_procs._process_ppid", + side_effect=lambda pid: ppids.get(pid), + ), + patch("os.kill", side_effect=fake_kill), + patch("sys.platform", "darwin"), + ): + os.environ.pop("HERMES_DESKTOP_CHILD_PID", None) + result = _reap_orphaned_desktop_local_serves( + sleep_fn=lambda _s: None, + signal_term=15, + signal_kill=9, + ) + + assert set(result["matched"]) == {111, 444} + assert set(terms) == {111, 444} + assert set(result["killed"]) == {111, 444} + assert 222 not in terms + assert 333 not in terms + + +def test_reap_passes_child_pid_exclude_to_scan(): + with ( + patch( + "hermes_cli.dashboard_procs._scan_dashboard_processes", + return_value=[], + ) as scan, + patch("sys.platform", "darwin"), + patch.dict(os.environ, {"HERMES_DESKTOP_CHILD_PID": "999,111"}, clear=False), + ): + result = _reap_orphaned_desktop_local_serves(sleep_fn=lambda _s: None) + + assert result["matched"] == [] + exclude = scan.call_args.kwargs["exclude_pids"] + assert 111 in exclude + assert 999 in exclude