fix(desktop): reap orphaned local serve backends on desktop boot
When Desktop exits uncleanly, leftover `hermes serve --host 127.0.0.1 --port 0` processes can be reparented to pid 1 and keep full MCP trees alive. The next boot then stacks another backend on top of the corpses until EMFILE kills sidebar/session APIs and tabs disappear. - Detect Desktop-local serve shape (loopback + ephemeral port 0) - Only reap processes whose ppid is 0/1 (true orphans) - Spare fixed-port remote serves (e.g. --port 9119) and HERMES_DESKTOP_CHILD_PID - Run at Desktop backend start (HERMES_DESKTOP=1) before parent-death watchdog Complements parent-death watchdog (prevents future orphans) and configurable nofile soft limit (capacity floor). Together these stop the multi-backend pile-up cascade observed on macOS Desktop SSH/local installs.
This commit is contained in:
Leon Phull
Teknium
parent
a9a0648f49
commit
6386c75306
|
|
@ -456,3 +456,188 @@ def _detect_concurrent_hermes_instances(
|
|||
matches.append((int(pid), str(name)))
|
||||
|
||||
return matches
|
||||
|
||||
|
||||
def _is_desktop_local_serve_cmdline(command: str) -> bool:
|
||||
"""True for the Desktop-local serve spawn shape (loopback + ephemeral port).
|
||||
|
||||
Desktop primary/pool backends launch as::
|
||||
|
||||
hermes serve --host 127.0.0.1 --port 0
|
||||
hermes serve --isolated --host 127.0.0.1 --port 0 ...
|
||||
|
||||
Intentional long-lived headless serves (e.g. ``--host <tailscale-ip>
|
||||
--port 9119``) must never match — those are operator-managed remote
|
||||
backends and may legitimately run with ppid 1 under launchd/nohup.
|
||||
"""
|
||||
cmd = command.lower()
|
||||
if "serve" not in cmd:
|
||||
return False
|
||||
if "hermes" not in cmd and "hermes_cli" not in cmd:
|
||||
return False
|
||||
# Ephemeral desktop bind: host loopback + port 0 (exact tokens).
|
||||
has_loopback = (
|
||||
"--host 127.0.0.1" in cmd
|
||||
or "--host=127.0.0.1" in cmd
|
||||
or "--host localhost" in cmd
|
||||
or "--host=localhost" in cmd
|
||||
)
|
||||
has_ephemeral = "--port 0" in cmd or "--port=0" in cmd
|
||||
if not (has_loopback and has_ephemeral):
|
||||
return False
|
||||
# Spare anything with a concrete non-zero port flag first (defensive).
|
||||
# (port 0 already required above.)
|
||||
return True
|
||||
|
||||
|
||||
def _process_ppid(pid: int) -> int | None:
|
||||
"""Best-effort parent pid lookup. None on failure."""
|
||||
try:
|
||||
if sys.platform == "win32":
|
||||
return None # Windows orphan reap is handled by desktop tree-kill.
|
||||
result = subprocess.run(
|
||||
["ps", "-o", "ppid=", "-p", str(pid)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=5,
|
||||
)
|
||||
if result.returncode != 0 or not result.stdout:
|
||||
return None
|
||||
return int(result.stdout.strip().split()[0])
|
||||
except (ValueError, FileNotFoundError, subprocess.TimeoutExpired, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def _exclude_pids_from_env() -> set[int]:
|
||||
"""PIDs Desktop marks as live backends (HERMES_DESKTOP_CHILD_PID)."""
|
||||
raw = os.environ.get("HERMES_DESKTOP_CHILD_PID", "")
|
||||
out: set[int] = set()
|
||||
for part in raw.split(","):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
try:
|
||||
out.add(int(part))
|
||||
except ValueError:
|
||||
continue
|
||||
return out
|
||||
|
||||
|
||||
def _reap_orphaned_desktop_local_serves(
|
||||
*,
|
||||
reason: str = "orphaned desktop-local hermes serve",
|
||||
signal_term=None,
|
||||
signal_kill=None,
|
||||
sleep_fn=None,
|
||||
) -> dict[str, list]:
|
||||
"""Kill leftover Desktop-local ``hermes serve`` backends with no parent.
|
||||
|
||||
When Electron dies uncleanly (crash / SIGKILL / update handoff), local
|
||||
``serve --host 127.0.0.1 --port 0`` children can be reparented to pid 1 and
|
||||
keep their full MCP trees alive. The next Desktop boot then stacks a fresh
|
||||
backend on top of the corpses until the machine hits EMFILE and the UI
|
||||
loses tabs/sidebar.
|
||||
|
||||
The parent-death watchdog prevents *future* orphans once a backend is
|
||||
running under HERMES_PARENT_PID; this helper clears *already* orphaned
|
||||
corpses at the start of a new Desktop backend.
|
||||
|
||||
Safety:
|
||||
- only the Desktop-local spawn shape (loopback + ``--port 0``)
|
||||
- only processes whose current ppid is 1 (or 0 on some supervisors)
|
||||
- never self / never HERMES_DESKTOP_CHILD_PID entries
|
||||
- never fixed-port remote serves (e.g. ``--port 9119``)
|
||||
- best-effort; failures never raise to the caller
|
||||
"""
|
||||
import signal as _signal
|
||||
import time as _time
|
||||
|
||||
if signal_term is None:
|
||||
signal_term = _signal.SIGTERM
|
||||
if signal_kill is None:
|
||||
signal_kill = getattr(_signal, "SIGKILL", _signal.SIGTERM)
|
||||
if sleep_fn is None:
|
||||
sleep_fn = _time.sleep
|
||||
|
||||
if sys.platform == "win32":
|
||||
# Windows desktop uses taskkill tree teardown; orphan scan here is POSIX.
|
||||
return {"matched": [], "killed": [], "failed": []}
|
||||
|
||||
exclude = _exclude_pids_from_env()
|
||||
exclude.add(os.getpid())
|
||||
# Also spare our direct parent (the desktop / sshd wrapper).
|
||||
try:
|
||||
exclude.add(os.getppid())
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
scanned = _scan_dashboard_processes(exclude_pids=exclude)
|
||||
except Exception:
|
||||
return {"matched": [], "killed": [], "failed": []}
|
||||
|
||||
targets: list[tuple[int, str]] = []
|
||||
for pid, cmd in scanned:
|
||||
if not _is_desktop_local_serve_cmdline(cmd):
|
||||
continue
|
||||
ppid = _process_ppid(pid)
|
||||
if ppid is None:
|
||||
continue
|
||||
# Orphaned under init/launchd.
|
||||
if ppid not in (0, 1):
|
||||
continue
|
||||
targets.append((pid, cmd))
|
||||
|
||||
if not targets:
|
||||
return {"matched": [], "killed": [], "failed": []}
|
||||
|
||||
matched = [pid for pid, _ in targets]
|
||||
killed: list[int] = []
|
||||
failed: list[int] = []
|
||||
|
||||
for pid, _cmd in targets:
|
||||
try:
|
||||
os.kill(pid, signal_term)
|
||||
except ProcessLookupError:
|
||||
continue
|
||||
except PermissionError:
|
||||
failed.append(pid)
|
||||
continue
|
||||
except OSError:
|
||||
failed.append(pid)
|
||||
continue
|
||||
|
||||
# Brief grace, then SIGKILL survivors.
|
||||
sleep_fn(1.5)
|
||||
for pid, _cmd in targets:
|
||||
if pid in failed:
|
||||
continue
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
except ProcessLookupError:
|
||||
killed.append(pid)
|
||||
continue
|
||||
except OSError:
|
||||
killed.append(pid)
|
||||
continue
|
||||
try:
|
||||
os.kill(pid, signal_kill)
|
||||
killed.append(pid)
|
||||
except ProcessLookupError:
|
||||
killed.append(pid)
|
||||
except OSError:
|
||||
failed.append(pid)
|
||||
|
||||
if matched:
|
||||
try:
|
||||
print(
|
||||
f"⟲ Reaped {len(killed)} orphaned desktop-local serve "
|
||||
f"backend(s) ({reason}): {killed or matched}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {"matched": matched, "killed": killed, "failed": failed}
|
||||
|
||||
|
|
|
|||
|
|
@ -17893,6 +17893,19 @@ def start_server(
|
|||
# serving — leaking the whole backend + its MCP child subtree
|
||||
# (each MCP watchdog is parented to THIS process, so os._exit here
|
||||
# cascades their teardown). Same pattern as
|
||||
# Clear corpses left by a previous unclean Desktop exit before we
|
||||
# stack another backend + MCP tree (EMFILE / missing tabs).
|
||||
# Parent-death watchdog only protects *this* process going forward.
|
||||
if os.getenv("HERMES_DESKTOP") == "1":
|
||||
try:
|
||||
from hermes_cli.dashboard_procs import (
|
||||
_reap_orphaned_desktop_local_serves,
|
||||
)
|
||||
|
||||
_reap_orphaned_desktop_local_serves()
|
||||
except Exception as exc:
|
||||
_log.debug("orphan desktop-local serve reap skipped: %s", exc)
|
||||
|
||||
# tui_gateway/slash_worker.py::_start_parent_death_watchdog. No-op
|
||||
# for standalone `hermes serve` (no HERMES_PARENT_PID env).
|
||||
_start_parent_death_watchdog()
|
||||
|
|
|
|||
|
|
@ -0,0 +1,110 @@
|
|||
"""Orphan Desktop-local ``hermes serve`` reap at backend start.
|
||||
|
||||
When Desktop dies uncleanly, local ``serve --host 127.0.0.1 --port 0``
|
||||
children can be reparented to pid 1 and keep full MCP trees alive. The next
|
||||
boot must clear those corpses without touching intentional fixed-port serves
|
||||
(e.g. ``--port 9119`` remote dashboards).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from hermes_cli.dashboard_procs import (
|
||||
_is_desktop_local_serve_cmdline,
|
||||
_reap_orphaned_desktop_local_serves,
|
||||
)
|
||||
|
||||
|
||||
def test_desktop_local_serve_shape_matches_ephemeral_loopback():
|
||||
assert _is_desktop_local_serve_cmdline(
|
||||
"python -m hermes_cli.main serve --host 127.0.0.1 --port 0"
|
||||
)
|
||||
assert _is_desktop_local_serve_cmdline(
|
||||
"hermes serve --isolated --host 127.0.0.1 --port 0 --ssh-owner-nonce abc"
|
||||
)
|
||||
assert _is_desktop_local_serve_cmdline(
|
||||
"/venv/bin/hermes serve --host=127.0.0.1 --port=0"
|
||||
)
|
||||
|
||||
|
||||
def test_desktop_local_serve_shape_spares_fixed_port_and_non_serve():
|
||||
assert not _is_desktop_local_serve_cmdline(
|
||||
"hermes serve --host 100.106.105.2 --port 9119 --skip-build"
|
||||
)
|
||||
assert not _is_desktop_local_serve_cmdline(
|
||||
"hermes serve --host 127.0.0.1 --port 9119"
|
||||
)
|
||||
assert not _is_desktop_local_serve_cmdline("hermes gateway run --replace")
|
||||
assert not _is_desktop_local_serve_cmdline(
|
||||
"vim notes about hermes serve --port 0"
|
||||
)
|
||||
|
||||
|
||||
def test_reap_only_kills_ppid1_local_serves():
|
||||
scanned = [
|
||||
(111, "hermes serve --host 127.0.0.1 --port 0"), # orphan local
|
||||
(222, "hermes serve --host 127.0.0.1 --port 0"), # still has parent
|
||||
(333, "hermes serve --host 100.1.2.3 --port 9119"), # fixed remote
|
||||
(444, "hermes serve --isolated --host 127.0.0.1 --port 0"), # orphan isolated
|
||||
]
|
||||
ppids = {111: 1, 222: 50, 333: 1, 444: 1}
|
||||
terms: list[int] = []
|
||||
live = {111, 222, 333, 444}
|
||||
|
||||
def fake_kill(pid, sig):
|
||||
if sig == 0:
|
||||
if pid in live:
|
||||
return None
|
||||
raise ProcessLookupError()
|
||||
if sig == 15:
|
||||
terms.append(pid)
|
||||
live.discard(pid)
|
||||
return None
|
||||
if sig == 9:
|
||||
live.discard(pid)
|
||||
return None
|
||||
return None
|
||||
|
||||
with (
|
||||
patch(
|
||||
"hermes_cli.dashboard_procs._scan_dashboard_processes",
|
||||
return_value=scanned,
|
||||
),
|
||||
patch(
|
||||
"hermes_cli.dashboard_procs._process_ppid",
|
||||
side_effect=lambda pid: ppids.get(pid),
|
||||
),
|
||||
patch("os.kill", side_effect=fake_kill),
|
||||
patch("sys.platform", "darwin"),
|
||||
):
|
||||
os.environ.pop("HERMES_DESKTOP_CHILD_PID", None)
|
||||
result = _reap_orphaned_desktop_local_serves(
|
||||
sleep_fn=lambda _s: None,
|
||||
signal_term=15,
|
||||
signal_kill=9,
|
||||
)
|
||||
|
||||
assert set(result["matched"]) == {111, 444}
|
||||
assert set(terms) == {111, 444}
|
||||
assert set(result["killed"]) == {111, 444}
|
||||
assert 222 not in terms
|
||||
assert 333 not in terms
|
||||
|
||||
|
||||
def test_reap_passes_child_pid_exclude_to_scan():
|
||||
with (
|
||||
patch(
|
||||
"hermes_cli.dashboard_procs._scan_dashboard_processes",
|
||||
return_value=[],
|
||||
) as scan,
|
||||
patch("sys.platform", "darwin"),
|
||||
patch.dict(os.environ, {"HERMES_DESKTOP_CHILD_PID": "999,111"}, clear=False),
|
||||
):
|
||||
result = _reap_orphaned_desktop_local_serves(sleep_fn=lambda _s: None)
|
||||
|
||||
assert result["matched"] == []
|
||||
exclude = scan.call_args.kwargs["exclude_pids"]
|
||||
assert 111 in exclude
|
||||
assert 999 in exclude
|
||||
Loading…
Reference in New Issue