From 59c273ba3ae27fb7f4a882e4c56b82675fea9623 Mon Sep 17 00:00:00 2001 From: Brooklyn Nicholson Date: Sat, 6 Jun 2026 10:21:27 -0500 Subject: [PATCH] fix(gateway): fall back to detached launch when launchd rejects domain (macOS 26) macOS 26+ broke launchctl management of the gui/ (and user/) domains: `bootstrap` returns error 5 and `kickstart` returns error 125 ("Domain does not support specified action"), so `hermes gateway start/install/restart` crashed with a cryptic traceback (#23387). Detect these codes and degrade gracefully: launch the gateway as a CLI-managed detached background process (the documented `nohup hermes gateway run --replace` workaround), with logs to gateway.log and the PID tracked via gateway.pid so stop/status/restart keep working. Print clear guidance that the service won't auto-start at login or auto-restart on crash on this macOS version. launchd_stop also tolerates 125/5 from bootout and falls through to the PID-based kill. --- hermes_cli/gateway.py | 195 +++++++++++++++++++---- tests/hermes_cli/test_gateway_service.py | 117 ++++++++++++++ 2 files changed, 279 insertions(+), 33 deletions(-) diff --git a/hermes_cli/gateway.py b/hermes_cli/gateway.py index c857ae71fd34a..75b2473012386 100644 --- a/hermes_cli/gateway.py +++ b/hermes_cli/gateway.py @@ -3003,6 +3003,99 @@ def _launchd_domain() -> str: return f"gui/{os.getuid()}" # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows +# macOS 26+ broke launchctl management of the per-user GUI domain: `bootstrap` +# returns error 5 ("Input/output error") and `kickstart` returns error 125 +# ("Domain does not support specified action"). When launchd refuses to manage +# the gateway we can't supervise it as a service, so we fall back to a detached +# background process (the documented `nohup hermes gateway run` workaround). +# See issue #23387. +_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125}) + + +def _launchctl_domain_unsupported(returncode: int) -> bool: + """True when launchctl rejected the action because the domain can't manage it. + + Codes 5 and 125 are emitted by macOS 26+ for `bootstrap`/`kickstart` against + the `gui/` (and `user/`) domains, which no longer support service + management. Treat these as "launchd unavailable" and degrade gracefully. + """ + return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES + + +def _gateway_run_command() -> list[str]: + """Build the `python -m hermes_cli.main [--profile X] gateway run --replace` argv. + + Profile-aware: honors the active HERMES_HOME via `_profile_arg()` so the + detached fallback launches into the same profile as the CLI invocation. + """ + cmd = [get_python_path(), "-m", "hermes_cli.main"] + profile_arg = _profile_arg() + if profile_arg: + cmd.extend(profile_arg.split()) + cmd.extend(["gateway", "run", "--replace"]) + return cmd + + +def _spawn_detached_gateway() -> bool: + """Launch the gateway as a detached background process (launchd fallback). + + Used when launchctl can no longer bootstrap/kickstart the gateway on + macOS 26+ (issue #23387). Mirrors the `nohup hermes gateway run --replace` + workaround but keeps it CLI-managed: stdout/stderr go to the profile's + gateway logs and the PID is tracked via the gateway.pid file that + `run_gateway` writes, so stop/status/restart keep working. + """ + from hermes_cli._subprocess_compat import windows_detach_popen_kwargs + + log_dir = get_hermes_home() / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + out_path = log_dir / "gateway.log" + err_path = log_dir / "gateway.error.log" + try: + out = open(out_path, "ab") + err = open(err_path, "ab") + except OSError: + return False + try: + with out, err: + subprocess.Popen( + _gateway_run_command(), + stdin=subprocess.DEVNULL, + stdout=out, + stderr=err, + **windows_detach_popen_kwargs(), + ) + except OSError: + return False + return True + + +def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool: + """Start the gateway detached when launchd can't manage it, with guidance. + + Returns True if the detached gateway was launched. When it can't be + launched, prints the manual workaround and (by default) exits non-zero so + the failure surfaces instead of silently doing nothing. + """ + from hermes_constants import display_hermes_home as _dhh + + print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).") + if _spawn_detached_gateway(): + print("✓ Started gateway as a background process instead") + print(" It will NOT auto-start at login or auto-restart on crash.") + print(f" Logs: {_dhh()}/logs/gateway.log") + print(" Stop it with: hermes gateway stop") + return True + print_error("Failed to start the gateway as a background process.") + print( + f" Try manually: nohup hermes gateway run --replace " + f"> {_dhh()}/logs/gateway.log 2>&1 &" + ) + if exit_on_failure: + sys.exit(1) + return False + + def generate_launchd_plist() -> str: python_path = get_python_path() # Stable cwd anchor — never the volatile source checkout. See @@ -3154,11 +3247,17 @@ def launchd_install(force: bool = False): print(f"Installing launchd service to: {plist_path}") plist_path.write_text(generate_launchd_plist()) - subprocess.run( - ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], - check=True, - timeout=30, - ) + try: + subprocess.run( + ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], + check=True, + timeout=30, + ) + except subprocess.CalledProcessError as e: + if not _launchctl_domain_unsupported(e.returncode): + raise + _launchd_fallback_to_detached(f"launchctl bootstrap exit {e.returncode}") + return print() print("✓ Service installed and loaded!") @@ -3195,16 +3294,22 @@ def launchd_start(): print("↻ launchd plist missing; regenerating service definition") plist_path.parent.mkdir(parents=True, exist_ok=True) plist_path.write_text(generate_launchd_plist(), encoding="utf-8") - subprocess.run( - ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], - check=True, - timeout=30, - ) - subprocess.run( - ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], - check=True, - timeout=30, - ) + try: + subprocess.run( + ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], + check=True, + timeout=30, + ) + subprocess.run( + ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], + check=True, + timeout=30, + ) + except subprocess.CalledProcessError as e: + if not _launchctl_domain_unsupported(e.returncode): + raise + _launchd_fallback_to_detached(f"launchctl exit {e.returncode}") + return print("✓ Service started") return @@ -3216,19 +3321,28 @@ def launchd_start(): timeout=30, ) except subprocess.CalledProcessError as e: + if _launchctl_domain_unsupported(e.returncode): + _launchd_fallback_to_detached(f"launchctl kickstart exit {e.returncode}") + return if e.returncode not in {3, 113}: raise print("↻ launchd job was unloaded; reloading service definition") - subprocess.run( - ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], - check=True, - timeout=30, - ) - subprocess.run( - ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], - check=True, - timeout=30, - ) + try: + subprocess.run( + ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], + check=True, + timeout=30, + ) + subprocess.run( + ["launchctl", "kickstart", f"{_launchd_domain()}/{label}"], + check=True, + timeout=30, + ) + except subprocess.CalledProcessError as e2: + if not _launchctl_domain_unsupported(e2.returncode): + raise + _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}") + return print("✓ Service started") @@ -3250,8 +3364,11 @@ def launchd_stop(): try: subprocess.run(["launchctl", "bootout", target], check=True, timeout=90) except subprocess.CalledProcessError as e: - if e.returncode in {3, 113}: - pass # Already unloaded — nothing to stop. + # 3/113: job already unloaded. 5/125: macOS 26+ can't manage the domain + # (issue #23387) — the gateway is a detached fallback process, so just + # fall through to the PID-based kill below. + if e.returncode in {3, 113} or _launchctl_domain_unsupported(e.returncode): + pass else: raise _wait_for_gateway_exit(timeout=10.0, force_after=5.0) @@ -3335,17 +3452,29 @@ def launchd_restart(): subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90) print("✓ Service restarted") except subprocess.CalledProcessError as e: + if _launchctl_domain_unsupported(e.returncode): + # macOS 26+ can't kickstart the domain (issue #23387). The old + # process was already drained/terminated above, so relaunch a + # fresh detached gateway. + _launchd_fallback_to_detached(f"launchctl kickstart exit {e.returncode}") + return if e.returncode not in {3, 113}: raise # Job not loaded — bootstrap and start fresh print("↻ launchd job was unloaded; reloading") plist_path = get_launchd_plist_path() - subprocess.run( - ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], - check=True, - timeout=30, - ) - subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30) + try: + subprocess.run( + ["launchctl", "bootstrap", _launchd_domain(), str(plist_path)], + check=True, + timeout=30, + ) + subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30) + except subprocess.CalledProcessError as e2: + if not _launchctl_domain_unsupported(e2.returncode): + raise + _launchd_fallback_to_detached(f"launchctl exit {e2.returncode}") + return print("✓ Service restarted") diff --git a/tests/hermes_cli/test_gateway_service.py b/tests/hermes_cli/test_gateway_service.py index b1d97920084cd..90ad6a1a1554f 100644 --- a/tests/hermes_cli/test_gateway_service.py +++ b/tests/hermes_cli/test_gateway_service.py @@ -679,6 +679,123 @@ class TestLaunchdServiceRecovery: assert "stale" in output.lower() assert "not loaded" in output.lower() + def test_launchctl_domain_unsupported_recognizes_macos26_codes(self): + # macOS 26+ rejects gui/ management with these codes (issue #23387). + assert gateway_cli._launchctl_domain_unsupported(5) is True + assert gateway_cli._launchctl_domain_unsupported(125) is True + # Codes that mean "job not loaded" are NOT domain-unsupported. + assert gateway_cli._launchctl_domain_unsupported(3) is False + assert gateway_cli._launchctl_domain_unsupported(113) is False + assert gateway_cli._launchctl_domain_unsupported(0) is False + + def test_launchd_start_falls_back_to_detached_on_kickstart_125(self, tmp_path, monkeypatch, capsys): + """macOS 26 kickstart error 125 should spawn a detached gateway, not crash.""" + plist_path = tmp_path / "ai.hermes.gateway.plist" + plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8") + label = gateway_cli.get_launchd_label() + target = f"{gateway_cli._launchd_domain()}/{label}" + + monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) + monkeypatch.setattr(gateway_cli, "refresh_launchd_plist_if_needed", lambda: False) + + def fake_run(cmd, check=False, **kwargs): + if cmd == ["launchctl", "kickstart", target]: + raise gateway_cli.subprocess.CalledProcessError( + 125, cmd, stderr="Domain does not support specified action" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run) + + spawned = [] + monkeypatch.setattr( + gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True + ) + + gateway_cli.launchd_start() + + assert spawned == [True] + out = capsys.readouterr().out.lower() + assert "background process" in out + + def test_launchd_install_falls_back_to_detached_on_bootstrap_5(self, tmp_path, monkeypatch, capsys): + """macOS 26 bootstrap error 5 should spawn a detached gateway, not crash.""" + plist_path = tmp_path / "ai.hermes.gateway.plist" + monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path) + + def fake_run(cmd, check=False, **kwargs): + if cmd[:2] == ["launchctl", "bootstrap"]: + raise gateway_cli.subprocess.CalledProcessError( + 5, cmd, stderr="Input/output error" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run) + + spawned = [] + monkeypatch.setattr( + gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True + ) + + gateway_cli.launchd_install(force=True) + + assert spawned == [True] + assert "Service installed and loaded" not in capsys.readouterr().out + + def test_launchd_restart_falls_back_to_detached_on_kickstart_125(self, monkeypatch, capsys): + """When kickstart -k returns 125, restart should relaunch detached.""" + target = f"{gateway_cli._launchd_domain()}/{gateway_cli.get_launchd_label()}" + + monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 5.0) + monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False) + monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda timeout, force_after=None: True) + monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None) + monkeypatch.setattr("gateway.status.get_running_pid", lambda: 321) + + def fake_run(cmd, check=False, **kwargs): + if cmd == ["launchctl", "kickstart", "-k", target]: + raise gateway_cli.subprocess.CalledProcessError( + 125, cmd, stderr="Domain does not support specified action" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run) + + spawned = [] + monkeypatch.setattr( + gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True + ) + + gateway_cli.launchd_restart() + + assert spawned == [True] + + def test_launchd_stop_tolerates_domain_unsupported_bootout(self, monkeypatch, capsys): + """bootout exit 125 (macOS 26) must fall through to PID-based kill, not raise.""" + def fake_run(cmd, check=False, **kwargs): + if "bootout" in cmd: + raise gateway_cli.subprocess.CalledProcessError( + 125, cmd, stderr="Domain does not support specified action" + ) + return SimpleNamespace(returncode=0, stdout="", stderr="") + + monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run) + monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda **kw: None) + + gateway_cli.launchd_stop() + + assert "stopped" in capsys.readouterr().out.lower() + + def test_launchd_fallback_exits_when_spawn_fails(self, monkeypatch, capsys): + """If the detached spawn fails, surface the manual workaround and exit 1.""" + monkeypatch.setattr(gateway_cli, "_spawn_detached_gateway", lambda: False) + + with pytest.raises(SystemExit) as exc: + gateway_cli._launchd_fallback_to_detached("test reason") + assert exc.value.code == 1 + out = capsys.readouterr().out + assert "nohup hermes gateway run" in out + class TestGatewayServiceDetection: def test_supports_systemd_services_requires_systemctl_binary(self, monkeypatch):