fix(gateway): fall back to detached launch when launchd rejects domain (macOS 26)
macOS 26+ broke launchctl management of the gui/<uid> (and user/<uid>)
domains: `bootstrap` returns error 5 and `kickstart` returns error 125
("Domain does not support specified action"), so `hermes gateway
start/install/restart` crashed with a cryptic traceback (#23387).
Detect these codes and degrade gracefully: launch the gateway as a
CLI-managed detached background process (the documented `nohup hermes
gateway run --replace` workaround), with logs to gateway.log and the PID
tracked via gateway.pid so stop/status/restart keep working. Print clear
guidance that the service won't auto-start at login or auto-restart on
crash on this macOS version. launchd_stop also tolerates 125/5 from
bootout and falls through to the PID-based kill.
This commit is contained in:
Brooklyn Nicholson
Teknium
parent
2666638192
commit
59c273ba3a
|
|
@ -3003,6 +3003,99 @@ def _launchd_domain() -> str:
|
|||
return f"gui/{os.getuid()}" # windows-footgun: ok — POSIX launchd (macOS) helper, never invoked on Windows
|
||||
|
||||
|
||||
# macOS 26+ broke launchctl management of the per-user GUI domain: `bootstrap`
|
||||
# returns error 5 ("Input/output error") and `kickstart` returns error 125
|
||||
# ("Domain does not support specified action"). When launchd refuses to manage
|
||||
# the gateway we can't supervise it as a service, so we fall back to a detached
|
||||
# background process (the documented `nohup hermes gateway run` workaround).
|
||||
# See issue #23387.
|
||||
_LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES = frozenset({5, 125})
|
||||
|
||||
|
||||
def _launchctl_domain_unsupported(returncode: int) -> bool:
|
||||
"""True when launchctl rejected the action because the domain can't manage it.
|
||||
|
||||
Codes 5 and 125 are emitted by macOS 26+ for `bootstrap`/`kickstart` against
|
||||
the `gui/<uid>` (and `user/<uid>`) domains, which no longer support service
|
||||
management. Treat these as "launchd unavailable" and degrade gracefully.
|
||||
"""
|
||||
return returncode in _LAUNCHCTL_DOMAIN_UNSUPPORTED_CODES
|
||||
|
||||
|
||||
def _gateway_run_command() -> list[str]:
|
||||
"""Build the `python -m hermes_cli.main [--profile X] gateway run --replace` argv.
|
||||
|
||||
Profile-aware: honors the active HERMES_HOME via `_profile_arg()` so the
|
||||
detached fallback launches into the same profile as the CLI invocation.
|
||||
"""
|
||||
cmd = [get_python_path(), "-m", "hermes_cli.main"]
|
||||
profile_arg = _profile_arg()
|
||||
if profile_arg:
|
||||
cmd.extend(profile_arg.split())
|
||||
cmd.extend(["gateway", "run", "--replace"])
|
||||
return cmd
|
||||
|
||||
|
||||
def _spawn_detached_gateway() -> bool:
|
||||
"""Launch the gateway as a detached background process (launchd fallback).
|
||||
|
||||
Used when launchctl can no longer bootstrap/kickstart the gateway on
|
||||
macOS 26+ (issue #23387). Mirrors the `nohup hermes gateway run --replace`
|
||||
workaround but keeps it CLI-managed: stdout/stderr go to the profile's
|
||||
gateway logs and the PID is tracked via the gateway.pid file that
|
||||
`run_gateway` writes, so stop/status/restart keep working.
|
||||
"""
|
||||
from hermes_cli._subprocess_compat import windows_detach_popen_kwargs
|
||||
|
||||
log_dir = get_hermes_home() / "logs"
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
out_path = log_dir / "gateway.log"
|
||||
err_path = log_dir / "gateway.error.log"
|
||||
try:
|
||||
out = open(out_path, "ab")
|
||||
err = open(err_path, "ab")
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
with out, err:
|
||||
subprocess.Popen(
|
||||
_gateway_run_command(),
|
||||
stdin=subprocess.DEVNULL,
|
||||
stdout=out,
|
||||
stderr=err,
|
||||
**windows_detach_popen_kwargs(),
|
||||
)
|
||||
except OSError:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _launchd_fallback_to_detached(reason: str, *, exit_on_failure: bool = True) -> bool:
|
||||
"""Start the gateway detached when launchd can't manage it, with guidance.
|
||||
|
||||
Returns True if the detached gateway was launched. When it can't be
|
||||
launched, prints the manual workaround and (by default) exits non-zero so
|
||||
the failure surfaces instead of silently doing nothing.
|
||||
"""
|
||||
from hermes_constants import display_hermes_home as _dhh
|
||||
|
||||
print(f"⚠ launchd cannot manage the gateway on this macOS version ({reason}).")
|
||||
if _spawn_detached_gateway():
|
||||
print("✓ Started gateway as a background process instead")
|
||||
print(" It will NOT auto-start at login or auto-restart on crash.")
|
||||
print(f" Logs: {_dhh()}/logs/gateway.log")
|
||||
print(" Stop it with: hermes gateway stop")
|
||||
return True
|
||||
print_error("Failed to start the gateway as a background process.")
|
||||
print(
|
||||
f" Try manually: nohup hermes gateway run --replace "
|
||||
f"> {_dhh()}/logs/gateway.log 2>&1 &"
|
||||
)
|
||||
if exit_on_failure:
|
||||
sys.exit(1)
|
||||
return False
|
||||
|
||||
|
||||
def generate_launchd_plist() -> str:
|
||||
python_path = get_python_path()
|
||||
# Stable cwd anchor — never the volatile source checkout. See
|
||||
|
|
@ -3154,11 +3247,17 @@ def launchd_install(force: bool = False):
|
|||
print(f"Installing launchd service to: {plist_path}")
|
||||
plist_path.write_text(generate_launchd_plist())
|
||||
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
try:
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if not _launchctl_domain_unsupported(e.returncode):
|
||||
raise
|
||||
_launchd_fallback_to_detached(f"launchctl bootstrap exit {e.returncode}")
|
||||
return
|
||||
|
||||
print()
|
||||
print("✓ Service installed and loaded!")
|
||||
|
|
@ -3195,16 +3294,22 @@ def launchd_start():
|
|||
print("↻ launchd plist missing; regenerating service definition")
|
||||
plist_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
plist_path.write_text(generate_launchd_plist(), encoding="utf-8")
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(
|
||||
["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
try:
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(
|
||||
["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if not _launchctl_domain_unsupported(e.returncode):
|
||||
raise
|
||||
_launchd_fallback_to_detached(f"launchctl exit {e.returncode}")
|
||||
return
|
||||
print("✓ Service started")
|
||||
return
|
||||
|
||||
|
|
@ -3216,19 +3321,28 @@ def launchd_start():
|
|||
timeout=30,
|
||||
)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if _launchctl_domain_unsupported(e.returncode):
|
||||
_launchd_fallback_to_detached(f"launchctl kickstart exit {e.returncode}")
|
||||
return
|
||||
if e.returncode not in {3, 113}:
|
||||
raise
|
||||
print("↻ launchd job was unloaded; reloading service definition")
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(
|
||||
["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
try:
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(
|
||||
["launchctl", "kickstart", f"{_launchd_domain()}/{label}"],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
except subprocess.CalledProcessError as e2:
|
||||
if not _launchctl_domain_unsupported(e2.returncode):
|
||||
raise
|
||||
_launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
|
||||
return
|
||||
print("✓ Service started")
|
||||
|
||||
|
||||
|
|
@ -3250,8 +3364,11 @@ def launchd_stop():
|
|||
try:
|
||||
subprocess.run(["launchctl", "bootout", target], check=True, timeout=90)
|
||||
except subprocess.CalledProcessError as e:
|
||||
if e.returncode in {3, 113}:
|
||||
pass # Already unloaded — nothing to stop.
|
||||
# 3/113: job already unloaded. 5/125: macOS 26+ can't manage the domain
|
||||
# (issue #23387) — the gateway is a detached fallback process, so just
|
||||
# fall through to the PID-based kill below.
|
||||
if e.returncode in {3, 113} or _launchctl_domain_unsupported(e.returncode):
|
||||
pass
|
||||
else:
|
||||
raise
|
||||
_wait_for_gateway_exit(timeout=10.0, force_after=5.0)
|
||||
|
|
@ -3335,17 +3452,29 @@ def launchd_restart():
|
|||
subprocess.run(["launchctl", "kickstart", "-k", target], check=True, timeout=90)
|
||||
print("✓ Service restarted")
|
||||
except subprocess.CalledProcessError as e:
|
||||
if _launchctl_domain_unsupported(e.returncode):
|
||||
# macOS 26+ can't kickstart the domain (issue #23387). The old
|
||||
# process was already drained/terminated above, so relaunch a
|
||||
# fresh detached gateway.
|
||||
_launchd_fallback_to_detached(f"launchctl kickstart exit {e.returncode}")
|
||||
return
|
||||
if e.returncode not in {3, 113}:
|
||||
raise
|
||||
# Job not loaded — bootstrap and start fresh
|
||||
print("↻ launchd job was unloaded; reloading")
|
||||
plist_path = get_launchd_plist_path()
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
|
||||
try:
|
||||
subprocess.run(
|
||||
["launchctl", "bootstrap", _launchd_domain(), str(plist_path)],
|
||||
check=True,
|
||||
timeout=30,
|
||||
)
|
||||
subprocess.run(["launchctl", "kickstart", target], check=True, timeout=30)
|
||||
except subprocess.CalledProcessError as e2:
|
||||
if not _launchctl_domain_unsupported(e2.returncode):
|
||||
raise
|
||||
_launchd_fallback_to_detached(f"launchctl exit {e2.returncode}")
|
||||
return
|
||||
print("✓ Service restarted")
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -679,6 +679,123 @@ class TestLaunchdServiceRecovery:
|
|||
assert "stale" in output.lower()
|
||||
assert "not loaded" in output.lower()
|
||||
|
||||
def test_launchctl_domain_unsupported_recognizes_macos26_codes(self):
|
||||
# macOS 26+ rejects gui/<uid> management with these codes (issue #23387).
|
||||
assert gateway_cli._launchctl_domain_unsupported(5) is True
|
||||
assert gateway_cli._launchctl_domain_unsupported(125) is True
|
||||
# Codes that mean "job not loaded" are NOT domain-unsupported.
|
||||
assert gateway_cli._launchctl_domain_unsupported(3) is False
|
||||
assert gateway_cli._launchctl_domain_unsupported(113) is False
|
||||
assert gateway_cli._launchctl_domain_unsupported(0) is False
|
||||
|
||||
def test_launchd_start_falls_back_to_detached_on_kickstart_125(self, tmp_path, monkeypatch, capsys):
|
||||
"""macOS 26 kickstart error 125 should spawn a detached gateway, not crash."""
|
||||
plist_path = tmp_path / "ai.hermes.gateway.plist"
|
||||
plist_path.write_text(gateway_cli.generate_launchd_plist(), encoding="utf-8")
|
||||
label = gateway_cli.get_launchd_label()
|
||||
target = f"{gateway_cli._launchd_domain()}/{label}"
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
|
||||
monkeypatch.setattr(gateway_cli, "refresh_launchd_plist_if_needed", lambda: False)
|
||||
|
||||
def fake_run(cmd, check=False, **kwargs):
|
||||
if cmd == ["launchctl", "kickstart", target]:
|
||||
raise gateway_cli.subprocess.CalledProcessError(
|
||||
125, cmd, stderr="Domain does not support specified action"
|
||||
)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
|
||||
|
||||
spawned = []
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
|
||||
)
|
||||
|
||||
gateway_cli.launchd_start()
|
||||
|
||||
assert spawned == [True]
|
||||
out = capsys.readouterr().out.lower()
|
||||
assert "background process" in out
|
||||
|
||||
def test_launchd_install_falls_back_to_detached_on_bootstrap_5(self, tmp_path, monkeypatch, capsys):
|
||||
"""macOS 26 bootstrap error 5 should spawn a detached gateway, not crash."""
|
||||
plist_path = tmp_path / "ai.hermes.gateway.plist"
|
||||
monkeypatch.setattr(gateway_cli, "get_launchd_plist_path", lambda: plist_path)
|
||||
|
||||
def fake_run(cmd, check=False, **kwargs):
|
||||
if cmd[:2] == ["launchctl", "bootstrap"]:
|
||||
raise gateway_cli.subprocess.CalledProcessError(
|
||||
5, cmd, stderr="Input/output error"
|
||||
)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
|
||||
|
||||
spawned = []
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
|
||||
)
|
||||
|
||||
gateway_cli.launchd_install(force=True)
|
||||
|
||||
assert spawned == [True]
|
||||
assert "Service installed and loaded" not in capsys.readouterr().out
|
||||
|
||||
def test_launchd_restart_falls_back_to_detached_on_kickstart_125(self, monkeypatch, capsys):
|
||||
"""When kickstart -k returns 125, restart should relaunch detached."""
|
||||
target = f"{gateway_cli._launchd_domain()}/{gateway_cli.get_launchd_label()}"
|
||||
|
||||
monkeypatch.setattr(gateway_cli, "_get_restart_drain_timeout", lambda: 5.0)
|
||||
monkeypatch.setattr(gateway_cli, "_request_gateway_self_restart", lambda pid: False)
|
||||
monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda timeout, force_after=None: True)
|
||||
monkeypatch.setattr(gateway_cli, "terminate_pid", lambda pid, force=False: None)
|
||||
monkeypatch.setattr("gateway.status.get_running_pid", lambda: 321)
|
||||
|
||||
def fake_run(cmd, check=False, **kwargs):
|
||||
if cmd == ["launchctl", "kickstart", "-k", target]:
|
||||
raise gateway_cli.subprocess.CalledProcessError(
|
||||
125, cmd, stderr="Domain does not support specified action"
|
||||
)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
|
||||
|
||||
spawned = []
|
||||
monkeypatch.setattr(
|
||||
gateway_cli, "_spawn_detached_gateway", lambda: spawned.append(True) or True
|
||||
)
|
||||
|
||||
gateway_cli.launchd_restart()
|
||||
|
||||
assert spawned == [True]
|
||||
|
||||
def test_launchd_stop_tolerates_domain_unsupported_bootout(self, monkeypatch, capsys):
|
||||
"""bootout exit 125 (macOS 26) must fall through to PID-based kill, not raise."""
|
||||
def fake_run(cmd, check=False, **kwargs):
|
||||
if "bootout" in cmd:
|
||||
raise gateway_cli.subprocess.CalledProcessError(
|
||||
125, cmd, stderr="Domain does not support specified action"
|
||||
)
|
||||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||||
|
||||
monkeypatch.setattr(gateway_cli.subprocess, "run", fake_run)
|
||||
monkeypatch.setattr(gateway_cli, "_wait_for_gateway_exit", lambda **kw: None)
|
||||
|
||||
gateway_cli.launchd_stop()
|
||||
|
||||
assert "stopped" in capsys.readouterr().out.lower()
|
||||
|
||||
def test_launchd_fallback_exits_when_spawn_fails(self, monkeypatch, capsys):
|
||||
"""If the detached spawn fails, surface the manual workaround and exit 1."""
|
||||
monkeypatch.setattr(gateway_cli, "_spawn_detached_gateway", lambda: False)
|
||||
|
||||
with pytest.raises(SystemExit) as exc:
|
||||
gateway_cli._launchd_fallback_to_detached("test reason")
|
||||
assert exc.value.code == 1
|
||||
out = capsys.readouterr().out
|
||||
assert "nohup hermes gateway run" in out
|
||||
|
||||
|
||||
class TestGatewayServiceDetection:
|
||||
def test_supports_systemd_services_requires_systemctl_binary(self, monkeypatch):
|
||||
|
|
|
|||
Loading…
Reference in New Issue