218 lines
8.2 KiB
Python
218 lines
8.2 KiB
Python
"""Bootstrap for the managed runtime: config -> installed binaries ->
|
|
running supervised server.
|
|
|
|
One public call, ``ensure_local_runtime(config)``, safe to call at any
|
|
session start:
|
|
- disabled or already-running (state file answers /health) -> no-op
|
|
- enabled -> install binaries if missing (idempotent), spawn supervisor
|
|
|
|
Kept import-light: callers gate on config before importing this module so
|
|
sessions with local_runtime disabled never pay the import.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from hermes_constants import get_hermes_home
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_SUPERVISOR = None # process-wide singleton; one router per Hermes process
|
|
|
|
|
|
def _detect_gpu_vendor() -> str | None:
|
|
"""Best-effort GPU vendor for backend selection. NVIDIA via nvidia-smi;
|
|
anything else defers to select_backend's fallback ladder."""
|
|
try:
|
|
out = subprocess.run(
|
|
["nvidia-smi", "--query-gpu=name", "--format=csv,noheader"],
|
|
capture_output=True, text=True, timeout=10)
|
|
if out.returncode == 0 and out.stdout.strip():
|
|
return "nvidia " + out.stdout.strip().splitlines()[0]
|
|
except (OSError, subprocess.TimeoutExpired):
|
|
pass
|
|
return None
|
|
|
|
|
|
def models_dir() -> Path:
|
|
return get_hermes_home() / "models"
|
|
|
|
|
|
def assets_dir() -> Path:
|
|
"""Non-model companion files (mmproj vision projectors, spec-decode
|
|
draft models). A subdirectory so the router's model listing — and our
|
|
staged_models() — never mistakes an asset for a servable model."""
|
|
return models_dir() / "assets"
|
|
|
|
|
|
def staged_models() -> "list[Path]":
|
|
"""Servable staged models: first-parts of split GGUFs count once,
|
|
continuation parts and assets/ don't count at all."""
|
|
import re
|
|
|
|
part = re.compile(r"-(\d{5})-of-\d{5}\.gguf$")
|
|
out = []
|
|
for p in sorted(models_dir().glob("*.gguf")):
|
|
m = part.search(p.name)
|
|
if m and m.group(1) != "00001":
|
|
continue
|
|
out.append(p)
|
|
return out
|
|
|
|
|
|
def staged_model_ids() -> "list[str]":
|
|
import re
|
|
|
|
return [re.sub(r"-\d{5}-of-\d{5}$", "", p.stem) for p in staged_models()]
|
|
|
|
|
|
def refresh_local_runtime() -> bool:
|
|
"""Restart the managed server so it rescans the models directory.
|
|
|
|
The router's model list is SPAWN-ONLY: a GGUF added after start is
|
|
invisible to GET /models and 400s on completion, so anything that
|
|
changes the staged set while the server runs must bounce it. Only
|
|
touches a server THIS process supervises; returns False when there is
|
|
nothing to refresh (next boot scans fresh anyway)."""
|
|
global _SUPERVISOR
|
|
if _SUPERVISOR is None:
|
|
return False
|
|
try:
|
|
from hermes_cli.config import load_config
|
|
|
|
shutdown_local_runtime()
|
|
return ensure_local_runtime(load_config(), force=True) is not None
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.warning("local runtime refresh failed: %s", exc)
|
|
return False
|
|
|
|
|
|
def ensure_local_runtime(config: dict, force: bool = False) -> "object | None":
|
|
"""Idempotent boot of the managed runtime. Returns the supervisor (or
|
|
None when disabled/unavailable). Never raises into a session start —
|
|
failures log and return None; chat falls back to configured providers.
|
|
|
|
``force=True`` skips the enabled gate — used by the explicit "Use this
|
|
model" action, where the click IS the opt-in (the caller records it in
|
|
config so future boots auto-start).
|
|
"""
|
|
global _SUPERVISOR
|
|
section = (config or {}).get("local_runtime") or {}
|
|
if not force and not section.get("enabled"):
|
|
return None
|
|
if _SUPERVISOR is not None:
|
|
return _SUPERVISOR
|
|
|
|
# Residency: no staged models means nothing to serve — don't boot an
|
|
# empty server. The walked-away story handled with zero configuration
|
|
# (delete your last model and boots stop); Use force-boots as ever.
|
|
if not force and not staged_models():
|
|
logger.info("local runtime enabled but no models staged; not booting")
|
|
return None
|
|
|
|
# Another Hermes process may already be supervising — reuse via state.
|
|
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
|
|
|
if _state_endpoint() is not None:
|
|
logger.info("managed llama-server already running (another process)")
|
|
return None
|
|
|
|
try:
|
|
from hermes_cli.local_runtime.binaries import (
|
|
ensure_runtime_installed,
|
|
select_backend,
|
|
)
|
|
from hermes_cli.local_runtime.hardware import probe_budget
|
|
from hermes_cli.local_runtime.presets import generate_presets
|
|
from hermes_cli.local_runtime.supervisor import LlamaServerSupervisor
|
|
|
|
backend = section.get("backend", "auto")
|
|
if backend == "auto":
|
|
backend = select_backend(_detect_gpu_vendor())
|
|
# Boot ladder: serve what is INSTALLED, never download here. The
|
|
# configured tag (config root-of-trust; deep-merge supplies the
|
|
# Hermes-release default when unpinned) is preferred; when it isn't
|
|
# installed yet, the newest installed tag serves and the status
|
|
# endpoint reports the pending update — the download is a deliberate
|
|
# button click in the pane, not a boot-path surprise (a multi-minute
|
|
# inline download here is exactly how the onboarding bounce returns).
|
|
from hermes_cli.local_runtime.binaries import default_tag, installed_tags
|
|
|
|
tag = section.get("tag") or default_tag()
|
|
have = installed_tags()
|
|
if tag not in have:
|
|
if not have:
|
|
logger.info("local runtime enabled but no build installed; "
|
|
"install happens in the Local Models pane")
|
|
return None
|
|
logger.info("configured tag %s not installed; serving %s "
|
|
"(update is a click in Local Models)", tag, have[0])
|
|
tag = have[0]
|
|
install_dir = ensure_runtime_installed(tag, backend)
|
|
|
|
mdir = models_dir()
|
|
mdir.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Context policy: one launch decision per staged model,
|
|
# carried to the router via the preset INI.
|
|
preset_path = get_hermes_home() / "runtimes" / "llamacpp" / "presets.ini"
|
|
try:
|
|
entries = generate_presets(mdir, probe_budget(), preset_path)
|
|
for entry in entries:
|
|
if entry.refusal:
|
|
logger.warning("model refused by physics check: %s", entry.refusal)
|
|
except Exception as exc: # noqa: BLE001 — policy failure must not block serving
|
|
logger.warning("preset generation failed (%s); router runs stock fit", exc)
|
|
preset_path = None
|
|
|
|
sup = LlamaServerSupervisor(
|
|
install_dir, mdir,
|
|
models_max=int(section.get("models_max", 4)),
|
|
port=int(section.get("port", 0)) or None,
|
|
preset_path=preset_path,
|
|
)
|
|
sup.start()
|
|
_SUPERVISOR = sup
|
|
logger.info("managed llama-server up at %s (backend=%s tag=%s)",
|
|
sup.base_url, backend, tag)
|
|
_start_idle_sweeper(sup)
|
|
return sup
|
|
except Exception as exc: # noqa: BLE001 — never break session start
|
|
logger.warning("managed local runtime unavailable: %s", exc)
|
|
return None
|
|
|
|
|
|
def shutdown_local_runtime() -> None:
|
|
global _SUPERVISOR
|
|
if _SUPERVISOR is not None:
|
|
_SUPERVISOR.stop()
|
|
_SUPERVISOR = None
|
|
|
|
|
|
def get_supervisor():
|
|
"""The process-local supervisor, or None (server may still be running
|
|
under another process — check the state file)."""
|
|
return _SUPERVISOR
|
|
|
|
|
|
def _start_idle_sweeper(sup) -> None:
|
|
"""Idle-residency loop: every couple of minutes, unload non-primary
|
|
models idle past the supervisor's threshold. Daemon thread tied to the
|
|
supervisor's lifetime — exits when the server stops."""
|
|
import threading
|
|
|
|
def _loop():
|
|
while sup.proc is not None and sup.proc.poll() is None:
|
|
time.sleep(120)
|
|
try:
|
|
sup.sweep_idle()
|
|
except Exception as exc: # noqa: BLE001
|
|
logger.debug("idle sweep skipped: %s", exc)
|
|
|
|
threading.Thread(target=_loop, daemon=True,
|
|
name="local-runtime-idle-sweep").start()
|