diff --git a/hermes_cli/inventory.py b/hermes_cli/inventory.py index 4e95665d481f8..2e6ac889a1d12 100644 --- a/hermes_cli/inventory.py +++ b/hermes_cli/inventory.py @@ -200,6 +200,17 @@ def build_models_payload( excluded_providers=ctx.excluded_providers or [], ) + # Managed local runtime: staged GGUFs are selectable like any provider's + # models. list_authenticated_providers can't know about them (no + # credential, no custom_providers entry — the credential is + # reachability), so inject the row here where every picker surface + # inherits it. Present whenever models are staged; picking one routes + # through the llamacpp alias -> managed/detected server resolution. + local_row = _local_runtime_row(ctx) + if local_row is not None: + rows = [r for r in rows if str(r.get("slug", "")).lower() != "llamacpp"] + rows.append(local_row) + moa_row = _moa_provider_row(ctx.current_provider) if moa_row is not None: rows = [moa_row] + [r for r in rows if str(r.get("slug", "")).lower() != "moa"] @@ -825,6 +836,54 @@ def _apply_pricing( row["unavailable_models"] = [] +def _local_runtime_row(ctx: "ConfigContext") -> dict | None: + """Build the ``llamacpp`` provider row from staged local models. + + Present whenever GGUFs are staged in the managed models directory — + downloaded models must be selectable even before the server is running + (selection starts it via the runtime_provider seam / activate flow). + Returns ``None`` when nothing is staged. + """ + try: + from hermes_cli.local_runtime.bootstrap import staged_model_ids + + staged = staged_model_ids() + if not staged: + return None + current = (ctx.current_provider or "").strip().lower() in ( + "llamacpp", "llama.cpp", "llama-cpp") + if not current: + # A LIVE session on the managed server reports provider "custom" + # (the resolution seam's label) with the managed base_url. Match + # on the endpoint so the picker still marks this row current — + # otherwise the session the user is chatting in shows no + # selection. + try: + from hermes_cli.local_runtime.endpoint import _state_endpoint + + managed = _state_endpoint() + current = bool( + managed + and (ctx.current_base_url or "").strip().rstrip("/") + == managed["base_url"].rstrip("/")) + except Exception: + current = False + return { + "slug": "llamacpp", + "name": "Local (llama.cpp)", + "is_current": current, + "is_user_defined": False, + "models": staged, + "total_models": len(staged), + "source": "local-runtime", + "authenticated": True, # the credential is reachability + "auth_type": "local", + "warning": None, + } + except Exception: + return None + + def _moa_provider_row(current_provider: str = "") -> dict | None: """Build the virtual ``moa`` provider row for model pickers. diff --git a/hermes_cli/local_runtime/catalog.py b/hermes_cli/local_runtime/catalog.py new file mode 100644 index 0000000000000..626a8c52f3e3c --- /dev/null +++ b/hermes_cli/local_runtime/catalog.py @@ -0,0 +1,378 @@ +"""Curated starter catalog for the managed local runtime. + +Small and honest: every entry carries the estimator inputs (measured on +real GGUFs) so the picker can price a model BEFORE the user downloads +gigabytes. Once a file is on disk, profile_from_gguf() is the authority +and the catalog numbers are only used for the download decision. Entries +whose base config is gated upstream carry a same-family conservative +prior (commented) — the GGUF header corrects it at load time. + +Each model ships a QUANT LADDER (variants, best quality first). The ladder +floors at UD-Q4_K_XL — below Q4 the quality loss is too severe to ship as +someone's first local-AI experience. Selection is hardware-aware: pick the +highest-quality variant that zero-spills at the 64K floor on this machine; +else Q4 spilled (priced honestly by the fit policy); refuse only when even +Q4 fails the physics check. + +Validation lifecycle: rungs proven end-to-end on real hardware are marked +validated. Day-0 entries ship before that proof under the "day-0" tag — +ensure_model_ready's touch generation still gates every first load at +runtime. The contract test requires every ladder floor to be Q4 AND +(validated OR day-0). + +Multi-file models: variants may carry split-GGUF parts (llama-server loads +from the first part; all parts download together, each sha-verified). +Entries may carry an mmproj (vision projector) and a speculative-decode +draft model — both download alongside the weights. Spec decode is enabled +only when the launch decision spills, where its speedup is largest. + +sha256s are pinned from HF LFS metadata (the lfs oid IS the file sha256), +reviewed like a version bump — parsed data, never executed commands. + +This is deliberately not a live registry feed: entries are reviewed like a +version bump (the same policy governs vendor recipe ingestion — parsed +data, never executed commands). + +Vendor recipes overlay: a per-SKU recipes repo may SUPPLEMENT these +entries where applicable — vendor SKUs only, never the base layer for +other platforms. A recipe may enrich identity (GGUF/quant/sha), perf +hints (-b/-ub, spec-decode), and sampling defaults; it never carries +context/slots/placement/serving flags (the fit policy owns those). +Resolution: exact SKU -> GPU-class bucket -> fit-only. Snapshot-synced, +reviewed like a tag bump. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import PurePosixPath + +from hermes_cli.local_runtime.context_policy import FLOOR +from hermes_cli.local_runtime.estimator import ( + HardwareBudget, + LayerKind, + ModelProfile, + ctx_bytes, +) + +_GIB = 1 << 30 +_PART_SUFFIX = re.compile(r"-\d{5}-of-\d{5}$") + + +@dataclass(frozen=True) +class AssetFile: + """One downloadable file: repo-relative path, exact bytes, sha256. + ``local`` overrides the on-disk name (repos reuse generic names like + mmproj-BF16.gguf across models). Non-model extras live under the + models dir's assets/ subdirectory so the router never lists them.""" + + path: str # repo-relative (may include a subdir) + size_bytes: int + sha256: str + local: str | None = None + + @property + def local_name(self) -> str: + return self.local or PurePosixPath(self.path).name + + +@dataclass(frozen=True) +class QuantVariant: + """One downloadable build of a model. Ordered best-quality-first in + CatalogEntry.variants. Split GGUFs list every part in files; the model + loads from the first part.""" + + quant: str # e.g. "UD-Q8_K_XL" + files: tuple # AssetFile, first = the load target + validated: bool = False # proven end-to-end on real hardware + + @property + def model_id(self) -> str: + stem = PurePosixPath(self.files[0].path).name.removesuffix(".gguf") + return _PART_SUFFIX.sub("", stem) + + @property + def size_bytes(self) -> int: + return sum(f.size_bytes for f in self.files) + + @property + def weights_bytes(self) -> int: + """Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a + small header (<2%) — a safe, slightly conservative stand-in until + profile_from_gguf reads the real table.""" + return self.size_bytes + + +@dataclass(frozen=True) +class CatalogEntry: + id: str # stable family id (variant-independent) + display_name: str + description: str # one line, plain language + repo: str # HF repo + variants: tuple # QuantVariant, best first + # Estimator inputs (measured or config-derived; quant changes weights, + # never KV). Entries with gated upstream configs carry a conservative + # same-family prior — the GGUF header is the authority after download. + n_ctx_train: int + full_layers: int + recurrent_layers: int + per_layer_f16: int # KV bytes/token per full-attention layer + swa_layers: int = 0 + swa_window: int = 0 + moe: bool = False + mtp: bool = False # ships MTP heads (spec decode iff spilled) + mmproj: "AssetFile | None" = None # vision projector, downloads with model + draft: "AssetFile | None" = None # spec-decode draft model (e.g. DSpark) + sampling: dict = field(default_factory=dict) # INI long-form launch defaults + tags: tuple = field(default_factory=tuple) + + def profile(self, variant: QuantVariant) -> ModelProfile: + layers = ([(LayerKind.FULL, self.per_layer_f16)] * self.full_layers + + [(LayerKind.SWA, self.per_layer_f16)] * self.swa_layers + + [(LayerKind.RECURRENT, 0)] * self.recurrent_layers) + return ModelProfile( + name=variant.model_id, weights_bytes=variant.weights_bytes, + embd_table_bytes=0, n_ctx_train=self.n_ctx_train, + layers=layers, swa_window=self.swa_window, moe=self.moe) + + def download_files(self, variant: QuantVariant) -> tuple: + """Everything a download job fetches for this variant, in order.""" + extras = tuple(a for a in (self.mmproj, self.draft) if a is not None) + return tuple(variant.files) + extras + + def download_bytes(self, variant: QuantVariant) -> int: + return sum(f.size_bytes for f in self.download_files(variant)) + + +@dataclass(frozen=True) +class VariantChoice: + """Selection result: which build this machine should download and why. + reason_key is a UI-copy discriminator, not display text.""" + + variant: QuantVariant + zero_spill: bool + reason_key: str # "best-fits" | "smallest-fits-spilled" + + +def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None: + """Best build for this hardware, per the ladder policy. + + 1. Highest-quality variant whose weights + 64K-floor KV zero-spill in + usable VRAM (quality is free when it fits — take it). + 2. Else Q4 spilled — the floor guarantee stands, fit spills weights, + picker prices it honestly (slow beats lobotomized). + 3. None: even Q4 fails physics (true refusal). + """ + floor_kv = None + for variant in entry.variants: + profile = entry.profile(variant) + if floor_kv is None: + floor_kv = ctx_bytes(profile, min(FLOOR, entry.n_ctx_train or FLOOR)) + if variant.weights_bytes + floor_kv <= budget.usable_vram_bytes: + return VariantChoice(variant=variant, zero_spill=True, + reason_key="best-fits") + + smallest = min(entry.variants, key=lambda v: v.size_bytes) + needed = smallest.weights_bytes + (floor_kv or 0) + if needed <= budget.usable_vram_bytes + budget.ram_available_bytes: + return VariantChoice(variant=smallest, zero_spill=False, + reason_key="smallest-fits-spilled") + return None + + +def _v(quant: str, *files, validated: bool = False) -> QuantVariant: + return QuantVariant(quant=quant, + files=tuple(AssetFile(*f) for f in files), + validated=validated) + + +# Ordered: recommended first. File bytes + sha256s pinned from HF LFS +# metadata. +CATALOG: tuple[CatalogEntry, ...] = ( + CatalogEntry( + id="qwen3.6-27b", + display_name="Qwen3.6 27B", + description="Best all-round agent model; sees images; long context stays fast", + repo="unsloth/Qwen3.6-27B-GGUF", + variants=( + _v("UD-Q8_K_XL", + ("Qwen3.6-27B-UD-Q8_K_XL.gguf", 35325163744, + "19a2f4733a863088bc06665bf307dca95f7d4370b4d8690340cdff9992fe48c6")), + _v("UD-Q6_K_XL", + ("Qwen3.6-27B-UD-Q6_K_XL.gguf", 25636485344, + "8746881d40f280b1b6b858c656a347c754ed3d9cc8d2e1ad46b3635b87f611f8")), + _v("UD-Q5_K_XL", + ("Qwen3.6-27B-UD-Q5_K_XL.gguf", 20038256864, + "ac310abf2895aa397121bad6c0be89466af41f0f1606a21c1131b110eeb19d0e")), + _v("UD-Q4_K_XL", + ("Qwen3.6-27B-UD-Q4_K_XL.gguf", 17612564704, + "ff6941ded525b34eb159496762c29dd0ec6e71dc31b74d57e75d871a03eec259"), + validated=True), + ), + n_ctx_train=262144, + full_layers=16, recurrent_layers=48, per_layer_f16=4096, + mmproj=AssetFile("mmproj-BF16.gguf", 931146304, + "05353347512982ee62317b9d8c89372bc815f4b4043580e7ef3ad411ec1a1cd3", + local="mmproj-Qwen3.6-27B-BF16.gguf"), + sampling={"temp": "1.0", "top-p": "0.95", "top-k": "20", "min-p": "0.0"}, + tags=("recommended", "hybrid", "reasoning", "vision"), + ), + CatalogEntry( + id="qwen3.6-35b-a3b", + display_name="Qwen3.6 35B-A3B", + description="Bigger mixture-of-experts with multi-token prediction; sees images", + repo="unsloth/Qwen3.6-35B-A3B-MTP-GGUF", + variants=( + _v("UD-Q8_K_XL", + ("Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf", 39099447584, + "6c6b816537abad90b250a0972b345466028d861ddfe316d5f0de31ca6440f781")), + _v("UD-Q6_K_XL", + ("Qwen3.6-35B-A3B-UD-Q6_K_XL.gguf", 32611711264, + "35fce994cd36104a7dc1bd8a4bdf13778145664c00fdef6773aebc9246e5019c")), + _v("UD-Q5_K_XL", + ("Qwen3.6-35B-A3B-UD-Q5_K_XL.gguf", 27159116064, + "9de9a9420f61a0bb59bb2ca1ea170a6a57f6821fa1deec915bcaef523730a919")), + # Validated on this repo's prior upload; upstream has since + # re-uploaded. Same model id + pipeline — re-verify at the + # next validation pass. + _v("UD-Q4_K_XL", + ("Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf", 22853663008, + "55983c5a75a1ab969824077b3bb3de4146e82a9234072b48ad4e8f92ad3fe9f1"), + validated=True), + ), + n_ctx_train=262144, + # Config-derived (Qwen/Qwen3.6-35B-A3B): 40 layers, 10 full-attn + # (interval 4) + 30 linear; KV heads 2 x head_dim 256. + full_layers=10, recurrent_layers=30, per_layer_f16=2048, + moe=True, mtp=True, + mmproj=AssetFile("mmproj-BF16.gguf", 902822528, + "da63cb47a76763c712393f8a017070188a304fa39f8aeea6edc629ed7b975cfa", + local="mmproj-Qwen3.6-35B-A3B-BF16.gguf"), + sampling={"temp": "1.0", "top-p": "0.95", "top-k": "20", "min-p": "0.0"}, + tags=("hybrid", "moe", "mtp", "vision"), + ), + CatalogEntry( + id="nemotron-3.5-lightning-30b", + display_name="Nemotron 3.5 Lightning 30B", + description="NVIDIA's fast tool-calling model; 1M-token context", + repo="unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF", + variants=( + _v("UD-Q8_K_XL", + ("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q8_K_XL.gguf", 38615380032, + "48140cfb8bb6a38553275d334345f2638c119d979ab8b6d4afdc4b62d26d8219")), + _v("UD-Q6_K_XL", + ("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q6_K_XL.gguf", 35004643392, + "36b8b0f882ec739f895fe56ab6b9b892e702ba33b7b368614ca4078331ddfc29")), + _v("UD-Q5_K_XL", + ("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q5_K_XL.gguf", 30414829632, + "92dcaf682faf39fef906db7bad4000782001d089f28c4d6aecec6d6f1b4697ca")), + _v("UD-Q4_K_XL", + ("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_XL.gguf", 25505724480, + "112bf957489a497c18f60bf8bd44ee1dfa05e87368b8ed7f68998a4e38f275c9")), + ), + n_ctx_train=1048576, + # Prior from Nemotron-3-Nano (same hybrid family; base config gated + # upstream): 6 full-attn + 46 recurrent, ~1 KiB/tok/full-layer. + # Prior from the same hybrid family (measured: 1M ctx in ~3.2 GiB + # KV on the predecessor). GGUF header is + # the authority after download. + full_layers=6, recurrent_layers=46, per_layer_f16=1024, + moe=True, mtp=True, + sampling={"temp": "0.6", "top-p": "0.95", "min-p": "0.01"}, + tags=("day-0", "long-context", "hybrid", "moe", "mtp"), + ), + CatalogEntry( + id="muse-glimmer-30b", + display_name="Muse Glimmer 30B", + description="Meta's open vision model for coding and agent work", + repo="unsloth/Muse-Glimmer-30B-GGUF", + variants=( + _v("UD-Q8_K_XL", + ("Muse-Glimmer-30B-UD-Q8_K_XL.gguf", 32300651040, + "e63bf23b7710ecdea2579e4b1de58980c4a2b446e8ecf48b782cfcefd2e31770")), + _v("UD-Q6_K_XL", + ("Muse-Glimmer-30B-UD-Q6_K_XL.gguf", 26265362976, + "fb5f80d110c4fa932cc652e70873c0bd12c0954009038aa675e65086104c2739")), + _v("UD-Q5_K_XL", + ("Muse-Glimmer-30B-UD-Q5_K_XL.gguf", 21789618976, + "97a66c4b41d9e778af7cdfa43508e08dbf765fb5049b740c69ad815e5191c637")), + _v("UD-Q4_K_XL", + ("Muse-Glimmer-30B-UD-Q4_K_XL.gguf", 15878222368, + "82bece304887a313ece08400bc030f6066c7bff5b906b0cd40308ec8a409fd38")), + ), + n_ctx_train=262144, + # Conservative dense prior (base config gated upstream): 30B-class + # dense, ~60 layers x 4 KiB/tok. Dense KV is the expensive shape — + # overestimating here keeps the zero-spill promise safe until the + # GGUF header corrects it. + full_layers=60, recurrent_layers=0, per_layer_f16=4096, + mmproj=AssetFile("mmproj-Muse-Glimmer-30B-BF16.gguf", 3849173728, + "d08cdcfa0b41d8e20554b52df404ba4f7b440d0bc502a90038508b6407df8ee1"), + sampling={"temp": "1.0", "top-p": "0.95", "top-k": "64"}, + tags=("day-0", "vision", "dense"), + ), + CatalogEntry( + id="deepseek-v4-flash", + display_name="DeepSeek V4 Flash", + description="Frontier-class model for machines with 128GB+ memory", + repo="unsloth/DeepSeek-V4-Flash-0731-GGUF", + variants=( + # Q8 is bit-lossless vs the official QAT checkpoint; Q4 keeps + # the MXFP4 experts bit-exact and only requants the other 4%. + _v("UD-Q8_K_XL", + ("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00001-of-00005.gguf", 5257408, + "d13ce8f90855547bdaebe7312f531a1f2c4f822178d3103951f27fe884395cfa"), + ("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00002-of-00005.gguf", 49215492960, + "3da2f2443063f83635986f9b67fa7e8e3d03c53b81a9a08d2007936612423610"), + ("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00003-of-00005.gguf", 49700372160, + "7d622a7760d359ec9257b3493ad531e3bf0bfbe6f6533267e16e6dde8153ddce"), + ("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00004-of-00005.gguf", 49466495968, + "6ed2bce452214f156b85e7c5f7d4fc242a3052f409d1b90a61422f60669c2de3"), + ("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00005-of-00005.gguf", 13481997024, + "ea4727af4888fdca0fff796ec81ac2f3ebb43c310b2feb4798f41d82744b42ea")), + _v("UD-Q4_K_XL", + ("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00001-of-00005.gguf", 5257408, + "d13ce8f90855547bdaebe7312f531a1f2c4f822178d3103951f27fe884395cfa"), + ("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00002-of-00005.gguf", 48935523072, + "d5b61668950f4743aacd677675d7fcf7507dbe1db6d304e8ff97ed1f00827bee"), + ("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00003-of-00005.gguf", 48980787136, + "9705db7e589f360685ca7bd48100b270d78d228d4f5aa980508f3b2778af5494"), + ("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00004-of-00005.gguf", 49999168416, + "7f13a68e3ca64208454c4ba32cc2757c0cbe78e3e5576c3142bf7007ca97da42"), + ("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00005-of-00005.gguf", 7174505088, + "ed0d93164d3784968d6ce40d6d201ba98337f16e7db1b31fe495b2b0f334cc09")), + ), + n_ctx_train=1048576, + # Config-derived (deepseek-ai/DeepSeek-V4-Flash-0731): 43 layers, + # MLA compressed KV (rank 512 + 64 rope) ~1.15 KiB/tok/layer f16. + # Config shows sliding_window=128 with no per-layer map — priced + # all-full (conservative); GGUF header decides after download. + full_layers=43, recurrent_layers=0, per_layer_f16=1152, + moe=True, + draft=AssetFile("dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf", 10896057440, + "2c7ac54b0b64a99df1f139a9f1371a00198265e1d6a614b77597d20a655a4249"), + sampling={"temp": "1.0", "top-p": "0.95", "min-p": "0.01"}, + tags=("day-0", "long-context", "moe", "frontier"), + ), +) + + +def catalog_by_id() -> dict[str, CatalogEntry]: + return {entry.id: entry for entry in CATALOG} + + +def find_variant(entry_id: str, model_id: str) -> QuantVariant | None: + entry = catalog_by_id().get(entry_id) + if entry is None: + return None + return next((v for v in entry.variants if v.model_id == model_id), None) + + +def find_entry_for_model(model_id: str) -> "tuple[CatalogEntry, QuantVariant] | None": + """Locate the entry + variant that owns a staged model id.""" + for entry in CATALOG: + for variant in entry.variants: + if variant.model_id == model_id: + return entry, variant + return None diff --git a/hermes_cli/web_routers/local_models.py b/hermes_cli/web_routers/local_models.py new file mode 100644 index 0000000000000..9e0374dc690c8 --- /dev/null +++ b/hermes_cli/web_routers/local_models.py @@ -0,0 +1,925 @@ +"""Local-models dashboard routes — the desktop's window into the managed +llama.cpp runtime. + +Everything here is designed for a first-run user on an RTX laptop: every +payload carries plain-language, pre-formatted facts the UI can show verbatim +(what will this model do ON THIS MACHINE, how big is the download, what is +the runtime doing right now), never raw internals the renderer would have to +interpret. + +Long jobs (runtime install, model download) follow the repo's job pattern: +start-POST -> {job_id} -> GET poll with byte progress. Downloads are +sha256-verified; a hash mismatch deletes the file and reports it plainly. +""" + +from __future__ import annotations + +import json +import logging +import threading +import time +import urllib.request +import uuid +from pathlib import Path +from typing import Any, Dict, Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +logger = logging.getLogger(__name__) + +router = APIRouter() + +_GIB = 1 << 30 +_JOBS: Dict[str, Dict[str, Any]] = {} +_JOBS_LOCK = threading.Lock() + + +def _human_gb(n: int | float) -> str: + return f"{n / _GIB:.1f} GB" + + +def _job(kind: str, target: str, model_id: str | None = None) -> Dict[str, Any]: + job = { + "job_id": uuid.uuid4().hex[:12], + "kind": kind, # "runtime-install" | "model-download" + "target": target, + "model_id": model_id, # catalog id for downloads; None otherwise + "status": "running", # running | done | error + "phase": "starting", # human-readable step name + "detail": "", + "total_bytes": None, + "done_bytes": 0, + "started_at": time.time(), + "error": None, + } + with _JOBS_LOCK: + _JOBS[job["job_id"]] = job + return job + + +# ── fast download: ranged parallel streams ─────────────────── + +# One TCP stream to a CDN rarely fills a fast line; 8 ranged connections +# writing into a preallocated file saturate consumer gigabit. sha256 is +# computed in a sequential pass afterwards (NVMe read is seconds, and it +# keeps the hash independent of write ordering). +_DOWNLOAD_CONNECTIONS = 8 +_CHUNK = 4 << 20 + + +def _probe_range_support(url: str) -> int: + """Total size when the server honors Range requests, else 0. + + Auth-shaped failures raise with a plain-language message — a 401/403 + from the CDN means the repo is gated or the catalog entry names a + wrong repo, and the user deserves better than a bare status code. + """ + req = urllib.request.Request(url, headers={"Range": "bytes=0-0"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: + if r.status == 206: + content_range = r.headers.get("Content-Range", "") + if "/" in content_range: + return int(content_range.rsplit("/", 1)[1]) + except urllib.error.HTTPError as exc: + if exc.code in (401, 403): + raise RuntimeError( + "The model host refused the download (gated or moved). " + "This is a catalog problem, not yours — please report it.") from exc + raise + except Exception: # noqa: BLE001 + pass + return 0 + + +def _model_id_for(gguf: Path) -> str: + """Variant model id for a staged file (strips split-part suffixes).""" + import re + + return re.sub(r"-\d{5}-of-\d{5}$", "", gguf.stem) + + +def _variant_files_on_disk(model_id: str) -> "list[Path]": + """Every local file belonging to a staged model: all split parts plus + its catalog-declared assets (mmproj/draft) when present.""" + from hermes_cli.local_runtime.bootstrap import assets_dir + from hermes_cli.local_runtime.catalog import find_entry_for_model + + mdir = _models_dir() + files = [p for p in mdir.glob("*.gguf") if _model_id_for(p) == model_id] + hit = find_entry_for_model(model_id) + if hit is not None: + entry, _variant = hit + for asset in (entry.mmproj, entry.draft): + if asset is not None: + p = assets_dir() / asset.local_name + if p.exists(): + files.append(p) + return files + + +def download_file(url: str, dest: Path, job: Dict[str, Any], + expected_sha256: str = "", *, + base_done: int = 0, keep_totals: bool = False) -> None: + """Download url -> dest with byte progress on ``job``. + + Ranged-parallel when the server supports it, single-stream fallback + otherwise. Verifies sha256 when given; a mismatch deletes the file and + raises with a plain-language message. Never leaves a .part behind. + + Multi-file variants: ``base_done`` offsets the progress so this file's + bytes accumulate onto the files before it, and ``keep_totals=True`` + stops the per-file size from overwriting the variant's total. + """ + import hashlib + import shutil + import threading as _threading + + tmp = dest.with_suffix(".part") + dest.parent.mkdir(parents=True, exist_ok=True) + file_done = [0] + progress_lock = _threading.Lock() + + def bump(n: int) -> None: + with progress_lock: + file_done[0] += n + job["done_bytes"] = base_done + file_done[0] + + try: + total = _probe_range_support(url) + if total: + if not keep_totals: + job["total_bytes"] = total + # Preallocate so each worker writes at its own offset. + with open(tmp, "wb") as f: + f.truncate(total) + errors: list[Exception] = [] + bounds = [(i * total // _DOWNLOAD_CONNECTIONS, + (i + 1) * total // _DOWNLOAD_CONNECTIONS - 1) + for i in range(_DOWNLOAD_CONNECTIONS)] + + def fetch_range(start: int, end: int) -> None: + try: + req = urllib.request.Request( + url, headers={"Range": f"bytes={start}-{end}"}) + with urllib.request.urlopen(req, timeout=120) as r, \ + open(tmp, "r+b") as f: + f.seek(start) + while True: + chunk = r.read(_CHUNK) + if not chunk: + break + f.write(chunk) + bump(len(chunk)) + except Exception as exc: # noqa: BLE001 + errors.append(exc) + + threads = [_threading.Thread(target=fetch_range, args=b, daemon=True, + name=f"lm-dl-{i}") + for i, b in enumerate(bounds)] + for t in threads: + t.start() + for t in threads: + t.join() + if errors: + raise errors[0] + if file_done[0] != total: + raise RuntimeError( + f"download incomplete ({file_done[0]} of {total} bytes)") + else: + # No range support: single stream, large chunks. + with urllib.request.urlopen(url, timeout=120) as r, open(tmp, "wb") as f: + length = int(r.headers.get("Content-Length") or 0) + if length and not keep_totals: + job["total_bytes"] = length + while True: + chunk = r.read(_CHUNK) + if not chunk: + break + f.write(chunk) + bump(len(chunk)) + + if expected_sha256: + job["phase"] = "verifying" + job["detail"] = "Checking file integrity" + digest = hashlib.sha256() + with open(tmp, "rb") as f: + for chunk in iter(lambda: f.read(16 << 20), b""): + digest.update(chunk) + if digest.hexdigest() != expected_sha256: + raise RuntimeError( + "Downloaded file failed its integrity check and was removed — try again") + shutil.move(str(tmp), str(dest)) + except Exception: + tmp.unlink(missing_ok=True) + raise + + +def _models_dir() -> Path: + from hermes_cli.local_runtime.bootstrap import models_dir + + return models_dir() + + +def _load_config() -> dict: + from hermes_cli.config import load_config + + try: + return load_config() + except Exception: # noqa: BLE001 + return {} + + +def _runtime_section() -> dict: + return (_load_config() or {}).get("local_runtime") or {} + + +# ── status: the one call the pane opens with ───────────────── + + +@router.get("/api/local-models/status") +async def local_models_status(): + """Cheap, immediate, never blocks on probes (responsiveness standard): + config state + installed runtime + staged models + supervisor state. + GPU facts come from /api/local-models/hardware (slower, polled).""" + from hermes_cli.local_runtime.binaries import ( + default_tag, + installed_tags, + runtimes_root, + server_binary, + ) + from hermes_cli.local_runtime.endpoint import _state_endpoint + + section = _runtime_section() + configured_tag = section.get("tag") or default_tag() + have = installed_tags() + + # The tag actually serving (boot ladder: configured if installed, else + # newest installed). Present tense for the pane header. + tag = configured_tag if configured_tag in have else (have[0] if have else configured_tag) + + # A pending engine update exists when the user runs the local engine + # (enabled + something installed) and the configured tag — pinned or + # the Hermes-release default — is newer than anything on disk. The + # download is a button click, never automatic. + update_available = bool( + section.get("enabled") and have and configured_tag not in have) + + runtime_installed = False + runtime_backend = None + root = runtimes_root() / tag + if root.exists(): + for backend_dir in sorted(p for p in root.iterdir() if p.is_dir()): + try: + server_binary(backend_dir) + runtime_installed = True + runtime_backend = backend_dir.name + break + except Exception: # noqa: BLE001 + continue + + staged = [] + mdir = _models_dir() + if mdir.exists(): + from hermes_cli.local_runtime.bootstrap import staged_models + + # Split models: report the whole variant's bytes, not one part's. + from hermes_cli.local_runtime.catalog import find_entry_for_model + + for gguf in staged_models(): + model_id = _model_id_for(gguf) + size = gguf.stat().st_size + hit = find_entry_for_model(model_id) + if hit is not None: + size = hit[1].size_bytes + staged.append({ + "id": model_id, + "size_bytes": size, + "size_label": _human_gb(size), + }) + + running = _state_endpoint() + + # Which staged models are resident right now (loaded in VRAM). Read + # from the live router when it's up; {} when down. Feeds the pane's + # Loaded pills and eject buttons. + loaded: Dict[str, str] = {} + if running is not None: + try: + import urllib.request as _url + + req = _url.Request( + running["base_url"].rsplit("/v1", 1)[0] + "/models", + headers={"Authorization": f"Bearer {running.get('api_key', '')}"}) + with _url.urlopen(req, timeout=3) as r: + data = json.loads(r.read()) + loaded = { + m["id"]: m.get("status", {}).get("value", "unknown") + for m in data.get("data", []) + # Everything resident or becoming resident: 'loading' renders + # as its own state in the pane (a 20-GB load in flight is the + # single most important thing the pane can show). + if m.get("status", {}).get("value") in ("loaded", "ready", "loading") + } + except Exception as exc: # noqa: BLE001 + # Never silent: an empty dict here renders as 'Not in memory' + # on a machine whose VRAM is visibly full. + logger.warning("loaded-models read failed: %r", exc) + loaded = {} + + # The active main model, when it is one of ours (config authority: the + # same model.provider + model.default that /api/model/set writes). + active_model_id = None + try: + config = _load_config() + model_section = (config or {}).get("model") or {} + if str(model_section.get("provider", "")).strip().lower() in ( + "llamacpp", "llama.cpp", "llama-cpp"): + active_model_id = str( + model_section.get("default") or model_section.get("name") or "" + ).strip() or None + except Exception: # noqa: BLE001 + pass + + return { + "enabled": bool(section.get("enabled")), + "tag": tag, + "configured_tag": configured_tag, + "update_available": update_available, + "runtime_installed": runtime_installed, + "runtime_backend": runtime_backend, + "server_running": running is not None, + "server_base_url": (running or {}).get("base_url"), + "active_model_id": active_model_id, + "loaded_models": loaded, + "models": staged, + "models_dir": str(mdir), + } + + +# ── hardware: what this machine can do ─────────────────────── + + +@router.get("/api/local-models/hardware") +async def local_models_hardware(): + """The budget as plain facts. Polled by the pane and the statusbar + resource item (throttled client-side).""" + from hermes_cli.local_runtime.hardware import probe_budget, _nvidia_vram, _ram_bytes + + budget = probe_budget() + ram_total, ram_avail = _ram_bytes() + out = { + "uma": budget.uma, + "vram_total_bytes": budget.total_device_bytes, + "vram_usable_bytes": budget.usable_vram_bytes, + "ram_total_bytes": ram_total, + "ram_available_bytes": ram_avail, + "vram_label": _human_gb(budget.total_device_bytes), + "gpu_name": None, + "gpu_util_percent": None, + "vram_used_bytes": None, + } + # GPU identity + live utilization (NVIDIA; other vendors degrade to None + # and the UI hides those readouts). + try: + import subprocess + + smi = subprocess.run( + ["nvidia-smi", "--query-gpu=name,utilization.gpu,memory.used", + "--format=csv,noheader,nounits"], + capture_output=True, text=True, timeout=5) + if smi.returncode == 0 and smi.stdout.strip(): + name, util, used_mib = (x.strip() for x in smi.stdout.strip().splitlines()[0].split(",")) + out["gpu_name"] = name + out["gpu_util_percent"] = int(util) + out["vram_used_bytes"] = int(used_mib) << 20 + except Exception: # noqa: BLE001 + pass + return out + + +# ── catalog: priced for THIS machine before download ───────── + + +@router.get("/api/local-models/catalog") +async def local_models_catalog(): + """Every entry answers the user's three questions up front: how big is + the download, will it fit, and what context/speed shape will I get — + computed from the catalog's measured numbers + this machine's + budget. Hardware-aware quant selection: the row advertises the BEST + build for this machine (highest quality that runs fully on the GPU at + the 64K floor; else the smallest that works, spilled and priced). No + entry is hidden; unaffordable models show WHY.""" + from hermes_cli.local_runtime.catalog import CATALOG, select_variant + from hermes_cli.local_runtime.context_policy import initial_window + from hermes_cli.local_runtime.estimator import PhysicsRefusal + from hermes_cli.local_runtime.hardware import probe_budget + + # Planning budget: price against machine capacity, not live-free VRAM. + # A loaded model must not make the catalog call every row unaffordable. + budget = probe_budget(planning=True) + mdir = _models_dir() + entries = [] + for entry in CATALOG: + choice = select_variant(entry, budget) + # Any variant of this family already on disk counts as downloaded + # (split variants stage under their first part). + staged_ids = {_model_id_for(p) for p in mdir.glob("*.gguf")} if mdir.exists() else set() + downloaded_variant = next( + (v for v in entry.variants if v.model_id in staged_ids), None) + row: Dict[str, Any] = { + "id": entry.id, + "display_name": entry.display_name, + "description": entry.description, + "native_context": entry.n_ctx_train, + "native_context_label": f"{entry.n_ctx_train // 1024}K", + "tags": list(entry.tags), + "downloaded": downloaded_variant is not None, + "downloaded_model_id": downloaded_variant.model_id if downloaded_variant else None, + "downloaded_quant": downloaded_variant.quant if downloaded_variant else None, + "mtp": entry.mtp, + "vision": entry.mmproj is not None, + } + if choice is None: + smallest = min(entry.variants, key=lambda v: v.size_bytes) + smallest_total = entry.download_bytes(smallest) + row.update({ + "fits": False, + "size_bytes": smallest_total, + "size_label": _human_gb(smallest_total), + "fit_summary": "Needs more memory than this machine has", + "fit_detail": (f"even the most compact build ({smallest.quant}, " + f"{_human_gb(smallest_total)}) exceeds GPU + system memory"), + }) + entries.append(row) + continue + + variant = choice.variant + profile = entry.profile(variant) + decision = initial_window(profile, budget) + download_total = entry.download_bytes(variant) + row.update({ + "fits": True, + "model_id": variant.model_id, + "quant": variant.quant, + "quant_validated": variant.validated, + "size_bytes": download_total, + "size_label": _human_gb(download_total), + "variant_count": len(entry.variants), + }) + if choice.reason_key == "best-fits": + best = entry.variants[0] + row["quant_reason"] = ( + "Best quality build — runs fully on your GPU" + if variant.quant == best.quant + else f"Highest quality that runs fully on your GPU ({variant.quant})") + else: + row["quant_reason"] = ( + f"Compact build sized for this machine ({variant.quant}) — " + "larger than GPU memory, runs slower") + if not isinstance(decision, PhysicsRefusal): + row["start_window"] = decision.window + row["start_window_label"] = f"{decision.window // 1024}K" + row["spilled"] = decision.spilled + if decision.window >= entry.n_ctx_train: + shape = f"runs at its full {row['native_context_label']} context" + else: + shape = (f"starts at {row['start_window_label']} and grows toward " + f"{row['native_context_label']} as you use it") + if decision.spilled: + shape += " (larger than your GPU memory — runs slower)" + row["fit_summary"] = shape + else: + row["fit_summary"] = row["quant_reason"] + entries.append(row) + return {"models": entries} + + +# ── runtime install (job) ──────────────────────────────────── + + +class RuntimeInstallBody(BaseModel): + backend: Optional[str] = None # None/auto -> detect + + +@router.post("/api/local-models/runtime/install") +async def local_models_runtime_install(body: RuntimeInstallBody): + from hermes_cli.local_runtime.binaries import ( + default_tag, + resolve_assets, + select_backend, + ) + from hermes_cli.local_runtime.bootstrap import _detect_gpu_vendor + + section = _runtime_section() + tag = section.get("tag") or default_tag() + backend = body.backend or section.get("backend", "auto") + if backend == "auto": + backend = select_backend(_detect_gpu_vendor()) + # Resolve first so an impossible combination fails the POST, not the job. + try: + plan = resolve_assets(tag, backend) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=400, detail=str(exc)) + + job = _job("runtime-install", f"llama.cpp {tag} ({backend})") + + def _run(): + try: + from hermes_cli.local_runtime.binaries import ( + ensure_runtime_installed, + installed_tags, + prune_old_tags, + ) + + previous = installed_tags() + job["phase"] = "downloading" + job["detail"] = f"Fetching {len(plan.assets)} package(s) for {backend}" + ensure_runtime_installed(tag, backend) + + # Engine update path: a server already running on an older tag + # moves to the new one now — the click was the consent. Fresh + # installs (no server) skip this; Use/boot handles their start. + restarted = False + try: + from hermes_cli.local_runtime.bootstrap import ( + ensure_local_runtime, + get_supervisor, + shutdown_local_runtime, + ) + + sup = get_supervisor() + if sup is not None and previous and tag not in previous: + job["phase"] = "restarting" + job["detail"] = "Switching the running server to the new build" + shutdown_local_runtime() + ensure_local_runtime(_load_config(), force=True) + restarted = True + except Exception as exc: # noqa: BLE001 + # The new build is installed either way; the next boot serves + # it. Never fail the job on the restart nicety. + logger.warning("post-update restart skipped: %s", exc) + + # N-1 retention, only after the new tag verified: keep it and the + # newest previous build as the rollback pin target. + try: + keep = [tag] + [t for t in previous if t != tag][:1] + prune_old_tags(keep) + except Exception as exc: # noqa: BLE001 + logger.warning("runtime prune skipped: %s", exc) + + job["phase"] = "done" + job["status"] = "done" + job["detail"] = (f"llama.cpp {tag} ready ({backend})" + + (" — server restarted on the new build" if restarted else "")) + except Exception as exc: # noqa: BLE001 + logger.warning("runtime install failed: %s", exc) + job["status"] = "error" + job["error"] = str(exc) + + threading.Thread(target=_run, daemon=True, name="lr-runtime-install").start() + return {"job_id": job["job_id"], "backend": backend, "tag": tag} + + +# ── model download (job with byte progress + sha256) ───────── + + +class ModelDownloadBody(BaseModel): + model_id: str + + +@router.post("/api/local-models/download") +async def local_models_download(body: ModelDownloadBody): + """Accepts either a family id (downloads this machine's selected + variant) or an exact variant model_id.""" + from hermes_cli.local_runtime.catalog import ( + CATALOG, + catalog_by_id, + select_variant, + ) + from hermes_cli.local_runtime.hardware import probe_budget + + entry = catalog_by_id().get(body.model_id) + variant = None + if entry is not None: + # Same planning budget as the catalog — the user downloads exactly + # the build the row advertised. + choice = select_variant(entry, probe_budget(planning=True)) + if choice is None: + raise HTTPException(status_code=409, + detail=f"no variant of {entry.id} fits this machine") + variant = choice.variant + else: + for candidate in CATALOG: + for v in candidate.variants: + if v.model_id == body.model_id: + entry, variant = candidate, v + break + if variant: + break + if entry is None or variant is None: + raise HTTPException(status_code=404, detail=f"unknown model {body.model_id}") + + from hermes_cli.local_runtime.bootstrap import assets_dir, staged_model_ids + + if variant.model_id in staged_model_ids(): + return {"job_id": None, "already_downloaded": True, "model_id": variant.model_id} + + # Everything this variant needs: split parts + mmproj/draft assets. + plan = [] # (url, dest, sha256, bytes) + for asset in variant.files: + plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}", + _models_dir() / asset.local_name, asset.sha256, asset.size_bytes)) + for asset in (entry.mmproj, entry.draft): + if asset is not None: + plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}", + assets_dir() / asset.local_name, asset.sha256, asset.size_bytes)) + + total = sum(p[3] for p in plan) + job = _job("model-download", f"{entry.display_name} ({variant.quant})", + model_id=entry.id) + job["total_bytes"] = total + + def _run(): + try: + job["phase"] = "downloading" + job["detail"] = f"{entry.display_name} — {_human_gb(total)}" + done_before = 0 + for url, dest, sha, size in plan: + if dest.exists(): + done_before += size + job["done_bytes"] = done_before + continue + download_file(url, dest, job, expected_sha256=sha, + base_done=done_before, keep_totals=True) + job["phase"] = "downloading" + done_before += size + job["done_bytes"] = done_before + job["phase"] = "done" + job["status"] = "done" + job["detail"] = f"{entry.display_name} ready" + # A running router only scans models at spawn — + # bounce it so the new model is servable + # immediately instead of 400ing until the next app restart. + try: + from hermes_cli.local_runtime.bootstrap import refresh_local_runtime + + refresh_local_runtime() + except Exception: # noqa: BLE001 + logger.debug("post-download runtime refresh skipped", exc_info=True) + except Exception as exc: # noqa: BLE001 + logger.warning("model download failed: %s", exc) + job["status"] = "error" + job["error"] = str(exc) + + threading.Thread(target=_run, daemon=True, name="lr-model-download").start() + return {"job_id": job["job_id"], "model_id": variant.model_id} + + +@router.delete("/api/local-models/models/{model_id}") +async def local_models_delete(model_id: str): + """Remove a staged model: every split part plus its private assets. + A running router keeps serving from its spawn-time scan, so bounce it + off the request thread — deleting the active file mid-serve is the + kind of stale state the refresh exists for.""" + files = _variant_files_on_disk(model_id) + if not files: + raise HTTPException(status_code=404, detail="model not found") + for path in files: + path.unlink(missing_ok=True) + # Growth state dies with the model: a re-download starts back at its + # zero-spill window instead of inheriting a stale grown one. + try: + from hermes_cli.local_runtime.growth import clear_window_override + + clear_window_override(model_id) + except Exception: # noqa: BLE001 + logger.debug("window-override clear skipped", exc_info=True) + + def _refresh(): + try: + from hermes_cli.local_runtime.bootstrap import refresh_local_runtime + + refresh_local_runtime() + except Exception: # noqa: BLE001 + logger.debug("post-delete runtime refresh skipped", exc_info=True) + + threading.Thread(target=_refresh, daemon=True, name="lr-post-delete").start() + return {"ok": True} + + +# ── server lifecycle: turn the engine on/off ───────────────── + + +class ServerActionBody(BaseModel): + action: str # "stop" | "start" + + +@router.post("/api/local-models/server") +async def local_models_server(body: ServerActionBody): + """Turn the local engine off (stop the server, free ALL GPU memory, + and disable auto-start) or back on. The off switch is the whole-engine + counterpart of per-model eject — and unlike eject it IS durable: the + user said off, so boots stay off until they say on.""" + import asyncio + + from hermes_cli.config import load_config, save_config + + action = (body.action or "").strip().lower() + if action not in ("stop", "start"): + raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'") + + def _stop(): + from hermes_cli.local_runtime.bootstrap import ( + get_supervisor, + shutdown_local_runtime, + ) + + sup = get_supervisor() + if sup is not None: + shutdown_local_runtime() + else: + # Server owned by another process (or an orphan): best-effort + # terminate via the state file's pid, then clear the state. + endpoint = _state_endpoint() + if endpoint is not None: + try: + import psutil # type: ignore + + from hermes_cli.local_runtime.supervisor import state_path + + state = json.loads(state_path().read_text(encoding="utf-8")) + pid = int(state.get("pid") or 0) + if pid > 0 and psutil.pid_exists(pid): + psutil.Process(pid).terminate() + state_path().unlink(missing_ok=True) + except Exception: # noqa: BLE001 + pass + config = load_config() + config.setdefault("local_runtime", {})["enabled"] = False + save_config(config) + + def _start(): + from hermes_cli.local_runtime.bootstrap import ensure_local_runtime + + config = load_config() + config.setdefault("local_runtime", {})["enabled"] = True + save_config(config) + sup = ensure_local_runtime(config, force=True) + if sup is None and _state_endpoint() is None: + raise RuntimeError("The local server could not start — check the " + "runtime is installed") + + try: + await asyncio.to_thread(_stop if action == "stop" else _start) + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=502, detail=str(exc)) from exc + return {"ok": True, "action": action} + + +# ── activate: make a downloaded model THE model ────────────── + + +class ModelEjectBody(BaseModel): + model_id: str + + +@router.post("/api/local-models/eject") +async def local_models_eject(body: ModelEjectBody): + """Free a loaded model's GPU memory now. Nothing reloads it except + demand — the next message to it (residency v2: no automatic loading + exists anywhere).""" + from hermes_cli.local_runtime.bootstrap import get_supervisor + + sup = get_supervisor() + if sup is not None: + try: + sup.unload_model(body.model_id) + return {"ok": True} + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=502, detail=str(exc)) from exc + + # Server owned by another process (or state-file only): drive the + # router directly with the persisted endpoint. + endpoint = _state_endpoint() + if endpoint is None: + raise HTTPException(status_code=409, detail="local server is not running") + try: + import urllib.request as _url + + req = _url.Request( + endpoint["base_url"].rsplit("/v1", 1)[0] + "/models/unload", + data=json.dumps({"model": body.model_id}).encode(), + headers={"Content-Type": "application/json", + "Authorization": f"Bearer {endpoint.get('api_key', '')}"}, + method="POST") + with _url.urlopen(req, timeout=120): + pass + return {"ok": True} + except Exception as exc: # noqa: BLE001 + raise HTTPException(status_code=502, detail=str(exc)) from exc + + +class ModelActivateBody(BaseModel): + model_id: str # exact variant id (a staged .gguf stem) + + +@router.post("/api/local-models/activate") +async def local_models_activate(body: ModelActivateBody): + """Make a downloaded model the default for new chats. Pure selection + (residency v2): a config write through the same machinery as + /api/model/set, plus making sure the server is up. NO model loading — + models load on first inference, always; an empty router costs nothing. + Fast enough to be synchronous-feeling, but kept as a job for UI + continuity.""" + # Split variants stage under their first part — resolve like the rest + # of the routes instead of assuming a single flat file. + from hermes_cli.local_runtime.bootstrap import staged_model_ids + + if body.model_id not in staged_model_ids(): + raise HTTPException(status_code=404, detail=f"{body.model_id} is not downloaded") + + job = _job("model-activate", body.model_id, model_id=body.model_id) + + def _run(): + try: + from hermes_cli.config import load_config, save_config + from hermes_cli.local_runtime.bootstrap import ( + ensure_local_runtime, + refresh_local_runtime, + ) + + job["phase"] = "starting-server" + job["detail"] = "Starting the local server" + config = load_config() + sup = ensure_local_runtime(config, force=True) + if sup is None: + from hermes_cli.local_runtime.endpoint import _state_endpoint as _se + + if _se() is None: + raise RuntimeError( + "The local server could not start — check the runtime is installed") + + # Self-heal a stale router: the model list is spawn-only, so a + # server started before this model finished downloading can't + # serve it. If the router doesn't know the model, bounce it. + if sup is not None: + try: + if body.model_id not in sup.models(): + job["detail"] = "Refreshing the local server" + refresh_local_runtime() + except Exception: # noqa: BLE001 + logger.debug("activate rescan check skipped", exc_info=True) + + job["phase"] = "setting-default" + job["detail"] = "Making it your default" + config = load_config() + config.setdefault("local_runtime", {})["enabled"] = True + save_config(config) + from hermes_cli.web_deps import late + + late("_apply_model_assignment_sync")( + "main", "llamacpp", body.model_id, "", "", "") + + job["phase"] = "done" + job["status"] = "done" + job["detail"] = f"{body.model_id} is the default for new chats" + except Exception as exc: # noqa: BLE001 + logger.warning("model activate failed: %s", exc) + job["status"] = "error" + job["error"] = str(exc) + + threading.Thread(target=_run, daemon=True, name="lr-model-activate").start() + return {"job_id": job["job_id"]} + + +# ── job polling ────────────────────────────────────────────── + + +@router.get("/api/local-models/jobs") +async def local_models_jobs(): + """All recent jobs, running first — the pane and the app-level poller + rediscover in-flight work here after a remount or app restart.""" + with _JOBS_LOCK: + jobs = sorted(_JOBS.values(), + key=lambda j: (j["status"] != "running", -j["started_at"])) + out = [] + for job in jobs[:20]: + entry = dict(job) + if entry["total_bytes"]: + entry["percent"] = min(100, round(entry["done_bytes"] / entry["total_bytes"] * 100)) + out.append(entry) + return {"jobs": out} + + +@router.get("/api/local-models/jobs/{job_id}") +async def local_models_job(job_id: str): + with _JOBS_LOCK: + job = _JOBS.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail="job not found") + out = dict(job) + if out["total_bytes"]: + out["percent"] = min(100, round(out["done_bytes"] / out["total_bytes"] * 100)) + return out diff --git a/tests/hermes_cli/test_catalog_reachability.py b/tests/hermes_cli/test_catalog_reachability.py new file mode 100644 index 0000000000000..defb88f62b146 --- /dev/null +++ b/tests/hermes_cli/test_catalog_reachability.py @@ -0,0 +1,50 @@ +"""Catalog integrity: every entry's files must exist on the host with +matching sha256s (HF LFS oids ARE the file hashes). + +Network-marked (skipped in hermetic CI unless explicitly enabled) — this is +the test that catches wrong repo names (the Nemotron 401), moved files, and +upstream re-uploads before a user's download does. Run before any catalog +commit: + + HERMES_TEST_NETWORK=1 scripts/run_tests.sh tests/hermes_cli/test_catalog_reachability.py +""" + +from __future__ import annotations + +import json +import os +import urllib.request + +import pytest + +pytestmark = pytest.mark.skipif( + not os.environ.get("HERMES_TEST_NETWORK"), + reason="network test; set HERMES_TEST_NETWORK=1 to run", +) + + +def test_every_catalog_file_resolves(): + from hermes_cli.local_runtime.catalog import CATALOG + + problems = [] + for entry in CATALOG: + url = f"https://huggingface.co/api/models/{entry.repo}/tree/main?recursive=true" + try: + with urllib.request.urlopen(url, timeout=30) as r: + files = {f["path"]: f.get("lfs") or {} for f in json.load(r)} + except Exception as exc: # noqa: BLE001 + problems.append(f"{entry.id}: repo {entry.repo} unreachable ({exc})") + continue + for variant in entry.variants: + for asset in entry.download_files(variant): + if asset.path not in files: + problems.append( + f"{entry.id}/{variant.quant}: {asset.path} not in {entry.repo}") + continue + live_sha = files[asset.path].get("oid", "") + if live_sha and live_sha != asset.sha256: + problems.append( + f"{entry.id}/{variant.quant}: sha drift on {asset.path} — " + f"catalog {asset.sha256[:12]} vs live {live_sha[:12]} " + f"(upstream re-uploaded; re-pin deliberately)") + assert not problems, "\n".join(problems) diff --git a/tests/hermes_cli/test_catalog_variants.py b/tests/hermes_cli/test_catalog_variants.py new file mode 100644 index 0000000000000..6b0ca803c148d --- /dev/null +++ b/tests/hermes_cli/test_catalog_variants.py @@ -0,0 +1,129 @@ +"""Variant-selection contracts: the quant ladder picks the best build for +the hardware, per the design's 'offer a smaller quant' remedy run +proactively, bounded by the Q4 quality floor. Pure decision-table tests +over synthetic budgets.""" + +from __future__ import annotations + +import pytest + +from hermes_cli.local_runtime.catalog import ( + CATALOG, + catalog_by_id, + find_entry_for_model, + select_variant, +) +from hermes_cli.local_runtime.estimator import HardwareBudget + +GIB = 1 << 30 + + +def budget(vram_gib: float, ram_gib: float = 64) -> HardwareBudget: + return HardwareBudget(usable_vram_bytes=int(vram_gib * GIB), + total_device_bytes=int(vram_gib * GIB), + ram_available_bytes=int(ram_gib * GIB)) + + +def test_variants_ordered_best_first_and_floor_is_q4(): + for entry in CATALOG: + sizes = [v.size_bytes for v in entry.variants] + assert sizes == sorted(sizes, reverse=True), f"{entry.id}: variants not best-first" + for v in entry.variants: + for asset in entry.download_files(v): + assert len(asset.sha256) == 64, f"{entry.id}/{v.quant}: unpinned {asset.path}" + # The quality floor: the ladder ends at Q4 — no Q3/Q2 builds ship + # (product decision, 2026-08-09). Every floor rung is either + # validated on real hardware or an explicit day-0 entry. + floor = entry.variants[-1] + assert floor.quant == "UD-Q4_K_XL", f"{entry.id}: ladder floor is {floor.quant}, not Q4" + assert floor.validated or "day-0" in entry.tags, ( + f"{entry.id}: floor rung neither validated nor tagged day-0") + + +def test_split_variants_have_coherent_parts(): + """Multi-file variants: same model_id from every part, exact sizes, + first file is the load target.""" + entry = catalog_by_id()["deepseek-v4-flash"] + for v in entry.variants: + assert len(v.files) >= 2, "deepseek ships split GGUFs" + assert "00001-of" in v.files[0].path, "first part must be the load target" + assert v.size_bytes == sum(f.size_bytes for f in v.files) + assert entry.draft is not None, "DSpark draft rides along" + + +def test_big_card_gets_best_quality(): + """A card with headroom takes the top rung — quality is free when it fits.""" + entry = catalog_by_id()["muse-glimmer-30b"] + choice = select_variant(entry, budget(48)) + assert choice is not None + assert choice.zero_spill + assert choice.variant.quant == entry.variants[0].quant # UD-Q8_K_XL + + +def test_quality_monotone_in_vram(): + """More VRAM never selects a smaller build.""" + entry = catalog_by_id()["qwen3.6-27b"] + sizes = [] + for vram in (8, 12, 16, 24, 32, 48): + choice = select_variant(entry, budget(vram)) + assert choice is not None + sizes.append(choice.variant.size_bytes) + assert sizes == sorted(sizes), f"quality not monotone in VRAM: {sizes}" + + +def test_small_card_gets_q4_spilled_never_below(): + """8 GiB card + 27B: nothing zero-spills. The floor holds — the + selector offers Q4 spilled (priced honestly), never a sub-Q4 build.""" + entry = catalog_by_id()["qwen3.6-27b"] + choice = select_variant(entry, budget(8)) + assert choice is not None + assert not choice.zero_spill + assert choice.reason_key == "smallest-fits-spilled" + assert choice.variant.quant == "UD-Q4_K_XL" + + +def test_frontier_model_refused_on_consumer_card_offered_on_big_ram(): + """DeepSeek V4 Flash (161 GB at Q4): refused outright on a 32 GiB-RAM + desktop; offered spilled on a 192 GiB-RAM workstation. The catalog + carries frontier hardware honestly instead of hiding the model.""" + entry = catalog_by_id()["deepseek-v4-flash"] + assert select_variant(entry, budget(32, ram_gib=32)) is None + big = select_variant(entry, budget(32, ram_gib=192)) + assert big is not None and not big.zero_spill + + +def test_selection_accounts_for_kv_not_just_weights(): + """The zero-spill check prices weights + 64K-floor KV, not weights + alone: give a machine exactly enough VRAM for the Q8 weights of a + dense model and it must step down a rung.""" + entry = catalog_by_id()["muse-glimmer-30b"] + q8 = entry.variants[0] + exactly_weights = HardwareBudget( + usable_vram_bytes=q8.size_bytes + (100 << 20), + total_device_bytes=q8.size_bytes + (100 << 20), + ram_available_bytes=64 * GIB) + choice = select_variant(entry, exactly_weights) + assert choice is not None + assert choice.variant.quant != q8.quant, "KV cost ignored — Q8 can't fit with floor KV" + + +def test_find_entry_for_model_resolves_split_ids(): + hit = find_entry_for_model("DeepSeek-V4-Flash-0731-UD-Q4_K_XL") + assert hit is not None + entry, variant = hit + assert entry.id == "deepseek-v4-flash" + assert variant.quant == "UD-Q4_K_XL" + + +def test_hybrid_long_context_stays_cheap(): + """The reason Nemotron/Qwen3.6 headline the catalog: their priced + 64K-floor KV must be a small fraction of a dense model's.""" + from hermes_cli.local_runtime.catalog import FLOOR + from hermes_cli.local_runtime.estimator import ctx_bytes + + dense = catalog_by_id()["muse-glimmer-30b"] + hybrid = catalog_by_id()["nemotron-3.5-lightning-30b"] + dense_kv = ctx_bytes(dense.profile(dense.variants[-1]), FLOOR) + hybrid_kv = ctx_bytes(hybrid.profile(hybrid.variants[-1]), FLOOR) + assert hybrid_kv * 5 < dense_kv, ( + f"hybrid KV ({hybrid_kv:,}) should be >5x cheaper than dense ({dense_kv:,})") diff --git a/tests/hermes_cli/test_local_models_routes.py b/tests/hermes_cli/test_local_models_routes.py new file mode 100644 index 0000000000000..6eb5217c41d00 --- /dev/null +++ b/tests/hermes_cli/test_local_models_routes.py @@ -0,0 +1,209 @@ +"""Contract tests for the local-models dashboard routes (Rollout 4). + +Real FastAPI TestClient against the real router; the runtime pieces +underneath are exercised against temp HERMES_HOME (autouse fixture). Network +downloads are stubbed at the urllib boundary — never live.""" + +from __future__ import annotations + +import io +import json +import time +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture +def client(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + from hermes_cli import web_server + + test_client = TestClient(web_server.app) + # Same auth pattern as the git-route tests: present the session token. + test_client.headers[web_server._SESSION_HEADER_NAME] = web_server._SESSION_TOKEN + return test_client + + +def test_local_models_routes_require_auth(tmp_path, monkeypatch): + monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes")) + from hermes_cli import web_server + + unauth = TestClient(web_server.app) + assert unauth.get("/api/local-models/status").status_code == 401 + + +def _write_fake_gguf(path: Path, size: int = 1024) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"GGUF" + b"\x00" * size) + + +# ── status ─────────────────────────────────────────────────── + + +def test_status_shape_and_defaults(client): + r = client.get("/api/local-models/status") + assert r.status_code == 200 + data = r.json() + # Contract: every key the pane's first paint needs, present and typed. + assert isinstance(data["enabled"], bool) + assert isinstance(data["tag"], str) and data["tag"].startswith("b") + assert isinstance(data["runtime_installed"], bool) + assert isinstance(data["server_running"], bool) + assert isinstance(data["models"], list) + + +def test_status_lists_staged_models_with_labels(client, tmp_path): + from hermes_cli.local_runtime.bootstrap import models_dir + + _write_fake_gguf(models_dir() / "Some-Model.gguf", size=2048) + data = client.get("/api/local-models/status").json() + ids = [m["id"] for m in data["models"]] + assert "Some-Model" in ids + row = data["models"][ids.index("Some-Model")] + assert row["size_bytes"] > 0 + assert row["size_label"].endswith("GB") + + +# ── hardware ───────────────────────────────────────────────── + + +def test_hardware_plain_facts(client): + data = client.get("/api/local-models/hardware").json() + assert isinstance(data["uma"], bool) + assert data["ram_total_bytes"] > 0 + assert data["vram_total_bytes"] >= 0 + # GPU fields are None-able (non-NVIDIA machines) but must exist. + assert "gpu_name" in data and "gpu_util_percent" in data and "vram_used_bytes" in data + + +# ── catalog ────────────────────────────────────────────────── + + +def test_catalog_prices_every_entry_for_this_machine(client): + data = client.get("/api/local-models/catalog").json() + assert len(data["models"]) >= 3 + for row in data["models"]: + # The three user questions, answered on every row: + assert row["size_label"].endswith("GB") # how big + assert isinstance(row["fits"], bool) # will it fit + assert row["fit_summary"] # what shape + if row["fits"]: + assert row["start_window"] >= 1 + assert row["start_window_label"].endswith("K") + else: + assert "memory" in row["fit_summary"].lower() + assert isinstance(row["downloaded"], bool) + + +def test_catalog_never_hides_unaffordable_models(client, monkeypatch): + """Unaffordable entries stay visible with a plain reason — hiding them + is how users conclude the feature is broken.""" + from hermes_cli.local_runtime.estimator import HardwareBudget + + tiny = HardwareBudget(usable_vram_bytes=1 << 30, total_device_bytes=1 << 30, + ram_available_bytes=1 << 30) + monkeypatch.setattr("hermes_cli.local_runtime.hardware.probe_budget", + lambda **kw: tiny) + data = client.get("/api/local-models/catalog").json() + from hermes_cli.local_runtime.catalog import CATALOG + + assert len(data["models"]) == len(CATALOG) + refused = [m for m in data["models"] if not m["fits"]] + assert refused, "a 1 GiB machine must refuse the 20 GB models" + for row in refused: + assert row["fit_detail"] or row["fit_summary"] + + +# ── downloads ──────────────────────────────────────────────── + + +def test_download_unknown_model_404s(client): + r = client.post("/api/local-models/download", json={"model_id": "nope"}) + assert r.status_code == 404 + + +def test_download_job_lifecycle_with_sha_failure(client, monkeypatch): + """Wrong-hash download must delete the file and error the job with a + human-readable message — never leave a corrupt GGUF staged.""" + + class FakeResponse(io.BytesIO): + headers = {"Content-Length": "16"} + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + monkeypatch.setattr("urllib.request.urlopen", + lambda *a, **k: FakeResponse(b"not the real body")) + + from hermes_cli.local_runtime.catalog import CATALOG + + entry_id = CATALOG[0].id + r = client.post("/api/local-models/download", json={"model_id": entry_id}) + assert r.status_code == 200 + job_id = r.json()["job_id"] + assert job_id + + deadline = time.time() + 10 + status = None + while time.time() < deadline: + status = client.get(f"/api/local-models/jobs/{job_id}").json() + if status["status"] in ("done", "error"): + break + time.sleep(0.05) + assert status is not None and status["status"] == "error" + assert "integrity" in status["error"].lower() + + from hermes_cli.local_runtime.bootstrap import models_dir + + assert not (models_dir() / f"{entry_id}.gguf").exists() + assert not (models_dir() / f"{entry_id}.part").exists() + + +def test_download_already_downloaded_short_circuits(client, monkeypatch): + from hermes_cli.local_runtime.bootstrap import models_dir + from hermes_cli.local_runtime.catalog import CATALOG, select_variant + from hermes_cli.local_runtime.estimator import HardwareBudget + + # Pin the budget so the selected variant is deterministic in the test. + budget = HardwareBudget(usable_vram_bytes=64 << 30, total_device_bytes=64 << 30, + ram_available_bytes=64 << 30) + monkeypatch.setattr("hermes_cli.local_runtime.hardware.probe_budget", + lambda **kw: budget) + choice = select_variant(CATALOG[0], budget) + assert choice is not None + _write_fake_gguf(models_dir() / choice.variant.files[0].local_name) + r = client.post("/api/local-models/download", json={"model_id": CATALOG[0].id}) + assert r.status_code == 200 + assert r.json()["already_downloaded"] is True + + +def test_delete_model(client): + from hermes_cli.local_runtime.bootstrap import models_dir + + _write_fake_gguf(models_dir() / "Doomed.gguf") + assert client.delete("/api/local-models/models/Doomed").status_code == 200 + assert not (models_dir() / "Doomed.gguf").exists() + assert client.delete("/api/local-models/models/Doomed").status_code == 404 + + +# ── runtime install ────────────────────────────────────────── + + +def test_runtime_install_rejects_impossible_combo(client, monkeypatch): + """Impossible platform/backend combos fail the POST itself with the + resolver's honest message — not a background job that dies silently. + (win-arm64-vulkan; the old cuda case became real upstream at ~b1036x.)""" + monkeypatch.setattr( + "hermes_cli.local_runtime.binaries._host_os_arch", lambda: ("win", "arm64")) + r = client.post("/api/local-models/runtime/install", json={"backend": "vulkan"}) + assert r.status_code == 400 + assert "arm64" in r.json()["detail"] + + +def test_job_poll_unknown_404s(client): + assert client.get("/api/local-models/jobs/deadbeef").status_code == 404 diff --git a/tests/hermes_cli/test_local_runtime_picker_row.py b/tests/hermes_cli/test_local_runtime_picker_row.py new file mode 100644 index 0000000000000..d906ba7d1457b --- /dev/null +++ b/tests/hermes_cli/test_local_runtime_picker_row.py @@ -0,0 +1,67 @@ +"""The llamacpp provider row in the model picker payload (Rollout 4). + +Contract: staged local GGUFs appear as a selectable provider row in +build_models_payload — the same payload /api/model/options and the desktop +picker consume — whenever models are staged, without any credential.""" + +from __future__ import annotations + +import pytest + + +@pytest.fixture +def hermes_home(tmp_path, monkeypatch): + home = tmp_path / ".hermes" + home.mkdir() + monkeypatch.setenv("HERMES_HOME", str(home)) + return home + + +def _stage(home, *names): + mdir = home / "models" + mdir.mkdir(exist_ok=True) + for name in names: + (mdir / f"{name}.gguf").write_bytes(b"GGUF" + b"\x00" * 64) + + +def test_no_staged_models_no_row(hermes_home): + from hermes_cli.inventory import _local_runtime_row, load_picker_context + + assert _local_runtime_row(load_picker_context()) is None + + +def test_staged_models_make_a_selectable_row(hermes_home): + from hermes_cli.inventory import _local_runtime_row, load_picker_context + + _stage(hermes_home, "Qwen3-4B-Instruct-2507-UD-Q8_K_XL", "Some-Other-Model") + row = _local_runtime_row(load_picker_context()) + assert row is not None + assert row["slug"] == "llamacpp" + assert row["authenticated"] is True + assert "Qwen3-4B-Instruct-2507-UD-Q8_K_XL" in row["models"] + assert row["total_models"] == 2 + + +def test_row_marks_current_when_config_points_at_llamacpp(hermes_home): + from hermes_cli.inventory import _local_runtime_row, load_picker_context + + _stage(hermes_home, "M") + ctx = load_picker_context().with_overrides(current_provider="llamacpp") + row = _local_runtime_row(ctx) + assert row is not None and row["is_current"] is True + + +def test_full_payload_includes_local_row(hermes_home): + """Through the REAL payload builder — the shape the desktop picker eats.""" + from hermes_cli.inventory import build_models_payload, load_picker_context + + _stage(hermes_home, "Local-Model-X") + payload = build_models_payload( + load_picker_context(), + probe_custom_providers=False, + probe_current_custom_provider=False, + ) + slugs = [p["slug"] for p in payload["providers"]] + assert "llamacpp" in slugs + row = payload["providers"][slugs.index("llamacpp")] + assert row["models"] == ["Local-Model-X"]