feat(local-runtime): model catalog and dashboard routes
A curated model catalog priced for the machine it's viewed on, and the /api/local-models/* routes the desktop consumes. - catalog: each model ships a quant ladder (Q8 down to Q4, best first) with exact sizes and sha256s pinned from Hugging Face LFS metadata. Selection picks the highest-quality build whose weights + 64K-floor KV fit GPU memory entirely; machines that can't get Q4 spilled to system RAM; refusal only when even Q4 exceeds GPU + RAM. Nothing below Q4 ships — the quality loss is too severe for a first local-AI experience. Split-GGUF variants, vision projectors, and spec-decode draft models download as a unit, each file verified. - routes: status (cheap, poll-safe), hardware, catalog (each row carries plain-language fit facts the UI shows verbatim), download jobs with aggregate byte progress that survive the pane unmounting, activate (start server, set as main model — the click is the opt-in), eject, delete (removes every staged file), and server on/off. Downloads run 8 parallel ranged connections and verify sha256 before use; a running router only scans models at spawn, so staging changes bounce it. - inventory: staged local models appear as a provider row in the model picker payload every surface consumes; no credential — a local server is authenticated by reachability. Catalog reachability (repos, filenames, live-sha drift) is covered by an opt-in network test gated on HERMES_TEST_NETWORK=1.
This commit is contained in:
emozilla
parent
b95e38757a
commit
aa4bf1ec2b
|
|
@ -200,6 +200,17 @@ def build_models_payload(
|
|||
excluded_providers=ctx.excluded_providers or [],
|
||||
)
|
||||
|
||||
# Managed local runtime: staged GGUFs are selectable like any provider's
|
||||
# models. list_authenticated_providers can't know about them (no
|
||||
# credential, no custom_providers entry — the credential is
|
||||
# reachability), so inject the row here where every picker surface
|
||||
# inherits it. Present whenever models are staged; picking one routes
|
||||
# through the llamacpp alias -> managed/detected server resolution.
|
||||
local_row = _local_runtime_row(ctx)
|
||||
if local_row is not None:
|
||||
rows = [r for r in rows if str(r.get("slug", "")).lower() != "llamacpp"]
|
||||
rows.append(local_row)
|
||||
|
||||
moa_row = _moa_provider_row(ctx.current_provider)
|
||||
if moa_row is not None:
|
||||
rows = [moa_row] + [r for r in rows if str(r.get("slug", "")).lower() != "moa"]
|
||||
|
|
@ -825,6 +836,54 @@ def _apply_pricing(
|
|||
row["unavailable_models"] = []
|
||||
|
||||
|
||||
def _local_runtime_row(ctx: "ConfigContext") -> dict | None:
|
||||
"""Build the ``llamacpp`` provider row from staged local models.
|
||||
|
||||
Present whenever GGUFs are staged in the managed models directory —
|
||||
downloaded models must be selectable even before the server is running
|
||||
(selection starts it via the runtime_provider seam / activate flow).
|
||||
Returns ``None`` when nothing is staged.
|
||||
"""
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
||||
|
||||
staged = staged_model_ids()
|
||||
if not staged:
|
||||
return None
|
||||
current = (ctx.current_provider or "").strip().lower() in (
|
||||
"llamacpp", "llama.cpp", "llama-cpp")
|
||||
if not current:
|
||||
# A LIVE session on the managed server reports provider "custom"
|
||||
# (the resolution seam's label) with the managed base_url. Match
|
||||
# on the endpoint so the picker still marks this row current —
|
||||
# otherwise the session the user is chatting in shows no
|
||||
# selection.
|
||||
try:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
managed = _state_endpoint()
|
||||
current = bool(
|
||||
managed
|
||||
and (ctx.current_base_url or "").strip().rstrip("/")
|
||||
== managed["base_url"].rstrip("/"))
|
||||
except Exception:
|
||||
current = False
|
||||
return {
|
||||
"slug": "llamacpp",
|
||||
"name": "Local (llama.cpp)",
|
||||
"is_current": current,
|
||||
"is_user_defined": False,
|
||||
"models": staged,
|
||||
"total_models": len(staged),
|
||||
"source": "local-runtime",
|
||||
"authenticated": True, # the credential is reachability
|
||||
"auth_type": "local",
|
||||
"warning": None,
|
||||
}
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _moa_provider_row(current_provider: str = "") -> dict | None:
|
||||
"""Build the virtual ``moa`` provider row for model pickers.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,378 @@
|
|||
"""Curated starter catalog for the managed local runtime.
|
||||
|
||||
Small and honest: every entry carries the estimator inputs (measured on
|
||||
real GGUFs) so the picker can price a model BEFORE the user downloads
|
||||
gigabytes. Once a file is on disk, profile_from_gguf() is the authority
|
||||
and the catalog numbers are only used for the download decision. Entries
|
||||
whose base config is gated upstream carry a same-family conservative
|
||||
prior (commented) — the GGUF header corrects it at load time.
|
||||
|
||||
Each model ships a QUANT LADDER (variants, best quality first). The ladder
|
||||
floors at UD-Q4_K_XL — below Q4 the quality loss is too severe to ship as
|
||||
someone's first local-AI experience. Selection is hardware-aware: pick the
|
||||
highest-quality variant that zero-spills at the 64K floor on this machine;
|
||||
else Q4 spilled (priced honestly by the fit policy); refuse only when even
|
||||
Q4 fails the physics check.
|
||||
|
||||
Validation lifecycle: rungs proven end-to-end on real hardware are marked
|
||||
validated. Day-0 entries ship before that proof under the "day-0" tag —
|
||||
ensure_model_ready's touch generation still gates every first load at
|
||||
runtime. The contract test requires every ladder floor to be Q4 AND
|
||||
(validated OR day-0).
|
||||
|
||||
Multi-file models: variants may carry split-GGUF parts (llama-server loads
|
||||
from the first part; all parts download together, each sha-verified).
|
||||
Entries may carry an mmproj (vision projector) and a speculative-decode
|
||||
draft model — both download alongside the weights. Spec decode is enabled
|
||||
only when the launch decision spills, where its speedup is largest.
|
||||
|
||||
sha256s are pinned from HF LFS metadata (the lfs oid IS the file sha256),
|
||||
reviewed like a version bump — parsed data, never executed commands.
|
||||
|
||||
This is deliberately not a live registry feed: entries are reviewed like a
|
||||
version bump (the same policy governs vendor recipe ingestion — parsed
|
||||
data, never executed commands).
|
||||
|
||||
Vendor recipes overlay: a per-SKU recipes repo may SUPPLEMENT these
|
||||
entries where applicable — vendor SKUs only, never the base layer for
|
||||
other platforms. A recipe may enrich identity (GGUF/quant/sha), perf
|
||||
hints (-b/-ub, spec-decode), and sampling defaults; it never carries
|
||||
context/slots/placement/serving flags (the fit policy owns those).
|
||||
Resolution: exact SKU -> GPU-class bucket -> fit-only. Snapshot-synced,
|
||||
reviewed like a tag bump.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
from hermes_cli.local_runtime.context_policy import FLOOR
|
||||
from hermes_cli.local_runtime.estimator import (
|
||||
HardwareBudget,
|
||||
LayerKind,
|
||||
ModelProfile,
|
||||
ctx_bytes,
|
||||
)
|
||||
|
||||
_GIB = 1 << 30
|
||||
_PART_SUFFIX = re.compile(r"-\d{5}-of-\d{5}$")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class AssetFile:
|
||||
"""One downloadable file: repo-relative path, exact bytes, sha256.
|
||||
``local`` overrides the on-disk name (repos reuse generic names like
|
||||
mmproj-BF16.gguf across models). Non-model extras live under the
|
||||
models dir's assets/ subdirectory so the router never lists them."""
|
||||
|
||||
path: str # repo-relative (may include a subdir)
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
local: str | None = None
|
||||
|
||||
@property
|
||||
def local_name(self) -> str:
|
||||
return self.local or PurePosixPath(self.path).name
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuantVariant:
|
||||
"""One downloadable build of a model. Ordered best-quality-first in
|
||||
CatalogEntry.variants. Split GGUFs list every part in files; the model
|
||||
loads from the first part."""
|
||||
|
||||
quant: str # e.g. "UD-Q8_K_XL"
|
||||
files: tuple # AssetFile, first = the load target
|
||||
validated: bool = False # proven end-to-end on real hardware
|
||||
|
||||
@property
|
||||
def model_id(self) -> str:
|
||||
stem = PurePosixPath(self.files[0].path).name.removesuffix(".gguf")
|
||||
return _PART_SUFFIX.sub("", stem)
|
||||
|
||||
@property
|
||||
def size_bytes(self) -> int:
|
||||
return sum(f.size_bytes for f in self.files)
|
||||
|
||||
@property
|
||||
def weights_bytes(self) -> int:
|
||||
"""Pre-download weights estimate: GGUF bytes ≈ tensor bytes + a
|
||||
small header (<2%) — a safe, slightly conservative stand-in until
|
||||
profile_from_gguf reads the real table."""
|
||||
return self.size_bytes
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CatalogEntry:
|
||||
id: str # stable family id (variant-independent)
|
||||
display_name: str
|
||||
description: str # one line, plain language
|
||||
repo: str # HF repo
|
||||
variants: tuple # QuantVariant, best first
|
||||
# Estimator inputs (measured or config-derived; quant changes weights,
|
||||
# never KV). Entries with gated upstream configs carry a conservative
|
||||
# same-family prior — the GGUF header is the authority after download.
|
||||
n_ctx_train: int
|
||||
full_layers: int
|
||||
recurrent_layers: int
|
||||
per_layer_f16: int # KV bytes/token per full-attention layer
|
||||
swa_layers: int = 0
|
||||
swa_window: int = 0
|
||||
moe: bool = False
|
||||
mtp: bool = False # ships MTP heads (spec decode iff spilled)
|
||||
mmproj: "AssetFile | None" = None # vision projector, downloads with model
|
||||
draft: "AssetFile | None" = None # spec-decode draft model (e.g. DSpark)
|
||||
sampling: dict = field(default_factory=dict) # INI long-form launch defaults
|
||||
tags: tuple = field(default_factory=tuple)
|
||||
|
||||
def profile(self, variant: QuantVariant) -> ModelProfile:
|
||||
layers = ([(LayerKind.FULL, self.per_layer_f16)] * self.full_layers
|
||||
+ [(LayerKind.SWA, self.per_layer_f16)] * self.swa_layers
|
||||
+ [(LayerKind.RECURRENT, 0)] * self.recurrent_layers)
|
||||
return ModelProfile(
|
||||
name=variant.model_id, weights_bytes=variant.weights_bytes,
|
||||
embd_table_bytes=0, n_ctx_train=self.n_ctx_train,
|
||||
layers=layers, swa_window=self.swa_window, moe=self.moe)
|
||||
|
||||
def download_files(self, variant: QuantVariant) -> tuple:
|
||||
"""Everything a download job fetches for this variant, in order."""
|
||||
extras = tuple(a for a in (self.mmproj, self.draft) if a is not None)
|
||||
return tuple(variant.files) + extras
|
||||
|
||||
def download_bytes(self, variant: QuantVariant) -> int:
|
||||
return sum(f.size_bytes for f in self.download_files(variant))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VariantChoice:
|
||||
"""Selection result: which build this machine should download and why.
|
||||
reason_key is a UI-copy discriminator, not display text."""
|
||||
|
||||
variant: QuantVariant
|
||||
zero_spill: bool
|
||||
reason_key: str # "best-fits" | "smallest-fits-spilled"
|
||||
|
||||
|
||||
def select_variant(entry: CatalogEntry, budget: HardwareBudget) -> VariantChoice | None:
|
||||
"""Best build for this hardware, per the ladder policy.
|
||||
|
||||
1. Highest-quality variant whose weights + 64K-floor KV zero-spill in
|
||||
usable VRAM (quality is free when it fits — take it).
|
||||
2. Else Q4 spilled — the floor guarantee stands, fit spills weights,
|
||||
picker prices it honestly (slow beats lobotomized).
|
||||
3. None: even Q4 fails physics (true refusal).
|
||||
"""
|
||||
floor_kv = None
|
||||
for variant in entry.variants:
|
||||
profile = entry.profile(variant)
|
||||
if floor_kv is None:
|
||||
floor_kv = ctx_bytes(profile, min(FLOOR, entry.n_ctx_train or FLOOR))
|
||||
if variant.weights_bytes + floor_kv <= budget.usable_vram_bytes:
|
||||
return VariantChoice(variant=variant, zero_spill=True,
|
||||
reason_key="best-fits")
|
||||
|
||||
smallest = min(entry.variants, key=lambda v: v.size_bytes)
|
||||
needed = smallest.weights_bytes + (floor_kv or 0)
|
||||
if needed <= budget.usable_vram_bytes + budget.ram_available_bytes:
|
||||
return VariantChoice(variant=smallest, zero_spill=False,
|
||||
reason_key="smallest-fits-spilled")
|
||||
return None
|
||||
|
||||
|
||||
def _v(quant: str, *files, validated: bool = False) -> QuantVariant:
|
||||
return QuantVariant(quant=quant,
|
||||
files=tuple(AssetFile(*f) for f in files),
|
||||
validated=validated)
|
||||
|
||||
|
||||
# Ordered: recommended first. File bytes + sha256s pinned from HF LFS
|
||||
# metadata.
|
||||
CATALOG: tuple[CatalogEntry, ...] = (
|
||||
CatalogEntry(
|
||||
id="qwen3.6-27b",
|
||||
display_name="Qwen3.6 27B",
|
||||
description="Best all-round agent model; sees images; long context stays fast",
|
||||
repo="unsloth/Qwen3.6-27B-GGUF",
|
||||
variants=(
|
||||
_v("UD-Q8_K_XL",
|
||||
("Qwen3.6-27B-UD-Q8_K_XL.gguf", 35325163744,
|
||||
"19a2f4733a863088bc06665bf307dca95f7d4370b4d8690340cdff9992fe48c6")),
|
||||
_v("UD-Q6_K_XL",
|
||||
("Qwen3.6-27B-UD-Q6_K_XL.gguf", 25636485344,
|
||||
"8746881d40f280b1b6b858c656a347c754ed3d9cc8d2e1ad46b3635b87f611f8")),
|
||||
_v("UD-Q5_K_XL",
|
||||
("Qwen3.6-27B-UD-Q5_K_XL.gguf", 20038256864,
|
||||
"ac310abf2895aa397121bad6c0be89466af41f0f1606a21c1131b110eeb19d0e")),
|
||||
_v("UD-Q4_K_XL",
|
||||
("Qwen3.6-27B-UD-Q4_K_XL.gguf", 17612564704,
|
||||
"ff6941ded525b34eb159496762c29dd0ec6e71dc31b74d57e75d871a03eec259"),
|
||||
validated=True),
|
||||
),
|
||||
n_ctx_train=262144,
|
||||
full_layers=16, recurrent_layers=48, per_layer_f16=4096,
|
||||
mmproj=AssetFile("mmproj-BF16.gguf", 931146304,
|
||||
"05353347512982ee62317b9d8c89372bc815f4b4043580e7ef3ad411ec1a1cd3",
|
||||
local="mmproj-Qwen3.6-27B-BF16.gguf"),
|
||||
sampling={"temp": "1.0", "top-p": "0.95", "top-k": "20", "min-p": "0.0"},
|
||||
tags=("recommended", "hybrid", "reasoning", "vision"),
|
||||
),
|
||||
CatalogEntry(
|
||||
id="qwen3.6-35b-a3b",
|
||||
display_name="Qwen3.6 35B-A3B",
|
||||
description="Bigger mixture-of-experts with multi-token prediction; sees images",
|
||||
repo="unsloth/Qwen3.6-35B-A3B-MTP-GGUF",
|
||||
variants=(
|
||||
_v("UD-Q8_K_XL",
|
||||
("Qwen3.6-35B-A3B-UD-Q8_K_XL.gguf", 39099447584,
|
||||
"6c6b816537abad90b250a0972b345466028d861ddfe316d5f0de31ca6440f781")),
|
||||
_v("UD-Q6_K_XL",
|
||||
("Qwen3.6-35B-A3B-UD-Q6_K_XL.gguf", 32611711264,
|
||||
"35fce994cd36104a7dc1bd8a4bdf13778145664c00fdef6773aebc9246e5019c")),
|
||||
_v("UD-Q5_K_XL",
|
||||
("Qwen3.6-35B-A3B-UD-Q5_K_XL.gguf", 27159116064,
|
||||
"9de9a9420f61a0bb59bb2ca1ea170a6a57f6821fa1deec915bcaef523730a919")),
|
||||
# Validated on this repo's prior upload; upstream has since
|
||||
# re-uploaded. Same model id + pipeline — re-verify at the
|
||||
# next validation pass.
|
||||
_v("UD-Q4_K_XL",
|
||||
("Qwen3.6-35B-A3B-UD-Q4_K_XL.gguf", 22853663008,
|
||||
"55983c5a75a1ab969824077b3bb3de4146e82a9234072b48ad4e8f92ad3fe9f1"),
|
||||
validated=True),
|
||||
),
|
||||
n_ctx_train=262144,
|
||||
# Config-derived (Qwen/Qwen3.6-35B-A3B): 40 layers, 10 full-attn
|
||||
# (interval 4) + 30 linear; KV heads 2 x head_dim 256.
|
||||
full_layers=10, recurrent_layers=30, per_layer_f16=2048,
|
||||
moe=True, mtp=True,
|
||||
mmproj=AssetFile("mmproj-BF16.gguf", 902822528,
|
||||
"da63cb47a76763c712393f8a017070188a304fa39f8aeea6edc629ed7b975cfa",
|
||||
local="mmproj-Qwen3.6-35B-A3B-BF16.gguf"),
|
||||
sampling={"temp": "1.0", "top-p": "0.95", "top-k": "20", "min-p": "0.0"},
|
||||
tags=("hybrid", "moe", "mtp", "vision"),
|
||||
),
|
||||
CatalogEntry(
|
||||
id="nemotron-3.5-lightning-30b",
|
||||
display_name="Nemotron 3.5 Lightning 30B",
|
||||
description="NVIDIA's fast tool-calling model; 1M-token context",
|
||||
repo="unsloth/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-GGUF",
|
||||
variants=(
|
||||
_v("UD-Q8_K_XL",
|
||||
("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q8_K_XL.gguf", 38615380032,
|
||||
"48140cfb8bb6a38553275d334345f2638c119d979ab8b6d4afdc4b62d26d8219")),
|
||||
_v("UD-Q6_K_XL",
|
||||
("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q6_K_XL.gguf", 35004643392,
|
||||
"36b8b0f882ec739f895fe56ab6b9b892e702ba33b7b368614ca4078331ddfc29")),
|
||||
_v("UD-Q5_K_XL",
|
||||
("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q5_K_XL.gguf", 30414829632,
|
||||
"92dcaf682faf39fef906db7bad4000782001d089f28c4d6aecec6d6f1b4697ca")),
|
||||
_v("UD-Q4_K_XL",
|
||||
("NVIDIA-Nemotron-3.5-Lightning-30B-A3B-UD-Q4_K_XL.gguf", 25505724480,
|
||||
"112bf957489a497c18f60bf8bd44ee1dfa05e87368b8ed7f68998a4e38f275c9")),
|
||||
),
|
||||
n_ctx_train=1048576,
|
||||
# Prior from Nemotron-3-Nano (same hybrid family; base config gated
|
||||
# upstream): 6 full-attn + 46 recurrent, ~1 KiB/tok/full-layer.
|
||||
# Prior from the same hybrid family (measured: 1M ctx in ~3.2 GiB
|
||||
# KV on the predecessor). GGUF header is
|
||||
# the authority after download.
|
||||
full_layers=6, recurrent_layers=46, per_layer_f16=1024,
|
||||
moe=True, mtp=True,
|
||||
sampling={"temp": "0.6", "top-p": "0.95", "min-p": "0.01"},
|
||||
tags=("day-0", "long-context", "hybrid", "moe", "mtp"),
|
||||
),
|
||||
CatalogEntry(
|
||||
id="muse-glimmer-30b",
|
||||
display_name="Muse Glimmer 30B",
|
||||
description="Meta's open vision model for coding and agent work",
|
||||
repo="unsloth/Muse-Glimmer-30B-GGUF",
|
||||
variants=(
|
||||
_v("UD-Q8_K_XL",
|
||||
("Muse-Glimmer-30B-UD-Q8_K_XL.gguf", 32300651040,
|
||||
"e63bf23b7710ecdea2579e4b1de58980c4a2b446e8ecf48b782cfcefd2e31770")),
|
||||
_v("UD-Q6_K_XL",
|
||||
("Muse-Glimmer-30B-UD-Q6_K_XL.gguf", 26265362976,
|
||||
"fb5f80d110c4fa932cc652e70873c0bd12c0954009038aa675e65086104c2739")),
|
||||
_v("UD-Q5_K_XL",
|
||||
("Muse-Glimmer-30B-UD-Q5_K_XL.gguf", 21789618976,
|
||||
"97a66c4b41d9e778af7cdfa43508e08dbf765fb5049b740c69ad815e5191c637")),
|
||||
_v("UD-Q4_K_XL",
|
||||
("Muse-Glimmer-30B-UD-Q4_K_XL.gguf", 15878222368,
|
||||
"82bece304887a313ece08400bc030f6066c7bff5b906b0cd40308ec8a409fd38")),
|
||||
),
|
||||
n_ctx_train=262144,
|
||||
# Conservative dense prior (base config gated upstream): 30B-class
|
||||
# dense, ~60 layers x 4 KiB/tok. Dense KV is the expensive shape —
|
||||
# overestimating here keeps the zero-spill promise safe until the
|
||||
# GGUF header corrects it.
|
||||
full_layers=60, recurrent_layers=0, per_layer_f16=4096,
|
||||
mmproj=AssetFile("mmproj-Muse-Glimmer-30B-BF16.gguf", 3849173728,
|
||||
"d08cdcfa0b41d8e20554b52df404ba4f7b440d0bc502a90038508b6407df8ee1"),
|
||||
sampling={"temp": "1.0", "top-p": "0.95", "top-k": "64"},
|
||||
tags=("day-0", "vision", "dense"),
|
||||
),
|
||||
CatalogEntry(
|
||||
id="deepseek-v4-flash",
|
||||
display_name="DeepSeek V4 Flash",
|
||||
description="Frontier-class model for machines with 128GB+ memory",
|
||||
repo="unsloth/DeepSeek-V4-Flash-0731-GGUF",
|
||||
variants=(
|
||||
# Q8 is bit-lossless vs the official QAT checkpoint; Q4 keeps
|
||||
# the MXFP4 experts bit-exact and only requants the other 4%.
|
||||
_v("UD-Q8_K_XL",
|
||||
("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00001-of-00005.gguf", 5257408,
|
||||
"d13ce8f90855547bdaebe7312f531a1f2c4f822178d3103951f27fe884395cfa"),
|
||||
("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00002-of-00005.gguf", 49215492960,
|
||||
"3da2f2443063f83635986f9b67fa7e8e3d03c53b81a9a08d2007936612423610"),
|
||||
("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00003-of-00005.gguf", 49700372160,
|
||||
"7d622a7760d359ec9257b3493ad531e3bf0bfbe6f6533267e16e6dde8153ddce"),
|
||||
("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00004-of-00005.gguf", 49466495968,
|
||||
"6ed2bce452214f156b85e7c5f7d4fc242a3052f409d1b90a61422f60669c2de3"),
|
||||
("UD-Q8_K_XL/DeepSeek-V4-Flash-0731-UD-Q8_K_XL-00005-of-00005.gguf", 13481997024,
|
||||
"ea4727af4888fdca0fff796ec81ac2f3ebb43c310b2feb4798f41d82744b42ea")),
|
||||
_v("UD-Q4_K_XL",
|
||||
("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00001-of-00005.gguf", 5257408,
|
||||
"d13ce8f90855547bdaebe7312f531a1f2c4f822178d3103951f27fe884395cfa"),
|
||||
("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00002-of-00005.gguf", 48935523072,
|
||||
"d5b61668950f4743aacd677675d7fcf7507dbe1db6d304e8ff97ed1f00827bee"),
|
||||
("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00003-of-00005.gguf", 48980787136,
|
||||
"9705db7e589f360685ca7bd48100b270d78d228d4f5aa980508f3b2778af5494"),
|
||||
("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00004-of-00005.gguf", 49999168416,
|
||||
"7f13a68e3ca64208454c4ba32cc2757c0cbe78e3e5576c3142bf7007ca97da42"),
|
||||
("UD-Q4_K_XL/DeepSeek-V4-Flash-0731-UD-Q4_K_XL-00005-of-00005.gguf", 7174505088,
|
||||
"ed0d93164d3784968d6ce40d6d201ba98337f16e7db1b31fe495b2b0f334cc09")),
|
||||
),
|
||||
n_ctx_train=1048576,
|
||||
# Config-derived (deepseek-ai/DeepSeek-V4-Flash-0731): 43 layers,
|
||||
# MLA compressed KV (rank 512 + 64 rope) ~1.15 KiB/tok/layer f16.
|
||||
# Config shows sliding_window=128 with no per-layer map — priced
|
||||
# all-full (conservative); GGUF header decides after download.
|
||||
full_layers=43, recurrent_layers=0, per_layer_f16=1152,
|
||||
moe=True,
|
||||
draft=AssetFile("dspark-DeepSeek-V4-Flash-0731-Q8_0.gguf", 10896057440,
|
||||
"2c7ac54b0b64a99df1f139a9f1371a00198265e1d6a614b77597d20a655a4249"),
|
||||
sampling={"temp": "1.0", "top-p": "0.95", "min-p": "0.01"},
|
||||
tags=("day-0", "long-context", "moe", "frontier"),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def catalog_by_id() -> dict[str, CatalogEntry]:
|
||||
return {entry.id: entry for entry in CATALOG}
|
||||
|
||||
|
||||
def find_variant(entry_id: str, model_id: str) -> QuantVariant | None:
|
||||
entry = catalog_by_id().get(entry_id)
|
||||
if entry is None:
|
||||
return None
|
||||
return next((v for v in entry.variants if v.model_id == model_id), None)
|
||||
|
||||
|
||||
def find_entry_for_model(model_id: str) -> "tuple[CatalogEntry, QuantVariant] | None":
|
||||
"""Locate the entry + variant that owns a staged model id."""
|
||||
for entry in CATALOG:
|
||||
for variant in entry.variants:
|
||||
if variant.model_id == model_id:
|
||||
return entry, variant
|
||||
return None
|
||||
|
|
@ -0,0 +1,925 @@
|
|||
"""Local-models dashboard routes — the desktop's window into the managed
|
||||
llama.cpp runtime.
|
||||
|
||||
Everything here is designed for a first-run user on an RTX laptop: every
|
||||
payload carries plain-language, pre-formatted facts the UI can show verbatim
|
||||
(what will this model do ON THIS MACHINE, how big is the download, what is
|
||||
the runtime doing right now), never raw internals the renderer would have to
|
||||
interpret.
|
||||
|
||||
Long jobs (runtime install, model download) follow the repo's job pattern:
|
||||
start-POST -> {job_id} -> GET poll with byte progress. Downloads are
|
||||
sha256-verified; a hash mismatch deletes the file and reports it plainly.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import urllib.request
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from pydantic import BaseModel
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
_GIB = 1 << 30
|
||||
_JOBS: Dict[str, Dict[str, Any]] = {}
|
||||
_JOBS_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def _human_gb(n: int | float) -> str:
|
||||
return f"{n / _GIB:.1f} GB"
|
||||
|
||||
|
||||
def _job(kind: str, target: str, model_id: str | None = None) -> Dict[str, Any]:
|
||||
job = {
|
||||
"job_id": uuid.uuid4().hex[:12],
|
||||
"kind": kind, # "runtime-install" | "model-download"
|
||||
"target": target,
|
||||
"model_id": model_id, # catalog id for downloads; None otherwise
|
||||
"status": "running", # running | done | error
|
||||
"phase": "starting", # human-readable step name
|
||||
"detail": "",
|
||||
"total_bytes": None,
|
||||
"done_bytes": 0,
|
||||
"started_at": time.time(),
|
||||
"error": None,
|
||||
}
|
||||
with _JOBS_LOCK:
|
||||
_JOBS[job["job_id"]] = job
|
||||
return job
|
||||
|
||||
|
||||
# ── fast download: ranged parallel streams ───────────────────
|
||||
|
||||
# One TCP stream to a CDN rarely fills a fast line; 8 ranged connections
|
||||
# writing into a preallocated file saturate consumer gigabit. sha256 is
|
||||
# computed in a sequential pass afterwards (NVMe read is seconds, and it
|
||||
# keeps the hash independent of write ordering).
|
||||
_DOWNLOAD_CONNECTIONS = 8
|
||||
_CHUNK = 4 << 20
|
||||
|
||||
|
||||
def _probe_range_support(url: str) -> int:
|
||||
"""Total size when the server honors Range requests, else 0.
|
||||
|
||||
Auth-shaped failures raise with a plain-language message — a 401/403
|
||||
from the CDN means the repo is gated or the catalog entry names a
|
||||
wrong repo, and the user deserves better than a bare status code.
|
||||
"""
|
||||
req = urllib.request.Request(url, headers={"Range": "bytes=0-0"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=60) as r:
|
||||
if r.status == 206:
|
||||
content_range = r.headers.get("Content-Range", "")
|
||||
if "/" in content_range:
|
||||
return int(content_range.rsplit("/", 1)[1])
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (401, 403):
|
||||
raise RuntimeError(
|
||||
"The model host refused the download (gated or moved). "
|
||||
"This is a catalog problem, not yours — please report it.") from exc
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
def _model_id_for(gguf: Path) -> str:
|
||||
"""Variant model id for a staged file (strips split-part suffixes)."""
|
||||
import re
|
||||
|
||||
return re.sub(r"-\d{5}-of-\d{5}$", "", gguf.stem)
|
||||
|
||||
|
||||
def _variant_files_on_disk(model_id: str) -> "list[Path]":
|
||||
"""Every local file belonging to a staged model: all split parts plus
|
||||
its catalog-declared assets (mmproj/draft) when present."""
|
||||
from hermes_cli.local_runtime.bootstrap import assets_dir
|
||||
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
||||
|
||||
mdir = _models_dir()
|
||||
files = [p for p in mdir.glob("*.gguf") if _model_id_for(p) == model_id]
|
||||
hit = find_entry_for_model(model_id)
|
||||
if hit is not None:
|
||||
entry, _variant = hit
|
||||
for asset in (entry.mmproj, entry.draft):
|
||||
if asset is not None:
|
||||
p = assets_dir() / asset.local_name
|
||||
if p.exists():
|
||||
files.append(p)
|
||||
return files
|
||||
|
||||
|
||||
def download_file(url: str, dest: Path, job: Dict[str, Any],
|
||||
expected_sha256: str = "", *,
|
||||
base_done: int = 0, keep_totals: bool = False) -> None:
|
||||
"""Download url -> dest with byte progress on ``job``.
|
||||
|
||||
Ranged-parallel when the server supports it, single-stream fallback
|
||||
otherwise. Verifies sha256 when given; a mismatch deletes the file and
|
||||
raises with a plain-language message. Never leaves a .part behind.
|
||||
|
||||
Multi-file variants: ``base_done`` offsets the progress so this file's
|
||||
bytes accumulate onto the files before it, and ``keep_totals=True``
|
||||
stops the per-file size from overwriting the variant's total.
|
||||
"""
|
||||
import hashlib
|
||||
import shutil
|
||||
import threading as _threading
|
||||
|
||||
tmp = dest.with_suffix(".part")
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
file_done = [0]
|
||||
progress_lock = _threading.Lock()
|
||||
|
||||
def bump(n: int) -> None:
|
||||
with progress_lock:
|
||||
file_done[0] += n
|
||||
job["done_bytes"] = base_done + file_done[0]
|
||||
|
||||
try:
|
||||
total = _probe_range_support(url)
|
||||
if total:
|
||||
if not keep_totals:
|
||||
job["total_bytes"] = total
|
||||
# Preallocate so each worker writes at its own offset.
|
||||
with open(tmp, "wb") as f:
|
||||
f.truncate(total)
|
||||
errors: list[Exception] = []
|
||||
bounds = [(i * total // _DOWNLOAD_CONNECTIONS,
|
||||
(i + 1) * total // _DOWNLOAD_CONNECTIONS - 1)
|
||||
for i in range(_DOWNLOAD_CONNECTIONS)]
|
||||
|
||||
def fetch_range(start: int, end: int) -> None:
|
||||
try:
|
||||
req = urllib.request.Request(
|
||||
url, headers={"Range": f"bytes={start}-{end}"})
|
||||
with urllib.request.urlopen(req, timeout=120) as r, \
|
||||
open(tmp, "r+b") as f:
|
||||
f.seek(start)
|
||||
while True:
|
||||
chunk = r.read(_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
bump(len(chunk))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
errors.append(exc)
|
||||
|
||||
threads = [_threading.Thread(target=fetch_range, args=b, daemon=True,
|
||||
name=f"lm-dl-{i}")
|
||||
for i, b in enumerate(bounds)]
|
||||
for t in threads:
|
||||
t.start()
|
||||
for t in threads:
|
||||
t.join()
|
||||
if errors:
|
||||
raise errors[0]
|
||||
if file_done[0] != total:
|
||||
raise RuntimeError(
|
||||
f"download incomplete ({file_done[0]} of {total} bytes)")
|
||||
else:
|
||||
# No range support: single stream, large chunks.
|
||||
with urllib.request.urlopen(url, timeout=120) as r, open(tmp, "wb") as f:
|
||||
length = int(r.headers.get("Content-Length") or 0)
|
||||
if length and not keep_totals:
|
||||
job["total_bytes"] = length
|
||||
while True:
|
||||
chunk = r.read(_CHUNK)
|
||||
if not chunk:
|
||||
break
|
||||
f.write(chunk)
|
||||
bump(len(chunk))
|
||||
|
||||
if expected_sha256:
|
||||
job["phase"] = "verifying"
|
||||
job["detail"] = "Checking file integrity"
|
||||
digest = hashlib.sha256()
|
||||
with open(tmp, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(16 << 20), b""):
|
||||
digest.update(chunk)
|
||||
if digest.hexdigest() != expected_sha256:
|
||||
raise RuntimeError(
|
||||
"Downloaded file failed its integrity check and was removed — try again")
|
||||
shutil.move(str(tmp), str(dest))
|
||||
except Exception:
|
||||
tmp.unlink(missing_ok=True)
|
||||
raise
|
||||
|
||||
|
||||
def _models_dir() -> Path:
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
|
||||
return models_dir()
|
||||
|
||||
|
||||
def _load_config() -> dict:
|
||||
from hermes_cli.config import load_config
|
||||
|
||||
try:
|
||||
return load_config()
|
||||
except Exception: # noqa: BLE001
|
||||
return {}
|
||||
|
||||
|
||||
def _runtime_section() -> dict:
|
||||
return (_load_config() or {}).get("local_runtime") or {}
|
||||
|
||||
|
||||
# ── status: the one call the pane opens with ─────────────────
|
||||
|
||||
|
||||
@router.get("/api/local-models/status")
|
||||
async def local_models_status():
|
||||
"""Cheap, immediate, never blocks on probes (responsiveness standard):
|
||||
config state + installed runtime + staged models + supervisor state.
|
||||
GPU facts come from /api/local-models/hardware (slower, polled)."""
|
||||
from hermes_cli.local_runtime.binaries import (
|
||||
default_tag,
|
||||
installed_tags,
|
||||
runtimes_root,
|
||||
server_binary,
|
||||
)
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint
|
||||
|
||||
section = _runtime_section()
|
||||
configured_tag = section.get("tag") or default_tag()
|
||||
have = installed_tags()
|
||||
|
||||
# The tag actually serving (boot ladder: configured if installed, else
|
||||
# newest installed). Present tense for the pane header.
|
||||
tag = configured_tag if configured_tag in have else (have[0] if have else configured_tag)
|
||||
|
||||
# A pending engine update exists when the user runs the local engine
|
||||
# (enabled + something installed) and the configured tag — pinned or
|
||||
# the Hermes-release default — is newer than anything on disk. The
|
||||
# download is a button click, never automatic.
|
||||
update_available = bool(
|
||||
section.get("enabled") and have and configured_tag not in have)
|
||||
|
||||
runtime_installed = False
|
||||
runtime_backend = None
|
||||
root = runtimes_root() / tag
|
||||
if root.exists():
|
||||
for backend_dir in sorted(p for p in root.iterdir() if p.is_dir()):
|
||||
try:
|
||||
server_binary(backend_dir)
|
||||
runtime_installed = True
|
||||
runtime_backend = backend_dir.name
|
||||
break
|
||||
except Exception: # noqa: BLE001
|
||||
continue
|
||||
|
||||
staged = []
|
||||
mdir = _models_dir()
|
||||
if mdir.exists():
|
||||
from hermes_cli.local_runtime.bootstrap import staged_models
|
||||
|
||||
# Split models: report the whole variant's bytes, not one part's.
|
||||
from hermes_cli.local_runtime.catalog import find_entry_for_model
|
||||
|
||||
for gguf in staged_models():
|
||||
model_id = _model_id_for(gguf)
|
||||
size = gguf.stat().st_size
|
||||
hit = find_entry_for_model(model_id)
|
||||
if hit is not None:
|
||||
size = hit[1].size_bytes
|
||||
staged.append({
|
||||
"id": model_id,
|
||||
"size_bytes": size,
|
||||
"size_label": _human_gb(size),
|
||||
})
|
||||
|
||||
running = _state_endpoint()
|
||||
|
||||
# Which staged models are resident right now (loaded in VRAM). Read
|
||||
# from the live router when it's up; {} when down. Feeds the pane's
|
||||
# Loaded pills and eject buttons.
|
||||
loaded: Dict[str, str] = {}
|
||||
if running is not None:
|
||||
try:
|
||||
import urllib.request as _url
|
||||
|
||||
req = _url.Request(
|
||||
running["base_url"].rsplit("/v1", 1)[0] + "/models",
|
||||
headers={"Authorization": f"Bearer {running.get('api_key', '')}"})
|
||||
with _url.urlopen(req, timeout=3) as r:
|
||||
data = json.loads(r.read())
|
||||
loaded = {
|
||||
m["id"]: m.get("status", {}).get("value", "unknown")
|
||||
for m in data.get("data", [])
|
||||
# Everything resident or becoming resident: 'loading' renders
|
||||
# as its own state in the pane (a 20-GB load in flight is the
|
||||
# single most important thing the pane can show).
|
||||
if m.get("status", {}).get("value") in ("loaded", "ready", "loading")
|
||||
}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# Never silent: an empty dict here renders as 'Not in memory'
|
||||
# on a machine whose VRAM is visibly full.
|
||||
logger.warning("loaded-models read failed: %r", exc)
|
||||
loaded = {}
|
||||
|
||||
# The active main model, when it is one of ours (config authority: the
|
||||
# same model.provider + model.default that /api/model/set writes).
|
||||
active_model_id = None
|
||||
try:
|
||||
config = _load_config()
|
||||
model_section = (config or {}).get("model") or {}
|
||||
if str(model_section.get("provider", "")).strip().lower() in (
|
||||
"llamacpp", "llama.cpp", "llama-cpp"):
|
||||
active_model_id = str(
|
||||
model_section.get("default") or model_section.get("name") or ""
|
||||
).strip() or None
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
|
||||
return {
|
||||
"enabled": bool(section.get("enabled")),
|
||||
"tag": tag,
|
||||
"configured_tag": configured_tag,
|
||||
"update_available": update_available,
|
||||
"runtime_installed": runtime_installed,
|
||||
"runtime_backend": runtime_backend,
|
||||
"server_running": running is not None,
|
||||
"server_base_url": (running or {}).get("base_url"),
|
||||
"active_model_id": active_model_id,
|
||||
"loaded_models": loaded,
|
||||
"models": staged,
|
||||
"models_dir": str(mdir),
|
||||
}
|
||||
|
||||
|
||||
# ── hardware: what this machine can do ───────────────────────
|
||||
|
||||
|
||||
@router.get("/api/local-models/hardware")
|
||||
async def local_models_hardware():
|
||||
"""The budget as plain facts. Polled by the pane and the statusbar
|
||||
resource item (throttled client-side)."""
|
||||
from hermes_cli.local_runtime.hardware import probe_budget, _nvidia_vram, _ram_bytes
|
||||
|
||||
budget = probe_budget()
|
||||
ram_total, ram_avail = _ram_bytes()
|
||||
out = {
|
||||
"uma": budget.uma,
|
||||
"vram_total_bytes": budget.total_device_bytes,
|
||||
"vram_usable_bytes": budget.usable_vram_bytes,
|
||||
"ram_total_bytes": ram_total,
|
||||
"ram_available_bytes": ram_avail,
|
||||
"vram_label": _human_gb(budget.total_device_bytes),
|
||||
"gpu_name": None,
|
||||
"gpu_util_percent": None,
|
||||
"vram_used_bytes": None,
|
||||
}
|
||||
# GPU identity + live utilization (NVIDIA; other vendors degrade to None
|
||||
# and the UI hides those readouts).
|
||||
try:
|
||||
import subprocess
|
||||
|
||||
smi = subprocess.run(
|
||||
["nvidia-smi", "--query-gpu=name,utilization.gpu,memory.used",
|
||||
"--format=csv,noheader,nounits"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
if smi.returncode == 0 and smi.stdout.strip():
|
||||
name, util, used_mib = (x.strip() for x in smi.stdout.strip().splitlines()[0].split(","))
|
||||
out["gpu_name"] = name
|
||||
out["gpu_util_percent"] = int(util)
|
||||
out["vram_used_bytes"] = int(used_mib) << 20
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
return out
|
||||
|
||||
|
||||
# ── catalog: priced for THIS machine before download ─────────
|
||||
|
||||
|
||||
@router.get("/api/local-models/catalog")
|
||||
async def local_models_catalog():
|
||||
"""Every entry answers the user's three questions up front: how big is
|
||||
the download, will it fit, and what context/speed shape will I get —
|
||||
computed from the catalog's measured numbers + this machine's
|
||||
budget. Hardware-aware quant selection: the row advertises the BEST
|
||||
build for this machine (highest quality that runs fully on the GPU at
|
||||
the 64K floor; else the smallest that works, spilled and priced). No
|
||||
entry is hidden; unaffordable models show WHY."""
|
||||
from hermes_cli.local_runtime.catalog import CATALOG, select_variant
|
||||
from hermes_cli.local_runtime.context_policy import initial_window
|
||||
from hermes_cli.local_runtime.estimator import PhysicsRefusal
|
||||
from hermes_cli.local_runtime.hardware import probe_budget
|
||||
|
||||
# Planning budget: price against machine capacity, not live-free VRAM.
|
||||
# A loaded model must not make the catalog call every row unaffordable.
|
||||
budget = probe_budget(planning=True)
|
||||
mdir = _models_dir()
|
||||
entries = []
|
||||
for entry in CATALOG:
|
||||
choice = select_variant(entry, budget)
|
||||
# Any variant of this family already on disk counts as downloaded
|
||||
# (split variants stage under their first part).
|
||||
staged_ids = {_model_id_for(p) for p in mdir.glob("*.gguf")} if mdir.exists() else set()
|
||||
downloaded_variant = next(
|
||||
(v for v in entry.variants if v.model_id in staged_ids), None)
|
||||
row: Dict[str, Any] = {
|
||||
"id": entry.id,
|
||||
"display_name": entry.display_name,
|
||||
"description": entry.description,
|
||||
"native_context": entry.n_ctx_train,
|
||||
"native_context_label": f"{entry.n_ctx_train // 1024}K",
|
||||
"tags": list(entry.tags),
|
||||
"downloaded": downloaded_variant is not None,
|
||||
"downloaded_model_id": downloaded_variant.model_id if downloaded_variant else None,
|
||||
"downloaded_quant": downloaded_variant.quant if downloaded_variant else None,
|
||||
"mtp": entry.mtp,
|
||||
"vision": entry.mmproj is not None,
|
||||
}
|
||||
if choice is None:
|
||||
smallest = min(entry.variants, key=lambda v: v.size_bytes)
|
||||
smallest_total = entry.download_bytes(smallest)
|
||||
row.update({
|
||||
"fits": False,
|
||||
"size_bytes": smallest_total,
|
||||
"size_label": _human_gb(smallest_total),
|
||||
"fit_summary": "Needs more memory than this machine has",
|
||||
"fit_detail": (f"even the most compact build ({smallest.quant}, "
|
||||
f"{_human_gb(smallest_total)}) exceeds GPU + system memory"),
|
||||
})
|
||||
entries.append(row)
|
||||
continue
|
||||
|
||||
variant = choice.variant
|
||||
profile = entry.profile(variant)
|
||||
decision = initial_window(profile, budget)
|
||||
download_total = entry.download_bytes(variant)
|
||||
row.update({
|
||||
"fits": True,
|
||||
"model_id": variant.model_id,
|
||||
"quant": variant.quant,
|
||||
"quant_validated": variant.validated,
|
||||
"size_bytes": download_total,
|
||||
"size_label": _human_gb(download_total),
|
||||
"variant_count": len(entry.variants),
|
||||
})
|
||||
if choice.reason_key == "best-fits":
|
||||
best = entry.variants[0]
|
||||
row["quant_reason"] = (
|
||||
"Best quality build — runs fully on your GPU"
|
||||
if variant.quant == best.quant
|
||||
else f"Highest quality that runs fully on your GPU ({variant.quant})")
|
||||
else:
|
||||
row["quant_reason"] = (
|
||||
f"Compact build sized for this machine ({variant.quant}) — "
|
||||
"larger than GPU memory, runs slower")
|
||||
if not isinstance(decision, PhysicsRefusal):
|
||||
row["start_window"] = decision.window
|
||||
row["start_window_label"] = f"{decision.window // 1024}K"
|
||||
row["spilled"] = decision.spilled
|
||||
if decision.window >= entry.n_ctx_train:
|
||||
shape = f"runs at its full {row['native_context_label']} context"
|
||||
else:
|
||||
shape = (f"starts at {row['start_window_label']} and grows toward "
|
||||
f"{row['native_context_label']} as you use it")
|
||||
if decision.spilled:
|
||||
shape += " (larger than your GPU memory — runs slower)"
|
||||
row["fit_summary"] = shape
|
||||
else:
|
||||
row["fit_summary"] = row["quant_reason"]
|
||||
entries.append(row)
|
||||
return {"models": entries}
|
||||
|
||||
|
||||
# ── runtime install (job) ────────────────────────────────────
|
||||
|
||||
|
||||
class RuntimeInstallBody(BaseModel):
|
||||
backend: Optional[str] = None # None/auto -> detect
|
||||
|
||||
|
||||
@router.post("/api/local-models/runtime/install")
|
||||
async def local_models_runtime_install(body: RuntimeInstallBody):
|
||||
from hermes_cli.local_runtime.binaries import (
|
||||
default_tag,
|
||||
resolve_assets,
|
||||
select_backend,
|
||||
)
|
||||
from hermes_cli.local_runtime.bootstrap import _detect_gpu_vendor
|
||||
|
||||
section = _runtime_section()
|
||||
tag = section.get("tag") or default_tag()
|
||||
backend = body.backend or section.get("backend", "auto")
|
||||
if backend == "auto":
|
||||
backend = select_backend(_detect_gpu_vendor())
|
||||
# Resolve first so an impossible combination fails the POST, not the job.
|
||||
try:
|
||||
plan = resolve_assets(tag, backend)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=400, detail=str(exc))
|
||||
|
||||
job = _job("runtime-install", f"llama.cpp {tag} ({backend})")
|
||||
|
||||
def _run():
|
||||
try:
|
||||
from hermes_cli.local_runtime.binaries import (
|
||||
ensure_runtime_installed,
|
||||
installed_tags,
|
||||
prune_old_tags,
|
||||
)
|
||||
|
||||
previous = installed_tags()
|
||||
job["phase"] = "downloading"
|
||||
job["detail"] = f"Fetching {len(plan.assets)} package(s) for {backend}"
|
||||
ensure_runtime_installed(tag, backend)
|
||||
|
||||
# Engine update path: a server already running on an older tag
|
||||
# moves to the new one now — the click was the consent. Fresh
|
||||
# installs (no server) skip this; Use/boot handles their start.
|
||||
restarted = False
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import (
|
||||
ensure_local_runtime,
|
||||
get_supervisor,
|
||||
shutdown_local_runtime,
|
||||
)
|
||||
|
||||
sup = get_supervisor()
|
||||
if sup is not None and previous and tag not in previous:
|
||||
job["phase"] = "restarting"
|
||||
job["detail"] = "Switching the running server to the new build"
|
||||
shutdown_local_runtime()
|
||||
ensure_local_runtime(_load_config(), force=True)
|
||||
restarted = True
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# The new build is installed either way; the next boot serves
|
||||
# it. Never fail the job on the restart nicety.
|
||||
logger.warning("post-update restart skipped: %s", exc)
|
||||
|
||||
# N-1 retention, only after the new tag verified: keep it and the
|
||||
# newest previous build as the rollback pin target.
|
||||
try:
|
||||
keep = [tag] + [t for t in previous if t != tag][:1]
|
||||
prune_old_tags(keep)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("runtime prune skipped: %s", exc)
|
||||
|
||||
job["phase"] = "done"
|
||||
job["status"] = "done"
|
||||
job["detail"] = (f"llama.cpp {tag} ready ({backend})"
|
||||
+ (" — server restarted on the new build" if restarted else ""))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("runtime install failed: %s", exc)
|
||||
job["status"] = "error"
|
||||
job["error"] = str(exc)
|
||||
|
||||
threading.Thread(target=_run, daemon=True, name="lr-runtime-install").start()
|
||||
return {"job_id": job["job_id"], "backend": backend, "tag": tag}
|
||||
|
||||
|
||||
# ── model download (job with byte progress + sha256) ─────────
|
||||
|
||||
|
||||
class ModelDownloadBody(BaseModel):
|
||||
model_id: str
|
||||
|
||||
|
||||
@router.post("/api/local-models/download")
|
||||
async def local_models_download(body: ModelDownloadBody):
|
||||
"""Accepts either a family id (downloads this machine's selected
|
||||
variant) or an exact variant model_id."""
|
||||
from hermes_cli.local_runtime.catalog import (
|
||||
CATALOG,
|
||||
catalog_by_id,
|
||||
select_variant,
|
||||
)
|
||||
from hermes_cli.local_runtime.hardware import probe_budget
|
||||
|
||||
entry = catalog_by_id().get(body.model_id)
|
||||
variant = None
|
||||
if entry is not None:
|
||||
# Same planning budget as the catalog — the user downloads exactly
|
||||
# the build the row advertised.
|
||||
choice = select_variant(entry, probe_budget(planning=True))
|
||||
if choice is None:
|
||||
raise HTTPException(status_code=409,
|
||||
detail=f"no variant of {entry.id} fits this machine")
|
||||
variant = choice.variant
|
||||
else:
|
||||
for candidate in CATALOG:
|
||||
for v in candidate.variants:
|
||||
if v.model_id == body.model_id:
|
||||
entry, variant = candidate, v
|
||||
break
|
||||
if variant:
|
||||
break
|
||||
if entry is None or variant is None:
|
||||
raise HTTPException(status_code=404, detail=f"unknown model {body.model_id}")
|
||||
|
||||
from hermes_cli.local_runtime.bootstrap import assets_dir, staged_model_ids
|
||||
|
||||
if variant.model_id in staged_model_ids():
|
||||
return {"job_id": None, "already_downloaded": True, "model_id": variant.model_id}
|
||||
|
||||
# Everything this variant needs: split parts + mmproj/draft assets.
|
||||
plan = [] # (url, dest, sha256, bytes)
|
||||
for asset in variant.files:
|
||||
plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
||||
_models_dir() / asset.local_name, asset.sha256, asset.size_bytes))
|
||||
for asset in (entry.mmproj, entry.draft):
|
||||
if asset is not None:
|
||||
plan.append((f"https://huggingface.co/{entry.repo}/resolve/main/{asset.path}",
|
||||
assets_dir() / asset.local_name, asset.sha256, asset.size_bytes))
|
||||
|
||||
total = sum(p[3] for p in plan)
|
||||
job = _job("model-download", f"{entry.display_name} ({variant.quant})",
|
||||
model_id=entry.id)
|
||||
job["total_bytes"] = total
|
||||
|
||||
def _run():
|
||||
try:
|
||||
job["phase"] = "downloading"
|
||||
job["detail"] = f"{entry.display_name} — {_human_gb(total)}"
|
||||
done_before = 0
|
||||
for url, dest, sha, size in plan:
|
||||
if dest.exists():
|
||||
done_before += size
|
||||
job["done_bytes"] = done_before
|
||||
continue
|
||||
download_file(url, dest, job, expected_sha256=sha,
|
||||
base_done=done_before, keep_totals=True)
|
||||
job["phase"] = "downloading"
|
||||
done_before += size
|
||||
job["done_bytes"] = done_before
|
||||
job["phase"] = "done"
|
||||
job["status"] = "done"
|
||||
job["detail"] = f"{entry.display_name} ready"
|
||||
# A running router only scans models at spawn —
|
||||
# bounce it so the new model is servable
|
||||
# immediately instead of 400ing until the next app restart.
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
||||
|
||||
refresh_local_runtime()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("post-download runtime refresh skipped", exc_info=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("model download failed: %s", exc)
|
||||
job["status"] = "error"
|
||||
job["error"] = str(exc)
|
||||
|
||||
threading.Thread(target=_run, daemon=True, name="lr-model-download").start()
|
||||
return {"job_id": job["job_id"], "model_id": variant.model_id}
|
||||
|
||||
|
||||
@router.delete("/api/local-models/models/{model_id}")
|
||||
async def local_models_delete(model_id: str):
|
||||
"""Remove a staged model: every split part plus its private assets.
|
||||
A running router keeps serving from its spawn-time scan, so bounce it
|
||||
off the request thread — deleting the active file mid-serve is the
|
||||
kind of stale state the refresh exists for."""
|
||||
files = _variant_files_on_disk(model_id)
|
||||
if not files:
|
||||
raise HTTPException(status_code=404, detail="model not found")
|
||||
for path in files:
|
||||
path.unlink(missing_ok=True)
|
||||
# Growth state dies with the model: a re-download starts back at its
|
||||
# zero-spill window instead of inheriting a stale grown one.
|
||||
try:
|
||||
from hermes_cli.local_runtime.growth import clear_window_override
|
||||
|
||||
clear_window_override(model_id)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("window-override clear skipped", exc_info=True)
|
||||
|
||||
def _refresh():
|
||||
try:
|
||||
from hermes_cli.local_runtime.bootstrap import refresh_local_runtime
|
||||
|
||||
refresh_local_runtime()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("post-delete runtime refresh skipped", exc_info=True)
|
||||
|
||||
threading.Thread(target=_refresh, daemon=True, name="lr-post-delete").start()
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
# ── server lifecycle: turn the engine on/off ─────────────────
|
||||
|
||||
|
||||
class ServerActionBody(BaseModel):
|
||||
action: str # "stop" | "start"
|
||||
|
||||
|
||||
@router.post("/api/local-models/server")
|
||||
async def local_models_server(body: ServerActionBody):
|
||||
"""Turn the local engine off (stop the server, free ALL GPU memory,
|
||||
and disable auto-start) or back on. The off switch is the whole-engine
|
||||
counterpart of per-model eject — and unlike eject it IS durable: the
|
||||
user said off, so boots stay off until they say on."""
|
||||
import asyncio
|
||||
|
||||
from hermes_cli.config import load_config, save_config
|
||||
|
||||
action = (body.action or "").strip().lower()
|
||||
if action not in ("stop", "start"):
|
||||
raise HTTPException(status_code=400, detail="action must be 'stop' or 'start'")
|
||||
|
||||
def _stop():
|
||||
from hermes_cli.local_runtime.bootstrap import (
|
||||
get_supervisor,
|
||||
shutdown_local_runtime,
|
||||
)
|
||||
|
||||
sup = get_supervisor()
|
||||
if sup is not None:
|
||||
shutdown_local_runtime()
|
||||
else:
|
||||
# Server owned by another process (or an orphan): best-effort
|
||||
# terminate via the state file's pid, then clear the state.
|
||||
endpoint = _state_endpoint()
|
||||
if endpoint is not None:
|
||||
try:
|
||||
import psutil # type: ignore
|
||||
|
||||
from hermes_cli.local_runtime.supervisor import state_path
|
||||
|
||||
state = json.loads(state_path().read_text(encoding="utf-8"))
|
||||
pid = int(state.get("pid") or 0)
|
||||
if pid > 0 and psutil.pid_exists(pid):
|
||||
psutil.Process(pid).terminate()
|
||||
state_path().unlink(missing_ok=True)
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
config = load_config()
|
||||
config.setdefault("local_runtime", {})["enabled"] = False
|
||||
save_config(config)
|
||||
|
||||
def _start():
|
||||
from hermes_cli.local_runtime.bootstrap import ensure_local_runtime
|
||||
|
||||
config = load_config()
|
||||
config.setdefault("local_runtime", {})["enabled"] = True
|
||||
save_config(config)
|
||||
sup = ensure_local_runtime(config, force=True)
|
||||
if sup is None and _state_endpoint() is None:
|
||||
raise RuntimeError("The local server could not start — check the "
|
||||
"runtime is installed")
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(_stop if action == "stop" else _start)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
return {"ok": True, "action": action}
|
||||
|
||||
|
||||
# ── activate: make a downloaded model THE model ──────────────
|
||||
|
||||
|
||||
class ModelEjectBody(BaseModel):
|
||||
model_id: str
|
||||
|
||||
|
||||
@router.post("/api/local-models/eject")
|
||||
async def local_models_eject(body: ModelEjectBody):
|
||||
"""Free a loaded model's GPU memory now. Nothing reloads it except
|
||||
demand — the next message to it (residency v2: no automatic loading
|
||||
exists anywhere)."""
|
||||
from hermes_cli.local_runtime.bootstrap import get_supervisor
|
||||
|
||||
sup = get_supervisor()
|
||||
if sup is not None:
|
||||
try:
|
||||
sup.unload_model(body.model_id)
|
||||
return {"ok": True}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
|
||||
# Server owned by another process (or state-file only): drive the
|
||||
# router directly with the persisted endpoint.
|
||||
endpoint = _state_endpoint()
|
||||
if endpoint is None:
|
||||
raise HTTPException(status_code=409, detail="local server is not running")
|
||||
try:
|
||||
import urllib.request as _url
|
||||
|
||||
req = _url.Request(
|
||||
endpoint["base_url"].rsplit("/v1", 1)[0] + "/models/unload",
|
||||
data=json.dumps({"model": body.model_id}).encode(),
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {endpoint.get('api_key', '')}"},
|
||||
method="POST")
|
||||
with _url.urlopen(req, timeout=120):
|
||||
pass
|
||||
return {"ok": True}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise HTTPException(status_code=502, detail=str(exc)) from exc
|
||||
|
||||
|
||||
class ModelActivateBody(BaseModel):
|
||||
model_id: str # exact variant id (a staged .gguf stem)
|
||||
|
||||
|
||||
@router.post("/api/local-models/activate")
|
||||
async def local_models_activate(body: ModelActivateBody):
|
||||
"""Make a downloaded model the default for new chats. Pure selection
|
||||
(residency v2): a config write through the same machinery as
|
||||
/api/model/set, plus making sure the server is up. NO model loading —
|
||||
models load on first inference, always; an empty router costs nothing.
|
||||
Fast enough to be synchronous-feeling, but kept as a job for UI
|
||||
continuity."""
|
||||
# Split variants stage under their first part — resolve like the rest
|
||||
# of the routes instead of assuming a single flat file.
|
||||
from hermes_cli.local_runtime.bootstrap import staged_model_ids
|
||||
|
||||
if body.model_id not in staged_model_ids():
|
||||
raise HTTPException(status_code=404, detail=f"{body.model_id} is not downloaded")
|
||||
|
||||
job = _job("model-activate", body.model_id, model_id=body.model_id)
|
||||
|
||||
def _run():
|
||||
try:
|
||||
from hermes_cli.config import load_config, save_config
|
||||
from hermes_cli.local_runtime.bootstrap import (
|
||||
ensure_local_runtime,
|
||||
refresh_local_runtime,
|
||||
)
|
||||
|
||||
job["phase"] = "starting-server"
|
||||
job["detail"] = "Starting the local server"
|
||||
config = load_config()
|
||||
sup = ensure_local_runtime(config, force=True)
|
||||
if sup is None:
|
||||
from hermes_cli.local_runtime.endpoint import _state_endpoint as _se
|
||||
|
||||
if _se() is None:
|
||||
raise RuntimeError(
|
||||
"The local server could not start — check the runtime is installed")
|
||||
|
||||
# Self-heal a stale router: the model list is spawn-only, so a
|
||||
# server started before this model finished downloading can't
|
||||
# serve it. If the router doesn't know the model, bounce it.
|
||||
if sup is not None:
|
||||
try:
|
||||
if body.model_id not in sup.models():
|
||||
job["detail"] = "Refreshing the local server"
|
||||
refresh_local_runtime()
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug("activate rescan check skipped", exc_info=True)
|
||||
|
||||
job["phase"] = "setting-default"
|
||||
job["detail"] = "Making it your default"
|
||||
config = load_config()
|
||||
config.setdefault("local_runtime", {})["enabled"] = True
|
||||
save_config(config)
|
||||
from hermes_cli.web_deps import late
|
||||
|
||||
late("_apply_model_assignment_sync")(
|
||||
"main", "llamacpp", body.model_id, "", "", "")
|
||||
|
||||
job["phase"] = "done"
|
||||
job["status"] = "done"
|
||||
job["detail"] = f"{body.model_id} is the default for new chats"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.warning("model activate failed: %s", exc)
|
||||
job["status"] = "error"
|
||||
job["error"] = str(exc)
|
||||
|
||||
threading.Thread(target=_run, daemon=True, name="lr-model-activate").start()
|
||||
return {"job_id": job["job_id"]}
|
||||
|
||||
|
||||
# ── job polling ──────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/api/local-models/jobs")
|
||||
async def local_models_jobs():
|
||||
"""All recent jobs, running first — the pane and the app-level poller
|
||||
rediscover in-flight work here after a remount or app restart."""
|
||||
with _JOBS_LOCK:
|
||||
jobs = sorted(_JOBS.values(),
|
||||
key=lambda j: (j["status"] != "running", -j["started_at"]))
|
||||
out = []
|
||||
for job in jobs[:20]:
|
||||
entry = dict(job)
|
||||
if entry["total_bytes"]:
|
||||
entry["percent"] = min(100, round(entry["done_bytes"] / entry["total_bytes"] * 100))
|
||||
out.append(entry)
|
||||
return {"jobs": out}
|
||||
|
||||
|
||||
@router.get("/api/local-models/jobs/{job_id}")
|
||||
async def local_models_job(job_id: str):
|
||||
with _JOBS_LOCK:
|
||||
job = _JOBS.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="job not found")
|
||||
out = dict(job)
|
||||
if out["total_bytes"]:
|
||||
out["percent"] = min(100, round(out["done_bytes"] / out["total_bytes"] * 100))
|
||||
return out
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
"""Catalog integrity: every entry's files must exist on the host with
|
||||
matching sha256s (HF LFS oids ARE the file hashes).
|
||||
|
||||
Network-marked (skipped in hermetic CI unless explicitly enabled) — this is
|
||||
the test that catches wrong repo names (the Nemotron 401), moved files, and
|
||||
upstream re-uploads before a user's download does. Run before any catalog
|
||||
commit:
|
||||
|
||||
HERMES_TEST_NETWORK=1 scripts/run_tests.sh tests/hermes_cli/test_catalog_reachability.py
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("HERMES_TEST_NETWORK"),
|
||||
reason="network test; set HERMES_TEST_NETWORK=1 to run",
|
||||
)
|
||||
|
||||
|
||||
def test_every_catalog_file_resolves():
|
||||
from hermes_cli.local_runtime.catalog import CATALOG
|
||||
|
||||
problems = []
|
||||
for entry in CATALOG:
|
||||
url = f"https://huggingface.co/api/models/{entry.repo}/tree/main?recursive=true"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=30) as r:
|
||||
files = {f["path"]: f.get("lfs") or {} for f in json.load(r)}
|
||||
except Exception as exc: # noqa: BLE001
|
||||
problems.append(f"{entry.id}: repo {entry.repo} unreachable ({exc})")
|
||||
continue
|
||||
for variant in entry.variants:
|
||||
for asset in entry.download_files(variant):
|
||||
if asset.path not in files:
|
||||
problems.append(
|
||||
f"{entry.id}/{variant.quant}: {asset.path} not in {entry.repo}")
|
||||
continue
|
||||
live_sha = files[asset.path].get("oid", "")
|
||||
if live_sha and live_sha != asset.sha256:
|
||||
problems.append(
|
||||
f"{entry.id}/{variant.quant}: sha drift on {asset.path} — "
|
||||
f"catalog {asset.sha256[:12]} vs live {live_sha[:12]} "
|
||||
f"(upstream re-uploaded; re-pin deliberately)")
|
||||
assert not problems, "\n".join(problems)
|
||||
|
|
@ -0,0 +1,129 @@
|
|||
"""Variant-selection contracts: the quant ladder picks the best build for
|
||||
the hardware, per the design's 'offer a smaller quant' remedy run
|
||||
proactively, bounded by the Q4 quality floor. Pure decision-table tests
|
||||
over synthetic budgets."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from hermes_cli.local_runtime.catalog import (
|
||||
CATALOG,
|
||||
catalog_by_id,
|
||||
find_entry_for_model,
|
||||
select_variant,
|
||||
)
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
|
||||
GIB = 1 << 30
|
||||
|
||||
|
||||
def budget(vram_gib: float, ram_gib: float = 64) -> HardwareBudget:
|
||||
return HardwareBudget(usable_vram_bytes=int(vram_gib * GIB),
|
||||
total_device_bytes=int(vram_gib * GIB),
|
||||
ram_available_bytes=int(ram_gib * GIB))
|
||||
|
||||
|
||||
def test_variants_ordered_best_first_and_floor_is_q4():
|
||||
for entry in CATALOG:
|
||||
sizes = [v.size_bytes for v in entry.variants]
|
||||
assert sizes == sorted(sizes, reverse=True), f"{entry.id}: variants not best-first"
|
||||
for v in entry.variants:
|
||||
for asset in entry.download_files(v):
|
||||
assert len(asset.sha256) == 64, f"{entry.id}/{v.quant}: unpinned {asset.path}"
|
||||
# The quality floor: the ladder ends at Q4 — no Q3/Q2 builds ship
|
||||
# (product decision, 2026-08-09). Every floor rung is either
|
||||
# validated on real hardware or an explicit day-0 entry.
|
||||
floor = entry.variants[-1]
|
||||
assert floor.quant == "UD-Q4_K_XL", f"{entry.id}: ladder floor is {floor.quant}, not Q4"
|
||||
assert floor.validated or "day-0" in entry.tags, (
|
||||
f"{entry.id}: floor rung neither validated nor tagged day-0")
|
||||
|
||||
|
||||
def test_split_variants_have_coherent_parts():
|
||||
"""Multi-file variants: same model_id from every part, exact sizes,
|
||||
first file is the load target."""
|
||||
entry = catalog_by_id()["deepseek-v4-flash"]
|
||||
for v in entry.variants:
|
||||
assert len(v.files) >= 2, "deepseek ships split GGUFs"
|
||||
assert "00001-of" in v.files[0].path, "first part must be the load target"
|
||||
assert v.size_bytes == sum(f.size_bytes for f in v.files)
|
||||
assert entry.draft is not None, "DSpark draft rides along"
|
||||
|
||||
|
||||
def test_big_card_gets_best_quality():
|
||||
"""A card with headroom takes the top rung — quality is free when it fits."""
|
||||
entry = catalog_by_id()["muse-glimmer-30b"]
|
||||
choice = select_variant(entry, budget(48))
|
||||
assert choice is not None
|
||||
assert choice.zero_spill
|
||||
assert choice.variant.quant == entry.variants[0].quant # UD-Q8_K_XL
|
||||
|
||||
|
||||
def test_quality_monotone_in_vram():
|
||||
"""More VRAM never selects a smaller build."""
|
||||
entry = catalog_by_id()["qwen3.6-27b"]
|
||||
sizes = []
|
||||
for vram in (8, 12, 16, 24, 32, 48):
|
||||
choice = select_variant(entry, budget(vram))
|
||||
assert choice is not None
|
||||
sizes.append(choice.variant.size_bytes)
|
||||
assert sizes == sorted(sizes), f"quality not monotone in VRAM: {sizes}"
|
||||
|
||||
|
||||
def test_small_card_gets_q4_spilled_never_below():
|
||||
"""8 GiB card + 27B: nothing zero-spills. The floor holds — the
|
||||
selector offers Q4 spilled (priced honestly), never a sub-Q4 build."""
|
||||
entry = catalog_by_id()["qwen3.6-27b"]
|
||||
choice = select_variant(entry, budget(8))
|
||||
assert choice is not None
|
||||
assert not choice.zero_spill
|
||||
assert choice.reason_key == "smallest-fits-spilled"
|
||||
assert choice.variant.quant == "UD-Q4_K_XL"
|
||||
|
||||
|
||||
def test_frontier_model_refused_on_consumer_card_offered_on_big_ram():
|
||||
"""DeepSeek V4 Flash (161 GB at Q4): refused outright on a 32 GiB-RAM
|
||||
desktop; offered spilled on a 192 GiB-RAM workstation. The catalog
|
||||
carries frontier hardware honestly instead of hiding the model."""
|
||||
entry = catalog_by_id()["deepseek-v4-flash"]
|
||||
assert select_variant(entry, budget(32, ram_gib=32)) is None
|
||||
big = select_variant(entry, budget(32, ram_gib=192))
|
||||
assert big is not None and not big.zero_spill
|
||||
|
||||
|
||||
def test_selection_accounts_for_kv_not_just_weights():
|
||||
"""The zero-spill check prices weights + 64K-floor KV, not weights
|
||||
alone: give a machine exactly enough VRAM for the Q8 weights of a
|
||||
dense model and it must step down a rung."""
|
||||
entry = catalog_by_id()["muse-glimmer-30b"]
|
||||
q8 = entry.variants[0]
|
||||
exactly_weights = HardwareBudget(
|
||||
usable_vram_bytes=q8.size_bytes + (100 << 20),
|
||||
total_device_bytes=q8.size_bytes + (100 << 20),
|
||||
ram_available_bytes=64 * GIB)
|
||||
choice = select_variant(entry, exactly_weights)
|
||||
assert choice is not None
|
||||
assert choice.variant.quant != q8.quant, "KV cost ignored — Q8 can't fit with floor KV"
|
||||
|
||||
|
||||
def test_find_entry_for_model_resolves_split_ids():
|
||||
hit = find_entry_for_model("DeepSeek-V4-Flash-0731-UD-Q4_K_XL")
|
||||
assert hit is not None
|
||||
entry, variant = hit
|
||||
assert entry.id == "deepseek-v4-flash"
|
||||
assert variant.quant == "UD-Q4_K_XL"
|
||||
|
||||
|
||||
def test_hybrid_long_context_stays_cheap():
|
||||
"""The reason Nemotron/Qwen3.6 headline the catalog: their priced
|
||||
64K-floor KV must be a small fraction of a dense model's."""
|
||||
from hermes_cli.local_runtime.catalog import FLOOR
|
||||
from hermes_cli.local_runtime.estimator import ctx_bytes
|
||||
|
||||
dense = catalog_by_id()["muse-glimmer-30b"]
|
||||
hybrid = catalog_by_id()["nemotron-3.5-lightning-30b"]
|
||||
dense_kv = ctx_bytes(dense.profile(dense.variants[-1]), FLOOR)
|
||||
hybrid_kv = ctx_bytes(hybrid.profile(hybrid.variants[-1]), FLOOR)
|
||||
assert hybrid_kv * 5 < dense_kv, (
|
||||
f"hybrid KV ({hybrid_kv:,}) should be >5x cheaper than dense ({dense_kv:,})")
|
||||
|
|
@ -0,0 +1,209 @@
|
|||
"""Contract tests for the local-models dashboard routes (Rollout 4).
|
||||
|
||||
Real FastAPI TestClient against the real router; the runtime pieces
|
||||
underneath are exercised against temp HERMES_HOME (autouse fixture). Network
|
||||
downloads are stubbed at the urllib boundary — never live."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
from hermes_cli import web_server
|
||||
|
||||
test_client = TestClient(web_server.app)
|
||||
# Same auth pattern as the git-route tests: present the session token.
|
||||
test_client.headers[web_server._SESSION_HEADER_NAME] = web_server._SESSION_TOKEN
|
||||
return test_client
|
||||
|
||||
|
||||
def test_local_models_routes_require_auth(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("HERMES_HOME", str(tmp_path / ".hermes"))
|
||||
from hermes_cli import web_server
|
||||
|
||||
unauth = TestClient(web_server.app)
|
||||
assert unauth.get("/api/local-models/status").status_code == 401
|
||||
|
||||
|
||||
def _write_fake_gguf(path: Path, size: int = 1024) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(b"GGUF" + b"\x00" * size)
|
||||
|
||||
|
||||
# ── status ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_status_shape_and_defaults(client):
|
||||
r = client.get("/api/local-models/status")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
# Contract: every key the pane's first paint needs, present and typed.
|
||||
assert isinstance(data["enabled"], bool)
|
||||
assert isinstance(data["tag"], str) and data["tag"].startswith("b")
|
||||
assert isinstance(data["runtime_installed"], bool)
|
||||
assert isinstance(data["server_running"], bool)
|
||||
assert isinstance(data["models"], list)
|
||||
|
||||
|
||||
def test_status_lists_staged_models_with_labels(client, tmp_path):
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
|
||||
_write_fake_gguf(models_dir() / "Some-Model.gguf", size=2048)
|
||||
data = client.get("/api/local-models/status").json()
|
||||
ids = [m["id"] for m in data["models"]]
|
||||
assert "Some-Model" in ids
|
||||
row = data["models"][ids.index("Some-Model")]
|
||||
assert row["size_bytes"] > 0
|
||||
assert row["size_label"].endswith("GB")
|
||||
|
||||
|
||||
# ── hardware ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_hardware_plain_facts(client):
|
||||
data = client.get("/api/local-models/hardware").json()
|
||||
assert isinstance(data["uma"], bool)
|
||||
assert data["ram_total_bytes"] > 0
|
||||
assert data["vram_total_bytes"] >= 0
|
||||
# GPU fields are None-able (non-NVIDIA machines) but must exist.
|
||||
assert "gpu_name" in data and "gpu_util_percent" in data and "vram_used_bytes" in data
|
||||
|
||||
|
||||
# ── catalog ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_catalog_prices_every_entry_for_this_machine(client):
|
||||
data = client.get("/api/local-models/catalog").json()
|
||||
assert len(data["models"]) >= 3
|
||||
for row in data["models"]:
|
||||
# The three user questions, answered on every row:
|
||||
assert row["size_label"].endswith("GB") # how big
|
||||
assert isinstance(row["fits"], bool) # will it fit
|
||||
assert row["fit_summary"] # what shape
|
||||
if row["fits"]:
|
||||
assert row["start_window"] >= 1
|
||||
assert row["start_window_label"].endswith("K")
|
||||
else:
|
||||
assert "memory" in row["fit_summary"].lower()
|
||||
assert isinstance(row["downloaded"], bool)
|
||||
|
||||
|
||||
def test_catalog_never_hides_unaffordable_models(client, monkeypatch):
|
||||
"""Unaffordable entries stay visible with a plain reason — hiding them
|
||||
is how users conclude the feature is broken."""
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
|
||||
tiny = HardwareBudget(usable_vram_bytes=1 << 30, total_device_bytes=1 << 30,
|
||||
ram_available_bytes=1 << 30)
|
||||
monkeypatch.setattr("hermes_cli.local_runtime.hardware.probe_budget",
|
||||
lambda **kw: tiny)
|
||||
data = client.get("/api/local-models/catalog").json()
|
||||
from hermes_cli.local_runtime.catalog import CATALOG
|
||||
|
||||
assert len(data["models"]) == len(CATALOG)
|
||||
refused = [m for m in data["models"] if not m["fits"]]
|
||||
assert refused, "a 1 GiB machine must refuse the 20 GB models"
|
||||
for row in refused:
|
||||
assert row["fit_detail"] or row["fit_summary"]
|
||||
|
||||
|
||||
# ── downloads ────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_download_unknown_model_404s(client):
|
||||
r = client.post("/api/local-models/download", json={"model_id": "nope"})
|
||||
assert r.status_code == 404
|
||||
|
||||
|
||||
def test_download_job_lifecycle_with_sha_failure(client, monkeypatch):
|
||||
"""Wrong-hash download must delete the file and error the job with a
|
||||
human-readable message — never leave a corrupt GGUF staged."""
|
||||
|
||||
class FakeResponse(io.BytesIO):
|
||||
headers = {"Content-Length": "16"}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *a):
|
||||
return False
|
||||
|
||||
monkeypatch.setattr("urllib.request.urlopen",
|
||||
lambda *a, **k: FakeResponse(b"not the real body"))
|
||||
|
||||
from hermes_cli.local_runtime.catalog import CATALOG
|
||||
|
||||
entry_id = CATALOG[0].id
|
||||
r = client.post("/api/local-models/download", json={"model_id": entry_id})
|
||||
assert r.status_code == 200
|
||||
job_id = r.json()["job_id"]
|
||||
assert job_id
|
||||
|
||||
deadline = time.time() + 10
|
||||
status = None
|
||||
while time.time() < deadline:
|
||||
status = client.get(f"/api/local-models/jobs/{job_id}").json()
|
||||
if status["status"] in ("done", "error"):
|
||||
break
|
||||
time.sleep(0.05)
|
||||
assert status is not None and status["status"] == "error"
|
||||
assert "integrity" in status["error"].lower()
|
||||
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
|
||||
assert not (models_dir() / f"{entry_id}.gguf").exists()
|
||||
assert not (models_dir() / f"{entry_id}.part").exists()
|
||||
|
||||
|
||||
def test_download_already_downloaded_short_circuits(client, monkeypatch):
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
from hermes_cli.local_runtime.catalog import CATALOG, select_variant
|
||||
from hermes_cli.local_runtime.estimator import HardwareBudget
|
||||
|
||||
# Pin the budget so the selected variant is deterministic in the test.
|
||||
budget = HardwareBudget(usable_vram_bytes=64 << 30, total_device_bytes=64 << 30,
|
||||
ram_available_bytes=64 << 30)
|
||||
monkeypatch.setattr("hermes_cli.local_runtime.hardware.probe_budget",
|
||||
lambda **kw: budget)
|
||||
choice = select_variant(CATALOG[0], budget)
|
||||
assert choice is not None
|
||||
_write_fake_gguf(models_dir() / choice.variant.files[0].local_name)
|
||||
r = client.post("/api/local-models/download", json={"model_id": CATALOG[0].id})
|
||||
assert r.status_code == 200
|
||||
assert r.json()["already_downloaded"] is True
|
||||
|
||||
|
||||
def test_delete_model(client):
|
||||
from hermes_cli.local_runtime.bootstrap import models_dir
|
||||
|
||||
_write_fake_gguf(models_dir() / "Doomed.gguf")
|
||||
assert client.delete("/api/local-models/models/Doomed").status_code == 200
|
||||
assert not (models_dir() / "Doomed.gguf").exists()
|
||||
assert client.delete("/api/local-models/models/Doomed").status_code == 404
|
||||
|
||||
|
||||
# ── runtime install ──────────────────────────────────────────
|
||||
|
||||
|
||||
def test_runtime_install_rejects_impossible_combo(client, monkeypatch):
|
||||
"""Impossible platform/backend combos fail the POST itself with the
|
||||
resolver's honest message — not a background job that dies silently.
|
||||
(win-arm64-vulkan; the old cuda case became real upstream at ~b1036x.)"""
|
||||
monkeypatch.setattr(
|
||||
"hermes_cli.local_runtime.binaries._host_os_arch", lambda: ("win", "arm64"))
|
||||
r = client.post("/api/local-models/runtime/install", json={"backend": "vulkan"})
|
||||
assert r.status_code == 400
|
||||
assert "arm64" in r.json()["detail"]
|
||||
|
||||
|
||||
def test_job_poll_unknown_404s(client):
|
||||
assert client.get("/api/local-models/jobs/deadbeef").status_code == 404
|
||||
|
|
@ -0,0 +1,67 @@
|
|||
"""The llamacpp provider row in the model picker payload (Rollout 4).
|
||||
|
||||
Contract: staged local GGUFs appear as a selectable provider row in
|
||||
build_models_payload — the same payload /api/model/options and the desktop
|
||||
picker consume — whenever models are staged, without any credential."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def hermes_home(tmp_path, monkeypatch):
|
||||
home = tmp_path / ".hermes"
|
||||
home.mkdir()
|
||||
monkeypatch.setenv("HERMES_HOME", str(home))
|
||||
return home
|
||||
|
||||
|
||||
def _stage(home, *names):
|
||||
mdir = home / "models"
|
||||
mdir.mkdir(exist_ok=True)
|
||||
for name in names:
|
||||
(mdir / f"{name}.gguf").write_bytes(b"GGUF" + b"\x00" * 64)
|
||||
|
||||
|
||||
def test_no_staged_models_no_row(hermes_home):
|
||||
from hermes_cli.inventory import _local_runtime_row, load_picker_context
|
||||
|
||||
assert _local_runtime_row(load_picker_context()) is None
|
||||
|
||||
|
||||
def test_staged_models_make_a_selectable_row(hermes_home):
|
||||
from hermes_cli.inventory import _local_runtime_row, load_picker_context
|
||||
|
||||
_stage(hermes_home, "Qwen3-4B-Instruct-2507-UD-Q8_K_XL", "Some-Other-Model")
|
||||
row = _local_runtime_row(load_picker_context())
|
||||
assert row is not None
|
||||
assert row["slug"] == "llamacpp"
|
||||
assert row["authenticated"] is True
|
||||
assert "Qwen3-4B-Instruct-2507-UD-Q8_K_XL" in row["models"]
|
||||
assert row["total_models"] == 2
|
||||
|
||||
|
||||
def test_row_marks_current_when_config_points_at_llamacpp(hermes_home):
|
||||
from hermes_cli.inventory import _local_runtime_row, load_picker_context
|
||||
|
||||
_stage(hermes_home, "M")
|
||||
ctx = load_picker_context().with_overrides(current_provider="llamacpp")
|
||||
row = _local_runtime_row(ctx)
|
||||
assert row is not None and row["is_current"] is True
|
||||
|
||||
|
||||
def test_full_payload_includes_local_row(hermes_home):
|
||||
"""Through the REAL payload builder — the shape the desktop picker eats."""
|
||||
from hermes_cli.inventory import build_models_payload, load_picker_context
|
||||
|
||||
_stage(hermes_home, "Local-Model-X")
|
||||
payload = build_models_payload(
|
||||
load_picker_context(),
|
||||
probe_custom_providers=False,
|
||||
probe_current_custom_provider=False,
|
||||
)
|
||||
slugs = [p["slug"] for p in payload["providers"]]
|
||||
assert "llamacpp" in slugs
|
||||
row = payload["providers"][slugs.index("llamacpp")]
|
||||
assert row["models"] == ["Local-Model-X"]
|
||||
Loading…
Reference in New Issue