mirror of https://github.com/razor-ai/soup.git
feat(eval,registry): live gate scoring + registry attach (v0.33.0 Part A wave 1)
Closes #32, #35. (#34 soup can run/publish deferred to Part A wave 2.) #32 Live model scoring for `soup eval gate` + `soup eval quant-check`: - gate.run_gate now dispatches judge / benchmark / custom task types, wrapping each scorer in try/except so a backend failure produces score=None + error=str(exc) instead of a silent score=1.0 pass. - New _parse_judge_url splits ollama:// / http(s):// judge_model URLs into (provider, model, api_base) for JudgeEvaluator. - New _run_judge_task / _run_benchmark_task plug into existing eval/judge.py and eval/forgetting.py runners. - New quant_check.make_model_generator(model_path) wraps transformers AutoTokenizer + AutoModelForCausalLM into a generate_fn callable; greedy by default for reproducible scores; lazy-imported. - gate_cmd / quant_check_cmd build live generators when --model is given; fall back to deterministic stub on load failure so CI without GPUs still runs the orchestration layer. - GateTaskResult.score is now Optional[float] with new error: Optional[str]. - _print_gate_result renders ERROR + reason cleanly. #35 Registry attach hooks: - registry/store.py _VALID_KINDS extended with eval_results, tensorrt. - New registry/attach.py: attach_artifact, write_eval_json (cwd-containment via realpath+commonpath), lookup_entry_by_output_dir. - `soup eval custom` gains --attach-to-registry + --output (paired); on success writes JSON results and adds eval_results artifact row. - `soup export` gains --registry-id with auto-match by source --model output dir; auto-attaches the produced GGUF artifact. Failures here are warnings, not hard exits — export already succeeded. Tests: +19 in tests/test_part_a_wave1.py covering URL parser, error propagation across all 3 task types, score=None semantics, generator factory bounds + transformers mocking, registry attach helpers (containment + missing entry), and CLI integration. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alpamys
parent
7f32a5e7c0
commit
ca799f6fd3
|
|
@ -122,6 +122,14 @@ def custom(
|
|||
None, "--run-id",
|
||||
help="Link results to an existing training run",
|
||||
),
|
||||
attach_to_registry: Optional[str] = typer.Option(
|
||||
None, "--attach-to-registry",
|
||||
help="Attach the eval JSON to a registry entry as kind=eval_results",
|
||||
),
|
||||
output: Optional[str] = typer.Option(
|
||||
None, "--output", "-o",
|
||||
help="Path for the eval JSON output (required with --attach-to-registry)",
|
||||
),
|
||||
):
|
||||
"""Run custom evaluation tasks from a JSONL file."""
|
||||
from soup_cli.eval.custom import load_eval_tasks
|
||||
|
|
@ -189,6 +197,36 @@ def custom(
|
|||
_save_custom_results(eval_results, str(model_path), run_id)
|
||||
console.print("\n[green]Results saved to experiment tracker.[/]")
|
||||
|
||||
# v0.33.0 #35: optional registry attach
|
||||
if attach_to_registry:
|
||||
if not output:
|
||||
console.print(
|
||||
"[red]--attach-to-registry requires --output <json-path>[/]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
from soup_cli.registry.attach import attach_artifact, write_eval_json
|
||||
|
||||
payload = {
|
||||
"model": str(model_path),
|
||||
"tasks": str(tasks_path),
|
||||
"total": eval_results.total,
|
||||
"correct": eval_results.correct,
|
||||
"accuracy": eval_results.accuracy,
|
||||
"category_scores": eval_results.category_scores,
|
||||
}
|
||||
try:
|
||||
json_path = write_eval_json(output, payload=payload)
|
||||
attach_artifact(
|
||||
attach_to_registry, path=str(json_path), kind="eval_results",
|
||||
)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
console.print(f"[red]Registry attach failed:[/] {exc}")
|
||||
raise typer.Exit(1) from exc
|
||||
console.print(
|
||||
f"[green]Attached eval results to registry entry "
|
||||
f"'{attach_to_registry}' as eval_results.[/]"
|
||||
)
|
||||
|
||||
|
||||
# ─── soup eval judge ───
|
||||
|
||||
|
|
@ -934,24 +972,28 @@ def gate_cmd(
|
|||
console.print(f"[red]Cannot resolve baseline:[/] {exc}")
|
||||
raise typer.Exit(1) from exc
|
||||
|
||||
# Without a live model we can't generate real completions; emit a stub
|
||||
# generator so the CLI is still testable. Wiring a real model is out of
|
||||
# scope for the v0.26.0 launch (see plan Part B).
|
||||
# When --model is provided, build a transformers-backed generator.
|
||||
# Otherwise fall back to an empty-string stub for smoke runs.
|
||||
if model is None:
|
||||
console.print(
|
||||
"[yellow]No --model given; using stub generator "
|
||||
"(empty output per prompt) for a smoke run.[/]"
|
||||
)
|
||||
|
||||
def _stub_generate(_: str) -> str:
|
||||
return ""
|
||||
|
||||
generate_fn = _stub_generate
|
||||
else:
|
||||
console.print(
|
||||
"[yellow]Live model scoring not yet wired; using stub. "
|
||||
"Subscribe to v0.26.1 for real inference support.[/]"
|
||||
)
|
||||
from soup_cli.eval.quant_check import make_model_generator
|
||||
|
||||
def _stub_generate(_: str) -> str:
|
||||
return ""
|
||||
|
||||
generate_fn = _stub_generate
|
||||
try:
|
||||
generate_fn = make_model_generator(model)
|
||||
except (OSError, ValueError, ImportError) as exc:
|
||||
console.print(
|
||||
f"[red]Failed to load --model '{model}':[/] {exc}"
|
||||
)
|
||||
raise typer.Exit(1) from exc
|
||||
|
||||
result = run_gate(
|
||||
eval_suite, generate_fn=generate_fn, baseline=baseline_scores,
|
||||
|
|
@ -1038,14 +1080,24 @@ def quant_check_cmd(
|
|||
console.print(f"[red]--after not found: {resolved_after}[/]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# Live model loading is post-v0.26.0; stub for the orchestration layer.
|
||||
console.print(
|
||||
"[yellow]Live model scoring not yet wired; using deterministic stub. "
|
||||
"v0.26.1+ will plug in transformers/GGUF/AWQ backends.[/]"
|
||||
)
|
||||
# Live model scoring: build transformers-backed generators per side.
|
||||
# Falls back to deterministic stubs if loading fails (e.g. missing deps),
|
||||
# so CI without GPUs can still smoke-test the orchestration layer.
|
||||
from soup_cli.eval.quant_check import make_model_generator
|
||||
|
||||
try:
|
||||
before_gen = make_model_generator(resolved_before)
|
||||
after_gen = make_model_generator(resolved_after)
|
||||
except (OSError, ValueError, ImportError) as exc:
|
||||
console.print(
|
||||
f"[yellow]Live model load failed ({exc}); using deterministic stub.[/]"
|
||||
)
|
||||
before_gen = stub_generator("before")
|
||||
after_gen = stub_generator("after")
|
||||
|
||||
result = run_quant_check(
|
||||
before_gen=stub_generator("before"),
|
||||
after_gen=stub_generator("after"),
|
||||
before_gen=before_gen,
|
||||
after_gen=after_gen,
|
||||
tasks_file=tasks,
|
||||
)
|
||||
rendered = render(result, fmt=fmt)
|
||||
|
|
@ -1067,13 +1119,21 @@ def _print_gate_result(result) -> None:
|
|||
table.add_column("Delta", justify="right")
|
||||
table.add_column("Verdict")
|
||||
for row in result.task_results:
|
||||
score_text = (
|
||||
f"{row.score:.3f}" if row.score is not None
|
||||
else "[red]ERROR[/]"
|
||||
)
|
||||
verdict = (
|
||||
"[green]PASS[/]" if row.passed
|
||||
else (f"[red]FAIL ({row.error})[/]" if row.error else "[red]FAIL[/]")
|
||||
)
|
||||
table.add_row(
|
||||
row.name,
|
||||
f"{row.score:.3f}",
|
||||
score_text,
|
||||
f"{row.threshold:.3f}",
|
||||
f"{row.baseline:.3f}" if row.baseline is not None else "-",
|
||||
f"{row.delta:+.3f}" if row.delta is not None else "-",
|
||||
"[green]PASS[/]" if row.passed else "[red]FAIL[/]",
|
||||
verdict,
|
||||
)
|
||||
console.print(table)
|
||||
verdict = "[green]GATE PASSED[/]" if result.passed else "[red]GATE FAILED[/]"
|
||||
|
|
|
|||
|
|
@ -91,6 +91,12 @@ def export(
|
|||
"--calibration-samples",
|
||||
help="Number of calibration samples for AWQ/GPTQ",
|
||||
),
|
||||
registry_id: Optional[str] = typer.Option(
|
||||
None,
|
||||
"--registry-id",
|
||||
help="Attach exported artifact to this registry entry "
|
||||
"(default: auto-match by source --model output dir)",
|
||||
),
|
||||
):
|
||||
"""Export a model to GGUF, ONNX, TensorRT-LLM, AWQ, or GPTQ format."""
|
||||
model_path = Path(model)
|
||||
|
|
@ -218,6 +224,12 @@ def export(
|
|||
console.print("[red]Export failed - output file not created.[/]")
|
||||
raise typer.Exit(1)
|
||||
|
||||
# v0.33.0 #35: optional auto-attach to registry entry
|
||||
_maybe_attach_export(
|
||||
artifact_path=str(output_path), kind="gguf",
|
||||
explicit_id=registry_id, source_model=str(Path(model)),
|
||||
)
|
||||
|
||||
file_size = output_path.stat().st_size
|
||||
size_str = _format_size(file_size)
|
||||
|
||||
|
|
@ -966,3 +978,36 @@ def _format_size(size_bytes: int) -> str:
|
|||
return f"{value:.1f} {unit}"
|
||||
value /= 1024.0
|
||||
return f"{value:.1f} TB"
|
||||
|
||||
|
||||
def _maybe_attach_export(
|
||||
*, artifact_path: str, kind: str,
|
||||
explicit_id: Optional[str], source_model: str,
|
||||
) -> None:
|
||||
"""Attach an exported artifact to a registry entry.
|
||||
|
||||
Resolution order:
|
||||
1. ``--registry-id`` (explicit override)
|
||||
2. Auto-match by source model output dir
|
||||
Silent no-op if no match is found and no explicit id was given. Failures
|
||||
are surfaced as warnings, never as a hard CLI exit (the export itself
|
||||
succeeded).
|
||||
"""
|
||||
from soup_cli.registry.attach import attach_artifact, lookup_entry_by_output_dir
|
||||
|
||||
entry_id = explicit_id
|
||||
if entry_id is None:
|
||||
entry_id = lookup_entry_by_output_dir(source_model)
|
||||
if entry_id is None:
|
||||
return
|
||||
try:
|
||||
attach_artifact(entry_id, path=artifact_path, kind=kind)
|
||||
except (ValueError, FileNotFoundError) as exc:
|
||||
console.print(
|
||||
f"[yellow]Could not attach export to registry "
|
||||
f"'{entry_id}':[/] {exc}"
|
||||
)
|
||||
return
|
||||
console.print(
|
||||
f"[green]Attached export to registry entry '{entry_id}' as {kind}.[/]"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -21,11 +21,12 @@ from soup_cli.utils.paths import is_under_cwd
|
|||
@dataclass(frozen=True)
|
||||
class GateTaskResult:
|
||||
name: str
|
||||
score: float
|
||||
score: Optional[float]
|
||||
threshold: float
|
||||
baseline: Optional[float]
|
||||
delta: Optional[float]
|
||||
passed: bool
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -159,6 +160,110 @@ def resolve_baseline(spec: Optional[str]) -> dict[str, float]:
|
|||
return {str(k): float(v) for k, v in data.items()}
|
||||
|
||||
|
||||
def _parse_judge_url(judge_model: str) -> tuple[str, str, Optional[str]]:
|
||||
"""Split a ``judge_model`` URL into ``(provider, model, api_base)``.
|
||||
|
||||
Examples:
|
||||
``ollama://llama3.1`` -> ("ollama", "llama3.1", None)
|
||||
``http://localhost:8000/Qwen2.5`` -> ("server", "Qwen2.5", "http://localhost:8000")
|
||||
``https://api.openai.com/gpt-4o-mini`` -> ("openai", "gpt-4o-mini", "https://api.openai.com")
|
||||
"""
|
||||
if judge_model.startswith("ollama://"):
|
||||
return ("ollama", judge_model[len("ollama://"):], None)
|
||||
# http(s):// — last path segment is the model id; the rest is api_base.
|
||||
for prefix, default_provider in (
|
||||
("http://localhost", "server"),
|
||||
("http://127.0.0.1", "server"),
|
||||
("https://", "openai"),
|
||||
("http://", "server"),
|
||||
):
|
||||
if judge_model.startswith(prefix):
|
||||
try:
|
||||
base, model = judge_model.rsplit("/", 1)
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"judge_model '{judge_model}' missing model id"
|
||||
) from exc
|
||||
if not model:
|
||||
raise ValueError(f"judge_model '{judge_model}' missing model id")
|
||||
return (default_provider, model, base)
|
||||
raise ValueError(f"judge_model '{judge_model}' uses unsupported scheme")
|
||||
|
||||
|
||||
def _run_judge_task(
|
||||
task: GateTask, generate_fn: Callable[[str], str],
|
||||
) -> float:
|
||||
"""Run a type=judge task. Generates a completion per prompt, then asks
|
||||
the judge model to score the (prompt, response) pair on a 1-10 scale.
|
||||
Aggregate score is normalised to [0, 1] (mean / 10).
|
||||
"""
|
||||
if not task.prompts:
|
||||
raise ValueError(f"task '{task.name}' is type=judge but 'prompts' is missing")
|
||||
if not task.judge_model:
|
||||
raise ValueError(
|
||||
f"task '{task.name}' is type=judge but 'judge_model' is missing"
|
||||
)
|
||||
|
||||
prompts_path = Path(task.prompts)
|
||||
if not is_under_cwd(prompts_path):
|
||||
raise ValueError(f"prompts file '{task.prompts}' is outside cwd")
|
||||
if not prompts_path.exists():
|
||||
raise FileNotFoundError(f"prompts file not found: {task.prompts}")
|
||||
|
||||
from soup_cli.eval.judge import JudgeEvaluator
|
||||
|
||||
provider, model, api_base = _parse_judge_url(task.judge_model)
|
||||
evaluator = JudgeEvaluator(provider=provider, model=model, api_base=api_base)
|
||||
|
||||
items: list[dict] = []
|
||||
with prompts_path.open("r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
row = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise ValueError(
|
||||
f"invalid JSONL in {task.prompts}: {exc}"
|
||||
) from exc
|
||||
prompt = row.get("prompt", "")
|
||||
response = generate_fn(prompt)
|
||||
items.append({
|
||||
"prompt": prompt,
|
||||
"response": response,
|
||||
"category": row.get("category", "default"),
|
||||
})
|
||||
|
||||
if not items:
|
||||
return 0.0
|
||||
|
||||
results = evaluator.evaluate_batch(items)
|
||||
# results.overall_score is on a 1-10 scale; normalise to [0, 1].
|
||||
overall = float(getattr(results, "overall_score", 0.0))
|
||||
return max(0.0, min(1.0, overall / 10.0))
|
||||
|
||||
|
||||
def _run_benchmark_task(
|
||||
task: GateTask, generate_fn: Callable[[str], str],
|
||||
) -> float:
|
||||
"""Run a type=benchmark task using the existing forgetting-mini-benchmark."""
|
||||
if not task.benchmark:
|
||||
raise ValueError(
|
||||
f"task '{task.name}' is type=benchmark but 'benchmark' is missing"
|
||||
)
|
||||
from soup_cli.eval import forgetting
|
||||
|
||||
runner = getattr(forgetting, "run_mini_benchmark", None)
|
||||
if runner is None:
|
||||
raise RuntimeError(
|
||||
"mini-benchmark runner unavailable - "
|
||||
"install [eval] extras or update soup-cli"
|
||||
)
|
||||
score = runner(benchmark=task.benchmark, generate_fn=generate_fn)
|
||||
return max(0.0, min(1.0, float(score)))
|
||||
|
||||
|
||||
def _run_custom_task(
|
||||
task: GateTask, generate_fn: Callable[[str], str],
|
||||
) -> float:
|
||||
|
|
@ -201,22 +306,41 @@ def run_gate(
|
|||
any_failed_threshold = False
|
||||
|
||||
for task in suite.tasks:
|
||||
if task.type == "custom":
|
||||
score = _run_custom_task(task, generate_fn)
|
||||
else:
|
||||
# Judge / benchmark are wired in v0.26.1+; treat as skipped with
|
||||
# score=1.0 here so we don't hard-fail valid configs. The CLI
|
||||
# surfaces a warning when these are encountered.
|
||||
score = 1.0
|
||||
score: Optional[float]
|
||||
error: Optional[str] = None
|
||||
try:
|
||||
if task.type == "custom":
|
||||
score = _run_custom_task(task, generate_fn)
|
||||
elif task.type == "judge":
|
||||
score = _run_judge_task(task, generate_fn)
|
||||
elif task.type == "benchmark":
|
||||
score = _run_benchmark_task(task, generate_fn)
|
||||
else:
|
||||
# Pydantic Literal already restricts task.type, so this is a
|
||||
# belt-and-braces fallthrough.
|
||||
raise ValueError(f"unknown task type: {task.type}")
|
||||
except (ValueError, FileNotFoundError, OSError, RuntimeError) as exc:
|
||||
score = None
|
||||
error = str(exc)
|
||||
except Exception as exc: # noqa: BLE001 — surface as score=None, never silent pass
|
||||
score = None
|
||||
error = f"{type(exc).__name__}: {exc}"
|
||||
|
||||
passed_threshold = score >= task.threshold
|
||||
base_score = baseline.get(task.name)
|
||||
delta = None
|
||||
regressed = False
|
||||
if base_score is not None:
|
||||
delta = score - base_score
|
||||
if delta < -abs(regression_threshold):
|
||||
regressed = True
|
||||
if score is None:
|
||||
# Failed evaluation never silently passes the gate.
|
||||
passed_threshold = False
|
||||
base_score = baseline.get(task.name)
|
||||
delta = None
|
||||
regressed = False
|
||||
else:
|
||||
passed_threshold = score >= task.threshold
|
||||
base_score = baseline.get(task.name)
|
||||
delta = None
|
||||
regressed = False
|
||||
if base_score is not None:
|
||||
delta = score - base_score
|
||||
if delta < -abs(regression_threshold):
|
||||
regressed = True
|
||||
|
||||
if not passed_threshold:
|
||||
any_failed_threshold = True
|
||||
|
|
@ -230,6 +354,7 @@ def run_gate(
|
|||
baseline=base_score,
|
||||
delta=delta,
|
||||
passed=passed_threshold and not regressed,
|
||||
error=error,
|
||||
))
|
||||
|
||||
return GateResult(
|
||||
|
|
|
|||
|
|
@ -149,6 +149,51 @@ def ensure_format(fmt: str) -> None:
|
|||
raise ValueError(f"unknown format '{fmt}'. Use table | json | markdown")
|
||||
|
||||
|
||||
def make_model_generator(
|
||||
model_path: str,
|
||||
*,
|
||||
max_new_tokens: int = 256,
|
||||
temperature: float = 0.0,
|
||||
) -> Callable[[str], str]:
|
||||
"""Return a ``generate_fn(prompt) -> str`` backed by a transformers model.
|
||||
|
||||
Lazy-loaded so the CLI stays cold-start fast. The model is loaded once
|
||||
and reused across calls. ``temperature=0`` enables greedy decoding for
|
||||
reproducible eval scores.
|
||||
"""
|
||||
if max_new_tokens < 1 or max_new_tokens > 16384:
|
||||
raise ValueError("max_new_tokens must be in [1, 16384]")
|
||||
|
||||
from transformers import (
|
||||
AutoModelForCausalLM,
|
||||
AutoTokenizer,
|
||||
)
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=False)
|
||||
model = AutoModelForCausalLM.from_pretrained(
|
||||
model_path, trust_remote_code=False
|
||||
)
|
||||
model.eval()
|
||||
|
||||
def _generate(prompt: str) -> str:
|
||||
if not prompt:
|
||||
return ""
|
||||
inputs = tokenizer(prompt, return_tensors="pt", truncation=True)
|
||||
do_sample = temperature > 0.0
|
||||
outputs = model.generate(
|
||||
**inputs,
|
||||
max_new_tokens=max_new_tokens,
|
||||
do_sample=do_sample,
|
||||
temperature=max(temperature, 1e-5),
|
||||
pad_token_id=tokenizer.eos_token_id,
|
||||
)
|
||||
# Strip the prompt prefix from the decoded text.
|
||||
new_tokens = outputs[0][inputs["input_ids"].shape[1]:]
|
||||
return tokenizer.decode(new_tokens, skip_special_tokens=True)
|
||||
|
||||
return _generate
|
||||
|
||||
|
||||
def stub_generator(label: str) -> Callable[[str], str]:
|
||||
"""Return a deterministic stub generator so the CLI has something runnable.
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,85 @@
|
|||
"""Attach eval/export artifacts to existing registry entries (v0.33.0 #35).
|
||||
|
||||
Thin wrappers around ``RegistryStore.add_artifact`` so the eval and export CLI
|
||||
commands have a single, consistent entry point for post-hoc artifact attachment.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def attach_artifact(
|
||||
entry_id: str, *, path: str, kind: str, enforce_cwd: bool = True,
|
||||
) -> Optional[int]:
|
||||
"""Attach a file at ``path`` to a registry entry as ``kind``.
|
||||
|
||||
Returns the inserted artifact rowid, or None on lookup failure. Raises
|
||||
``ValueError`` / ``FileNotFoundError`` for explicit user-actionable errors
|
||||
so the CLI layer can render them.
|
||||
"""
|
||||
from soup_cli.registry.store import RegistryStore
|
||||
|
||||
artifact_path = Path(path)
|
||||
if not artifact_path.exists():
|
||||
raise FileNotFoundError(f"artifact not found: {path}")
|
||||
|
||||
with RegistryStore() as store:
|
||||
resolved_id = store.resolve(entry_id)
|
||||
if resolved_id is None:
|
||||
raise ValueError(f"registry entry not found: {entry_id}")
|
||||
return store.add_artifact(
|
||||
entry_id=resolved_id,
|
||||
kind=kind,
|
||||
path=str(artifact_path),
|
||||
enforce_cwd=enforce_cwd,
|
||||
)
|
||||
|
||||
|
||||
def write_eval_json(
|
||||
output_path: str, *, payload: dict[str, Any],
|
||||
) -> Path:
|
||||
"""Write an eval payload as JSON, returning the resolved path.
|
||||
|
||||
Writes are confined to cwd via realpath + commonpath check.
|
||||
"""
|
||||
cwd_real = os.path.realpath(os.getcwd())
|
||||
out_real = os.path.realpath(output_path)
|
||||
try:
|
||||
common = os.path.commonpath([cwd_real, out_real])
|
||||
except ValueError as exc:
|
||||
raise ValueError(
|
||||
f"output path '{output_path}' is outside cwd"
|
||||
) from exc
|
||||
if common != cwd_real:
|
||||
raise ValueError(f"output path '{output_path}' is outside cwd")
|
||||
|
||||
out = Path(out_real)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8")
|
||||
return out
|
||||
|
||||
|
||||
def lookup_entry_by_output_dir(output_dir: str) -> Optional[str]:
|
||||
"""Find a registry entry whose stored ``output`` directory matches.
|
||||
|
||||
Used by ``soup export`` to auto-attach artifacts when the user did not
|
||||
pass ``--registry-id`` explicitly. Returns None if no match.
|
||||
"""
|
||||
from soup_cli.registry.store import RegistryStore
|
||||
|
||||
target_real = os.path.realpath(output_dir)
|
||||
with RegistryStore() as store:
|
||||
for entry in store.list(limit=1000):
|
||||
stored = entry.get("output") or ""
|
||||
if not stored:
|
||||
continue
|
||||
try:
|
||||
if os.path.realpath(stored) == target_real:
|
||||
return entry.get("id")
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
|
@ -37,7 +37,10 @@ class AmbiguousRefError(ValueError):
|
|||
REGISTRY_DB_FILENAME = "registry.db"
|
||||
|
||||
_VALID_KINDS = frozenset(
|
||||
{"adapter", "merged", "gguf", "awq", "gptq", "onnx", "dataset", "config"}
|
||||
{
|
||||
"adapter", "merged", "gguf", "awq", "gptq", "onnx", "dataset", "config",
|
||||
"eval_results", "tensorrt",
|
||||
}
|
||||
)
|
||||
_VALID_RELATIONS = frozenset(
|
||||
{"forked_from", "merged_from", "evaluated_with", "promoted_from"}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,327 @@
|
|||
"""Part A wave 1 — v0.26.1 follow-ups (#32, #35) for v0.33.0.
|
||||
|
||||
Covers:
|
||||
- #32 Live model scoring: judge / benchmark task dispatch in run_gate,
|
||||
score=None + error propagation, _parse_judge_url helper, generator
|
||||
factory shape.
|
||||
- #35 Registry attach: --attach-to-registry on `soup eval custom`,
|
||||
--registry-id auto-attach in `soup export`, registry artifact kind
|
||||
extensions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from typer.testing import CliRunner
|
||||
|
||||
runner = CliRunner()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #32 — gate: judge URL parser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestParseJudgeURL:
|
||||
def test_ollama_scheme(self):
|
||||
from soup_cli.eval.gate import _parse_judge_url
|
||||
|
||||
provider, model, base = _parse_judge_url("ollama://llama3.1")
|
||||
assert provider == "ollama"
|
||||
assert model == "llama3.1"
|
||||
assert base is None
|
||||
|
||||
def test_https_openai(self):
|
||||
from soup_cli.eval.gate import _parse_judge_url
|
||||
|
||||
provider, model, base = _parse_judge_url(
|
||||
"https://api.openai.com/gpt-4o-mini"
|
||||
)
|
||||
assert provider == "openai"
|
||||
assert model == "gpt-4o-mini"
|
||||
assert base == "https://api.openai.com"
|
||||
|
||||
def test_http_localhost_server(self):
|
||||
from soup_cli.eval.gate import _parse_judge_url
|
||||
|
||||
provider, model, base = _parse_judge_url(
|
||||
"http://localhost:8000/Qwen2.5"
|
||||
)
|
||||
assert provider == "server"
|
||||
assert model == "Qwen2.5"
|
||||
assert base == "http://localhost:8000"
|
||||
|
||||
def test_rejects_unsupported_scheme(self):
|
||||
from soup_cli.eval.gate import _parse_judge_url
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported scheme"):
|
||||
_parse_judge_url("ftp://example.com/model")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #32 — run_gate: error → score=None propagation
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRunGateErrorPropagation:
|
||||
def test_judge_task_failure_surfaces_score_none(self, tmp_path, monkeypatch):
|
||||
"""Exception from judge backend must produce score=None, error=str(exc),
|
||||
passed=False — never a silent score=1.0."""
|
||||
from soup_cli.eval.gate import EvalSuite, GateTask, run_gate
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
prompts = tmp_path / "prompts.jsonl"
|
||||
prompts.write_text(
|
||||
json.dumps({"prompt": "Hi"}) + "\n", encoding="utf-8",
|
||||
)
|
||||
|
||||
suite = EvalSuite(suite="t", tasks=[GateTask(
|
||||
type="judge", name="quality", threshold=0.5,
|
||||
prompts="prompts.jsonl",
|
||||
judge_model="ollama://llama3.1",
|
||||
)])
|
||||
|
||||
# Inject a JudgeEvaluator that explodes on construction.
|
||||
with patch("soup_cli.eval.judge.JudgeEvaluator") as mock_judge:
|
||||
mock_judge.side_effect = OSError("connection refused")
|
||||
result = run_gate(
|
||||
suite, generate_fn=lambda _p: "stub",
|
||||
regression_threshold=0.05,
|
||||
)
|
||||
|
||||
assert len(result.task_results) == 1
|
||||
row = result.task_results[0]
|
||||
assert row.score is None
|
||||
assert row.error and "connection refused" in row.error
|
||||
assert row.passed is False
|
||||
assert result.passed is False
|
||||
|
||||
def test_custom_task_unknown_file_error(self, tmp_path, monkeypatch):
|
||||
from soup_cli.eval.gate import EvalSuite, GateTask, run_gate
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
suite = EvalSuite(suite="t", tasks=[GateTask(
|
||||
type="custom", name="cust", threshold=0.5,
|
||||
tasks="missing.jsonl", scorer="exact",
|
||||
)])
|
||||
|
||||
result = run_gate(suite, generate_fn=lambda _p: "out")
|
||||
row = result.task_results[0]
|
||||
assert row.score is None
|
||||
assert row.error
|
||||
assert row.passed is False
|
||||
|
||||
def test_benchmark_task_unavailable(self, tmp_path, monkeypatch):
|
||||
from soup_cli.eval import forgetting
|
||||
from soup_cli.eval.gate import EvalSuite, GateTask, run_gate
|
||||
|
||||
# Strip the runner attr to force the RuntimeError branch
|
||||
monkeypatch.setattr(
|
||||
forgetting, "run_mini_benchmark", None, raising=False,
|
||||
)
|
||||
# Ensure attribute lookup returns None
|
||||
if hasattr(forgetting, "run_mini_benchmark"):
|
||||
monkeypatch.delattr(
|
||||
forgetting, "run_mini_benchmark", raising=False,
|
||||
)
|
||||
|
||||
suite = EvalSuite(suite="t", tasks=[GateTask(
|
||||
type="benchmark", name="bench", threshold=0.3,
|
||||
benchmark="mini_mmlu",
|
||||
)])
|
||||
result = run_gate(suite, generate_fn=lambda _p: "")
|
||||
row = result.task_results[0]
|
||||
assert row.score is None
|
||||
assert row.error and "unavailable" in row.error
|
||||
assert row.passed is False
|
||||
|
||||
|
||||
class TestGateTaskResultSchema:
|
||||
def test_error_field_default_none(self):
|
||||
from soup_cli.eval.gate import GateTaskResult
|
||||
|
||||
row = GateTaskResult(
|
||||
name="x", score=0.7, threshold=0.5,
|
||||
baseline=None, delta=None, passed=True,
|
||||
)
|
||||
assert row.error is None
|
||||
|
||||
def test_score_optional(self):
|
||||
from soup_cli.eval.gate import GateTaskResult
|
||||
|
||||
row = GateTaskResult(
|
||||
name="x", score=None, threshold=0.5,
|
||||
baseline=None, delta=None, passed=False,
|
||||
error="boom",
|
||||
)
|
||||
assert row.score is None
|
||||
assert row.error == "boom"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #32 — quant_check.make_model_generator
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestMakeModelGenerator:
|
||||
def test_max_new_tokens_bounds(self):
|
||||
from soup_cli.eval.quant_check import make_model_generator
|
||||
|
||||
with pytest.raises(ValueError, match="max_new_tokens"):
|
||||
make_model_generator("/tmp/x", max_new_tokens=0)
|
||||
with pytest.raises(ValueError, match="max_new_tokens"):
|
||||
make_model_generator("/tmp/x", max_new_tokens=99_999)
|
||||
|
||||
def test_returns_callable_with_mocked_transformers(self):
|
||||
from soup_cli.eval import quant_check
|
||||
|
||||
fake_tokenizer = MagicMock()
|
||||
fake_tokenizer.eos_token_id = 0
|
||||
fake_inputs = {"input_ids": MagicMock()}
|
||||
fake_inputs["input_ids"].shape = (1, 3)
|
||||
fake_tokenizer.return_value = fake_inputs
|
||||
fake_tokenizer.decode.return_value = "out"
|
||||
|
||||
fake_model = MagicMock()
|
||||
fake_model.generate.return_value = [[1, 2, 3, 4, 5, 6]]
|
||||
|
||||
with patch.dict("sys.modules", {"transformers": MagicMock(
|
||||
AutoTokenizer=MagicMock(from_pretrained=MagicMock(
|
||||
return_value=fake_tokenizer,
|
||||
)),
|
||||
AutoModelForCausalLM=MagicMock(from_pretrained=MagicMock(
|
||||
return_value=fake_model,
|
||||
)),
|
||||
)}):
|
||||
gen = quant_check.make_model_generator(
|
||||
"/fake/model", max_new_tokens=8,
|
||||
)
|
||||
out = gen("hello")
|
||||
assert out == "out"
|
||||
|
||||
def test_empty_prompt_returns_empty(self):
|
||||
from soup_cli.eval import quant_check
|
||||
|
||||
fake_tok = MagicMock()
|
||||
fake_tok.eos_token_id = 0
|
||||
fake_model = MagicMock()
|
||||
with patch.dict("sys.modules", {"transformers": MagicMock(
|
||||
AutoTokenizer=MagicMock(from_pretrained=MagicMock(return_value=fake_tok)),
|
||||
AutoModelForCausalLM=MagicMock(from_pretrained=MagicMock(return_value=fake_model)),
|
||||
)}):
|
||||
gen = quant_check.make_model_generator("/fake/model")
|
||||
assert gen("") == ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #35 — registry attach helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestRegistryAttachHelpers:
|
||||
def test_write_eval_json_containment(self, tmp_path, monkeypatch):
|
||||
from soup_cli.registry.attach import write_eval_json
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
out = write_eval_json(
|
||||
"results.json", payload={"score": 0.7},
|
||||
)
|
||||
assert out.exists()
|
||||
data = json.loads(out.read_text(encoding="utf-8"))
|
||||
assert data["score"] == 0.7
|
||||
|
||||
def test_write_eval_json_rejects_outside_cwd(self, tmp_path, monkeypatch):
|
||||
from soup_cli.registry.attach import write_eval_json
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
outside = str(tmp_path.parent / "evil.json")
|
||||
with pytest.raises(ValueError, match="outside cwd"):
|
||||
write_eval_json(outside, payload={})
|
||||
|
||||
def test_attach_artifact_unknown_entry(self, tmp_path, monkeypatch):
|
||||
from soup_cli.registry.attach import attach_artifact
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# Use an isolated registry DB
|
||||
db = tmp_path / "reg.db"
|
||||
monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(db))
|
||||
|
||||
target = tmp_path / "results.json"
|
||||
target.write_text("{}", encoding="utf-8")
|
||||
with pytest.raises(ValueError, match="not found"):
|
||||
attach_artifact("nonexistent-id", path=str(target), kind="eval_results")
|
||||
|
||||
def test_attach_artifact_missing_file(self, tmp_path, monkeypatch):
|
||||
from soup_cli.registry.attach import attach_artifact
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(FileNotFoundError):
|
||||
attach_artifact(
|
||||
"any", path=str(tmp_path / "missing.json"), kind="eval_results",
|
||||
)
|
||||
|
||||
|
||||
class TestRegistryArtifactKindsExtended:
|
||||
def test_eval_results_kind_accepted(self):
|
||||
from soup_cli.registry.store import _VALID_KINDS
|
||||
|
||||
assert "eval_results" in _VALID_KINDS
|
||||
assert "tensorrt" in _VALID_KINDS
|
||||
|
||||
|
||||
class TestLookupEntryByOutputDir:
|
||||
def test_lookup_returns_none_when_no_match(self, tmp_path, monkeypatch):
|
||||
from soup_cli.registry.attach import lookup_entry_by_output_dir
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
db = tmp_path / "reg.db"
|
||||
monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(db))
|
||||
|
||||
result = lookup_entry_by_output_dir(str(tmp_path / "no-such-output"))
|
||||
assert result is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# #35 — `soup eval custom --attach-to-registry` CLI integration
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestEvalCustomAttachCLI:
|
||||
def test_attach_to_unknown_entry_errors(self, tmp_path, monkeypatch):
|
||||
"""--attach-to-registry pointing at a missing entry produces a clean
|
||||
error and exits non-zero rather than silently passing."""
|
||||
from soup_cli.cli import app
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
# Isolated registry DB so we don't pollute the user's ~/.soup
|
||||
monkeypatch.setenv(
|
||||
"SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db"),
|
||||
)
|
||||
(tmp_path / "tasks.jsonl").write_text(
|
||||
json.dumps({"prompt": "p", "expected": "x"}) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "model").mkdir()
|
||||
|
||||
with patch(
|
||||
"soup_cli.eval.custom._create_default_generator",
|
||||
return_value=lambda _p: "x",
|
||||
):
|
||||
result = runner.invoke(
|
||||
app,
|
||||
[
|
||||
"eval", "custom",
|
||||
"--tasks", "tasks.jsonl",
|
||||
"--model", "model",
|
||||
"--attach-to-registry", "no-such-id",
|
||||
"--output", "results.json",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 1, (result.output, repr(result.exception))
|
||||
assert (
|
||||
"registry entry not found" in result.output
|
||||
or "not found" in result.output
|
||||
)
|
||||
Loading…
Reference in New Issue