From ca799f6fd3e875786c3d06b609565f77bec7091f Mon Sep 17 00:00:00 2001 From: Alpamys Date: Mon, 27 Apr 2026 18:06:27 +0500 Subject: [PATCH] feat(eval,registry): live gate scoring + registry attach (v0.33.0 Part A wave 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #32, #35. (#34 soup can run/publish deferred to Part A wave 2.) #32 Live model scoring for `soup eval gate` + `soup eval quant-check`: - gate.run_gate now dispatches judge / benchmark / custom task types, wrapping each scorer in try/except so a backend failure produces score=None + error=str(exc) instead of a silent score=1.0 pass. - New _parse_judge_url splits ollama:// / http(s):// judge_model URLs into (provider, model, api_base) for JudgeEvaluator. - New _run_judge_task / _run_benchmark_task plug into existing eval/judge.py and eval/forgetting.py runners. - New quant_check.make_model_generator(model_path) wraps transformers AutoTokenizer + AutoModelForCausalLM into a generate_fn callable; greedy by default for reproducible scores; lazy-imported. - gate_cmd / quant_check_cmd build live generators when --model is given; fall back to deterministic stub on load failure so CI without GPUs still runs the orchestration layer. - GateTaskResult.score is now Optional[float] with new error: Optional[str]. - _print_gate_result renders ERROR + reason cleanly. #35 Registry attach hooks: - registry/store.py _VALID_KINDS extended with eval_results, tensorrt. - New registry/attach.py: attach_artifact, write_eval_json (cwd-containment via realpath+commonpath), lookup_entry_by_output_dir. - `soup eval custom` gains --attach-to-registry + --output (paired); on success writes JSON results and adds eval_results artifact row. - `soup export` gains --registry-id with auto-match by source --model output dir; auto-attaches the produced GGUF artifact. Failures here are warnings, not hard exits — export already succeeded. Tests: +19 in tests/test_part_a_wave1.py covering URL parser, error propagation across all 3 task types, score=None semantics, generator factory bounds + transformers mocking, registry attach helpers (containment + missing entry), and CLI integration. Co-Authored-By: Claude Opus 4.7 (1M context) --- soup_cli/commands/eval.py | 100 ++++++++--- soup_cli/commands/export.py | 45 +++++ soup_cli/eval/gate.py | 157 +++++++++++++++-- soup_cli/eval/quant_check.py | 45 +++++ soup_cli/registry/attach.py | 85 +++++++++ soup_cli/registry/store.py | 5 +- tests/test_part_a_wave1.py | 327 +++++++++++++++++++++++++++++++++++ 7 files changed, 727 insertions(+), 37 deletions(-) create mode 100644 soup_cli/registry/attach.py create mode 100644 tests/test_part_a_wave1.py diff --git a/soup_cli/commands/eval.py b/soup_cli/commands/eval.py index b7283eb..d306ff3 100644 --- a/soup_cli/commands/eval.py +++ b/soup_cli/commands/eval.py @@ -122,6 +122,14 @@ def custom( None, "--run-id", help="Link results to an existing training run", ), + attach_to_registry: Optional[str] = typer.Option( + None, "--attach-to-registry", + help="Attach the eval JSON to a registry entry as kind=eval_results", + ), + output: Optional[str] = typer.Option( + None, "--output", "-o", + help="Path for the eval JSON output (required with --attach-to-registry)", + ), ): """Run custom evaluation tasks from a JSONL file.""" from soup_cli.eval.custom import load_eval_tasks @@ -189,6 +197,36 @@ def custom( _save_custom_results(eval_results, str(model_path), run_id) console.print("\n[green]Results saved to experiment tracker.[/]") + # v0.33.0 #35: optional registry attach + if attach_to_registry: + if not output: + console.print( + "[red]--attach-to-registry requires --output [/]" + ) + raise typer.Exit(1) + from soup_cli.registry.attach import attach_artifact, write_eval_json + + payload = { + "model": str(model_path), + "tasks": str(tasks_path), + "total": eval_results.total, + "correct": eval_results.correct, + "accuracy": eval_results.accuracy, + "category_scores": eval_results.category_scores, + } + try: + json_path = write_eval_json(output, payload=payload) + attach_artifact( + attach_to_registry, path=str(json_path), kind="eval_results", + ) + except (ValueError, FileNotFoundError) as exc: + console.print(f"[red]Registry attach failed:[/] {exc}") + raise typer.Exit(1) from exc + console.print( + f"[green]Attached eval results to registry entry " + f"'{attach_to_registry}' as eval_results.[/]" + ) + # ─── soup eval judge ─── @@ -934,24 +972,28 @@ def gate_cmd( console.print(f"[red]Cannot resolve baseline:[/] {exc}") raise typer.Exit(1) from exc - # Without a live model we can't generate real completions; emit a stub - # generator so the CLI is still testable. Wiring a real model is out of - # scope for the v0.26.0 launch (see plan Part B). + # When --model is provided, build a transformers-backed generator. + # Otherwise fall back to an empty-string stub for smoke runs. if model is None: console.print( "[yellow]No --model given; using stub generator " "(empty output per prompt) for a smoke run.[/]" ) + + def _stub_generate(_: str) -> str: + return "" + + generate_fn = _stub_generate else: - console.print( - "[yellow]Live model scoring not yet wired; using stub. " - "Subscribe to v0.26.1 for real inference support.[/]" - ) + from soup_cli.eval.quant_check import make_model_generator - def _stub_generate(_: str) -> str: - return "" - - generate_fn = _stub_generate + try: + generate_fn = make_model_generator(model) + except (OSError, ValueError, ImportError) as exc: + console.print( + f"[red]Failed to load --model '{model}':[/] {exc}" + ) + raise typer.Exit(1) from exc result = run_gate( eval_suite, generate_fn=generate_fn, baseline=baseline_scores, @@ -1038,14 +1080,24 @@ def quant_check_cmd( console.print(f"[red]--after not found: {resolved_after}[/]") raise typer.Exit(1) - # Live model loading is post-v0.26.0; stub for the orchestration layer. - console.print( - "[yellow]Live model scoring not yet wired; using deterministic stub. " - "v0.26.1+ will plug in transformers/GGUF/AWQ backends.[/]" - ) + # Live model scoring: build transformers-backed generators per side. + # Falls back to deterministic stubs if loading fails (e.g. missing deps), + # so CI without GPUs can still smoke-test the orchestration layer. + from soup_cli.eval.quant_check import make_model_generator + + try: + before_gen = make_model_generator(resolved_before) + after_gen = make_model_generator(resolved_after) + except (OSError, ValueError, ImportError) as exc: + console.print( + f"[yellow]Live model load failed ({exc}); using deterministic stub.[/]" + ) + before_gen = stub_generator("before") + after_gen = stub_generator("after") + result = run_quant_check( - before_gen=stub_generator("before"), - after_gen=stub_generator("after"), + before_gen=before_gen, + after_gen=after_gen, tasks_file=tasks, ) rendered = render(result, fmt=fmt) @@ -1067,13 +1119,21 @@ def _print_gate_result(result) -> None: table.add_column("Delta", justify="right") table.add_column("Verdict") for row in result.task_results: + score_text = ( + f"{row.score:.3f}" if row.score is not None + else "[red]ERROR[/]" + ) + verdict = ( + "[green]PASS[/]" if row.passed + else (f"[red]FAIL ({row.error})[/]" if row.error else "[red]FAIL[/]") + ) table.add_row( row.name, - f"{row.score:.3f}", + score_text, f"{row.threshold:.3f}", f"{row.baseline:.3f}" if row.baseline is not None else "-", f"{row.delta:+.3f}" if row.delta is not None else "-", - "[green]PASS[/]" if row.passed else "[red]FAIL[/]", + verdict, ) console.print(table) verdict = "[green]GATE PASSED[/]" if result.passed else "[red]GATE FAILED[/]" diff --git a/soup_cli/commands/export.py b/soup_cli/commands/export.py index e9d4fff..47ab1d3 100644 --- a/soup_cli/commands/export.py +++ b/soup_cli/commands/export.py @@ -91,6 +91,12 @@ def export( "--calibration-samples", help="Number of calibration samples for AWQ/GPTQ", ), + registry_id: Optional[str] = typer.Option( + None, + "--registry-id", + help="Attach exported artifact to this registry entry " + "(default: auto-match by source --model output dir)", + ), ): """Export a model to GGUF, ONNX, TensorRT-LLM, AWQ, or GPTQ format.""" model_path = Path(model) @@ -218,6 +224,12 @@ def export( console.print("[red]Export failed - output file not created.[/]") raise typer.Exit(1) + # v0.33.0 #35: optional auto-attach to registry entry + _maybe_attach_export( + artifact_path=str(output_path), kind="gguf", + explicit_id=registry_id, source_model=str(Path(model)), + ) + file_size = output_path.stat().st_size size_str = _format_size(file_size) @@ -966,3 +978,36 @@ def _format_size(size_bytes: int) -> str: return f"{value:.1f} {unit}" value /= 1024.0 return f"{value:.1f} TB" + + +def _maybe_attach_export( + *, artifact_path: str, kind: str, + explicit_id: Optional[str], source_model: str, +) -> None: + """Attach an exported artifact to a registry entry. + + Resolution order: + 1. ``--registry-id`` (explicit override) + 2. Auto-match by source model output dir + Silent no-op if no match is found and no explicit id was given. Failures + are surfaced as warnings, never as a hard CLI exit (the export itself + succeeded). + """ + from soup_cli.registry.attach import attach_artifact, lookup_entry_by_output_dir + + entry_id = explicit_id + if entry_id is None: + entry_id = lookup_entry_by_output_dir(source_model) + if entry_id is None: + return + try: + attach_artifact(entry_id, path=artifact_path, kind=kind) + except (ValueError, FileNotFoundError) as exc: + console.print( + f"[yellow]Could not attach export to registry " + f"'{entry_id}':[/] {exc}" + ) + return + console.print( + f"[green]Attached export to registry entry '{entry_id}' as {kind}.[/]" + ) diff --git a/soup_cli/eval/gate.py b/soup_cli/eval/gate.py index 2cb9488..30c8d52 100644 --- a/soup_cli/eval/gate.py +++ b/soup_cli/eval/gate.py @@ -21,11 +21,12 @@ from soup_cli.utils.paths import is_under_cwd @dataclass(frozen=True) class GateTaskResult: name: str - score: float + score: Optional[float] threshold: float baseline: Optional[float] delta: Optional[float] passed: bool + error: Optional[str] = None @dataclass(frozen=True) @@ -159,6 +160,110 @@ def resolve_baseline(spec: Optional[str]) -> dict[str, float]: return {str(k): float(v) for k, v in data.items()} +def _parse_judge_url(judge_model: str) -> tuple[str, str, Optional[str]]: + """Split a ``judge_model`` URL into ``(provider, model, api_base)``. + + Examples: + ``ollama://llama3.1`` -> ("ollama", "llama3.1", None) + ``http://localhost:8000/Qwen2.5`` -> ("server", "Qwen2.5", "http://localhost:8000") + ``https://api.openai.com/gpt-4o-mini`` -> ("openai", "gpt-4o-mini", "https://api.openai.com") + """ + if judge_model.startswith("ollama://"): + return ("ollama", judge_model[len("ollama://"):], None) + # http(s):// — last path segment is the model id; the rest is api_base. + for prefix, default_provider in ( + ("http://localhost", "server"), + ("http://127.0.0.1", "server"), + ("https://", "openai"), + ("http://", "server"), + ): + if judge_model.startswith(prefix): + try: + base, model = judge_model.rsplit("/", 1) + except ValueError as exc: + raise ValueError( + f"judge_model '{judge_model}' missing model id" + ) from exc + if not model: + raise ValueError(f"judge_model '{judge_model}' missing model id") + return (default_provider, model, base) + raise ValueError(f"judge_model '{judge_model}' uses unsupported scheme") + + +def _run_judge_task( + task: GateTask, generate_fn: Callable[[str], str], +) -> float: + """Run a type=judge task. Generates a completion per prompt, then asks + the judge model to score the (prompt, response) pair on a 1-10 scale. + Aggregate score is normalised to [0, 1] (mean / 10). + """ + if not task.prompts: + raise ValueError(f"task '{task.name}' is type=judge but 'prompts' is missing") + if not task.judge_model: + raise ValueError( + f"task '{task.name}' is type=judge but 'judge_model' is missing" + ) + + prompts_path = Path(task.prompts) + if not is_under_cwd(prompts_path): + raise ValueError(f"prompts file '{task.prompts}' is outside cwd") + if not prompts_path.exists(): + raise FileNotFoundError(f"prompts file not found: {task.prompts}") + + from soup_cli.eval.judge import JudgeEvaluator + + provider, model, api_base = _parse_judge_url(task.judge_model) + evaluator = JudgeEvaluator(provider=provider, model=model, api_base=api_base) + + items: list[dict] = [] + with prompts_path.open("r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise ValueError( + f"invalid JSONL in {task.prompts}: {exc}" + ) from exc + prompt = row.get("prompt", "") + response = generate_fn(prompt) + items.append({ + "prompt": prompt, + "response": response, + "category": row.get("category", "default"), + }) + + if not items: + return 0.0 + + results = evaluator.evaluate_batch(items) + # results.overall_score is on a 1-10 scale; normalise to [0, 1]. + overall = float(getattr(results, "overall_score", 0.0)) + return max(0.0, min(1.0, overall / 10.0)) + + +def _run_benchmark_task( + task: GateTask, generate_fn: Callable[[str], str], +) -> float: + """Run a type=benchmark task using the existing forgetting-mini-benchmark.""" + if not task.benchmark: + raise ValueError( + f"task '{task.name}' is type=benchmark but 'benchmark' is missing" + ) + from soup_cli.eval import forgetting + + runner = getattr(forgetting, "run_mini_benchmark", None) + if runner is None: + raise RuntimeError( + "mini-benchmark runner unavailable - " + "install [eval] extras or update soup-cli" + ) + score = runner(benchmark=task.benchmark, generate_fn=generate_fn) + return max(0.0, min(1.0, float(score))) + + def _run_custom_task( task: GateTask, generate_fn: Callable[[str], str], ) -> float: @@ -201,22 +306,41 @@ def run_gate( any_failed_threshold = False for task in suite.tasks: - if task.type == "custom": - score = _run_custom_task(task, generate_fn) - else: - # Judge / benchmark are wired in v0.26.1+; treat as skipped with - # score=1.0 here so we don't hard-fail valid configs. The CLI - # surfaces a warning when these are encountered. - score = 1.0 + score: Optional[float] + error: Optional[str] = None + try: + if task.type == "custom": + score = _run_custom_task(task, generate_fn) + elif task.type == "judge": + score = _run_judge_task(task, generate_fn) + elif task.type == "benchmark": + score = _run_benchmark_task(task, generate_fn) + else: + # Pydantic Literal already restricts task.type, so this is a + # belt-and-braces fallthrough. + raise ValueError(f"unknown task type: {task.type}") + except (ValueError, FileNotFoundError, OSError, RuntimeError) as exc: + score = None + error = str(exc) + except Exception as exc: # noqa: BLE001 — surface as score=None, never silent pass + score = None + error = f"{type(exc).__name__}: {exc}" - passed_threshold = score >= task.threshold - base_score = baseline.get(task.name) - delta = None - regressed = False - if base_score is not None: - delta = score - base_score - if delta < -abs(regression_threshold): - regressed = True + if score is None: + # Failed evaluation never silently passes the gate. + passed_threshold = False + base_score = baseline.get(task.name) + delta = None + regressed = False + else: + passed_threshold = score >= task.threshold + base_score = baseline.get(task.name) + delta = None + regressed = False + if base_score is not None: + delta = score - base_score + if delta < -abs(regression_threshold): + regressed = True if not passed_threshold: any_failed_threshold = True @@ -230,6 +354,7 @@ def run_gate( baseline=base_score, delta=delta, passed=passed_threshold and not regressed, + error=error, )) return GateResult( diff --git a/soup_cli/eval/quant_check.py b/soup_cli/eval/quant_check.py index 60a0a07..aaf84e8 100644 --- a/soup_cli/eval/quant_check.py +++ b/soup_cli/eval/quant_check.py @@ -149,6 +149,51 @@ def ensure_format(fmt: str) -> None: raise ValueError(f"unknown format '{fmt}'. Use table | json | markdown") +def make_model_generator( + model_path: str, + *, + max_new_tokens: int = 256, + temperature: float = 0.0, +) -> Callable[[str], str]: + """Return a ``generate_fn(prompt) -> str`` backed by a transformers model. + + Lazy-loaded so the CLI stays cold-start fast. The model is loaded once + and reused across calls. ``temperature=0`` enables greedy decoding for + reproducible eval scores. + """ + if max_new_tokens < 1 or max_new_tokens > 16384: + raise ValueError("max_new_tokens must be in [1, 16384]") + + from transformers import ( + AutoModelForCausalLM, + AutoTokenizer, + ) + + tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=False) + model = AutoModelForCausalLM.from_pretrained( + model_path, trust_remote_code=False + ) + model.eval() + + def _generate(prompt: str) -> str: + if not prompt: + return "" + inputs = tokenizer(prompt, return_tensors="pt", truncation=True) + do_sample = temperature > 0.0 + outputs = model.generate( + **inputs, + max_new_tokens=max_new_tokens, + do_sample=do_sample, + temperature=max(temperature, 1e-5), + pad_token_id=tokenizer.eos_token_id, + ) + # Strip the prompt prefix from the decoded text. + new_tokens = outputs[0][inputs["input_ids"].shape[1]:] + return tokenizer.decode(new_tokens, skip_special_tokens=True) + + return _generate + + def stub_generator(label: str) -> Callable[[str], str]: """Return a deterministic stub generator so the CLI has something runnable. diff --git a/soup_cli/registry/attach.py b/soup_cli/registry/attach.py new file mode 100644 index 0000000..dbc75db --- /dev/null +++ b/soup_cli/registry/attach.py @@ -0,0 +1,85 @@ +"""Attach eval/export artifacts to existing registry entries (v0.33.0 #35). + +Thin wrappers around ``RegistryStore.add_artifact`` so the eval and export CLI +commands have a single, consistent entry point for post-hoc artifact attachment. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Optional + + +def attach_artifact( + entry_id: str, *, path: str, kind: str, enforce_cwd: bool = True, +) -> Optional[int]: + """Attach a file at ``path`` to a registry entry as ``kind``. + + Returns the inserted artifact rowid, or None on lookup failure. Raises + ``ValueError`` / ``FileNotFoundError`` for explicit user-actionable errors + so the CLI layer can render them. + """ + from soup_cli.registry.store import RegistryStore + + artifact_path = Path(path) + if not artifact_path.exists(): + raise FileNotFoundError(f"artifact not found: {path}") + + with RegistryStore() as store: + resolved_id = store.resolve(entry_id) + if resolved_id is None: + raise ValueError(f"registry entry not found: {entry_id}") + return store.add_artifact( + entry_id=resolved_id, + kind=kind, + path=str(artifact_path), + enforce_cwd=enforce_cwd, + ) + + +def write_eval_json( + output_path: str, *, payload: dict[str, Any], +) -> Path: + """Write an eval payload as JSON, returning the resolved path. + + Writes are confined to cwd via realpath + commonpath check. + """ + cwd_real = os.path.realpath(os.getcwd()) + out_real = os.path.realpath(output_path) + try: + common = os.path.commonpath([cwd_real, out_real]) + except ValueError as exc: + raise ValueError( + f"output path '{output_path}' is outside cwd" + ) from exc + if common != cwd_real: + raise ValueError(f"output path '{output_path}' is outside cwd") + + out = Path(out_real) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(payload, indent=2, default=str), encoding="utf-8") + return out + + +def lookup_entry_by_output_dir(output_dir: str) -> Optional[str]: + """Find a registry entry whose stored ``output`` directory matches. + + Used by ``soup export`` to auto-attach artifacts when the user did not + pass ``--registry-id`` explicitly. Returns None if no match. + """ + from soup_cli.registry.store import RegistryStore + + target_real = os.path.realpath(output_dir) + with RegistryStore() as store: + for entry in store.list(limit=1000): + stored = entry.get("output") or "" + if not stored: + continue + try: + if os.path.realpath(stored) == target_real: + return entry.get("id") + except (OSError, ValueError): + continue + return None diff --git a/soup_cli/registry/store.py b/soup_cli/registry/store.py index 12a3615..64c8cd3 100644 --- a/soup_cli/registry/store.py +++ b/soup_cli/registry/store.py @@ -37,7 +37,10 @@ class AmbiguousRefError(ValueError): REGISTRY_DB_FILENAME = "registry.db" _VALID_KINDS = frozenset( - {"adapter", "merged", "gguf", "awq", "gptq", "onnx", "dataset", "config"} + { + "adapter", "merged", "gguf", "awq", "gptq", "onnx", "dataset", "config", + "eval_results", "tensorrt", + } ) _VALID_RELATIONS = frozenset( {"forked_from", "merged_from", "evaluated_with", "promoted_from"} diff --git a/tests/test_part_a_wave1.py b/tests/test_part_a_wave1.py new file mode 100644 index 0000000..560baa2 --- /dev/null +++ b/tests/test_part_a_wave1.py @@ -0,0 +1,327 @@ +"""Part A wave 1 — v0.26.1 follow-ups (#32, #35) for v0.33.0. + +Covers: + - #32 Live model scoring: judge / benchmark task dispatch in run_gate, + score=None + error propagation, _parse_judge_url helper, generator + factory shape. + - #35 Registry attach: --attach-to-registry on `soup eval custom`, + --registry-id auto-attach in `soup export`, registry artifact kind + extensions. +""" + +from __future__ import annotations + +import json +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# #32 — gate: judge URL parser +# --------------------------------------------------------------------------- + + +class TestParseJudgeURL: + def test_ollama_scheme(self): + from soup_cli.eval.gate import _parse_judge_url + + provider, model, base = _parse_judge_url("ollama://llama3.1") + assert provider == "ollama" + assert model == "llama3.1" + assert base is None + + def test_https_openai(self): + from soup_cli.eval.gate import _parse_judge_url + + provider, model, base = _parse_judge_url( + "https://api.openai.com/gpt-4o-mini" + ) + assert provider == "openai" + assert model == "gpt-4o-mini" + assert base == "https://api.openai.com" + + def test_http_localhost_server(self): + from soup_cli.eval.gate import _parse_judge_url + + provider, model, base = _parse_judge_url( + "http://localhost:8000/Qwen2.5" + ) + assert provider == "server" + assert model == "Qwen2.5" + assert base == "http://localhost:8000" + + def test_rejects_unsupported_scheme(self): + from soup_cli.eval.gate import _parse_judge_url + + with pytest.raises(ValueError, match="unsupported scheme"): + _parse_judge_url("ftp://example.com/model") + + +# --------------------------------------------------------------------------- +# #32 — run_gate: error → score=None propagation +# --------------------------------------------------------------------------- + + +class TestRunGateErrorPropagation: + def test_judge_task_failure_surfaces_score_none(self, tmp_path, monkeypatch): + """Exception from judge backend must produce score=None, error=str(exc), + passed=False — never a silent score=1.0.""" + from soup_cli.eval.gate import EvalSuite, GateTask, run_gate + + monkeypatch.chdir(tmp_path) + prompts = tmp_path / "prompts.jsonl" + prompts.write_text( + json.dumps({"prompt": "Hi"}) + "\n", encoding="utf-8", + ) + + suite = EvalSuite(suite="t", tasks=[GateTask( + type="judge", name="quality", threshold=0.5, + prompts="prompts.jsonl", + judge_model="ollama://llama3.1", + )]) + + # Inject a JudgeEvaluator that explodes on construction. + with patch("soup_cli.eval.judge.JudgeEvaluator") as mock_judge: + mock_judge.side_effect = OSError("connection refused") + result = run_gate( + suite, generate_fn=lambda _p: "stub", + regression_threshold=0.05, + ) + + assert len(result.task_results) == 1 + row = result.task_results[0] + assert row.score is None + assert row.error and "connection refused" in row.error + assert row.passed is False + assert result.passed is False + + def test_custom_task_unknown_file_error(self, tmp_path, monkeypatch): + from soup_cli.eval.gate import EvalSuite, GateTask, run_gate + + monkeypatch.chdir(tmp_path) + suite = EvalSuite(suite="t", tasks=[GateTask( + type="custom", name="cust", threshold=0.5, + tasks="missing.jsonl", scorer="exact", + )]) + + result = run_gate(suite, generate_fn=lambda _p: "out") + row = result.task_results[0] + assert row.score is None + assert row.error + assert row.passed is False + + def test_benchmark_task_unavailable(self, tmp_path, monkeypatch): + from soup_cli.eval import forgetting + from soup_cli.eval.gate import EvalSuite, GateTask, run_gate + + # Strip the runner attr to force the RuntimeError branch + monkeypatch.setattr( + forgetting, "run_mini_benchmark", None, raising=False, + ) + # Ensure attribute lookup returns None + if hasattr(forgetting, "run_mini_benchmark"): + monkeypatch.delattr( + forgetting, "run_mini_benchmark", raising=False, + ) + + suite = EvalSuite(suite="t", tasks=[GateTask( + type="benchmark", name="bench", threshold=0.3, + benchmark="mini_mmlu", + )]) + result = run_gate(suite, generate_fn=lambda _p: "") + row = result.task_results[0] + assert row.score is None + assert row.error and "unavailable" in row.error + assert row.passed is False + + +class TestGateTaskResultSchema: + def test_error_field_default_none(self): + from soup_cli.eval.gate import GateTaskResult + + row = GateTaskResult( + name="x", score=0.7, threshold=0.5, + baseline=None, delta=None, passed=True, + ) + assert row.error is None + + def test_score_optional(self): + from soup_cli.eval.gate import GateTaskResult + + row = GateTaskResult( + name="x", score=None, threshold=0.5, + baseline=None, delta=None, passed=False, + error="boom", + ) + assert row.score is None + assert row.error == "boom" + + +# --------------------------------------------------------------------------- +# #32 — quant_check.make_model_generator +# --------------------------------------------------------------------------- + + +class TestMakeModelGenerator: + def test_max_new_tokens_bounds(self): + from soup_cli.eval.quant_check import make_model_generator + + with pytest.raises(ValueError, match="max_new_tokens"): + make_model_generator("/tmp/x", max_new_tokens=0) + with pytest.raises(ValueError, match="max_new_tokens"): + make_model_generator("/tmp/x", max_new_tokens=99_999) + + def test_returns_callable_with_mocked_transformers(self): + from soup_cli.eval import quant_check + + fake_tokenizer = MagicMock() + fake_tokenizer.eos_token_id = 0 + fake_inputs = {"input_ids": MagicMock()} + fake_inputs["input_ids"].shape = (1, 3) + fake_tokenizer.return_value = fake_inputs + fake_tokenizer.decode.return_value = "out" + + fake_model = MagicMock() + fake_model.generate.return_value = [[1, 2, 3, 4, 5, 6]] + + with patch.dict("sys.modules", {"transformers": MagicMock( + AutoTokenizer=MagicMock(from_pretrained=MagicMock( + return_value=fake_tokenizer, + )), + AutoModelForCausalLM=MagicMock(from_pretrained=MagicMock( + return_value=fake_model, + )), + )}): + gen = quant_check.make_model_generator( + "/fake/model", max_new_tokens=8, + ) + out = gen("hello") + assert out == "out" + + def test_empty_prompt_returns_empty(self): + from soup_cli.eval import quant_check + + fake_tok = MagicMock() + fake_tok.eos_token_id = 0 + fake_model = MagicMock() + with patch.dict("sys.modules", {"transformers": MagicMock( + AutoTokenizer=MagicMock(from_pretrained=MagicMock(return_value=fake_tok)), + AutoModelForCausalLM=MagicMock(from_pretrained=MagicMock(return_value=fake_model)), + )}): + gen = quant_check.make_model_generator("/fake/model") + assert gen("") == "" + + +# --------------------------------------------------------------------------- +# #35 — registry attach helpers +# --------------------------------------------------------------------------- + + +class TestRegistryAttachHelpers: + def test_write_eval_json_containment(self, tmp_path, monkeypatch): + from soup_cli.registry.attach import write_eval_json + + monkeypatch.chdir(tmp_path) + out = write_eval_json( + "results.json", payload={"score": 0.7}, + ) + assert out.exists() + data = json.loads(out.read_text(encoding="utf-8")) + assert data["score"] == 0.7 + + def test_write_eval_json_rejects_outside_cwd(self, tmp_path, monkeypatch): + from soup_cli.registry.attach import write_eval_json + + monkeypatch.chdir(tmp_path) + outside = str(tmp_path.parent / "evil.json") + with pytest.raises(ValueError, match="outside cwd"): + write_eval_json(outside, payload={}) + + def test_attach_artifact_unknown_entry(self, tmp_path, monkeypatch): + from soup_cli.registry.attach import attach_artifact + + monkeypatch.chdir(tmp_path) + # Use an isolated registry DB + db = tmp_path / "reg.db" + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(db)) + + target = tmp_path / "results.json" + target.write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="not found"): + attach_artifact("nonexistent-id", path=str(target), kind="eval_results") + + def test_attach_artifact_missing_file(self, tmp_path, monkeypatch): + from soup_cli.registry.attach import attach_artifact + + monkeypatch.chdir(tmp_path) + with pytest.raises(FileNotFoundError): + attach_artifact( + "any", path=str(tmp_path / "missing.json"), kind="eval_results", + ) + + +class TestRegistryArtifactKindsExtended: + def test_eval_results_kind_accepted(self): + from soup_cli.registry.store import _VALID_KINDS + + assert "eval_results" in _VALID_KINDS + assert "tensorrt" in _VALID_KINDS + + +class TestLookupEntryByOutputDir: + def test_lookup_returns_none_when_no_match(self, tmp_path, monkeypatch): + from soup_cli.registry.attach import lookup_entry_by_output_dir + + monkeypatch.chdir(tmp_path) + db = tmp_path / "reg.db" + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(db)) + + result = lookup_entry_by_output_dir(str(tmp_path / "no-such-output")) + assert result is None + + +# --------------------------------------------------------------------------- +# #35 — `soup eval custom --attach-to-registry` CLI integration +# --------------------------------------------------------------------------- + + +class TestEvalCustomAttachCLI: + def test_attach_to_unknown_entry_errors(self, tmp_path, monkeypatch): + """--attach-to-registry pointing at a missing entry produces a clean + error and exits non-zero rather than silently passing.""" + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + # Isolated registry DB so we don't pollute the user's ~/.soup + monkeypatch.setenv( + "SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db"), + ) + (tmp_path / "tasks.jsonl").write_text( + json.dumps({"prompt": "p", "expected": "x"}) + "\n", + encoding="utf-8", + ) + (tmp_path / "model").mkdir() + + with patch( + "soup_cli.eval.custom._create_default_generator", + return_value=lambda _p: "x", + ): + result = runner.invoke( + app, + [ + "eval", "custom", + "--tasks", "tasks.jsonl", + "--model", "model", + "--attach-to-registry", "no-such-id", + "--output", "results.json", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert ( + "registry entry not found" in result.output + or "not found" in result.output + )