mirror of https://github.com/razor-ai/soup.git
870 lines
34 KiB
Python
870 lines
34 KiB
Python
"""v0.71.7 "Eval live runners" — closes #161, #162, #208, #211, #212, #165.
|
|
|
|
Live model-loading is mocked at the ``soup_cli.utils.live_eval`` boundary (and
|
|
``lm_eval`` is faked via ``sys.modules``) so every orchestration path is
|
|
exercised on CPU without a GPU or a model download. The real model load is
|
|
covered by the release-step-6 smoke on SmolLM2-135M.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
import sys
|
|
import types
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def _clean_help(text: str) -> str:
|
|
"""Strip ANSI + remove all whitespace so flag substrings survive CI color.
|
|
|
|
Under CI (``FORCE_COLOR``) Rich renders a long flag like ``--base-model``
|
|
with ANSI escapes between styled segments AND may line-wrap at the hyphen
|
|
(``--base-`` newline ``model``). Stripping the escapes and removing every
|
|
whitespace char concatenates the literal flag back together.
|
|
"""
|
|
return re.sub(r"\s+", "", re.sub(r"\x1b\[[0-9;]*m", "", text))
|
|
|
|
|
|
def _write_jsonl(path: Path, rows: list[dict]) -> None:
|
|
path.write_text(
|
|
"\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# Shared utils/live_eval.py
|
|
# ===========================================================================
|
|
class TestLiveEvalCore:
|
|
def test_resolve_device_explicit(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
assert live_eval.resolve_device("cpu") == "cpu"
|
|
assert live_eval.resolve_device("cuda") == "cuda"
|
|
|
|
def test_resolve_device_rejects_empty(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.resolve_device("")
|
|
|
|
def test_resolve_device_auto_returns_str(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
assert live_eval.resolve_device(None) in {"cpu", "cuda"}
|
|
|
|
def test_apply_prompt_template_no_template(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
class _Tok:
|
|
chat_template = None
|
|
|
|
assert live_eval._apply_prompt_template(_Tok(), "hi") == "hi"
|
|
|
|
def test_apply_prompt_template_uses_chat_template(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
class _Tok:
|
|
chat_template = "x"
|
|
|
|
def apply_chat_template(self, msgs, tokenize, add_generation_prompt):
|
|
return "TEMPLATED:" + msgs[0]["content"]
|
|
|
|
assert live_eval._apply_prompt_template(_Tok(), "hi") == "TEMPLATED:hi"
|
|
|
|
def test_make_generator_rejects_bad_max_new_tokens(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_generator("m", max_new_tokens=0)
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_generator("m", max_new_tokens=True)
|
|
|
|
def test_make_generator_rejects_bad_loaded(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_generator("m", loaded=("only", "two"))
|
|
|
|
def test_make_multi_generator_rejects_bad_args(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_multi_generator("m", max_new_tokens=0)
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_multi_generator("m", max_new_tokens=True)
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_multi_generator("m", temperature=0)
|
|
with pytest.raises(ValueError):
|
|
live_eval.make_multi_generator("m", temperature=True)
|
|
|
|
def test_load_model_rejects_empty_id(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.load_model_and_tokenizer("")
|
|
|
|
def test_build_pairs_skips_missing_sides(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
rows = [
|
|
{"prompt": "p1", "response": "t1"},
|
|
{"prompt": "", "response": "t2"}, # no prompt
|
|
{"prompt": "p3", "response": ""}, # no target
|
|
"not-a-mapping",
|
|
]
|
|
pairs = live_eval._build_pairs(
|
|
rows,
|
|
input_extractor=lambda r: str(r.get("prompt", "")),
|
|
output_extractor=lambda r: str(r.get("response", "")),
|
|
)
|
|
assert pairs == [("p1", "t1")]
|
|
|
|
def test_lora_probe_rejects_bad_steps(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.lora_probe(
|
|
"m", [], input_extractor=str, output_extractor=str, n_steps=0
|
|
)
|
|
|
|
def test_measure_logit_agreement_rejects_bad_max_pairs(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.measure_logit_agreement(
|
|
"m", [], input_extractor=str, output_extractor=str, max_pairs=0
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# live_eval primitives — real torch tensors on CPU, fake model/tokenizer
|
|
# (covers the masking / loss / logit-agreement math without a download).
|
|
# ===========================================================================
|
|
class _FakeTok:
|
|
"""Word-length tokeniser: each whitespace word -> an int id == its length."""
|
|
|
|
chat_template = None
|
|
|
|
def __init__(self, eos: int | None = 99) -> None:
|
|
self.eos_token_id = eos
|
|
self.pad_token_id = eos
|
|
|
|
def __call__(self, text, add_special_tokens=False, **kwargs):
|
|
return {"input_ids": [len(w) for w in text.split()]}
|
|
|
|
|
|
class TestLiveEvalPrimitives:
|
|
def test_tokenize_pair_masks_prompt(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
ids, labels = live_eval._tokenize_pair(
|
|
_FakeTok(eos=99), "a bb", "ccc dddd", max_length=256
|
|
)
|
|
assert ids[0].tolist() == [1, 2, 3, 4, 99]
|
|
assert labels[0].tolist() == [-100, -100, 3, 4, 99]
|
|
|
|
def test_tokenize_pair_truncates(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
ids, _ = live_eval._tokenize_pair(
|
|
_FakeTok(eos=99), "a bb ccc", "dddd", max_length=2
|
|
)
|
|
assert ids.shape[1] == 2
|
|
|
|
def test_compute_eval_loss_mean(self) -> None:
|
|
import types
|
|
|
|
import torch
|
|
|
|
from soup_cli.utils import live_eval
|
|
|
|
class FakeModel:
|
|
def eval(self):
|
|
return self
|
|
|
|
def __call__(self, input_ids, labels):
|
|
return types.SimpleNamespace(loss=torch.tensor(1.5))
|
|
|
|
loss = live_eval.compute_eval_loss(
|
|
FakeModel(), _FakeTok(eos=99), [("a", "bb"), ("ccc", "dddd")],
|
|
device="cpu",
|
|
)
|
|
assert loss == pytest.approx(1.5)
|
|
|
|
def test_compute_eval_loss_all_empty_targets_nan(self) -> None:
|
|
import types
|
|
|
|
import torch
|
|
|
|
from soup_cli.utils import live_eval
|
|
|
|
class FakeModel:
|
|
def eval(self):
|
|
return self
|
|
|
|
def __call__(self, input_ids, labels):
|
|
return types.SimpleNamespace(loss=torch.tensor(0.0))
|
|
|
|
# eos=None + empty targets → labels are all -100 → skipped → NaN.
|
|
loss = live_eval.compute_eval_loss(
|
|
FakeModel(), _FakeTok(eos=None), [("a", ""), ("bb", "")], device="cpu",
|
|
)
|
|
assert loss != loss # NaN
|
|
|
|
def test_measure_logit_agreement_perfect(self, monkeypatch) -> None:
|
|
import types
|
|
|
|
import torch
|
|
|
|
from soup_cli.utils import live_eval
|
|
|
|
class FakeLogitModel:
|
|
vocab = 200
|
|
|
|
def eval(self):
|
|
return self
|
|
|
|
def __call__(self, input_ids):
|
|
seq = input_ids.shape[1]
|
|
logits = torch.zeros((1, seq, self.vocab))
|
|
ids = input_ids[0].tolist()
|
|
for t in range(seq - 1):
|
|
logits[0, t, ids[t + 1]] = 10.0 # predict the real next token
|
|
return types.SimpleNamespace(logits=logits)
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "load_model_and_tokenizer",
|
|
lambda *a, **k: (FakeLogitModel(), _FakeTok(eos=99), "cpu"),
|
|
)
|
|
score = live_eval.measure_logit_agreement(
|
|
"m", [{"p": "a bb", "t": "ccc dddd"}],
|
|
input_extractor=lambda r: r["p"], output_extractor=lambda r: r["t"],
|
|
)
|
|
assert score == pytest.approx(1.0)
|
|
|
|
def test_measure_logit_agreement_wrong(self, monkeypatch) -> None:
|
|
import types
|
|
|
|
import torch
|
|
|
|
from soup_cli.utils import live_eval
|
|
|
|
class WrongModel:
|
|
def eval(self):
|
|
return self
|
|
|
|
def __call__(self, input_ids):
|
|
seq = input_ids.shape[1]
|
|
logits = torch.zeros((1, seq, 200))
|
|
logits[0, :, 0] = 10.0 # always predict token 0
|
|
return types.SimpleNamespace(logits=logits)
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "load_model_and_tokenizer",
|
|
lambda *a, **k: (WrongModel(), _FakeTok(eos=99), "cpu"),
|
|
)
|
|
score = live_eval.measure_logit_agreement(
|
|
"m", [{"p": "a bb", "t": "ccc dddd"}],
|
|
input_extractor=lambda r: r["p"], output_extractor=lambda r: r["t"],
|
|
)
|
|
assert score == pytest.approx(0.0)
|
|
|
|
def test_lora_probe_rejects_bad_lr_and_max_length(self) -> None:
|
|
from soup_cli.utils import live_eval
|
|
|
|
with pytest.raises(ValueError):
|
|
live_eval.lora_probe(
|
|
"m", [], input_extractor=str, output_extractor=str,
|
|
n_steps=1, lr=0,
|
|
)
|
|
with pytest.raises(ValueError):
|
|
live_eval.lora_probe(
|
|
"m", [], input_extractor=str, output_extractor=str,
|
|
n_steps=1, max_length=0,
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# #161 — advise live probe (synth_probe_baselines / synth_probe_lora_delta)
|
|
# ===========================================================================
|
|
class TestAdviseLiveProbe:
|
|
def test_baselines_heuristic_when_no_model(self) -> None:
|
|
from soup_cli.utils.advise import synth_probe_baselines
|
|
|
|
rows = [{"prompt": "q", "response": "a short answer"} for _ in range(5)]
|
|
out = synth_probe_baselines(rows)
|
|
assert set(out) == {"zero_shot", "few_shot", "rag"}
|
|
|
|
def test_token_f1(self) -> None:
|
|
from soup_cli.utils.live_eval import token_f1
|
|
|
|
assert token_f1("hello world", "hello world") == pytest.approx(1.0)
|
|
assert token_f1("foo", "bar") == 0.0
|
|
assert token_f1("", "x") == 0.0
|
|
|
|
def test_baselines_live_when_model(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import synth_probe_baselines
|
|
|
|
def fake_make_generator(model_id, **kwargs):
|
|
return lambda prompt: "the capital is paris"
|
|
|
|
monkeypatch.setattr(live_eval, "make_generator", fake_make_generator)
|
|
rows = [
|
|
{"prompt": "capital of france?", "response": "the capital is paris"}
|
|
for _ in range(4)
|
|
]
|
|
out = synth_probe_baselines(rows, model="HuggingFaceTB/SmolLM2-135M")
|
|
# Perfect match → high F1.
|
|
assert out["zero_shot"] > 0.8
|
|
assert out["few_shot"] >= out["zero_shot"]
|
|
|
|
def test_baselines_falls_back_when_live_raises(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import synth_probe_baselines
|
|
|
|
def boom(*a, **k):
|
|
raise RuntimeError("no gpu")
|
|
|
|
monkeypatch.setattr(live_eval, "make_generator", boom)
|
|
rows = [{"prompt": "q", "response": "a"} for _ in range(5)]
|
|
out = synth_probe_baselines(rows, model="m") # falls back, no raise
|
|
assert set(out) == {"zero_shot", "few_shot", "rag"}
|
|
|
|
def test_lora_delta_heuristic_when_no_model(self) -> None:
|
|
from soup_cli.utils.advise import synth_probe_lora_delta
|
|
|
|
rows = [{"prompt": "q", "response": "a"} for _ in range(200)]
|
|
delta, wall = synth_probe_lora_delta(rows)
|
|
assert -0.2 <= delta <= 0.7
|
|
assert wall > 0
|
|
|
|
def test_lora_delta_live(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import synth_probe_lora_delta
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (2.0, 1.0, 12.5)
|
|
)
|
|
rows = [{"prompt": "q", "response": "a"} for _ in range(10)]
|
|
delta, wall = synth_probe_lora_delta(rows, model="m")
|
|
assert delta == pytest.approx(0.5) # (2-1)/2
|
|
assert wall == pytest.approx(12.5)
|
|
|
|
def test_lora_delta_live_nan_falls_back(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import synth_probe_lora_delta
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (float("nan"), 1.0, 5.0)
|
|
)
|
|
rows = [{"prompt": "q", "response": "a"} for _ in range(200)]
|
|
delta, wall = synth_probe_lora_delta(rows, model="m")
|
|
# NaN base_loss → fallback heuristic path.
|
|
assert -0.2 <= delta <= 0.7
|
|
|
|
|
|
# ===========================================================================
|
|
# #162 — base_model_proximity via logit agreement
|
|
# ===========================================================================
|
|
class TestBaseModelProximity:
|
|
def test_measure_proximity_live(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import measure_base_model_proximity
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "measure_logit_agreement", lambda *a, **k: 0.42
|
|
)
|
|
rows = [{"prompt": "q", "response": "a"}]
|
|
assert measure_base_model_proximity(rows, model="m") == pytest.approx(0.42)
|
|
|
|
def test_measure_proximity_nan_returns_none(self, monkeypatch) -> None:
|
|
from soup_cli.utils import live_eval
|
|
from soup_cli.utils.advise import measure_base_model_proximity
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "measure_logit_agreement", lambda *a, **k: float("nan")
|
|
)
|
|
assert measure_base_model_proximity([{"prompt": "q", "response": "a"}], model="m") is None
|
|
|
|
def test_measure_proximity_rejects_empty_model(self) -> None:
|
|
from soup_cli.utils.advise import measure_base_model_proximity
|
|
|
|
with pytest.raises(ValueError):
|
|
measure_base_model_proximity([], model="")
|
|
|
|
def test_profile_stores_proximity(self) -> None:
|
|
from soup_cli.utils.advise import compute_dataset_profile
|
|
|
|
rows = [{"prompt": "q", "response": "a"}]
|
|
prof = compute_dataset_profile(rows, base_model_proximity=0.7)
|
|
assert prof.base_model_proximity == pytest.approx(0.7)
|
|
|
|
def test_advise_cli_probe_no_model_still_heuristic(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "a"} for _ in range(60)])
|
|
result = runner.invoke(app, ["advise", "run", str(data), "--probe"])
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
|
|
def test_advise_cli_probe_model_live(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import live_eval
|
|
|
|
# Mock the live boundary: proximity + generator + lora probe.
|
|
monkeypatch.setattr(
|
|
live_eval, "measure_logit_agreement", lambda *a, **k: 0.55
|
|
)
|
|
monkeypatch.setattr(
|
|
live_eval, "make_generator", lambda *a, **k: (lambda p: "a")
|
|
)
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (2.0, 1.0, 9.0)
|
|
)
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": f"q{i}", "response": "a"} for i in range(60)])
|
|
result = runner.invoke(
|
|
app, ["advise", "run", str(data), "--probe-model", "m"]
|
|
)
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
|
|
|
|
# ===========================================================================
|
|
# #208 — tunability live LoRA probe
|
|
# ===========================================================================
|
|
class TestTunabilityLiveProbe:
|
|
def test_live_lora_probe(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import live_eval, tunability
|
|
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "a"} for _ in range(10)])
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (3.0, 2.0, 30.0)
|
|
)
|
|
cand = tunability.CandidateBase(
|
|
name="tiny", repo_id="HuggingFaceTB/SmolLM2-135M",
|
|
params_b=0.135, license_id="apache-2.0",
|
|
)
|
|
res = tunability.live_lora_probe(
|
|
cand, str(data), probe_steps=5, holdout_size=2
|
|
)
|
|
assert isinstance(res, tunability.TunabilityResult)
|
|
assert res.base_loss == pytest.approx(3.0)
|
|
assert res.delta == pytest.approx(1.0) # base - probe
|
|
assert res.wall_clock_seconds == pytest.approx(30.0)
|
|
|
|
def test_live_lora_probe_nan_neutral(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import live_eval, tunability
|
|
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "a"} for _ in range(4)])
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (float("nan"), float("nan"), 1.0)
|
|
)
|
|
cand = tunability.CandidateBase(
|
|
name="t", repo_id="r", params_b=0.1, license_id="mit",
|
|
)
|
|
res = tunability.live_lora_probe(cand, str(data), probe_steps=1, holdout_size=1)
|
|
assert res.delta == 0.0
|
|
|
|
def test_live_lora_probe_one_nan(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import live_eval, tunability
|
|
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "a"} for _ in range(4)])
|
|
# base_loss NaN but probe_loss finite → still neutral 0 delta.
|
|
monkeypatch.setattr(
|
|
live_eval, "lora_probe", lambda *a, **k: (float("nan"), 1.0, 2.0)
|
|
)
|
|
cand = tunability.CandidateBase(
|
|
name="t", repo_id="r", params_b=0.1, license_id="mit",
|
|
)
|
|
res = tunability.live_lora_probe(cand, str(data), probe_steps=1, holdout_size=1)
|
|
assert res.delta == 0.0
|
|
|
|
def test_load_jsonl_rows_outside_cwd_rejected(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import tunability
|
|
|
|
outside = tmp_path.parent / "outside.jsonl"
|
|
_write_jsonl(outside, [{"prompt": "q", "response": "a"}])
|
|
with pytest.raises((ValueError, OSError)):
|
|
tunability._load_jsonl_rows(str(outside))
|
|
|
|
def test_run_tunability_uses_injected_probe(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils import tunability
|
|
|
|
cand = tunability.CandidateBase(
|
|
name="t", repo_id="r", params_b=0.1, license_id="mit",
|
|
)
|
|
calls = []
|
|
|
|
def fake_probe(c, ds, *, probe_steps, holdout_size):
|
|
calls.append(c.name)
|
|
return tunability.TunabilityResult(
|
|
candidate=c, base_loss=2.0, probe_loss=1.0, delta=1.0,
|
|
wall_clock_seconds=10.0, estimated_cost_usd=0.5,
|
|
)
|
|
|
|
report = tunability.run_tunability(
|
|
candidates=[cand], dataset_path="d.jsonl", probe_fn=fake_probe
|
|
)
|
|
assert calls == ["t"]
|
|
assert report.results[0].delta == pytest.approx(1.0)
|
|
|
|
def test_cli_plan_only_live_text(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "a"}])
|
|
result = runner.invoke(
|
|
app, ["tunability", "--dataset", str(data), "--plan-only", "--live"]
|
|
)
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
assert "LIVE LoRA probe" in result.output
|
|
|
|
def test_cli_live_flag_in_help(self) -> None:
|
|
result = runner.invoke(app, ["tunability", "--help"])
|
|
assert "--live" in _clean_help(result.output)
|
|
|
|
|
|
# ===========================================================================
|
|
# #211 — capability live lm-eval runner
|
|
# ===========================================================================
|
|
def _install_fake_lm_eval(monkeypatch, *, results: dict, raise_for: set | None = None):
|
|
raise_for = raise_for or set()
|
|
|
|
def simple_evaluate(*, model, tasks, limit=None):
|
|
task = tasks[0]
|
|
if task in raise_for:
|
|
raise RuntimeError(f"task {task} unavailable")
|
|
return {"results": {task: results.get(task, {"acc,none": 0.5})}}
|
|
|
|
class HFLM:
|
|
def __init__(self, *, pretrained, device, batch_size):
|
|
self.pretrained = pretrained
|
|
|
|
fake = types.ModuleType("lm_eval")
|
|
fake.simple_evaluate = simple_evaluate
|
|
fake_models = types.ModuleType("lm_eval.models")
|
|
fake_hf = types.ModuleType("lm_eval.models.huggingface")
|
|
fake_hf.HFLM = HFLM
|
|
monkeypatch.setitem(sys.modules, "lm_eval", fake)
|
|
monkeypatch.setitem(sys.modules, "lm_eval.models", fake_models)
|
|
monkeypatch.setitem(sys.modules, "lm_eval.models.huggingface", fake_hf)
|
|
|
|
|
|
class TestCapabilityLiveRunner:
|
|
def test_primary_metric_prefers_acc_norm(self) -> None:
|
|
from soup_cli.utils.capability_suite import _primary_metric
|
|
|
|
key, val = _primary_metric({"acc,none": 0.4, "acc_norm,none": 0.6})
|
|
assert key.startswith("acc_norm")
|
|
assert val == pytest.approx(0.6)
|
|
|
|
def test_primary_metric_skips_stderr(self) -> None:
|
|
from soup_cli.utils.capability_suite import _primary_metric
|
|
|
|
key, val = _primary_metric({"exact_match,none": 0.3, "exact_match_stderr,none": 0.1})
|
|
assert "stderr" not in key
|
|
assert val == pytest.approx(0.3)
|
|
|
|
def test_run_capability_suite_tasks(self, monkeypatch) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
_install_fake_lm_eval(monkeypatch, results={"arc_easy": {"acc,none": 0.71}})
|
|
out = run_capability_suite(
|
|
run_id="r1", model_id="m", tasks=["arc_easy"], limit=2
|
|
)
|
|
assert out["run_id"] == "r1"
|
|
assert out["results"][0]["benchmark"] == "arc_easy"
|
|
assert out["results"][0]["score"] == pytest.approx(0.71)
|
|
|
|
def test_run_capability_suite_per_task_error_isolation(self, monkeypatch) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
_install_fake_lm_eval(
|
|
monkeypatch,
|
|
results={"arc_easy": {"acc,none": 0.7}},
|
|
raise_for={"bbeh_unregistered"},
|
|
)
|
|
out = run_capability_suite(
|
|
run_id="r", model_id="m", tasks=["arc_easy", "bbeh_unregistered"]
|
|
)
|
|
assert "score" in out["results"][0]
|
|
assert "error" in out["results"][1]
|
|
|
|
def test_run_capability_suite_resolves_profile(self, monkeypatch) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
_install_fake_lm_eval(monkeypatch, results={})
|
|
out = run_capability_suite(run_id="r", model_id="m", suite="fast")
|
|
names = {r["benchmark"] for r in out["results"]}
|
|
assert names == {"mmlu-pro", "humaneval-plus"}
|
|
|
|
def test_run_capability_empty_result_is_error(self, monkeypatch) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
_install_fake_lm_eval(monkeypatch, results={"arc_easy": {}})
|
|
out = run_capability_suite(run_id="r", model_id="m", tasks=["arc_easy"])
|
|
assert out["results"][0]["error"] == "no scalar metric in task result"
|
|
|
|
def test_run_capability_missing_lm_eval(self, monkeypatch) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
# Force the import to fail.
|
|
monkeypatch.setitem(sys.modules, "lm_eval", None)
|
|
with pytest.raises(RuntimeError, match="lm-eval"):
|
|
run_capability_suite(run_id="r", model_id="m", tasks=["arc_easy"])
|
|
|
|
def test_run_capability_validation(self) -> None:
|
|
from soup_cli.utils.capability_suite import run_capability_suite
|
|
|
|
with pytest.raises(ValueError):
|
|
run_capability_suite(run_id="", model_id="m", tasks=["x"])
|
|
with pytest.raises(ValueError):
|
|
run_capability_suite(run_id="r", model_id="", tasks=["x"])
|
|
with pytest.raises(ValueError):
|
|
run_capability_suite(run_id="r", model_id="m", tasks=["x"], limit=0)
|
|
with pytest.raises(ValueError):
|
|
run_capability_suite(run_id="r", model_id="m") # neither suite nor tasks
|
|
|
|
def test_cli_live_requires_model(self) -> None:
|
|
result = runner.invoke(app, ["eval", "capability", "r", "--live"])
|
|
assert result.exit_code == 2
|
|
assert "--model" in result.output
|
|
|
|
def test_cli_live_runs(self, monkeypatch, tmp_path) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
_install_fake_lm_eval(monkeypatch, results={"arc_easy": {"acc,none": 0.66}})
|
|
result = runner.invoke(
|
|
app,
|
|
["eval", "capability", "r", "--live", "--model", "m",
|
|
"--tasks", "arc_easy", "--limit", "1"],
|
|
)
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
assert "arc_easy" in result.output
|
|
|
|
def test_cli_manifest_default(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
result = runner.invoke(
|
|
app, ["eval", "capability", "r", "--suite", "fast", "--output", "out.json"]
|
|
)
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
payload = json.loads(Path("out.json").read_text(encoding="utf-8"))
|
|
assert "Manifest only" in payload["note"]
|
|
|
|
|
|
# ===========================================================================
|
|
# #212 — behavior live diff
|
|
# ===========================================================================
|
|
class TestBehaviorLive:
|
|
def _patch_generators(self, monkeypatch, base_text: str, post_text: str):
|
|
from soup_cli.utils import live_eval
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "load_model_and_tokenizer",
|
|
lambda *a, **k: ("model", "tok", "cpu"),
|
|
)
|
|
|
|
def fake_make_generator(model_id, *, adapter=None, **kwargs):
|
|
text = post_text if adapter else base_text
|
|
return lambda prompt: text
|
|
|
|
monkeypatch.setattr(live_eval, "make_generator", fake_make_generator)
|
|
|
|
def test_run_behavior_live_returns_report(self, monkeypatch) -> None:
|
|
from soup_cli.utils.behavior_battery import BehaviorDiffReport, run_behavior_live
|
|
|
|
self._patch_generators(monkeypatch, "safe answer", "safe answer")
|
|
report = run_behavior_live(
|
|
run_id="r", battery="xstest", base_model="m", adapter="adp",
|
|
)
|
|
assert isinstance(report, BehaviorDiffReport)
|
|
assert report.battery == "xstest"
|
|
|
|
def test_run_behavior_live_rejects_empty_model(self) -> None:
|
|
from soup_cli.utils.behavior_battery import run_behavior_live
|
|
|
|
with pytest.raises(ValueError):
|
|
run_behavior_live(run_id="r", battery="xstest", base_model="")
|
|
|
|
def test_run_behavior_live_rejects_bad_max_probes(self, monkeypatch) -> None:
|
|
from soup_cli.utils.behavior_battery import run_behavior_live
|
|
|
|
self._patch_generators(monkeypatch, "x", "x")
|
|
with pytest.raises(ValueError):
|
|
run_behavior_live(
|
|
run_id="r", battery="xstest", base_model="m", max_probes=0
|
|
)
|
|
|
|
def test_cli_live_diff(self, monkeypatch, tmp_path) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
self._patch_generators(monkeypatch, "ok", "ok")
|
|
result = runner.invoke(
|
|
app, ["eval", "behavior", "r", "--battery", "xstest", "--base-model", "m"]
|
|
)
|
|
# Identical pre/post "ok" responses fail the xstest "safe" oracle →
|
|
# MAJOR → exit 2 (the live diff ran and rendered before the gate).
|
|
assert result.exit_code == 2, (result.output, result.exception)
|
|
assert "live" in result.output.lower()
|
|
|
|
def test_cli_live_diff_ok(self, monkeypatch, tmp_path) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
# Responses containing the "safe" oracle word → agreement 1.0 → OK.
|
|
self._patch_generators(monkeypatch, "this is safe", "this is safe")
|
|
result = runner.invoke(
|
|
app, ["eval", "behavior", "r", "--battery", "xstest", "--base-model", "m"]
|
|
)
|
|
assert result.exit_code == 0, (result.output, result.exception)
|
|
|
|
def test_cli_base_model_in_help(self) -> None:
|
|
result = runner.invoke(app, ["eval", "behavior", "--help"])
|
|
assert "--base-model" in _clean_help(result.output)
|
|
|
|
|
|
# ===========================================================================
|
|
# #165 — diagnose live runners
|
|
# ===========================================================================
|
|
class TestDiagnoseLive:
|
|
def _patch_live_eval(self, monkeypatch, base_text="hello", post_text="hello"):
|
|
from soup_cli.utils import live_eval
|
|
|
|
monkeypatch.setattr(
|
|
live_eval, "load_model_and_tokenizer",
|
|
lambda *a, **k: ("model", "tok", "cpu"),
|
|
)
|
|
|
|
def fake_make_generator(model_id, *, adapter=None, **kwargs):
|
|
text = post_text if adapter else base_text
|
|
return lambda prompt: text
|
|
|
|
def fake_make_multi(model_id, *, adapter=None, **kwargs):
|
|
return lambda prompt, k: [f"sample {i}" for i in range(k)]
|
|
|
|
monkeypatch.setattr(live_eval, "make_generator", fake_make_generator)
|
|
monkeypatch.setattr(live_eval, "make_multi_generator", fake_make_multi)
|
|
|
|
def test_token_f1_helper(self) -> None:
|
|
from soup_cli.utils.live_eval import token_f1
|
|
|
|
assert token_f1("a b c", "a b c") == pytest.approx(1.0)
|
|
assert token_f1("x", "y") == 0.0
|
|
|
|
def test_looks_like_json_dataset(self) -> None:
|
|
from soup_cli.utils.diagnose.live import _looks_like_json_dataset
|
|
|
|
json_rows = [{"response": '{"k": 1}'} for _ in range(5)]
|
|
text_rows = [{"response": "plain text"} for _ in range(5)]
|
|
assert _looks_like_json_dataset(json_rows) is True
|
|
assert _looks_like_json_dataset(text_rows) is False
|
|
|
|
def test_load_dataset_rows_outside_cwd(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
from soup_cli.utils.diagnose.live import _load_dataset_rows
|
|
|
|
outside = tmp_path.parent / "x.jsonl"
|
|
_write_jsonl(outside, [{"prompt": "q", "response": "a"}])
|
|
with pytest.raises((ValueError, OSError)):
|
|
_load_dataset_rows(str(outside))
|
|
|
|
def test_load_adapter_pair_aliases_when_no_adapter(self, monkeypatch) -> None:
|
|
self._patch_live_eval(monkeypatch)
|
|
from soup_cli.utils.diagnose.live import load_adapter_pair
|
|
|
|
closures = load_adapter_pair("m", None)
|
|
assert set(closures) == {"base_gen", "adapter_gen", "base_multi", "adapter_multi"}
|
|
assert closures["base_gen"] is closures["adapter_gen"]
|
|
|
|
def test_run_live_diagnose_returns_report(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
self._patch_live_eval(monkeypatch)
|
|
from soup_cli.utils.diagnose.live import run_live_diagnose
|
|
from soup_cli.utils.diagnose.report import FAILURE_MODES
|
|
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": f"q{i}", "response": f"a{i}"} for i in range(8)])
|
|
report = run_live_diagnose(
|
|
run_id="r", base="m", adapter=None, dataset_path=str(data)
|
|
)
|
|
# All 6 modes present.
|
|
assert set(report.scores) >= set(FAILURE_MODES)
|
|
assert report.overall in {"OK", "MINOR", "MAJOR"}
|
|
|
|
def test_run_live_diagnose_format_neutral_when_not_json(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
self._patch_live_eval(monkeypatch)
|
|
from soup_cli.utils.diagnose.live import run_live_diagnose
|
|
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": "q", "response": "plain text"} for _ in range(6)])
|
|
report = run_live_diagnose(run_id="r", base="m", dataset_path=str(data))
|
|
assert "not JSON" in report.scores["format"].evidence
|
|
|
|
def test_run_live_diagnose_no_dataset(self, monkeypatch) -> None:
|
|
self._patch_live_eval(monkeypatch)
|
|
from soup_cli.utils.diagnose.live import run_live_diagnose
|
|
from soup_cli.utils.diagnose.report import FAILURE_MODES
|
|
|
|
report = run_live_diagnose(run_id="r", base="m")
|
|
# refusal still runs; dataset-driven probes neutral but present.
|
|
assert set(report.scores) >= set(FAILURE_MODES)
|
|
|
|
def test_run_live_diagnose_rejects_empty_base(self) -> None:
|
|
from soup_cli.utils.diagnose.live import run_live_diagnose
|
|
|
|
with pytest.raises(ValueError):
|
|
run_live_diagnose(run_id="r", base="")
|
|
|
|
def test_cli_live_diagnose(self, tmp_path, monkeypatch) -> None:
|
|
monkeypatch.chdir(tmp_path)
|
|
self._patch_live_eval(monkeypatch)
|
|
data = tmp_path / "d.jsonl"
|
|
_write_jsonl(data, [{"prompt": f"q{i}", "response": f"a{i}"} for i in range(8)])
|
|
result = runner.invoke(
|
|
app, ["diagnose", "r1", "--base-model", "m", "--dataset", str(data)]
|
|
)
|
|
assert result.exit_code in (0, 2), (result.output, result.exception)
|
|
assert "overall" in result.output.lower()
|
|
|
|
def test_cli_base_model_in_help(self) -> None:
|
|
result = runner.invoke(app, ["diagnose", "--help"])
|
|
cleaned = _clean_help(result.output)
|
|
assert "--base-model" in cleaned
|
|
assert "--tokenizer" in cleaned
|
|
|
|
|
|
# ===========================================================================
|
|
# Patch invariants
|
|
# ===========================================================================
|
|
class TestPatchInvariants:
|
|
def test_version_bumped(self) -> None:
|
|
from soup_cli import __version__
|
|
|
|
major_minor = tuple(int(x) for x in __version__.split(".")[:3])
|
|
assert major_minor >= (0, 71, 7)
|
|
|
|
def test_no_top_level_heavy_imports(self) -> None:
|
|
for mod in (
|
|
"live_eval.py",
|
|
"diagnose/live.py",
|
|
):
|
|
src = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "utils" / mod
|
|
).read_text(encoding="utf-8")
|
|
assert "\nimport torch" not in src, mod
|
|
assert "\nimport transformers" not in src, mod
|
|
assert "\nimport peft" not in src, mod
|