mirror of https://github.com/razor-ai/soup.git
493 lines
16 KiB
Python
493 lines
16 KiB
Python
"""Tests for v0.43.0 Part B — Eval metrics."""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from soup_cli.eval.arena import (
|
|
DEFAULT_BASE_RATING,
|
|
Tournament,
|
|
expected_score,
|
|
update_elo,
|
|
)
|
|
from soup_cli.eval.benchmarks_v0_43 import (
|
|
NEW_BENCHMARKS_V0_43,
|
|
benchmark_metadata,
|
|
is_v0_43_benchmark,
|
|
lm_eval_task_for,
|
|
)
|
|
from soup_cli.eval.calibrate import (
|
|
CalibrationReport,
|
|
classify_kl_delta,
|
|
kl_divergence,
|
|
run_calibration,
|
|
)
|
|
from soup_cli.utils.nlg_metrics import (
|
|
NLG_METRICS,
|
|
bleu_score,
|
|
compute_nlg_metric,
|
|
effective_tokens_per_second,
|
|
rouge_l_score,
|
|
rouge_n_score,
|
|
)
|
|
|
|
# ----------------- BLEU / ROUGE -----------------
|
|
|
|
class TestBleuScore:
|
|
def test_perfect_match(self):
|
|
score = bleu_score(["the cat sat on the mat"], ["the cat sat on the mat"])
|
|
assert score == pytest.approx(1.0, abs=1e-6)
|
|
|
|
def test_zero_overlap_no_smoothing(self):
|
|
# Without smoothing, any zero n-gram precision -> BLEU 0.
|
|
score = bleu_score(
|
|
["alpha beta gamma delta"],
|
|
["one two three four"],
|
|
smooth=False,
|
|
)
|
|
assert score == 0.0
|
|
|
|
def test_zero_overlap_smoothed(self):
|
|
# With smoothing (the default), Chen & Cherry assigns small mass
|
|
# to each zero bucket, so the score is small but positive.
|
|
score = bleu_score(["alpha beta gamma delta"], ["one two three four"])
|
|
assert 0.0 < score < 0.2
|
|
|
|
def test_brevity_penalty_one_when_pred_longer(self):
|
|
# Prediction longer than reference: BP = 1.0 (no penalty).
|
|
# Score reflects modified precision only.
|
|
# pred unigrams the=2,cat=1,sat=1,on=1,the=2,mat=1 (6 total);
|
|
# ref the=1,cat=1,sat=1.
|
|
# min-clipped overlap = the(min(2,1)) + cat + sat = 3, total=6 → 0.5
|
|
score = bleu_score(
|
|
["the cat sat on the mat"], ["the cat sat"], max_n=1
|
|
)
|
|
assert score == pytest.approx(0.5)
|
|
|
|
def test_partial_overlap(self):
|
|
# BLEU-2 partial: shorter prediction with all 4 unigram + 3 bigram
|
|
# overlaps — guarantees nonzero standard BLEU.
|
|
score = bleu_score(
|
|
["the quick brown fox"],
|
|
["the quick brown fox jumped"],
|
|
max_n=2,
|
|
)
|
|
assert 0.0 < score < 1.0
|
|
|
|
def test_empty_corpus(self):
|
|
assert bleu_score([], []) == 0.0
|
|
|
|
def test_length_mismatch(self):
|
|
with pytest.raises(ValueError, match="same length"):
|
|
bleu_score(["a"], ["a", "b"])
|
|
|
|
def test_invalid_max_n(self):
|
|
with pytest.raises(ValueError):
|
|
bleu_score(["a"], ["a"], max_n=0)
|
|
with pytest.raises(ValueError):
|
|
bleu_score(["a"], ["a"], max_n=10)
|
|
with pytest.raises(ValueError):
|
|
bleu_score(["a"], ["a"], max_n=True) # type: ignore[arg-type]
|
|
|
|
def test_null_byte_rejected(self):
|
|
with pytest.raises(ValueError, match="null"):
|
|
bleu_score(["a\x00"], ["a"])
|
|
|
|
|
|
class TestRougeNScore:
|
|
def test_perfect_match(self):
|
|
score = rouge_n_score(["alpha beta gamma"], ["alpha beta gamma"])
|
|
assert score == pytest.approx(1.0)
|
|
|
|
def test_zero_overlap(self):
|
|
score = rouge_n_score(["alpha beta"], ["one two"])
|
|
assert score == 0.0
|
|
|
|
def test_n_2(self):
|
|
score = rouge_n_score(["the cat sat"], ["the cat sat"], n=2)
|
|
assert score == pytest.approx(1.0)
|
|
|
|
def test_invalid_n(self):
|
|
with pytest.raises(ValueError):
|
|
rouge_n_score(["a"], ["a"], n=0)
|
|
with pytest.raises(ValueError):
|
|
rouge_n_score(["a"], ["a"], n=True) # type: ignore[arg-type]
|
|
|
|
def test_too_short_for_n(self):
|
|
# Single-token strings have no bigrams.
|
|
score = rouge_n_score(["a"], ["a"], n=2)
|
|
assert score == 0.0
|
|
|
|
def test_length_mismatch_message(self):
|
|
with pytest.raises(ValueError, match="same length"):
|
|
rouge_n_score(["a"], ["a", "b"])
|
|
|
|
|
|
class TestRougeLScore:
|
|
def test_perfect_match(self):
|
|
score = rouge_l_score(["alpha beta gamma"], ["alpha beta gamma"])
|
|
assert score == pytest.approx(1.0)
|
|
|
|
def test_lcs_partial(self):
|
|
score = rouge_l_score(
|
|
["the quick brown fox"],
|
|
["a quick brown dog"],
|
|
)
|
|
# LCS = "quick brown" (2 tokens)
|
|
# P = 2/4, R = 2/4 (4 tokens each), F1 = 0.5
|
|
assert score == pytest.approx(0.5)
|
|
|
|
def test_zero_overlap(self):
|
|
score = rouge_l_score(["alpha"], ["beta"])
|
|
assert score == 0.0
|
|
|
|
def test_empty(self):
|
|
assert rouge_l_score([], []) == 0.0
|
|
|
|
def test_length_mismatch(self):
|
|
with pytest.raises(ValueError, match="same length"):
|
|
rouge_l_score(["a"], [])
|
|
|
|
|
|
class TestComputeNlgMetric:
|
|
@pytest.mark.parametrize("metric", list(NLG_METRICS))
|
|
def test_dispatch(self, metric):
|
|
score = compute_nlg_metric(metric, ["a b c"], ["a b c"])
|
|
assert 0.0 <= score <= 1.0
|
|
|
|
def test_unknown(self):
|
|
with pytest.raises(ValueError, match="unknown nlg metric"):
|
|
compute_nlg_metric("meteor", ["a"], ["a"])
|
|
|
|
def test_non_string(self):
|
|
with pytest.raises(ValueError):
|
|
compute_nlg_metric(None, ["a"], ["a"]) # type: ignore[arg-type]
|
|
|
|
def test_case_insensitive(self):
|
|
assert compute_nlg_metric("BLEU", ["a"], ["a"]) >= 0.0
|
|
|
|
|
|
# ----------------- effective_tokens_per_second -----------------
|
|
|
|
class TestEffectiveTokensPerSecond:
|
|
def test_happy(self):
|
|
assert effective_tokens_per_second(
|
|
unmasked_tokens=10000, wall_clock_seconds=10.0
|
|
) == 1000.0
|
|
|
|
def test_zero_wall_clock_returns_none(self):
|
|
assert effective_tokens_per_second(
|
|
unmasked_tokens=100, wall_clock_seconds=0.0
|
|
) is None
|
|
|
|
def test_negative_wall_clock_returns_none(self):
|
|
assert effective_tokens_per_second(
|
|
unmasked_tokens=100, wall_clock_seconds=-1.0
|
|
) is None
|
|
|
|
def test_negative_tokens_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
effective_tokens_per_second(
|
|
unmasked_tokens=-1, wall_clock_seconds=1.0
|
|
)
|
|
|
|
def test_bool_tokens_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
effective_tokens_per_second(
|
|
unmasked_tokens=True, wall_clock_seconds=1.0 # type: ignore[arg-type]
|
|
)
|
|
|
|
def test_bool_wall_clock_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
effective_tokens_per_second(
|
|
unmasked_tokens=100, wall_clock_seconds=True # type: ignore[arg-type]
|
|
)
|
|
|
|
def test_nonfinite_wall_clock_rejected(self):
|
|
with pytest.raises(ValueError, match="finite"):
|
|
effective_tokens_per_second(
|
|
unmasked_tokens=100, wall_clock_seconds=float("inf")
|
|
)
|
|
|
|
|
|
# ----------------- KL Calibration -----------------
|
|
|
|
class TestKlDivergence:
|
|
def test_identical_distributions(self):
|
|
kl = kl_divergence([1.0, 2.0, 3.0], [1.0, 2.0, 3.0])
|
|
assert kl == pytest.approx(0.0, abs=1e-9)
|
|
|
|
def test_different_distributions_positive(self):
|
|
kl = kl_divergence([1.0, 2.0, 3.0], [3.0, 2.0, 1.0])
|
|
assert kl > 0.0
|
|
|
|
def test_length_mismatch(self):
|
|
with pytest.raises(ValueError, match="same length"):
|
|
kl_divergence([1.0, 2.0], [1.0])
|
|
|
|
def test_empty(self):
|
|
with pytest.raises(ValueError):
|
|
kl_divergence([], [])
|
|
|
|
def test_non_finite_rejected(self):
|
|
with pytest.raises(ValueError, match="finite"):
|
|
kl_divergence([1.0, float("inf")], [1.0, 2.0])
|
|
|
|
def test_bool_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
kl_divergence([True, False], [1.0, 2.0]) # type: ignore[list-item]
|
|
|
|
|
|
class TestClassifyKlDelta:
|
|
@pytest.mark.parametrize(
|
|
"kl,status",
|
|
[
|
|
(0.0, "OK"),
|
|
(0.04, "OK"),
|
|
(0.05, "MINOR"),
|
|
(0.10, "MINOR"),
|
|
(0.19, "MINOR"),
|
|
(0.20, "MAJOR"),
|
|
(1.0, "MAJOR"),
|
|
],
|
|
)
|
|
def test_thresholds(self, kl, status):
|
|
assert classify_kl_delta(kl) == status
|
|
|
|
def test_negative_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
classify_kl_delta(-0.01)
|
|
|
|
def test_nan_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
classify_kl_delta(float("nan"))
|
|
|
|
def test_bool_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
classify_kl_delta(True) # type: ignore[arg-type]
|
|
|
|
|
|
class TestRunCalibration:
|
|
def test_perfect_match_ok(self):
|
|
baseline = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
|
|
quant = [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]
|
|
report = run_calibration(baseline, quant)
|
|
assert isinstance(report, CalibrationReport)
|
|
assert report.mean_kl == pytest.approx(0.0, abs=1e-9)
|
|
assert report.delta_status == "OK"
|
|
assert report.num_prompts == 2
|
|
|
|
def test_diverged_quant_major(self):
|
|
baseline = [[10.0, 0.0, 0.0]]
|
|
quant = [[0.0, 0.0, 10.0]]
|
|
report = run_calibration(baseline, quant)
|
|
# Large divergence — guaranteed MAJOR.
|
|
assert report.delta_status == "MAJOR"
|
|
|
|
def test_length_mismatch(self):
|
|
with pytest.raises(ValueError, match="same length"):
|
|
run_calibration([[1.0]], [])
|
|
|
|
def test_empty(self):
|
|
with pytest.raises(ValueError):
|
|
run_calibration([], [])
|
|
|
|
def test_too_many_prompts(self):
|
|
big = [[1.0]] * 10_001
|
|
with pytest.raises(ValueError, match="too many"):
|
|
run_calibration(big, big)
|
|
|
|
def test_report_frozen(self):
|
|
from dataclasses import FrozenInstanceError
|
|
|
|
report = run_calibration([[1.0, 2.0]], [[1.0, 2.0]])
|
|
with pytest.raises(FrozenInstanceError):
|
|
report.mean_kl = 0.5 # type: ignore[misc]
|
|
|
|
|
|
# ----------------- Arena -----------------
|
|
|
|
class TestExpectedScore:
|
|
def test_equal_ratings_50_50(self):
|
|
assert expected_score(1500.0, 1500.0) == pytest.approx(0.5)
|
|
|
|
def test_higher_rated_favored(self):
|
|
assert expected_score(1700.0, 1500.0) > 0.5
|
|
|
|
def test_lower_rated_disfavored(self):
|
|
assert expected_score(1300.0, 1500.0) < 0.5
|
|
|
|
def test_non_finite_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
expected_score(float("nan"), 1500.0)
|
|
|
|
def test_bool_rejected(self):
|
|
with pytest.raises(ValueError):
|
|
expected_score(True, 1500.0) # type: ignore[arg-type]
|
|
|
|
|
|
class TestUpdateElo:
|
|
def test_winner_gains(self):
|
|
new_a, new_b = update_elo(1500.0, 1500.0, score_a=1.0)
|
|
assert new_a > 1500.0
|
|
assert new_b < 1500.0
|
|
|
|
def test_draw_no_change_at_equal(self):
|
|
new_a, new_b = update_elo(1500.0, 1500.0, score_a=0.5)
|
|
assert new_a == pytest.approx(1500.0)
|
|
assert new_b == pytest.approx(1500.0)
|
|
|
|
def test_zero_sum(self):
|
|
new_a, new_b = update_elo(1500.0, 1700.0, score_a=1.0)
|
|
# Symmetric Elo: total rating mass conserved.
|
|
assert (new_a - 1500.0) == pytest.approx(-(new_b - 1700.0), abs=1e-6)
|
|
|
|
def test_invalid_score_a(self):
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=-0.1)
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=1.1)
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=True) # type: ignore[arg-type]
|
|
|
|
def test_invalid_k(self):
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=1.0, k=0)
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=1.0, k=-32)
|
|
|
|
|
|
class TestTournament:
|
|
def test_register_and_record(self):
|
|
t = Tournament()
|
|
t.register("alpha")
|
|
t.register("beta")
|
|
t.record("alpha", "beta", winner="a")
|
|
assert t.ratings["alpha"] > DEFAULT_BASE_RATING
|
|
assert t.ratings["beta"] < DEFAULT_BASE_RATING
|
|
|
|
def test_register_idempotent(self):
|
|
t = Tournament()
|
|
t.register("alpha")
|
|
t.register("alpha")
|
|
assert t.ratings["alpha"] == DEFAULT_BASE_RATING
|
|
|
|
def test_ratings_immutable_view(self):
|
|
t = Tournament()
|
|
t.register("alpha")
|
|
# Ratings property must not allow mutation through the returned view.
|
|
with pytest.raises(TypeError):
|
|
t.ratings["alpha"] = 9999.0 # type: ignore[index]
|
|
|
|
def test_implicit_register_on_record(self):
|
|
t = Tournament()
|
|
t.record("alpha", "beta", winner="draw")
|
|
assert "alpha" in t.ratings
|
|
assert "beta" in t.ratings
|
|
|
|
def test_self_play_rejected(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError, match="must differ"):
|
|
t.record("alpha", "alpha", winner="a")
|
|
|
|
def test_invalid_winner(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError, match="winner"):
|
|
t.record("a", "b", winner="c")
|
|
|
|
def test_empty_name_rejected(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError):
|
|
t.register("")
|
|
|
|
def test_null_byte_name_rejected(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError, match="null"):
|
|
t.register("foo\x00")
|
|
|
|
def test_oversize_name_rejected(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError):
|
|
t.register("a" * 256)
|
|
|
|
def test_rich_markup_metacharacter_rejected(self):
|
|
t = Tournament()
|
|
with pytest.raises(ValueError, match="markup"):
|
|
t.register("[red]evil[/red]")
|
|
with pytest.raises(ValueError, match="markup"):
|
|
t.register("foo]bar")
|
|
|
|
def test_model_cap_exceeded(self):
|
|
from soup_cli.eval.arena import _MAX_MODELS
|
|
t = Tournament()
|
|
for i in range(_MAX_MODELS):
|
|
t.register(f"model_{i}")
|
|
with pytest.raises(ValueError, match="model cap"):
|
|
t.register("one_too_many")
|
|
|
|
def test_invalid_k_nan(self):
|
|
with pytest.raises(ValueError):
|
|
update_elo(1500.0, 1500.0, score_a=1.0, k=float("nan"))
|
|
|
|
def test_invalid_base_rating(self):
|
|
with pytest.raises(ValueError):
|
|
Tournament(base_rating=float("nan"))
|
|
|
|
def test_invalid_k(self):
|
|
with pytest.raises(ValueError):
|
|
Tournament(k=0)
|
|
|
|
def test_leaderboard_sorted(self):
|
|
t = Tournament()
|
|
t.record("alpha", "beta", winner="a")
|
|
t.record("alpha", "gamma", winner="a")
|
|
board = t.leaderboard()
|
|
assert board[0]["model"] == "alpha"
|
|
assert board[0]["wins"] == 2
|
|
assert board[0]["losses"] == 0
|
|
# Highest rating first.
|
|
for i in range(len(board) - 1):
|
|
assert board[i]["rating"] >= board[i + 1]["rating"]
|
|
|
|
def test_draw_records(self):
|
|
t = Tournament()
|
|
t.record("alpha", "beta", winner="draw")
|
|
for name in ("alpha", "beta"):
|
|
row = next(r for r in t.leaderboard() if r["model"] == name)
|
|
assert row["draws"] == 1
|
|
assert row["wins"] == 0
|
|
assert row["losses"] == 0
|
|
|
|
|
|
# ----------------- Benchmarks v0.43 -----------------
|
|
|
|
class TestBenchmarksV043:
|
|
@pytest.mark.parametrize("name", list(NEW_BENCHMARKS_V0_43))
|
|
def test_recognised(self, name):
|
|
assert is_v0_43_benchmark(name) is True
|
|
meta = benchmark_metadata(name)
|
|
assert meta is not None
|
|
assert "description" in meta
|
|
|
|
def test_case_insensitive(self):
|
|
assert is_v0_43_benchmark("CEval") is True
|
|
|
|
def test_unknown_returns_false(self):
|
|
assert is_v0_43_benchmark("mmlu") is False
|
|
assert is_v0_43_benchmark("garbage") is False
|
|
|
|
def test_non_string_returns_false(self):
|
|
assert is_v0_43_benchmark(None) is False # type: ignore[arg-type]
|
|
assert is_v0_43_benchmark(123) is False # type: ignore[arg-type]
|
|
|
|
def test_metadata_immutable(self):
|
|
meta = benchmark_metadata("ceval")
|
|
with pytest.raises(TypeError):
|
|
meta["description"] = "evil" # type: ignore[index]
|
|
|
|
def test_lm_eval_task(self):
|
|
assert lm_eval_task_for("ceval") == "ceval-valid"
|
|
assert lm_eval_task_for("cmmlu") == "cmmlu"
|
|
# Aider Polyglot has no lm-eval mapping.
|
|
assert lm_eval_task_for("aider_polyglot") is None
|
|
assert lm_eval_task_for("garbage") is None
|