mirror of https://github.com/razor-ai/soup.git
fix(active-sample): variance-based diversity score for K>2 reward models (#206)
v0.63.0 `score_uncertainty` raised on K>2 and `_row_uncertainty` fell back to a monotone-broken `max(scores) - min(scores)`. Now generalises to K<=32 via population variance scaled by 4 — adding a fresh RM score equal to the running mean strictly decreases uncertainty (the new contribution to the sum-of-squares is zero while the denominator grows), so consensus on redundant evidence can never spike the score. K=1 max-entropy and K=2 disagreement formulas preserved verbatim (existing operator dashboards depend on the |s1 - s2| value). Cap stays at K=32 for DoS defence. _row_uncertainty K>2 path now routes through score_uncertainty inside an isolated try/except — bad rows return 0.0 instead of crashing the batch. PEP 585 modernisation: collections.abc imports + list[...] annotations (safe because `from __future__ import annotations` is in scope). math.fsum used for the variance accumulation to keep rounding error sub-ULP at K=32. Tests: +32 net (25 in new tests/test_v0631_206.py + 7 TDD review-fix followups). Full suite 11941 -> 11973 pass. Closes #206.
This commit is contained in:
Alpamys
parent
525a0e1114
commit
148cb0c125
|
|
@ -4,15 +4,24 @@ v0.63.0 Part C — picks the rows the model is *least confident* about so
|
|||
humans only review what the policy itself thinks is borderline. Reduces
|
||||
human-eval cost by 5-10x in practice.
|
||||
|
||||
Two modes via the input data shape:
|
||||
1. Single reward-model score (`rm_score`) — uncertainty via max-entropy:
|
||||
Three modes via the input data shape:
|
||||
|
||||
1. Single reward-model score (``rm_score``) — uncertainty via max-entropy:
|
||||
``1 - |2 * score - 1|``. Score 0.5 -> uncertainty 1.0 (peak),
|
||||
scores 0.0 or 1.0 -> uncertainty 0.0.
|
||||
2. Two reward-model scores (`rm_scores: [s1, s2]`) — disagreement via
|
||||
2. Two reward-model scores (``rm_scores: [s1, s2]``) — disagreement via
|
||||
``|s1 - s2|``. Bigger gap -> higher uncertainty.
|
||||
3. K reward-model scores (``rm_scores: [s1, ..., sK]``) for ``3 <= K <= 32``
|
||||
— population variance scaled by 4 (max disagreement = 1.0 when half the
|
||||
RMs score 0 and half score 1). Monotone-correct: adding a fresh RM score
|
||||
equal to the running mean *decreases* the score (the new contribution to
|
||||
sum-of-squares is zero while the denominator grows), so consensus on
|
||||
redundant evidence can never spike uncertainty. Replaces the v0.63.0
|
||||
``max(scores) - min(scores)`` fallback which was monotone-broken — closes
|
||||
#206.
|
||||
|
||||
Composes with v0.19 human eval (the output JSONL is a drop-in human-eval
|
||||
prompt set) and v0.58 `soup loop watch` (which can run this nightly).
|
||||
prompt set) and v0.58 ``soup loop watch`` (which can run this nightly).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -20,13 +29,19 @@ from __future__ import annotations
|
|||
import json
|
||||
import math
|
||||
import os
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Iterable, List, Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from soup_cli.utils.paths import is_under_cwd
|
||||
|
||||
_MAX_BUDGET = 100_000
|
||||
_MAX_INPUT_ROWS = 10_000_000 # 10M — production-scale day of traces
|
||||
_MAX_BUDGET: Final[int] = 100_000
|
||||
_MAX_INPUT_ROWS: Final[int] = 10_000_000 # 10M — production-scale day of traces
|
||||
# Cap on K reward-model scores per row. 32 is generous (most ensembles use
|
||||
# 3-8 RMs) and bounds the inner O(K) variance loop at well under a
|
||||
# microsecond per row. K>32 raises ValueError (DoS defence) — closes #206.
|
||||
# Final[int] so a caller cannot silently disable the cap by rebinding it.
|
||||
_MAX_RM_SCORES: Final[int] = 32
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
|
|
@ -94,11 +109,20 @@ def _validate_score(score: object, *, idx: int) -> float:
|
|||
return f_score
|
||||
|
||||
|
||||
def score_uncertainty(*, scores: Sequence[float]) -> float:
|
||||
"""Compute uncertainty from one or two reward-model scores.
|
||||
def score_uncertainty(*, scores: Sequence[float | int]) -> float:
|
||||
"""Compute uncertainty from K reward-model scores (``1 <= K <= 32``).
|
||||
|
||||
1 score: max-entropy distance from 0.5 (peak at 0.5 -> uncertainty 1.0)
|
||||
2 scores: pairwise disagreement (|s1 - s2|)
|
||||
- K=1: max-entropy distance from 0.5 (peak at 0.5 -> uncertainty 1.0)
|
||||
- K=2: pairwise disagreement (``|s1 - s2|``)
|
||||
- K>=3: population variance scaled by 4, clamped to ``[0, 1]``
|
||||
|
||||
For scores in ``[0, 1]`` the population variance is bounded by 0.25
|
||||
(achieved when half the RMs score 0 and half score 1), so the 4x scale
|
||||
keeps the K>=3 path inside the unit interval and consistent with the
|
||||
K<=2 forms.
|
||||
|
||||
Validation rejects bool-as-int, non-finite values (NaN / +/-Inf), and
|
||||
out-of-range scores at every K — see ``_validate_score``.
|
||||
"""
|
||||
if not isinstance(scores, Sequence) or isinstance(scores, str):
|
||||
raise TypeError(
|
||||
|
|
@ -106,17 +130,32 @@ def score_uncertainty(*, scores: Sequence[float]) -> float:
|
|||
)
|
||||
if len(scores) == 0:
|
||||
return 0.0
|
||||
if len(scores) > 2:
|
||||
if len(scores) > _MAX_RM_SCORES:
|
||||
raise ValueError(
|
||||
"score_uncertainty supports 1 or 2 RM scores (v0.63.0). "
|
||||
"K>2 RMs deferred to a future release."
|
||||
f"score_uncertainty supports at most {_MAX_RM_SCORES} RM scores, "
|
||||
f"got {len(scores)} (DoS cap)"
|
||||
)
|
||||
validated = [_validate_score(s, idx=i) for i, s in enumerate(scores)]
|
||||
if len(validated) == 1:
|
||||
k = len(validated)
|
||||
if k == 1:
|
||||
# 1 - |2*s - 1| -> peak at s=0.5
|
||||
return 1.0 - abs(2.0 * validated[0] - 1.0)
|
||||
# len == 2 -> pairwise disagreement
|
||||
return abs(validated[0] - validated[1])
|
||||
if k == 2:
|
||||
# Disagreement closed-form. Preserved verbatim for K=2 even though
|
||||
# variance gives the same answer up to scaling — existing operator
|
||||
# dashboards / thresholds depend on the |s1 - s2| value.
|
||||
return abs(validated[0] - validated[1])
|
||||
# K>=3: 4 * population variance, clamped into the unit interval.
|
||||
# Population variance (divide by N, not N-1) keeps the bound tight at
|
||||
# 0.25 for scores in [0, 1] and gives the monotonicity invariant
|
||||
# described in the module docstring. ``math.fsum`` uses compensated
|
||||
# summation to keep accumulated rounding error sub-ULP even at K=32 —
|
||||
# the variance formula is sensitive to it because we square the deltas.
|
||||
mean = math.fsum(validated) / k
|
||||
var = math.fsum((s - mean) ** 2 for s in validated) / k
|
||||
# max(0.0, ...) is defensive: floating-point subtraction can yield -epsilon
|
||||
# even though population variance is mathematically non-negative.
|
||||
return max(0.0, min(1.0, 4.0 * var))
|
||||
|
||||
|
||||
def _row_uncertainty(row: Mapping[str, object]) -> float:
|
||||
|
|
@ -137,16 +176,21 @@ def _row_uncertainty(row: Mapping[str, object]) -> float:
|
|||
return max(0.0, min(1.0, f_val))
|
||||
scores_field = row.get("rm_scores")
|
||||
if isinstance(scores_field, Sequence) and not isinstance(scores_field, str):
|
||||
scores_list: List[float] = []
|
||||
scores_list: list[float] = []
|
||||
try:
|
||||
for i, s in enumerate(scores_field):
|
||||
scores_list.append(_validate_score(s, idx=i))
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
if len(scores_list) <= 2:
|
||||
# Route every K through ``score_uncertainty`` so the variance path
|
||||
# (K>=3) and the closed forms (K=1, K=2) share a single source of
|
||||
# truth. K>32 raises ValueError on the cap — isolate the row by
|
||||
# returning 0.0 instead of breaking the whole batch (matches the
|
||||
# existing _validate_score isolation policy two lines above).
|
||||
try:
|
||||
return score_uncertainty(scores=scores_list)
|
||||
# >2 scores: silently fall back to disagreement = max - min
|
||||
return max(scores_list) - min(scores_list)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
scalar = row.get("rm_score")
|
||||
if isinstance(scalar, (int, float)) and not isinstance(scalar, bool):
|
||||
try:
|
||||
|
|
@ -161,13 +205,13 @@ def pick_top_uncertain(
|
|||
rows: Iterable[Mapping[str, object]],
|
||||
*,
|
||||
budget: int,
|
||||
) -> List[Mapping[str, object]]:
|
||||
) -> list[Mapping[str, object]]:
|
||||
"""Pick the top-N rows by uncertainty.
|
||||
|
||||
Stable on ties — earlier rows win to make the output deterministic.
|
||||
"""
|
||||
n_budget = validate_budget(budget)
|
||||
materialised: List[Mapping[str, object]] = []
|
||||
materialised: list[Mapping[str, object]] = []
|
||||
for row in rows:
|
||||
if not isinstance(row, Mapping):
|
||||
raise TypeError(
|
||||
|
|
@ -213,7 +257,7 @@ def sample_uncertain_rows(
|
|||
if not os.path.isfile(input_path):
|
||||
raise FileNotFoundError(input_path)
|
||||
|
||||
rows: List[Mapping[str, object]] = []
|
||||
rows: list[Mapping[str, object]] = []
|
||||
with open(input_path, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
stripped = line.strip()
|
||||
|
|
|
|||
|
|
@ -104,12 +104,13 @@ def test_score_uncertainty_rejects_out_of_range():
|
|||
score_uncertainty(scores=[-0.1])
|
||||
|
||||
|
||||
def test_score_uncertainty_rejects_too_many_rms():
|
||||
def test_score_uncertainty_rejects_above_cap():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
# >2 RMs not yet supported (avg pairwise std would be the v0.63.1+ shape)
|
||||
# K=3..32 is the variance path (see #206 / test_v0631_206.py).
|
||||
# K>32 is the DoS cap.
|
||||
with pytest.raises(ValueError):
|
||||
score_uncertainty(scores=[0.5, 0.6, 0.7])
|
||||
score_uncertainty(scores=[0.5] * 33)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -0,0 +1,426 @@
|
|||
"""Closes #206 — Variance-based diversity score for K>2 reward models in soup data active-sample.
|
||||
|
||||
v0.63.0 ``score_uncertainty`` raised ValueError on len(scores) > 2 and
|
||||
``_row_uncertainty`` fell back to a broken ``max(scores) - min(scores)``
|
||||
range (monotone-broken — a third score equal to the existing mean still
|
||||
spiked the score, so adding redundant evidence inflated uncertainty).
|
||||
|
||||
This change generalises ``score_uncertainty`` to up to K=32 RMs via
|
||||
population-variance with a 4x scaling so the formula stays in [0, 1] for
|
||||
scores in [0, 1]:
|
||||
|
||||
K=0 -> 0.0 (unchanged)
|
||||
K=1 -> 1 - |2*s - 1| (max-entropy) (unchanged)
|
||||
K=2 -> |s1 - s2| (disagreement) (unchanged)
|
||||
K>=3 -> 4 * pop_variance (clamped [0,1]) (NEW)
|
||||
K>32 -> ValueError (DoS cap, was K>2)
|
||||
|
||||
Monotonicity invariant: adding a fresh score equal to the running mean
|
||||
*decreases* population variance (the new contribution to the sum-of-squares
|
||||
is zero while the denominator grows), so the uncertainty estimate cannot
|
||||
spike when the K-th RM just agrees with the consensus.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# K>=3 variance path
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_k_equals_3_returns_finite_unit_value():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
s = score_uncertainty(scores=[0.1, 0.5, 0.9])
|
||||
assert math.isfinite(s)
|
||||
assert 0.0 <= s <= 1.0
|
||||
# 4 * pop_var([0.1, 0.5, 0.9]) = 4 * (0.16 + 0 + 0.16) / 3 = 0.4266666...
|
||||
# Tight tolerance (1e-9) catches K vs K-1 denominator drift.
|
||||
assert s == pytest.approx(4.0 * 0.32 / 3.0, abs=1e-9)
|
||||
|
||||
|
||||
def test_k_equals_4_returns_finite_unit_value():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
s = score_uncertainty(scores=[0.2, 0.4, 0.6, 0.8])
|
||||
assert 0.0 <= s <= 1.0
|
||||
# mean=0.5, var = ((-.3)^2 + (-.1)^2 + (.1)^2 + (.3)^2)/4 = 0.20/4 = 0.05
|
||||
# 4*var = 0.20
|
||||
assert s == pytest.approx(0.20, abs=0.001)
|
||||
|
||||
|
||||
def test_k_equals_8_returns_finite_unit_value():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
scores = [0.0, 0.2, 0.4, 0.5, 0.5, 0.6, 0.8, 1.0]
|
||||
s = score_uncertainty(scores=scores)
|
||||
assert 0.0 <= s <= 1.0
|
||||
assert math.isfinite(s)
|
||||
|
||||
|
||||
def test_max_uncertainty_when_half_zero_half_one():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
# Half-0 half-1: pop variance = 0.25, 4*var = 1.0 — maximum disagreement
|
||||
s = score_uncertainty(scores=[0.0, 0.0, 1.0, 1.0])
|
||||
assert s == pytest.approx(1.0, abs=1e-9)
|
||||
|
||||
|
||||
def test_zero_uncertainty_when_all_agree():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
s = score_uncertainty(scores=[0.5, 0.5, 0.5, 0.5, 0.5])
|
||||
assert s == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Monotonicity — adding a score at the mean must decrease/hold variance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_adding_score_at_mean_decreases_uncertainty():
|
||||
"""Critical correctness invariant — see acceptance criteria in #206."""
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
base = [0.1, 0.5, 0.9]
|
||||
mean = sum(base) / len(base)
|
||||
u_before = score_uncertainty(scores=base)
|
||||
u_after = score_uncertainty(scores=base + [mean])
|
||||
assert u_after < u_before, (
|
||||
f"adding the mean ({mean}) must reduce variance — "
|
||||
f"before={u_before}, after={u_after}"
|
||||
)
|
||||
|
||||
|
||||
def test_adding_score_at_mean_holds_when_already_constant():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
# All-equal -> variance=0 already; adding the same value keeps it at 0.
|
||||
# Float reality: (0.4-0.4)^2 / N is ~1e-32 not literally 0.0 because
|
||||
# the mean computation introduces ULP noise. The math is correct;
|
||||
# the assertion just has to tolerate that noise.
|
||||
base = [0.4, 0.4, 0.4]
|
||||
u_before = score_uncertainty(scores=base)
|
||||
u_after = score_uncertainty(scores=base + [0.4])
|
||||
assert u_before == pytest.approx(0.0, abs=1e-12)
|
||||
assert u_after == pytest.approx(0.0, abs=1e-12)
|
||||
assert u_after <= u_before # monotonicity invariant still holds
|
||||
|
||||
|
||||
def test_adding_disagreeing_score_increases_uncertainty():
|
||||
"""Sanity counter-test: an outlier RM should INCREASE uncertainty."""
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
base = [0.5, 0.5, 0.5]
|
||||
u_before = score_uncertainty(scores=base)
|
||||
u_after = score_uncertainty(scores=base + [0.0])
|
||||
assert u_after > u_before
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# K=1 and K=2 formulas preserved
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_k_equals_1_max_entropy_formula_preserved():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
assert score_uncertainty(scores=[0.5]) == 1.0
|
||||
assert score_uncertainty(scores=[0.0]) == 0.0
|
||||
assert score_uncertainty(scores=[1.0]) == 0.0
|
||||
|
||||
|
||||
def test_k_equals_2_disagreement_formula_preserved():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
# The issue calls out "variance gives the same answer for K=2 up to
|
||||
# scaling" — but the literal pairwise-disagreement |s1 - s2| formula
|
||||
# is what existing operators depend on. Don't drift it.
|
||||
assert score_uncertainty(scores=[0.1, 0.9]) == pytest.approx(0.8)
|
||||
assert score_uncertainty(scores=[0.5, 0.5]) == 0.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Cap at K=32 — DoS defence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_k_equals_32_accepted_at_boundary():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
s = score_uncertainty(scores=[0.5] * 32)
|
||||
assert s == 0.0
|
||||
|
||||
|
||||
def test_k_equals_32_with_disagreement_accepted():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
scores = [0.0] * 16 + [1.0] * 16
|
||||
s = score_uncertainty(scores=scores)
|
||||
assert s == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_k_equals_33_rejected_at_boundary():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
with pytest.raises(ValueError, match="32"):
|
||||
score_uncertainty(scores=[0.5] * 33)
|
||||
|
||||
|
||||
def test_max_rm_scores_constant_is_32():
|
||||
"""Lock the cap so future drift fails loudly."""
|
||||
from soup_cli.utils.active_sampler import _MAX_RM_SCORES
|
||||
|
||||
assert _MAX_RM_SCORES == 32
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-element validation still propagates at K>2
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_k3_rejects_bool_entry():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
with pytest.raises(TypeError, match="bool"):
|
||||
score_uncertainty(scores=[0.5, 0.5, True])
|
||||
|
||||
|
||||
def test_k3_rejects_non_finite_entry():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
score_uncertainty(scores=[0.5, 0.5, float("nan")])
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
score_uncertainty(scores=[0.5, 0.5, float("inf")])
|
||||
|
||||
|
||||
def test_k3_rejects_out_of_range_entry():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"):
|
||||
score_uncertainty(scores=[0.5, 0.5, 1.5])
|
||||
with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"):
|
||||
score_uncertainty(scores=[0.5, 0.5, -0.1])
|
||||
|
||||
|
||||
def test_k3_rejects_non_numeric_entry():
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
with pytest.raises(TypeError, match="number"):
|
||||
score_uncertainty(scores=[0.5, 0.5, "0.5"])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _row_uncertainty K>2 path delegates to variance (no more max-min fallback)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_row_uncertainty_k3_uses_variance_not_max_minus_min():
|
||||
"""Regression: the old max-min fallback was monotone-broken.
|
||||
|
||||
Two rows with the SAME (min, max) but different middle scores must
|
||||
now score differently when the inner scores' spread differs.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
# Both rows have min=0.1, max=0.9 -> old max-min = 0.8 for both.
|
||||
row_consensus = {"rm_scores": [0.1, 0.5, 0.9]}
|
||||
row_polarised = {"rm_scores": [0.1, 0.1, 0.9]}
|
||||
u_consensus = _row_uncertainty(row_consensus)
|
||||
u_polarised = _row_uncertainty(row_polarised)
|
||||
# New behaviour: variance picks up the structural difference.
|
||||
assert u_consensus != u_polarised
|
||||
# Polarised (more mass at extremes) has higher variance.
|
||||
assert u_polarised > u_consensus
|
||||
|
||||
|
||||
def test_row_uncertainty_k4_returns_unit_value():
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
row = {"rm_scores": [0.2, 0.4, 0.6, 0.8]}
|
||||
u = _row_uncertainty(row)
|
||||
assert 0.0 <= u <= 1.0
|
||||
assert math.isfinite(u)
|
||||
|
||||
|
||||
def test_row_uncertainty_k33_returns_zero_isolated():
|
||||
"""K>32 in row data: do NOT crash the loop — fall through to 0.0.
|
||||
|
||||
Matches existing isolation policy: bad data on one row never breaks
|
||||
the whole batch (see ``_row_uncertainty`` try/except around _validate_score).
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
row = {"rm_scores": [0.5] * 33}
|
||||
assert _row_uncertainty(row) == 0.0
|
||||
|
||||
|
||||
def test_row_uncertainty_k1_and_k2_paths_preserved():
|
||||
"""K=1 and K=2 rows still route to their existing closed-form formulas."""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
# K=1
|
||||
assert _row_uncertainty({"rm_scores": [0.5]}) == 1.0
|
||||
assert _row_uncertainty({"rm_scores": [0.0]}) == 0.0
|
||||
# K=2
|
||||
assert _row_uncertainty({"rm_scores": [0.1, 0.9]}) == pytest.approx(0.8)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# sample_uncertain_rows end-to-end with K=3 rows
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sample_uncertain_rows_triple_rm(tmp_path, monkeypatch):
|
||||
"""Closes #206 — pick rows where 3 RMs disagree the most."""
|
||||
import json
|
||||
|
||||
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
inp = tmp_path / "in.jsonl"
|
||||
out = tmp_path / "out.jsonl"
|
||||
rows = [
|
||||
{"id": "a", "rm_scores": [0.0, 0.5, 1.0]}, # high variance
|
||||
{"id": "b", "rm_scores": [0.5, 0.5, 0.5]}, # zero variance
|
||||
{"id": "c", "rm_scores": [0.4, 0.5, 0.6]}, # low variance
|
||||
]
|
||||
inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
||||
|
||||
plan = sample_uncertain_rows(str(inp), output_path=str(out), budget=2)
|
||||
assert plan.rows_selected == 2
|
||||
out_rows = [json.loads(ln) for ln in out.read_text(encoding="utf-8").splitlines()]
|
||||
ids = [r["id"] for r in out_rows]
|
||||
# Highest variance "a" first; "c" beats "b" (zero variance) for the second slot.
|
||||
assert ids == ["a", "c"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Source-grep regression guards
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_max_minus_min_fallback_removed():
|
||||
"""The v0.63.0 max-min fallback is a monotone-broken anti-pattern.
|
||||
|
||||
If this guard ever fires, someone re-introduced the broken K>2 path —
|
||||
they must use variance via score_uncertainty instead.
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py"
|
||||
text = src.read_text(encoding="utf-8")
|
||||
# The exact broken line; if a comment mentions max/min that's fine.
|
||||
assert "return max(scores_list) - min(scores_list)" not in text
|
||||
|
||||
|
||||
def test_score_uncertainty_no_top_level_heavy_imports():
|
||||
from pathlib import Path
|
||||
|
||||
src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py"
|
||||
text = src.read_text(encoding="utf-8")
|
||||
for forbidden in ("import torch", "import numpy", "import statistics"):
|
||||
assert f"\n{forbidden}" not in text, (
|
||||
f"active_sampler.py must stay pure-stdlib (no top-level {forbidden!r})"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# TDD review follow-ups
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_k_equals_31_accepted_below_cap():
|
||||
"""HIGH: project convention is N-1, N, N+1 boundary coverage.
|
||||
|
||||
A regression that set ``_MAX_RM_SCORES = 30`` would otherwise pass
|
||||
every K=32/K=33 test in this file.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
s = score_uncertainty(scores=[0.5] * 31)
|
||||
assert 0.0 <= s <= 1.0
|
||||
assert math.isfinite(s)
|
||||
|
||||
|
||||
def test_row_uncertainty_explicit_field_overrides_k3_rm_scores():
|
||||
"""HIGH: ``_row_uncertainty`` field-priority contract.
|
||||
|
||||
Explicit ``uncertainty`` field MUST win over rm_scores even when
|
||||
rm_scores would also produce a valid K>=3 variance score. A regression
|
||||
that flipped the priority order could otherwise pass silently because
|
||||
rm_scores=[0.5,0.5,0.5] returns 0.0 — same as a missing field.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
assert _row_uncertainty(
|
||||
{"uncertainty": 0.9, "rm_scores": [0.5, 0.5, 0.5]}
|
||||
) == pytest.approx(0.9)
|
||||
|
||||
|
||||
def test_old_k2_deferred_error_message_removed():
|
||||
"""HIGH: source-grep regression guard for the broken K>2 ValueError text.
|
||||
|
||||
The v0.63.0 code raised ``ValueError("...K>2 RMs deferred...")``. We've
|
||||
lifted that. Without this guard, a future revert could re-introduce the
|
||||
deferred-message text and only the K=33 boundary test would catch it
|
||||
(matching only the literal "32" — not the deferred phrasing).
|
||||
"""
|
||||
from pathlib import Path
|
||||
|
||||
src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py"
|
||||
text = src.read_text(encoding="utf-8")
|
||||
assert "K>2 RMs deferred" not in text
|
||||
assert "deferred to a future release" not in text
|
||||
|
||||
|
||||
def test_empty_scores_returns_zero():
|
||||
"""MEDIUM: K=0 contract explicit (was implied, never asserted in v0.63.1)."""
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
assert score_uncertainty(scores=[]) == 0.0
|
||||
|
||||
|
||||
def test_row_uncertainty_scalar_rm_score_path_preserved():
|
||||
"""MEDIUM: scalar ``rm_score`` (Priority-3 path) preserved.
|
||||
|
||||
The K>2 routing rewrite could plausibly have broken the scalar branch
|
||||
without any new test catching it.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
assert _row_uncertainty({"rm_score": 1.0}) == pytest.approx(0.0)
|
||||
assert _row_uncertainty({"rm_score": 0.0}) == pytest.approx(0.0)
|
||||
assert _row_uncertainty({"rm_score": 0.5}) == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_row_uncertainty_k3_with_nan_score_isolates_to_zero():
|
||||
"""MEDIUM: K>=3 row with NaN entry must isolate to 0.0, not crash batch.
|
||||
|
||||
Matches the K>cap and bool/oversize isolation policy already in
|
||||
``_row_uncertainty`` — bad data on one row never breaks the batch.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import _row_uncertainty
|
||||
|
||||
assert _row_uncertainty({"rm_scores": [0.5, 0.5, float("nan")]}) == 0.0
|
||||
assert _row_uncertainty({"rm_scores": [0.5, 0.5, float("inf")]}) == 0.0
|
||||
|
||||
|
||||
def test_k2_equal_scores_strict_zero():
|
||||
"""MEDIUM: K=2 with equal scores returns *strict* 0.0.
|
||||
|
||||
The K=2 path uses IEEE 754 subtraction (no accumulated error) so we
|
||||
can assert strict equality. Pairs with the K>=3 monotonicity-with-
|
||||
constant-input test (which uses ``abs=1e-12`` tolerance) to document
|
||||
where strict-zero vs sub-ULP residual applies.
|
||||
"""
|
||||
from soup_cli.utils.active_sampler import score_uncertainty
|
||||
|
||||
assert score_uncertainty(scores=[0.4, 0.4]) == 0.0
|
||||
assert score_uncertainty(scores=[0.0, 0.0]) == 0.0
|
||||
assert score_uncertainty(scores=[1.0, 1.0]) == 0.0
|
||||
Loading…
Reference in New Issue