diff --git a/soup_cli/utils/active_sampler.py b/soup_cli/utils/active_sampler.py index 5fc1bb1..77d36a1 100644 --- a/soup_cli/utils/active_sampler.py +++ b/soup_cli/utils/active_sampler.py @@ -4,15 +4,24 @@ v0.63.0 Part C — picks the rows the model is *least confident* about so humans only review what the policy itself thinks is borderline. Reduces human-eval cost by 5-10x in practice. -Two modes via the input data shape: -1. Single reward-model score (`rm_score`) — uncertainty via max-entropy: +Three modes via the input data shape: + +1. Single reward-model score (``rm_score``) — uncertainty via max-entropy: ``1 - |2 * score - 1|``. Score 0.5 -> uncertainty 1.0 (peak), scores 0.0 or 1.0 -> uncertainty 0.0. -2. Two reward-model scores (`rm_scores: [s1, s2]`) — disagreement via +2. Two reward-model scores (``rm_scores: [s1, s2]``) — disagreement via ``|s1 - s2|``. Bigger gap -> higher uncertainty. +3. K reward-model scores (``rm_scores: [s1, ..., sK]``) for ``3 <= K <= 32`` + — population variance scaled by 4 (max disagreement = 1.0 when half the + RMs score 0 and half score 1). Monotone-correct: adding a fresh RM score + equal to the running mean *decreases* the score (the new contribution to + sum-of-squares is zero while the denominator grows), so consensus on + redundant evidence can never spike uncertainty. Replaces the v0.63.0 + ``max(scores) - min(scores)`` fallback which was monotone-broken — closes + #206. Composes with v0.19 human eval (the output JSONL is a drop-in human-eval -prompt set) and v0.58 `soup loop watch` (which can run this nightly). +prompt set) and v0.58 ``soup loop watch`` (which can run this nightly). """ from __future__ import annotations @@ -20,13 +29,19 @@ from __future__ import annotations import json import math import os +from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass -from typing import Iterable, List, Mapping, Sequence +from typing import Final from soup_cli.utils.paths import is_under_cwd -_MAX_BUDGET = 100_000 -_MAX_INPUT_ROWS = 10_000_000 # 10M — production-scale day of traces +_MAX_BUDGET: Final[int] = 100_000 +_MAX_INPUT_ROWS: Final[int] = 10_000_000 # 10M — production-scale day of traces +# Cap on K reward-model scores per row. 32 is generous (most ensembles use +# 3-8 RMs) and bounds the inner O(K) variance loop at well under a +# microsecond per row. K>32 raises ValueError (DoS defence) — closes #206. +# Final[int] so a caller cannot silently disable the cap by rebinding it. +_MAX_RM_SCORES: Final[int] = 32 @dataclass(frozen=True) @@ -94,11 +109,20 @@ def _validate_score(score: object, *, idx: int) -> float: return f_score -def score_uncertainty(*, scores: Sequence[float]) -> float: - """Compute uncertainty from one or two reward-model scores. +def score_uncertainty(*, scores: Sequence[float | int]) -> float: + """Compute uncertainty from K reward-model scores (``1 <= K <= 32``). - 1 score: max-entropy distance from 0.5 (peak at 0.5 -> uncertainty 1.0) - 2 scores: pairwise disagreement (|s1 - s2|) + - K=1: max-entropy distance from 0.5 (peak at 0.5 -> uncertainty 1.0) + - K=2: pairwise disagreement (``|s1 - s2|``) + - K>=3: population variance scaled by 4, clamped to ``[0, 1]`` + + For scores in ``[0, 1]`` the population variance is bounded by 0.25 + (achieved when half the RMs score 0 and half score 1), so the 4x scale + keeps the K>=3 path inside the unit interval and consistent with the + K<=2 forms. + + Validation rejects bool-as-int, non-finite values (NaN / +/-Inf), and + out-of-range scores at every K — see ``_validate_score``. """ if not isinstance(scores, Sequence) or isinstance(scores, str): raise TypeError( @@ -106,17 +130,32 @@ def score_uncertainty(*, scores: Sequence[float]) -> float: ) if len(scores) == 0: return 0.0 - if len(scores) > 2: + if len(scores) > _MAX_RM_SCORES: raise ValueError( - "score_uncertainty supports 1 or 2 RM scores (v0.63.0). " - "K>2 RMs deferred to a future release." + f"score_uncertainty supports at most {_MAX_RM_SCORES} RM scores, " + f"got {len(scores)} (DoS cap)" ) validated = [_validate_score(s, idx=i) for i, s in enumerate(scores)] - if len(validated) == 1: + k = len(validated) + if k == 1: # 1 - |2*s - 1| -> peak at s=0.5 return 1.0 - abs(2.0 * validated[0] - 1.0) - # len == 2 -> pairwise disagreement - return abs(validated[0] - validated[1]) + if k == 2: + # Disagreement closed-form. Preserved verbatim for K=2 even though + # variance gives the same answer up to scaling — existing operator + # dashboards / thresholds depend on the |s1 - s2| value. + return abs(validated[0] - validated[1]) + # K>=3: 4 * population variance, clamped into the unit interval. + # Population variance (divide by N, not N-1) keeps the bound tight at + # 0.25 for scores in [0, 1] and gives the monotonicity invariant + # described in the module docstring. ``math.fsum`` uses compensated + # summation to keep accumulated rounding error sub-ULP even at K=32 — + # the variance formula is sensitive to it because we square the deltas. + mean = math.fsum(validated) / k + var = math.fsum((s - mean) ** 2 for s in validated) / k + # max(0.0, ...) is defensive: floating-point subtraction can yield -epsilon + # even though population variance is mathematically non-negative. + return max(0.0, min(1.0, 4.0 * var)) def _row_uncertainty(row: Mapping[str, object]) -> float: @@ -137,16 +176,21 @@ def _row_uncertainty(row: Mapping[str, object]) -> float: return max(0.0, min(1.0, f_val)) scores_field = row.get("rm_scores") if isinstance(scores_field, Sequence) and not isinstance(scores_field, str): - scores_list: List[float] = [] + scores_list: list[float] = [] try: for i, s in enumerate(scores_field): scores_list.append(_validate_score(s, idx=i)) except (TypeError, ValueError): return 0.0 - if len(scores_list) <= 2: + # Route every K through ``score_uncertainty`` so the variance path + # (K>=3) and the closed forms (K=1, K=2) share a single source of + # truth. K>32 raises ValueError on the cap — isolate the row by + # returning 0.0 instead of breaking the whole batch (matches the + # existing _validate_score isolation policy two lines above). + try: return score_uncertainty(scores=scores_list) - # >2 scores: silently fall back to disagreement = max - min - return max(scores_list) - min(scores_list) + except (TypeError, ValueError): + return 0.0 scalar = row.get("rm_score") if isinstance(scalar, (int, float)) and not isinstance(scalar, bool): try: @@ -161,13 +205,13 @@ def pick_top_uncertain( rows: Iterable[Mapping[str, object]], *, budget: int, -) -> List[Mapping[str, object]]: +) -> list[Mapping[str, object]]: """Pick the top-N rows by uncertainty. Stable on ties — earlier rows win to make the output deterministic. """ n_budget = validate_budget(budget) - materialised: List[Mapping[str, object]] = [] + materialised: list[Mapping[str, object]] = [] for row in rows: if not isinstance(row, Mapping): raise TypeError( @@ -213,7 +257,7 @@ def sample_uncertain_rows( if not os.path.isfile(input_path): raise FileNotFoundError(input_path) - rows: List[Mapping[str, object]] = [] + rows: list[Mapping[str, object]] = [] with open(input_path, encoding="utf-8") as fh: for line in fh: stripped = line.strip() diff --git a/tests/test_v0630_part_c.py b/tests/test_v0630_part_c.py index 8a1dd11..32e195a 100644 --- a/tests/test_v0630_part_c.py +++ b/tests/test_v0630_part_c.py @@ -104,12 +104,13 @@ def test_score_uncertainty_rejects_out_of_range(): score_uncertainty(scores=[-0.1]) -def test_score_uncertainty_rejects_too_many_rms(): +def test_score_uncertainty_rejects_above_cap(): from soup_cli.utils.active_sampler import score_uncertainty - # >2 RMs not yet supported (avg pairwise std would be the v0.63.1+ shape) + # K=3..32 is the variance path (see #206 / test_v0631_206.py). + # K>32 is the DoS cap. with pytest.raises(ValueError): - score_uncertainty(scores=[0.5, 0.6, 0.7]) + score_uncertainty(scores=[0.5] * 33) # --------------------------------------------------------------------------- diff --git a/tests/test_v0631_206.py b/tests/test_v0631_206.py new file mode 100644 index 0000000..7ccb872 --- /dev/null +++ b/tests/test_v0631_206.py @@ -0,0 +1,426 @@ +"""Closes #206 — Variance-based diversity score for K>2 reward models in soup data active-sample. + +v0.63.0 ``score_uncertainty`` raised ValueError on len(scores) > 2 and +``_row_uncertainty`` fell back to a broken ``max(scores) - min(scores)`` +range (monotone-broken — a third score equal to the existing mean still +spiked the score, so adding redundant evidence inflated uncertainty). + +This change generalises ``score_uncertainty`` to up to K=32 RMs via +population-variance with a 4x scaling so the formula stays in [0, 1] for +scores in [0, 1]: + + K=0 -> 0.0 (unchanged) + K=1 -> 1 - |2*s - 1| (max-entropy) (unchanged) + K=2 -> |s1 - s2| (disagreement) (unchanged) + K>=3 -> 4 * pop_variance (clamped [0,1]) (NEW) + K>32 -> ValueError (DoS cap, was K>2) + +Monotonicity invariant: adding a fresh score equal to the running mean +*decreases* population variance (the new contribution to the sum-of-squares +is zero while the denominator grows), so the uncertainty estimate cannot +spike when the K-th RM just agrees with the consensus. +""" + +from __future__ import annotations + +import math + +import pytest + +# --------------------------------------------------------------------------- +# K>=3 variance path +# --------------------------------------------------------------------------- + + +def test_k_equals_3_returns_finite_unit_value(): + from soup_cli.utils.active_sampler import score_uncertainty + + s = score_uncertainty(scores=[0.1, 0.5, 0.9]) + assert math.isfinite(s) + assert 0.0 <= s <= 1.0 + # 4 * pop_var([0.1, 0.5, 0.9]) = 4 * (0.16 + 0 + 0.16) / 3 = 0.4266666... + # Tight tolerance (1e-9) catches K vs K-1 denominator drift. + assert s == pytest.approx(4.0 * 0.32 / 3.0, abs=1e-9) + + +def test_k_equals_4_returns_finite_unit_value(): + from soup_cli.utils.active_sampler import score_uncertainty + + s = score_uncertainty(scores=[0.2, 0.4, 0.6, 0.8]) + assert 0.0 <= s <= 1.0 + # mean=0.5, var = ((-.3)^2 + (-.1)^2 + (.1)^2 + (.3)^2)/4 = 0.20/4 = 0.05 + # 4*var = 0.20 + assert s == pytest.approx(0.20, abs=0.001) + + +def test_k_equals_8_returns_finite_unit_value(): + from soup_cli.utils.active_sampler import score_uncertainty + + scores = [0.0, 0.2, 0.4, 0.5, 0.5, 0.6, 0.8, 1.0] + s = score_uncertainty(scores=scores) + assert 0.0 <= s <= 1.0 + assert math.isfinite(s) + + +def test_max_uncertainty_when_half_zero_half_one(): + from soup_cli.utils.active_sampler import score_uncertainty + + # Half-0 half-1: pop variance = 0.25, 4*var = 1.0 — maximum disagreement + s = score_uncertainty(scores=[0.0, 0.0, 1.0, 1.0]) + assert s == pytest.approx(1.0, abs=1e-9) + + +def test_zero_uncertainty_when_all_agree(): + from soup_cli.utils.active_sampler import score_uncertainty + + s = score_uncertainty(scores=[0.5, 0.5, 0.5, 0.5, 0.5]) + assert s == 0.0 + + +# --------------------------------------------------------------------------- +# Monotonicity — adding a score at the mean must decrease/hold variance +# --------------------------------------------------------------------------- + + +def test_adding_score_at_mean_decreases_uncertainty(): + """Critical correctness invariant — see acceptance criteria in #206.""" + from soup_cli.utils.active_sampler import score_uncertainty + + base = [0.1, 0.5, 0.9] + mean = sum(base) / len(base) + u_before = score_uncertainty(scores=base) + u_after = score_uncertainty(scores=base + [mean]) + assert u_after < u_before, ( + f"adding the mean ({mean}) must reduce variance — " + f"before={u_before}, after={u_after}" + ) + + +def test_adding_score_at_mean_holds_when_already_constant(): + from soup_cli.utils.active_sampler import score_uncertainty + + # All-equal -> variance=0 already; adding the same value keeps it at 0. + # Float reality: (0.4-0.4)^2 / N is ~1e-32 not literally 0.0 because + # the mean computation introduces ULP noise. The math is correct; + # the assertion just has to tolerate that noise. + base = [0.4, 0.4, 0.4] + u_before = score_uncertainty(scores=base) + u_after = score_uncertainty(scores=base + [0.4]) + assert u_before == pytest.approx(0.0, abs=1e-12) + assert u_after == pytest.approx(0.0, abs=1e-12) + assert u_after <= u_before # monotonicity invariant still holds + + +def test_adding_disagreeing_score_increases_uncertainty(): + """Sanity counter-test: an outlier RM should INCREASE uncertainty.""" + from soup_cli.utils.active_sampler import score_uncertainty + + base = [0.5, 0.5, 0.5] + u_before = score_uncertainty(scores=base) + u_after = score_uncertainty(scores=base + [0.0]) + assert u_after > u_before + + +# --------------------------------------------------------------------------- +# K=1 and K=2 formulas preserved +# --------------------------------------------------------------------------- + + +def test_k_equals_1_max_entropy_formula_preserved(): + from soup_cli.utils.active_sampler import score_uncertainty + + assert score_uncertainty(scores=[0.5]) == 1.0 + assert score_uncertainty(scores=[0.0]) == 0.0 + assert score_uncertainty(scores=[1.0]) == 0.0 + + +def test_k_equals_2_disagreement_formula_preserved(): + from soup_cli.utils.active_sampler import score_uncertainty + + # The issue calls out "variance gives the same answer for K=2 up to + # scaling" — but the literal pairwise-disagreement |s1 - s2| formula + # is what existing operators depend on. Don't drift it. + assert score_uncertainty(scores=[0.1, 0.9]) == pytest.approx(0.8) + assert score_uncertainty(scores=[0.5, 0.5]) == 0.0 + + +# --------------------------------------------------------------------------- +# Cap at K=32 — DoS defence +# --------------------------------------------------------------------------- + + +def test_k_equals_32_accepted_at_boundary(): + from soup_cli.utils.active_sampler import score_uncertainty + + s = score_uncertainty(scores=[0.5] * 32) + assert s == 0.0 + + +def test_k_equals_32_with_disagreement_accepted(): + from soup_cli.utils.active_sampler import score_uncertainty + + scores = [0.0] * 16 + [1.0] * 16 + s = score_uncertainty(scores=scores) + assert s == pytest.approx(1.0) + + +def test_k_equals_33_rejected_at_boundary(): + from soup_cli.utils.active_sampler import score_uncertainty + + with pytest.raises(ValueError, match="32"): + score_uncertainty(scores=[0.5] * 33) + + +def test_max_rm_scores_constant_is_32(): + """Lock the cap so future drift fails loudly.""" + from soup_cli.utils.active_sampler import _MAX_RM_SCORES + + assert _MAX_RM_SCORES == 32 + + +# --------------------------------------------------------------------------- +# Per-element validation still propagates at K>2 +# --------------------------------------------------------------------------- + + +def test_k3_rejects_bool_entry(): + from soup_cli.utils.active_sampler import score_uncertainty + + with pytest.raises(TypeError, match="bool"): + score_uncertainty(scores=[0.5, 0.5, True]) + + +def test_k3_rejects_non_finite_entry(): + from soup_cli.utils.active_sampler import score_uncertainty + + with pytest.raises(ValueError, match="finite"): + score_uncertainty(scores=[0.5, 0.5, float("nan")]) + with pytest.raises(ValueError, match="finite"): + score_uncertainty(scores=[0.5, 0.5, float("inf")]) + + +def test_k3_rejects_out_of_range_entry(): + from soup_cli.utils.active_sampler import score_uncertainty + + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"): + score_uncertainty(scores=[0.5, 0.5, 1.5]) + with pytest.raises(ValueError, match=r"\[0\.0, 1\.0\]"): + score_uncertainty(scores=[0.5, 0.5, -0.1]) + + +def test_k3_rejects_non_numeric_entry(): + from soup_cli.utils.active_sampler import score_uncertainty + + with pytest.raises(TypeError, match="number"): + score_uncertainty(scores=[0.5, 0.5, "0.5"]) + + +# --------------------------------------------------------------------------- +# _row_uncertainty K>2 path delegates to variance (no more max-min fallback) +# --------------------------------------------------------------------------- + + +def test_row_uncertainty_k3_uses_variance_not_max_minus_min(): + """Regression: the old max-min fallback was monotone-broken. + + Two rows with the SAME (min, max) but different middle scores must + now score differently when the inner scores' spread differs. + """ + from soup_cli.utils.active_sampler import _row_uncertainty + + # Both rows have min=0.1, max=0.9 -> old max-min = 0.8 for both. + row_consensus = {"rm_scores": [0.1, 0.5, 0.9]} + row_polarised = {"rm_scores": [0.1, 0.1, 0.9]} + u_consensus = _row_uncertainty(row_consensus) + u_polarised = _row_uncertainty(row_polarised) + # New behaviour: variance picks up the structural difference. + assert u_consensus != u_polarised + # Polarised (more mass at extremes) has higher variance. + assert u_polarised > u_consensus + + +def test_row_uncertainty_k4_returns_unit_value(): + from soup_cli.utils.active_sampler import _row_uncertainty + + row = {"rm_scores": [0.2, 0.4, 0.6, 0.8]} + u = _row_uncertainty(row) + assert 0.0 <= u <= 1.0 + assert math.isfinite(u) + + +def test_row_uncertainty_k33_returns_zero_isolated(): + """K>32 in row data: do NOT crash the loop — fall through to 0.0. + + Matches existing isolation policy: bad data on one row never breaks + the whole batch (see ``_row_uncertainty`` try/except around _validate_score). + """ + from soup_cli.utils.active_sampler import _row_uncertainty + + row = {"rm_scores": [0.5] * 33} + assert _row_uncertainty(row) == 0.0 + + +def test_row_uncertainty_k1_and_k2_paths_preserved(): + """K=1 and K=2 rows still route to their existing closed-form formulas.""" + from soup_cli.utils.active_sampler import _row_uncertainty + + # K=1 + assert _row_uncertainty({"rm_scores": [0.5]}) == 1.0 + assert _row_uncertainty({"rm_scores": [0.0]}) == 0.0 + # K=2 + assert _row_uncertainty({"rm_scores": [0.1, 0.9]}) == pytest.approx(0.8) + + +# --------------------------------------------------------------------------- +# sample_uncertain_rows end-to-end with K=3 rows +# --------------------------------------------------------------------------- + + +def test_sample_uncertain_rows_triple_rm(tmp_path, monkeypatch): + """Closes #206 — pick rows where 3 RMs disagree the most.""" + import json + + from soup_cli.utils.active_sampler import sample_uncertain_rows + + monkeypatch.chdir(tmp_path) + inp = tmp_path / "in.jsonl" + out = tmp_path / "out.jsonl" + rows = [ + {"id": "a", "rm_scores": [0.0, 0.5, 1.0]}, # high variance + {"id": "b", "rm_scores": [0.5, 0.5, 0.5]}, # zero variance + {"id": "c", "rm_scores": [0.4, 0.5, 0.6]}, # low variance + ] + inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + plan = sample_uncertain_rows(str(inp), output_path=str(out), budget=2) + assert plan.rows_selected == 2 + out_rows = [json.loads(ln) for ln in out.read_text(encoding="utf-8").splitlines()] + ids = [r["id"] for r in out_rows] + # Highest variance "a" first; "c" beats "b" (zero variance) for the second slot. + assert ids == ["a", "c"] + + +# --------------------------------------------------------------------------- +# Source-grep regression guards +# --------------------------------------------------------------------------- + + +def test_max_minus_min_fallback_removed(): + """The v0.63.0 max-min fallback is a monotone-broken anti-pattern. + + If this guard ever fires, someone re-introduced the broken K>2 path — + they must use variance via score_uncertainty instead. + """ + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py" + text = src.read_text(encoding="utf-8") + # The exact broken line; if a comment mentions max/min that's fine. + assert "return max(scores_list) - min(scores_list)" not in text + + +def test_score_uncertainty_no_top_level_heavy_imports(): + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py" + text = src.read_text(encoding="utf-8") + for forbidden in ("import torch", "import numpy", "import statistics"): + assert f"\n{forbidden}" not in text, ( + f"active_sampler.py must stay pure-stdlib (no top-level {forbidden!r})" + ) + + +# --------------------------------------------------------------------------- +# TDD review follow-ups +# --------------------------------------------------------------------------- + + +def test_k_equals_31_accepted_below_cap(): + """HIGH: project convention is N-1, N, N+1 boundary coverage. + + A regression that set ``_MAX_RM_SCORES = 30`` would otherwise pass + every K=32/K=33 test in this file. + """ + from soup_cli.utils.active_sampler import score_uncertainty + + s = score_uncertainty(scores=[0.5] * 31) + assert 0.0 <= s <= 1.0 + assert math.isfinite(s) + + +def test_row_uncertainty_explicit_field_overrides_k3_rm_scores(): + """HIGH: ``_row_uncertainty`` field-priority contract. + + Explicit ``uncertainty`` field MUST win over rm_scores even when + rm_scores would also produce a valid K>=3 variance score. A regression + that flipped the priority order could otherwise pass silently because + rm_scores=[0.5,0.5,0.5] returns 0.0 — same as a missing field. + """ + from soup_cli.utils.active_sampler import _row_uncertainty + + assert _row_uncertainty( + {"uncertainty": 0.9, "rm_scores": [0.5, 0.5, 0.5]} + ) == pytest.approx(0.9) + + +def test_old_k2_deferred_error_message_removed(): + """HIGH: source-grep regression guard for the broken K>2 ValueError text. + + The v0.63.0 code raised ``ValueError("...K>2 RMs deferred...")``. We've + lifted that. Without this guard, a future revert could re-introduce the + deferred-message text and only the K=33 boundary test would catch it + (matching only the literal "32" — not the deferred phrasing). + """ + from pathlib import Path + + src = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "active_sampler.py" + text = src.read_text(encoding="utf-8") + assert "K>2 RMs deferred" not in text + assert "deferred to a future release" not in text + + +def test_empty_scores_returns_zero(): + """MEDIUM: K=0 contract explicit (was implied, never asserted in v0.63.1).""" + from soup_cli.utils.active_sampler import score_uncertainty + + assert score_uncertainty(scores=[]) == 0.0 + + +def test_row_uncertainty_scalar_rm_score_path_preserved(): + """MEDIUM: scalar ``rm_score`` (Priority-3 path) preserved. + + The K>2 routing rewrite could plausibly have broken the scalar branch + without any new test catching it. + """ + from soup_cli.utils.active_sampler import _row_uncertainty + + assert _row_uncertainty({"rm_score": 1.0}) == pytest.approx(0.0) + assert _row_uncertainty({"rm_score": 0.0}) == pytest.approx(0.0) + assert _row_uncertainty({"rm_score": 0.5}) == pytest.approx(1.0) + + +def test_row_uncertainty_k3_with_nan_score_isolates_to_zero(): + """MEDIUM: K>=3 row with NaN entry must isolate to 0.0, not crash batch. + + Matches the K>cap and bool/oversize isolation policy already in + ``_row_uncertainty`` — bad data on one row never breaks the batch. + """ + from soup_cli.utils.active_sampler import _row_uncertainty + + assert _row_uncertainty({"rm_scores": [0.5, 0.5, float("nan")]}) == 0.0 + assert _row_uncertainty({"rm_scores": [0.5, 0.5, float("inf")]}) == 0.0 + + +def test_k2_equal_scores_strict_zero(): + """MEDIUM: K=2 with equal scores returns *strict* 0.0. + + The K=2 path uses IEEE 754 subtraction (no accumulated error) so we + can assert strict equality. Pairs with the K>=3 monotonicity-with- + constant-input test (which uses ``abs=1e-12`` tolerance) to document + where strict-zero vs sub-ULP residual applies. + """ + from soup_cli.utils.active_sampler import score_uncertainty + + assert score_uncertainty(scores=[0.4, 0.4]) == 0.0 + assert score_uncertainty(scores=[0.0, 0.0]) == 0.0 + assert score_uncertainty(scores=[1.0, 1.0]) == 0.0