mirror of https://github.com/razor-ai/soup.git
334 lines
11 KiB
Python
334 lines
11 KiB
Python
"""v0.63.0 Part C — Active-learning sampler tests."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import json
|
|
import math
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def test_module_imports():
|
|
from soup_cli.utils import active_sampler
|
|
|
|
assert hasattr(active_sampler, "ActiveLearningPlan")
|
|
assert hasattr(active_sampler, "score_uncertainty")
|
|
assert hasattr(active_sampler, "pick_top_uncertain")
|
|
assert hasattr(active_sampler, "validate_budget")
|
|
assert hasattr(active_sampler, "sample_uncertain_rows")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_budget
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize("value", [1, 10, 100, 10_000])
|
|
def test_validate_budget_happy(value):
|
|
from soup_cli.utils.active_sampler import validate_budget
|
|
|
|
assert validate_budget(value) == value
|
|
|
|
|
|
@pytest.mark.parametrize("bad", [True, False, None, "10", -5, 0, 100_001, 1.5])
|
|
def test_validate_budget_rejects(bad):
|
|
from soup_cli.utils.active_sampler import validate_budget
|
|
|
|
with pytest.raises((TypeError, ValueError)):
|
|
validate_budget(bad)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# score_uncertainty
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_score_uncertainty_max_entropy():
|
|
"""Single-RM: uncertainty = 1 - |2*score - 1|. Score 0.5 -> max entropy."""
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
# Single RM with score 0.5 should yield maximum uncertainty
|
|
s = score_uncertainty(scores=[0.5])
|
|
assert math.isclose(s, 1.0, abs_tol=1e-6)
|
|
|
|
s = score_uncertainty(scores=[0.0])
|
|
assert math.isclose(s, 0.0, abs_tol=1e-6)
|
|
|
|
s = score_uncertainty(scores=[1.0])
|
|
assert math.isclose(s, 0.0, abs_tol=1e-6)
|
|
|
|
|
|
def test_score_uncertainty_two_rms_disagreement():
|
|
"""Two RMs: uncertainty = |s1 - s2|. Big gap -> max disagreement."""
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
s = score_uncertainty(scores=[0.1, 0.9])
|
|
assert math.isclose(s, 0.8, abs_tol=1e-6)
|
|
|
|
s = score_uncertainty(scores=[0.5, 0.5])
|
|
assert math.isclose(s, 0.0, abs_tol=1e-6)
|
|
|
|
|
|
def test_score_uncertainty_empty():
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
assert score_uncertainty(scores=[]) == 0.0
|
|
|
|
|
|
def test_score_uncertainty_rejects_non_finite():
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
with pytest.raises(ValueError):
|
|
score_uncertainty(scores=[float("nan")])
|
|
with pytest.raises(ValueError):
|
|
score_uncertainty(scores=[float("inf")])
|
|
|
|
|
|
def test_score_uncertainty_rejects_bool():
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
with pytest.raises(TypeError):
|
|
score_uncertainty(scores=[True, False])
|
|
|
|
|
|
def test_score_uncertainty_rejects_out_of_range():
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
with pytest.raises(ValueError):
|
|
score_uncertainty(scores=[1.5])
|
|
with pytest.raises(ValueError):
|
|
score_uncertainty(scores=[-0.1])
|
|
|
|
|
|
def test_score_uncertainty_rejects_above_cap():
|
|
from soup_cli.utils.active_sampler import score_uncertainty
|
|
|
|
# K=3..32 is the variance path (see #206 / test_v0631_206.py).
|
|
# K>32 is the DoS cap.
|
|
with pytest.raises(ValueError):
|
|
score_uncertainty(scores=[0.5] * 33)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# pick_top_uncertain
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_pick_top_uncertain_orders_descending():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
rows = [
|
|
{"id": "a", "uncertainty": 0.2},
|
|
{"id": "b", "uncertainty": 0.9},
|
|
{"id": "c", "uncertainty": 0.5},
|
|
]
|
|
top = pick_top_uncertain(rows, budget=2)
|
|
assert [r["id"] for r in top] == ["b", "c"]
|
|
|
|
|
|
def test_pick_top_uncertain_budget_caps_output():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
rows = [{"id": str(i), "uncertainty": i / 100} for i in range(50)]
|
|
top = pick_top_uncertain(rows, budget=5)
|
|
assert len(top) == 5
|
|
|
|
|
|
def test_pick_top_uncertain_handles_missing_uncertainty():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
rows = [{"id": "a"}, {"id": "b", "uncertainty": 0.7}]
|
|
top = pick_top_uncertain(rows, budget=2)
|
|
assert top[0]["id"] == "b" # 'a' treated as 0 uncertainty
|
|
|
|
|
|
def test_pick_top_uncertain_empty():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
assert pick_top_uncertain([], budget=5) == []
|
|
|
|
|
|
def test_pick_top_uncertain_invalid_budget():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
rows = [{"id": "a", "uncertainty": 0.5}]
|
|
with pytest.raises((TypeError, ValueError)):
|
|
pick_top_uncertain(rows, budget=True)
|
|
with pytest.raises((TypeError, ValueError)):
|
|
pick_top_uncertain(rows, budget=0)
|
|
|
|
|
|
def test_pick_top_uncertain_rejects_non_mapping():
|
|
from soup_cli.utils.active_sampler import pick_top_uncertain
|
|
|
|
with pytest.raises(TypeError):
|
|
pick_top_uncertain([1, 2, 3], budget=2)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ActiveLearningPlan
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_active_learning_plan_frozen():
|
|
from soup_cli.utils.active_sampler import ActiveLearningPlan
|
|
|
|
plan = ActiveLearningPlan(
|
|
rows_in=100,
|
|
rows_selected=10,
|
|
budget=10,
|
|
mean_uncertainty=0.85,
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
plan.rows_in = 99 # type: ignore[misc]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# sample_uncertain_rows
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_sample_uncertain_rows_happy(tmp_path, monkeypatch):
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
inp = tmp_path / "in.jsonl"
|
|
out = tmp_path / "out.jsonl"
|
|
rows = [
|
|
{"id": str(i), "prompt": f"q{i}", "output": f"a{i}", "rm_score": s}
|
|
for i, s in enumerate([0.1, 0.5, 0.9, 0.6, 0.05])
|
|
]
|
|
inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
|
|
|
plan = sample_uncertain_rows(str(inp), output_path=str(out), budget=2)
|
|
assert plan.rows_in == 5
|
|
assert plan.rows_selected == 2
|
|
out_rows = [json.loads(ln) for ln in out.read_text(encoding="utf-8").splitlines()]
|
|
assert len(out_rows) == 2
|
|
# Top uncertainty rows should be the 0.5 and 0.6 ones (closest to 0.5)
|
|
ids = [r["id"] for r in out_rows]
|
|
assert "1" in ids # rm=0.5 -> uncertainty=1.0
|
|
assert "3" in ids # rm=0.6 -> uncertainty=0.8
|
|
|
|
|
|
def test_sample_uncertain_rows_dual_rm(tmp_path, monkeypatch):
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
inp = tmp_path / "in.jsonl"
|
|
out = tmp_path / "out.jsonl"
|
|
rows = [
|
|
{"id": "a", "rm_scores": [0.1, 0.9]}, # disagreement 0.8
|
|
{"id": "b", "rm_scores": [0.5, 0.5]}, # disagreement 0.0
|
|
{"id": "c", "rm_scores": [0.3, 0.7]}, # disagreement 0.4
|
|
]
|
|
inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
|
|
|
plan = sample_uncertain_rows(str(inp), output_path=str(out), budget=2)
|
|
assert plan.rows_selected == 2
|
|
out_rows = [json.loads(ln) for ln in out.read_text(encoding="utf-8").splitlines()]
|
|
ids = [r["id"] for r in out_rows]
|
|
assert ids == ["a", "c"]
|
|
|
|
|
|
def test_sample_uncertain_rows_rejects_outside_cwd(tmp_path, monkeypatch):
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
outside = tmp_path.parent / "stray.jsonl"
|
|
outside.write_text('{"id":"x","rm_score":0.5}\n', encoding="utf-8")
|
|
out = tmp_path / "out.jsonl"
|
|
try:
|
|
with pytest.raises(ValueError, match="outside"):
|
|
sample_uncertain_rows(str(outside), output_path=str(out), budget=1)
|
|
finally:
|
|
if outside.exists():
|
|
outside.unlink()
|
|
|
|
|
|
def test_sample_uncertain_rows_missing_input(tmp_path, monkeypatch):
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(FileNotFoundError):
|
|
sample_uncertain_rows(
|
|
str(tmp_path / "missing.jsonl"),
|
|
output_path=str(tmp_path / "o.jsonl"),
|
|
budget=1,
|
|
)
|
|
|
|
|
|
def test_sample_uncertain_rows_budget_bigger_than_input(tmp_path, monkeypatch):
|
|
"""Selecting more than input has — output capped to input size."""
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
inp = tmp_path / "in.jsonl"
|
|
out = tmp_path / "out.jsonl"
|
|
rows = [
|
|
{"id": "1", "rm_score": 0.5},
|
|
{"id": "2", "rm_score": 0.7},
|
|
]
|
|
inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
|
|
|
plan = sample_uncertain_rows(str(inp), output_path=str(out), budget=100)
|
|
assert plan.rows_in == 2
|
|
assert plan.rows_selected == 2
|
|
|
|
|
|
def test_sample_uncertain_rows_rejects_null_byte():
|
|
from soup_cli.utils.active_sampler import sample_uncertain_rows
|
|
|
|
with pytest.raises(ValueError):
|
|
sample_uncertain_rows("bad\x00path.jsonl", output_path="o.jsonl", budget=1)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI smoke
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_cli_active_sample_help():
|
|
from soup_cli.cli import app
|
|
|
|
result = runner.invoke(app, ["data", "active-sample", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert "budget" in result.output.lower()
|
|
|
|
|
|
def test_cli_active_sample_happy(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
inp = tmp_path / "in.jsonl"
|
|
out = tmp_path / "out.jsonl"
|
|
rows = [{"id": str(i), "rm_score": s} for i, s in enumerate([0.5, 0.95, 0.1])]
|
|
inp.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
|
|
|
result = runner.invoke(
|
|
app,
|
|
["data", "active-sample", "--input", str(inp), "--output", str(out), "--budget", "1"],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert out.exists()
|
|
out_rows = [json.loads(ln) for ln in out.read_text(encoding="utf-8").splitlines()]
|
|
assert len(out_rows) == 1
|
|
assert out_rows[0]["id"] == "0" # rm=0.5 has highest uncertainty
|
|
|
|
|
|
def test_cli_active_sample_invalid_budget(tmp_path, monkeypatch):
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
inp = tmp_path / "in.jsonl"
|
|
inp.write_text('{"id":"x","rm_score":0.5}\n', encoding="utf-8")
|
|
result = runner.invoke(
|
|
app,
|
|
["data", "active-sample", "--input", str(inp), "--budget", "0"],
|
|
)
|
|
assert result.exit_code != 0
|