mirror of https://github.com/razor-ai/soup.git
357 lines
13 KiB
Python
357 lines
13 KiB
Python
"""Tests for training intelligence — forgetting detection + checkpoint intel (Part G)."""
|
|
|
|
import pytest
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config validation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestForgettingConfig:
|
|
def test_defaults(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig()
|
|
assert cfg.forgetting_detection is False
|
|
assert cfg.forgetting_eval_steps == 100
|
|
assert cfg.forgetting_threshold == 0.10
|
|
assert cfg.forgetting_benchmark == "mini_mmlu"
|
|
assert cfg.forgetting_stop is False
|
|
|
|
def test_threshold_bounded(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(forgetting_threshold=0.8) # > 0.50 max
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(forgetting_threshold=0.001) # < 0.01 min
|
|
|
|
def test_eval_steps_bounded(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(forgetting_eval_steps=5)
|
|
|
|
def test_benchmark_literal(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(forgetting_benchmark="evil-benchmark")
|
|
|
|
|
|
class TestCheckpointIntelConfig:
|
|
def test_defaults(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig()
|
|
assert cfg.checkpoint_intelligence is False
|
|
assert cfg.checkpoint_eval_steps == 200
|
|
assert cfg.checkpoint_eval_metric == "composite"
|
|
assert cfg.checkpoint_keep_top == 3
|
|
assert cfg.early_stop_on_regression is False
|
|
assert cfg.early_stop_patience == 2
|
|
|
|
def test_keep_top_bounded(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(checkpoint_keep_top=0)
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(checkpoint_keep_top=21)
|
|
|
|
def test_patience_bounded(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(early_stop_patience=0)
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(early_stop_patience=11)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ForgettingDetector
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestForgettingDetector:
|
|
def test_built_in_benchmarks_exist(self):
|
|
from soup_cli.eval.forgetting import MINI_BENCHMARKS
|
|
|
|
assert "mini_mmlu" in MINI_BENCHMARKS
|
|
assert "mini_common_sense" in MINI_BENCHMARKS
|
|
assert "mini_instruction" in MINI_BENCHMARKS
|
|
|
|
# v0.25.0 ships each mini benchmark with 5 questions as a starter set.
|
|
# Future releases can expand to the 100-question target from the plan.
|
|
for name, bench in MINI_BENCHMARKS.items():
|
|
assert len(bench) == 5, f"{name} expected 5 items, got {len(bench)}"
|
|
for item in bench:
|
|
assert "question" in item
|
|
assert "answer" in item
|
|
assert isinstance(item["question"], str)
|
|
assert isinstance(item["answer"], str)
|
|
|
|
def test_baseline_mocked(self):
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
def fake_gen(prompt: str) -> str:
|
|
# Always returns the correct answer for mini_mmlu first item
|
|
return "A"
|
|
|
|
detector = ForgettingDetector(
|
|
generate_fn=fake_gen, benchmark="mini_mmlu",
|
|
)
|
|
baseline = detector.run_baseline()
|
|
assert 0.0 <= baseline <= 1.0
|
|
|
|
def test_check_forgetting_level_green(self):
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
detector = ForgettingDetector(
|
|
generate_fn=lambda p: "dummy", benchmark="mini_mmlu",
|
|
threshold=0.10,
|
|
)
|
|
# Manually set baseline and current accuracy
|
|
detector._baseline_accuracy = 0.9
|
|
result = detector._build_result(step=100, accuracy=0.88)
|
|
assert result.warning_level == "green"
|
|
|
|
def test_check_forgetting_level_yellow(self):
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
detector = ForgettingDetector(
|
|
generate_fn=lambda p: "dummy", threshold=0.10,
|
|
)
|
|
detector._baseline_accuracy = 0.9
|
|
result = detector._build_result(step=100, accuracy=0.75)
|
|
assert result.warning_level == "yellow"
|
|
|
|
def test_check_forgetting_level_red(self):
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
detector = ForgettingDetector(
|
|
generate_fn=lambda p: "dummy", threshold=0.10,
|
|
)
|
|
detector._baseline_accuracy = 0.9
|
|
result = detector._build_result(step=100, accuracy=0.60)
|
|
assert result.warning_level == "red"
|
|
|
|
def test_check_forgetting_integration(self):
|
|
"""Full check_forgetting() integration: baseline then eval."""
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
# Model that returns the right answer for the first 3 questions
|
|
calls = {"n": 0}
|
|
|
|
def gen(prompt: str) -> str:
|
|
calls["n"] += 1
|
|
return "A" if calls["n"] <= 3 else "Z"
|
|
|
|
detector = ForgettingDetector(
|
|
generate_fn=gen, benchmark="mini_mmlu", threshold=0.10,
|
|
)
|
|
# First call implicitly computes baseline then re-evaluates
|
|
result = detector.check_forgetting(step=100)
|
|
assert result.step == 100
|
|
assert result.warning_level in ("green", "yellow", "red")
|
|
assert detector._baseline_accuracy is not None
|
|
|
|
def test_unknown_benchmark_rejected(self):
|
|
from soup_cli.eval.forgetting import ForgettingDetector
|
|
|
|
with pytest.raises(ValueError):
|
|
ForgettingDetector(
|
|
generate_fn=lambda p: "x",
|
|
benchmark="evil_benchmark",
|
|
)
|
|
|
|
def test_forgetting_stop_schema(self):
|
|
"""forgetting_stop is a proper TrainingConfig bool with False default."""
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
default_cfg = TrainingConfig()
|
|
assert default_cfg.forgetting_stop is False
|
|
|
|
enabled = TrainingConfig(forgetting_detection=True, forgetting_stop=True)
|
|
assert enabled.forgetting_stop is True
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CheckpointTracker
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestCheckpointTracker:
|
|
def test_initial_best_none(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite")
|
|
assert tracker.best is None
|
|
|
|
def test_record_becomes_best(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite")
|
|
tracker.record(step=100, score=0.8)
|
|
assert tracker.best is not None
|
|
assert tracker.best.score == 0.8
|
|
assert tracker.best.step == 100
|
|
|
|
def test_record_better_replaces_best(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite")
|
|
tracker.record(step=100, score=0.6)
|
|
tracker.record(step=200, score=0.9)
|
|
assert tracker.best.score == 0.9
|
|
assert tracker.best.step == 200
|
|
|
|
def test_record_worse_keeps_best(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite")
|
|
tracker.record(step=100, score=0.9)
|
|
tracker.record(step=200, score=0.5)
|
|
assert tracker.best.score == 0.9
|
|
assert tracker.best.step == 100
|
|
|
|
def test_should_early_stop_no_regression(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite", patience=2)
|
|
tracker.record(step=100, score=0.7)
|
|
tracker.record(step=200, score=0.8)
|
|
tracker.record(step=300, score=0.9)
|
|
assert tracker.should_early_stop() is False
|
|
|
|
def test_should_early_stop_on_patience(self):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite", patience=2)
|
|
tracker.record(step=100, score=0.9)
|
|
tracker.record(step=200, score=0.8)
|
|
tracker.record(step=300, score=0.7)
|
|
assert tracker.should_early_stop() is True
|
|
|
|
def test_prune_keeps_top_n(self, tmp_path):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite", keep_top=2)
|
|
|
|
# Create fake checkpoint dirs
|
|
for step, score in [(100, 0.6), (200, 0.9), (300, 0.75), (400, 0.5)]:
|
|
ckpt = tmp_path / f"checkpoint-{step}"
|
|
ckpt.mkdir()
|
|
(ckpt / "dummy.txt").write_text("x", encoding="utf-8")
|
|
tracker.record(step=step, score=score)
|
|
|
|
removed = tracker.prune_checkpoints(tmp_path)
|
|
|
|
surviving = sorted(
|
|
p.name for p in tmp_path.iterdir() if p.name.startswith("checkpoint-")
|
|
)
|
|
# Top 2 by score: 200 (0.9) and 300 (0.75)
|
|
assert surviving == ["checkpoint-200", "checkpoint-300"]
|
|
# Deleted the other two and reported them
|
|
assert sorted(removed) == [100, 400]
|
|
assert not (tmp_path / "checkpoint-100").exists()
|
|
assert not (tmp_path / "checkpoint-400").exists()
|
|
|
|
def test_prune_refuses_non_checkpoint_dirs(self, tmp_path):
|
|
from soup_cli.eval.checkpoint_intelligence import CheckpointTracker
|
|
|
|
tracker = CheckpointTracker(metric="composite", keep_top=1)
|
|
tracker.record(step=100, score=0.9)
|
|
|
|
# A sibling dir that should never be touched
|
|
sibling = tmp_path / "user_data"
|
|
sibling.mkdir()
|
|
(sibling / "file.txt").write_text("keep", encoding="utf-8")
|
|
|
|
ckpt = tmp_path / "checkpoint-100"
|
|
ckpt.mkdir()
|
|
|
|
tracker.prune_checkpoints(tmp_path)
|
|
|
|
assert sibling.exists()
|
|
assert (sibling / "file.txt").exists()
|
|
|
|
def test_composite_metric_weights(self):
|
|
from soup_cli.eval.checkpoint_intelligence import compute_composite
|
|
|
|
composite = compute_composite(judge=0.8, mmlu=0.6, custom=0.9)
|
|
assert 0.6 < composite < 0.9
|
|
|
|
def test_composite_all_zero(self):
|
|
from soup_cli.eval.checkpoint_intelligence import compute_composite
|
|
|
|
assert compute_composite(judge=0.0, mmlu=0.0, custom=0.0) == 0.0
|
|
|
|
def test_composite_all_ones(self):
|
|
from soup_cli.eval.checkpoint_intelligence import compute_composite
|
|
|
|
assert compute_composite(judge=1.0, mmlu=1.0, custom=1.0) == 1.0
|
|
|
|
def test_composite_missing_metrics(self):
|
|
"""None-valued metrics drop out of the weighted average."""
|
|
from soup_cli.eval.checkpoint_intelligence import compute_composite
|
|
|
|
# Only judge supplied — composite equals judge score exactly.
|
|
assert compute_composite(judge=0.7, mmlu=None, custom=None) == 0.7
|
|
|
|
def test_composite_all_none_returns_zero(self):
|
|
from soup_cli.eval.checkpoint_intelligence import compute_composite
|
|
|
|
assert compute_composite() == 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# SQLite tracker extension
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class TestTrackerSchema:
|
|
def test_checkpoint_quality_table_created(self, tmp_path):
|
|
from soup_cli.experiment.tracker import ExperimentTracker
|
|
|
|
db_path = tmp_path / "experiments.db"
|
|
tracker = ExperimentTracker(db_path=db_path)
|
|
tracker.init_db()
|
|
|
|
import sqlite3
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
"AND name='checkpoint_quality'"
|
|
)
|
|
assert cursor.fetchone() is not None
|
|
conn.close()
|
|
|
|
def test_forgetting_eval_table_created(self, tmp_path):
|
|
from soup_cli.experiment.tracker import ExperimentTracker
|
|
|
|
db_path = tmp_path / "experiments.db"
|
|
tracker = ExperimentTracker(db_path=db_path)
|
|
tracker.init_db()
|
|
|
|
import sqlite3
|
|
|
|
conn = sqlite3.connect(db_path)
|
|
cursor = conn.execute(
|
|
"SELECT name FROM sqlite_master WHERE type='table' "
|
|
"AND name='forgetting_eval'"
|
|
)
|
|
assert cursor.fetchone() is not None
|
|
conn.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
pytest.main([__file__, "-v"])
|