mirror of https://github.com/razor-ai/soup.git
104 lines
3.9 KiB
Python
104 lines
3.9 KiB
Python
"""Tests for v0.34.0 Part C — soup why explainer."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
|
|
from soup_cli.utils.why import Finding, diagnose
|
|
|
|
|
|
def _row(step, loss, grad_norm=1.0):
|
|
return {"step": step, "loss": loss, "grad_norm": grad_norm}
|
|
|
|
|
|
class TestDiagnose:
|
|
def test_empty_metrics(self):
|
|
result = diagnose([])
|
|
assert len(result) == 1
|
|
assert result[0].category == "no_metrics"
|
|
|
|
def test_short_run(self):
|
|
result = diagnose([_row(i, 2.0 - i * 0.01) for i in range(5)])
|
|
assert any(finding.category == "too_few_steps" for finding in result)
|
|
|
|
def test_healthy_run_no_findings(self):
|
|
# Decreasing loss across 100 steps with healthy gradients
|
|
rows = [_row(i, 2.0 * 0.99 ** i, grad_norm=2.0) for i in range(100)]
|
|
result = diagnose(rows)
|
|
assert result == []
|
|
|
|
def test_nan_detected(self):
|
|
rows = [_row(0, 2.0), _row(10, float("nan"))]
|
|
result = diagnose(rows)
|
|
assert any(finding.category == "nan_loss" for finding in result)
|
|
assert result[0].severity == "critical"
|
|
|
|
def test_inf_detected(self):
|
|
rows = [_row(0, 2.0), _row(10, math.inf)]
|
|
result = diagnose(rows)
|
|
assert any(finding.category == "nan_loss" for finding in result)
|
|
|
|
def test_plateau_detected(self):
|
|
rows = [_row(i, 2.0 + 0.0001 * i) for i in range(50)]
|
|
result = diagnose(rows)
|
|
assert any(finding.category == "loss_flat" for finding in result)
|
|
|
|
def test_explosion_detected(self):
|
|
rows = [_row(i, 1.0 + i * 0.5) for i in range(20)]
|
|
result = diagnose(rows)
|
|
cats = {finding.category for finding in result}
|
|
assert "loss_diverged" in cats
|
|
|
|
def test_high_grad_norm_detected(self):
|
|
rows = [_row(i, 2.0 * 0.99 ** i, grad_norm=100.0) for i in range(30)]
|
|
result = diagnose(rows)
|
|
assert any(finding.category == "grad_norm_high" for finding in result)
|
|
|
|
def test_lr_too_low_warning(self):
|
|
rows = [_row(i, 2.0 * 0.99 ** i) for i in range(30)]
|
|
config = {"training": {"lr": 1e-9}}
|
|
result = diagnose(rows, config)
|
|
assert any(finding.category == "lr_too_low" for finding in result)
|
|
|
|
def test_lr_too_high_warning(self):
|
|
rows = [_row(i, 2.0 * 0.99 ** i) for i in range(30)]
|
|
config = {"training": {"lr": 0.1}}
|
|
result = diagnose(rows, config)
|
|
assert any(finding.category == "lr_too_high" for finding in result)
|
|
|
|
def test_severity_ordering(self):
|
|
# Mix critical (nan) + warning (lr_too_low). Critical first.
|
|
rows = [_row(0, 2.0), _row(1, float("nan"))]
|
|
config = {"training": {"lr": 1e-9}}
|
|
result = diagnose(rows, config)
|
|
assert result[0].severity == "critical"
|
|
# Subsequent items must be warning or info
|
|
for item in result[1:]:
|
|
assert item.severity != "critical"
|
|
|
|
def test_too_few_steps_suppressed_when_other_findings_present(self):
|
|
# NaN at step 3 of a 4-step run: NaN should fire but too_few_steps must NOT.
|
|
rows = [_row(0, 2.0), _row(1, 1.5), _row(2, 1.0), _row(3, float("nan"))]
|
|
result = diagnose(rows)
|
|
cats = {finding.category for finding in result}
|
|
assert "nan_loss" in cats
|
|
assert "too_few_steps" not in cats
|
|
|
|
def test_plateau_first_zero_skipped(self):
|
|
# If initial loss is 0, plateau detector must short-circuit, not divide by zero.
|
|
rows = [_row(i, 0.0) for i in range(50)]
|
|
# Should not raise
|
|
result = diagnose(rows)
|
|
cats = {finding.category for finding in result}
|
|
assert "loss_flat" not in cats
|
|
|
|
def test_finding_is_frozen(self):
|
|
finding = Finding(
|
|
category="x", severity="info", message="m", suggestion="s",
|
|
)
|
|
try:
|
|
finding.category = "y" # type: ignore[misc]
|
|
except Exception:
|
|
return
|
|
raise AssertionError("Finding should be frozen")
|