mirror of https://github.com/razor-ai/soup.git
862 lines
32 KiB
Python
862 lines
32 KiB
Python
"""Tests for v0.32.0 — Training Stability & Auto-Tuning.
|
|
|
|
Covers:
|
|
- Part A: LR range finder (utils/lr_finder.py)
|
|
- Part B: Live grad-accum auto-tuning (utils/grad_accum.py)
|
|
- Part C: Auto mixed-precision picker (utils/mixed_precision.py)
|
|
- Part D: Warmup auto-schedule (utils/warmup.py)
|
|
- Part E: Loss spike auto-recovery (utils/spike_recovery.py + callback wiring)
|
|
- Part F: Convergence detector (utils/convergence.py)
|
|
- Part G: Autopilot integration (autopilot/decisions.py extensions)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import math
|
|
import re
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import ValidationError
|
|
from typer.testing import CliRunner
|
|
|
|
# Rich/Typer emits per-character ANSI escapes in CI ("--\x1b[m-find\x1b[m-lr"),
|
|
# so substring assertions on the raw `result.output` fail on Linux runners
|
|
# even though they pass on Windows where Rich auto-disables colour. Strip
|
|
# escapes before comparing — same pattern used by test_hf_integration.py
|
|
# and test_eval_platform.py.
|
|
_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
|
|
|
|
|
|
def _strip_ansi(text: str) -> str:
|
|
return _ANSI_RE.sub("", text)
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part A — LR range finder #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestLRFinderSchedule:
|
|
"""compute_lr_schedule produces a logarithmic LR sweep."""
|
|
|
|
def test_log_schedule_endpoints(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
lrs = compute_lr_schedule(start_lr=1e-7, end_lr=1e-1, num_steps=50)
|
|
|
|
assert len(lrs) == 50
|
|
assert lrs[0] == pytest.approx(1e-7, rel=1e-6)
|
|
assert lrs[-1] == pytest.approx(1e-1, rel=1e-6)
|
|
|
|
def test_log_schedule_monotonic(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
lrs = compute_lr_schedule(1e-6, 1e-1, 30)
|
|
for left, right in zip(lrs, lrs[1:]):
|
|
assert right > left
|
|
|
|
def test_log_schedule_geometric(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
lrs = compute_lr_schedule(1e-6, 1e-1, 11)
|
|
ratios = [lrs[i + 1] / lrs[i] for i in range(len(lrs) - 1)]
|
|
assert all(math.isclose(ratios[0], r, rel_tol=1e-6) for r in ratios)
|
|
|
|
def test_schedule_rejects_non_positive_start(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
with pytest.raises(ValueError, match="start_lr"):
|
|
compute_lr_schedule(0.0, 1e-1, 10)
|
|
|
|
def test_schedule_rejects_inverted_range(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
with pytest.raises(ValueError, match="end_lr"):
|
|
compute_lr_schedule(1e-2, 1e-4, 10)
|
|
|
|
def test_schedule_rejects_too_few_steps(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
with pytest.raises(ValueError, match="num_steps"):
|
|
compute_lr_schedule(1e-7, 1e-1, 1)
|
|
|
|
def test_schedule_caps_steps(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
with pytest.raises(ValueError, match="num_steps"):
|
|
compute_lr_schedule(1e-7, 1e-1, 100_000)
|
|
|
|
def test_schedule_min_steps_accepted(self):
|
|
from soup_cli.utils.lr_finder import compute_lr_schedule
|
|
|
|
lrs = compute_lr_schedule(1e-7, 1e-1, 2)
|
|
assert len(lrs) == 2
|
|
assert lrs[0] == pytest.approx(1e-7, rel=1e-6)
|
|
assert lrs[-1] == pytest.approx(1e-1, rel=1e-6)
|
|
|
|
|
|
class TestLRFinderRecommendation:
|
|
"""find_optimal_lr picks the LR with steepest negative gradient
|
|
(excluding the explosion tail)."""
|
|
|
|
def test_picks_steepest_descent(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
# Loss decreases through 1e-3 then explodes
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
|
|
losses = [3.0, 2.8, 2.4, 1.5, 4.0, 12.0]
|
|
|
|
result = find_optimal_lr(lrs, losses)
|
|
|
|
assert "recommended_lr" in result
|
|
assert "min_loss_lr" in result
|
|
assert "diverged_at" in result
|
|
# Recommended must be <= min_loss_lr (steepest descent comes earlier)
|
|
assert result["recommended_lr"] <= result["min_loss_lr"]
|
|
# Recommended LR must be in the descent region, not the explosion tail
|
|
assert result["recommended_lr"] < 1e-2
|
|
|
|
def test_returns_smoothed_curve(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1]
|
|
losses = [3.0, 2.8, 2.4, 1.5, 4.0, 12.0]
|
|
|
|
result = find_optimal_lr(lrs, losses)
|
|
assert "smoothed_losses" in result
|
|
assert len(result["smoothed_losses"]) == len(losses)
|
|
|
|
def test_detects_divergence(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3, 1e-2]
|
|
losses = [3.0, 2.8, 2.4, 1.5, 100.0]
|
|
|
|
result = find_optimal_lr(lrs, losses)
|
|
assert result["diverged_at"] is not None
|
|
assert result["diverged_at"] <= 1e-2
|
|
|
|
def test_no_divergence_when_loss_stable(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3]
|
|
losses = [3.0, 2.8, 2.6, 2.4]
|
|
|
|
result = find_optimal_lr(lrs, losses)
|
|
assert result["diverged_at"] is None
|
|
|
|
def test_mismatched_lengths_rejected(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
with pytest.raises(ValueError, match="length"):
|
|
find_optimal_lr([1e-6, 1e-5], [3.0, 2.8, 2.6])
|
|
|
|
def test_too_few_points_rejected(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
with pytest.raises(ValueError, match="at least"):
|
|
find_optimal_lr([1e-6, 1e-5], [3.0, 2.8])
|
|
|
|
def test_monotonic_increase_falls_back_to_first_lr(self):
|
|
from soup_cli.utils.lr_finder import find_optimal_lr
|
|
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3]
|
|
losses = [1.0, 2.0, 4.0, 8.0] # explodes immediately
|
|
result = find_optimal_lr(lrs, losses)
|
|
assert result["recommended_lr"] == pytest.approx(lrs[0], rel=1e-6)
|
|
|
|
|
|
class TestLRFinderCLI:
|
|
"""`soup train --find-lr` flag is registered and surfaces in help."""
|
|
|
|
def test_flag_in_help(self):
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["train", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert "--find-lr" in _strip_ansi(result.output)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part B — Live grad-accum auto-tuning #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestGradAccumMonitor:
|
|
def test_should_adjust_when_high_pressure(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0, threshold=0.92)
|
|
# 23.5/24 = 0.979 → above 0.92 threshold
|
|
assert mon.should_adjust(used_vram_gb=23.5) is True
|
|
|
|
def test_should_not_adjust_when_low_pressure(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0, threshold=0.92)
|
|
assert mon.should_adjust(used_vram_gb=10.0) is False
|
|
|
|
def test_recommend_doubles_grad_accum(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
new_batch, new_accum = mon.recommend(current_batch=8, current_accum=2)
|
|
assert new_batch == 4
|
|
assert new_accum == 4
|
|
|
|
def test_recommend_keeps_effective_batch(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
for batch, accum in [(16, 1), (8, 4), (32, 1)]:
|
|
new_batch, new_accum = mon.recommend(batch, accum)
|
|
assert new_batch * new_accum == batch * accum
|
|
|
|
def test_recommend_floor_at_one(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
new_batch, new_accum = mon.recommend(current_batch=1, current_accum=8)
|
|
assert new_batch == 1
|
|
assert new_accum == 8
|
|
|
|
def test_recommend_caps_accum_at_max(self):
|
|
from soup_cli.utils.grad_accum import MAX_ACCUM, GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
# Push accum past the cap; new should equal MAX_ACCUM.
|
|
_, new_accum = mon.recommend(current_batch=2, current_accum=MAX_ACCUM)
|
|
assert new_accum == MAX_ACCUM
|
|
|
|
def test_observe_updates_peak(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
mon.observe(10.0)
|
|
mon.observe(15.0)
|
|
mon.observe(12.0)
|
|
assert mon.peak_used_gb == pytest.approx(15.0)
|
|
|
|
def test_observe_rejects_negative(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
with pytest.raises(ValueError, match="used_vram_gb"):
|
|
mon.observe(-1.0)
|
|
|
|
def test_should_adjust_rejects_negative(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
with pytest.raises(ValueError, match="used_vram_gb"):
|
|
mon.should_adjust(-0.1)
|
|
|
|
def test_recommend_rejects_zero_inputs(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
mon = GradAccumMonitor(total_vram_gb=24.0)
|
|
with pytest.raises(ValueError, match="current_batch"):
|
|
mon.recommend(current_batch=0, current_accum=2)
|
|
with pytest.raises(ValueError, match="current_batch"):
|
|
mon.recommend(current_batch=2, current_accum=0)
|
|
|
|
def test_threshold_bounds(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
with pytest.raises(ValueError, match="threshold"):
|
|
GradAccumMonitor(total_vram_gb=24.0, threshold=1.5)
|
|
with pytest.raises(ValueError, match="threshold"):
|
|
GradAccumMonitor(total_vram_gb=24.0, threshold=0.0)
|
|
|
|
def test_total_vram_must_be_positive(self):
|
|
from soup_cli.utils.grad_accum import GradAccumMonitor
|
|
|
|
with pytest.raises(ValueError, match="total_vram_gb"):
|
|
GradAccumMonitor(total_vram_gb=-1.0)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part C — Auto mixed-precision picker #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestMixedPrecisionPicker:
|
|
def test_llama_ampere_picks_bf16(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
assert pick_mixed_precision("meta-llama/Llama-3-8B", 8.0) == "bf16"
|
|
|
|
def test_qwen2_picks_fp16(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
assert pick_mixed_precision("Qwen/Qwen2-7B", 8.0) == "fp16"
|
|
|
|
def test_pre_ampere_drops_to_fp16(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
# Turing T4 (cc=7.5) — no bf16
|
|
assert pick_mixed_precision("meta-llama/Llama-3-8B", 7.5) == "fp16"
|
|
|
|
def test_pre_pascal_returns_no(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
# cc<6.0 doesn't support fp16 reliably
|
|
assert pick_mixed_precision("meta-llama/Llama-3-8B", 5.0) == "no"
|
|
|
|
def test_unknown_model_default_bf16_on_ampere(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
assert pick_mixed_precision("some/unknown-model", 9.0) == "bf16"
|
|
|
|
def test_invalid_model_name_rejected(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
with pytest.raises(ValueError, match="model"):
|
|
pick_mixed_precision("", 8.0)
|
|
with pytest.raises(ValueError, match="model"):
|
|
pick_mixed_precision("a\x00b", 8.0)
|
|
with pytest.raises(ValueError, match="200"):
|
|
pick_mixed_precision("a" * 201, 8.0)
|
|
|
|
def test_qwen25_picks_fp16_not_qwen2_default(self):
|
|
"""Longer 'qwen2.5' substring must win over 'qwen2'."""
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
# Both entries map to fp16 today, but the test guards the iteration
|
|
# order: if someone changes qwen2.5 to bf16, this catches it.
|
|
assert pick_mixed_precision("Qwen/Qwen2.5-7B", 8.0) == "fp16"
|
|
|
|
def test_negative_cc_rejected(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
with pytest.raises(ValueError, match="compute_capability"):
|
|
pick_mixed_precision("meta-llama/Llama-3-8B", -1.0)
|
|
|
|
def test_known_quirk_mapping_includes_qwen_and_phi(self):
|
|
from soup_cli.utils.mixed_precision import KNOWN_PRECISION_QUIRKS
|
|
|
|
keys = [k.lower() for k in KNOWN_PRECISION_QUIRKS]
|
|
assert any("qwen" in k for k in keys)
|
|
assert any("phi" in k for k in keys)
|
|
|
|
def test_cc_exactly_fp16_boundary(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
# cc == 6.0 should accept fp16 (the comparison is `< FP16_MIN_CC`)
|
|
assert pick_mixed_precision("meta-llama/Llama-3-8B", 6.0) == "fp16"
|
|
|
|
def test_cc_exactly_bf16_boundary(self):
|
|
from soup_cli.utils.mixed_precision import pick_mixed_precision
|
|
|
|
# cc == 8.0 should pick bf16 (the comparison is `< BF16_MIN_CC`)
|
|
assert pick_mixed_precision("meta-llama/Llama-3-8B", 8.0) == "bf16"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part D — Warmup auto-schedule #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestWarmupAutoSchedule:
|
|
def test_basic_formula(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
# 10000 examples / batch 4 / accum 2 / 3 epochs = 3750 update steps
|
|
# 3% = 112 steps
|
|
steps = compute_warmup_steps(
|
|
num_examples=10000,
|
|
batch_size=4,
|
|
grad_accum=2,
|
|
epochs=3,
|
|
ratio=0.03,
|
|
)
|
|
assert 100 <= steps <= 130
|
|
|
|
def test_clamps_to_min(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
steps = compute_warmup_steps(
|
|
num_examples=10, batch_size=1, grad_accum=1, epochs=1, ratio=0.03,
|
|
)
|
|
assert steps >= 10 # MIN_WARMUP
|
|
|
|
def test_clamps_to_max(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
steps = compute_warmup_steps(
|
|
num_examples=10_000_000,
|
|
batch_size=1,
|
|
grad_accum=1,
|
|
epochs=1,
|
|
ratio=0.03,
|
|
)
|
|
assert steps <= 1000 # MAX_WARMUP
|
|
|
|
def test_invalid_ratio_rejected(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
for bad_ratio in [-0.01, 0.51]:
|
|
with pytest.raises(ValueError, match="ratio"):
|
|
compute_warmup_steps(
|
|
num_examples=1000, batch_size=1, grad_accum=1, epochs=1,
|
|
ratio=bad_ratio,
|
|
)
|
|
|
|
def test_ratio_zero_means_no_warmup(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
steps = compute_warmup_steps(
|
|
num_examples=1000, batch_size=1, grad_accum=1, epochs=1, ratio=0.0,
|
|
)
|
|
assert steps == 0
|
|
|
|
def test_invalid_inputs_rejected(self):
|
|
from soup_cli.utils.warmup import compute_warmup_steps
|
|
|
|
with pytest.raises(ValueError, match="num_examples"):
|
|
compute_warmup_steps(num_examples=0, batch_size=1, grad_accum=1, epochs=1)
|
|
with pytest.raises(ValueError, match="batch_size"):
|
|
compute_warmup_steps(num_examples=10, batch_size=0, grad_accum=1, epochs=1)
|
|
with pytest.raises(ValueError, match="grad_accum"):
|
|
compute_warmup_steps(num_examples=10, batch_size=1, grad_accum=0, epochs=1)
|
|
with pytest.raises(ValueError, match="epochs"):
|
|
compute_warmup_steps(num_examples=10, batch_size=1, grad_accum=1, epochs=0)
|
|
|
|
def test_ratio_at_max_accepted(self):
|
|
from soup_cli.utils.warmup import MAX_WARMUP, compute_warmup_steps
|
|
|
|
# ratio == MAX_RATIO (0.5) is the inclusive upper bound.
|
|
steps = compute_warmup_steps(
|
|
num_examples=100_000, batch_size=1, grad_accum=1, epochs=1, ratio=0.5,
|
|
)
|
|
assert steps == MAX_WARMUP
|
|
|
|
|
|
class TestWarmupConfigField:
|
|
def test_warmup_auto_default_false(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig()
|
|
assert cfg.warmup_auto is False
|
|
|
|
def test_warmup_auto_can_be_set(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig(warmup_auto=True)
|
|
assert cfg.warmup_auto is True
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part E — Loss spike auto-recovery #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestSpikeRecoveryStrategy:
|
|
def test_should_recover_within_budget(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
strat = SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5)
|
|
assert strat.should_recover(attempts=0) is True
|
|
assert strat.should_recover(attempts=2) is True
|
|
|
|
def test_should_not_recover_at_limit(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
strat = SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5)
|
|
assert strat.should_recover(attempts=3) is False
|
|
assert strat.should_recover(attempts=4) is False
|
|
|
|
def test_compute_new_lr_decays(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
strat = SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5)
|
|
assert strat.compute_new_lr(2e-4) == pytest.approx(1e-4)
|
|
assert strat.compute_new_lr(1e-3) == pytest.approx(5e-4)
|
|
|
|
def test_lr_decay_bounds(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
with pytest.raises(ValueError, match="lr_decay"):
|
|
SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.0)
|
|
with pytest.raises(ValueError, match="lr_decay"):
|
|
SpikeRecoveryStrategy(max_attempts=3, lr_decay=1.0)
|
|
|
|
def test_max_attempts_bounds(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
with pytest.raises(ValueError, match="max_attempts"):
|
|
SpikeRecoveryStrategy(max_attempts=0)
|
|
with pytest.raises(ValueError, match="max_attempts"):
|
|
SpikeRecoveryStrategy(max_attempts=100)
|
|
|
|
def test_minimum_lr_floor(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
strat = SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5, min_lr=1e-7)
|
|
# New LR is below floor → return floor
|
|
assert strat.compute_new_lr(1e-8) == pytest.approx(1e-7)
|
|
|
|
def test_compute_new_lr_rejects_non_positive(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
strat = SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5)
|
|
with pytest.raises(ValueError, match="current_lr"):
|
|
strat.compute_new_lr(0.0)
|
|
with pytest.raises(ValueError, match="current_lr"):
|
|
strat.compute_new_lr(-1e-4)
|
|
|
|
def test_min_lr_must_be_positive(self):
|
|
from soup_cli.utils.spike_recovery import SpikeRecoveryStrategy
|
|
|
|
with pytest.raises(ValueError, match="min_lr"):
|
|
SpikeRecoveryStrategy(max_attempts=3, lr_decay=0.5, min_lr=0.0)
|
|
|
|
|
|
class TestSpikeRecoveryConfig:
|
|
def test_field_default(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig()
|
|
assert cfg.loss_spike_recovery is False
|
|
assert cfg.loss_spike_recovery_max_attempts == 3
|
|
|
|
def test_max_attempts_bounds(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(loss_spike_recovery_max_attempts=0)
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(loss_spike_recovery_max_attempts=100)
|
|
|
|
def test_recovery_requires_watchdog(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
# Recovery without watchdog enabled is rejected.
|
|
with pytest.raises(ValidationError) as exc_info:
|
|
TrainingConfig(loss_spike_recovery=True, loss_watchdog=False)
|
|
messages = [err["msg"] for err in exc_info.value.errors()]
|
|
assert any(
|
|
"loss_watchdog" in msg and "loss_spike_recovery" in msg
|
|
for msg in messages
|
|
), messages
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part F — Convergence detector #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestConvergenceDetector:
|
|
def test_detects_plateau(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
# Last 50 losses essentially flat
|
|
losses = [3.0 - 0.001 * i for i in range(150)] + [2.85] * 50
|
|
assert detect_plateau(losses, window=50, rel_tol=0.005) is True
|
|
|
|
def test_does_not_detect_when_decreasing(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
losses = [3.0 - 0.005 * i for i in range(200)]
|
|
assert detect_plateau(losses, window=50, rel_tol=0.005) is False
|
|
|
|
def test_too_few_points(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
losses = [3.0, 2.9, 2.8]
|
|
assert detect_plateau(losses, window=50, rel_tol=0.005) is False
|
|
|
|
def test_window_bounds(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
with pytest.raises(ValueError, match="window"):
|
|
detect_plateau([3.0] * 100, window=0)
|
|
with pytest.raises(ValueError, match="window"):
|
|
detect_plateau([3.0] * 100, window=10001)
|
|
|
|
def test_rel_tol_bounds(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
with pytest.raises(ValueError, match="rel_tol"):
|
|
detect_plateau([3.0] * 100, window=50, rel_tol=-0.001)
|
|
with pytest.raises(ValueError, match="rel_tol"):
|
|
detect_plateau([3.0] * 100, window=50, rel_tol=2.0)
|
|
|
|
|
|
class TestRecommendAction:
|
|
def test_recommends_continue_for_decreasing(self):
|
|
from soup_cli.utils.convergence import recommend_action
|
|
|
|
losses = [3.0 - 0.005 * i for i in range(200)]
|
|
assert recommend_action(losses) == "continue"
|
|
|
|
def test_recommends_early_stop_for_long_plateau(self):
|
|
from soup_cli.utils.convergence import recommend_action
|
|
|
|
losses = [3.0 - 0.001 * i for i in range(150)] + [2.85] * 100
|
|
assert recommend_action(losses) == "early_stop"
|
|
|
|
def test_recommends_lower_lr_for_oscillation(self):
|
|
# Oscillating losses (high variance, no trend)
|
|
import random
|
|
|
|
from soup_cli.utils.convergence import recommend_action
|
|
|
|
rng = random.Random(42)
|
|
losses = [2.5 + rng.uniform(-0.4, 0.4) for _ in range(200)]
|
|
action = recommend_action(losses)
|
|
assert action in {"lower_lr", "continue"}
|
|
|
|
def test_too_few_points_returns_continue(self):
|
|
from soup_cli.utils.convergence import recommend_action
|
|
|
|
assert recommend_action([3.0, 2.9, 2.8]) == "continue"
|
|
|
|
|
|
class TestConvergenceConfig:
|
|
def test_field_defaults(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
cfg = TrainingConfig()
|
|
assert cfg.convergence_detection is False
|
|
|
|
def test_convergence_window_lower_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(convergence_window=4)
|
|
|
|
def test_convergence_window_upper_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(convergence_window=10_001)
|
|
|
|
def test_convergence_rel_tol_upper_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(convergence_rel_tol=1.1)
|
|
|
|
def test_convergence_rel_tol_zero_rejected(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(convergence_rel_tol=0.0)
|
|
|
|
|
|
class TestRecoveryFieldBounds:
|
|
def test_lr_decay_upper_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(loss_spike_recovery_lr_decay=1.0)
|
|
|
|
def test_lr_decay_lower_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(loss_spike_recovery_lr_decay=0.0)
|
|
|
|
|
|
class TestGradAccumThresholdField:
|
|
def test_pressure_threshold_upper_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(grad_accum_pressure_threshold=0.99)
|
|
|
|
def test_pressure_threshold_lower_bound(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(grad_accum_pressure_threshold=0.05)
|
|
|
|
|
|
class TestPlateauNonPositiveMean:
|
|
def test_plateau_non_positive_mean_returns_false(self):
|
|
from soup_cli.utils.convergence import detect_plateau
|
|
|
|
# All-negative losses → mean < 0 → refuse to assess.
|
|
losses = [-2.0] * 60
|
|
assert detect_plateau(losses, window=50, rel_tol=0.005) is False
|
|
|
|
def test_recommend_action_non_positive_mean_returns_continue(self):
|
|
from soup_cli.utils.convergence import recommend_action
|
|
|
|
losses = [-3.0 - 0.001 * i for i in range(200)]
|
|
assert recommend_action(losses) == "continue"
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Part G — Autopilot integration #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestAutopilotIntegration:
|
|
def test_decide_warmup_returns_int(self):
|
|
from soup_cli.autopilot.decisions import decide_warmup
|
|
|
|
steps = decide_warmup(
|
|
num_examples=10000, batch_size=4, grad_accum=2, epochs=3,
|
|
)
|
|
assert isinstance(steps, int)
|
|
assert steps > 0
|
|
|
|
def test_decide_mixed_precision_routes_to_picker(self):
|
|
from soup_cli.autopilot.decisions import decide_mixed_precision
|
|
|
|
prec = decide_mixed_precision("meta-llama/Llama-3-8B", 8.0)
|
|
assert prec in {"bf16", "fp16", "no"}
|
|
|
|
def test_decide_mixed_precision_invalid_inputs(self):
|
|
from soup_cli.autopilot.decisions import decide_mixed_precision
|
|
|
|
with pytest.raises(ValueError):
|
|
decide_mixed_precision("", 8.0)
|
|
|
|
|
|
class TestAutopilotConfigEmission:
|
|
def test_generated_config_includes_warmup_auto_when_set(
|
|
self, tmp_path, monkeypatch,
|
|
):
|
|
"""Generated config has warmup_auto=true so train.py picks it up."""
|
|
from soup_cli.autopilot.generate_config import generate_config
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
decisions = {
|
|
"task": "sft",
|
|
"format": "alpaca",
|
|
"max_length": 2048,
|
|
"quantization": "4bit",
|
|
"lora": {"r": 16, "alpha": 32, "use_dora": False},
|
|
"lr": 2e-4,
|
|
"epochs": 3,
|
|
"batch_size": 4,
|
|
"grad_accum": 2,
|
|
"perf": {
|
|
"use_flash_attn": True, "use_liger": True,
|
|
"gradient_checkpointing": False,
|
|
},
|
|
"warmup_auto": True,
|
|
"mixed_precision": "bf16",
|
|
}
|
|
out = Path("soup.yaml")
|
|
generate_config(
|
|
base="meta-llama/Llama-3-8B",
|
|
data_path="data.jsonl",
|
|
decisions=decisions,
|
|
output_path=out,
|
|
)
|
|
text = out.read_text(encoding="utf-8")
|
|
cfg = yaml.safe_load(text)
|
|
assert cfg["training"].get("warmup_auto") is True
|
|
|
|
def test_decisions_output_must_stay_under_cwd(self, tmp_path, monkeypatch):
|
|
from soup_cli.autopilot.generate_config import generate_config
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
decisions = {
|
|
"task": "sft", "format": "alpaca", "max_length": 1024,
|
|
"quantization": "4bit",
|
|
"lora": {"r": 8, "alpha": 16, "use_dora": False},
|
|
"lr": 2e-4, "epochs": 1, "batch_size": 1, "grad_accum": 1,
|
|
"perf": {
|
|
"use_flash_attn": False, "use_liger": False,
|
|
"gradient_checkpointing": False,
|
|
},
|
|
"output": "/tmp/escape",
|
|
}
|
|
with pytest.raises(ValueError, match="under cwd"):
|
|
generate_config(
|
|
base="meta-llama/Llama-3-8B",
|
|
data_path="data.jsonl",
|
|
decisions=decisions,
|
|
output_path=Path("soup.yaml"),
|
|
)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Integration: lr-finder CLI smoke (offline-safe) #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestLRFinderRunner:
|
|
"""save_lr_finder_report writes a JSON report users can plot."""
|
|
|
|
def test_save_report(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.lr_finder import save_lr_finder_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
lrs = [1e-6, 1e-5, 1e-4, 1e-3]
|
|
losses = [3.0, 2.8, 2.4, 2.6]
|
|
out = Path("lr_report.json")
|
|
save_lr_finder_report(lrs, losses, out)
|
|
|
|
assert out.exists()
|
|
data = json.loads(out.read_text(encoding="utf-8"))
|
|
assert data["lrs"] == lrs
|
|
assert data["losses"] == losses
|
|
assert "recommended_lr" in data
|
|
|
|
def test_save_report_rejects_nan(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.lr_finder import save_lr_finder_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="non-finite"):
|
|
save_lr_finder_report(
|
|
[1e-6, 1e-5, 1e-4, 1e-3],
|
|
[3.0, 2.8, float("nan"), 2.4],
|
|
Path("report.json"),
|
|
)
|
|
|
|
def test_save_report_rejects_infinity(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.lr_finder import save_lr_finder_report
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="non-finite"):
|
|
save_lr_finder_report(
|
|
[1e-6, 1e-5, 1e-4, 1e-3],
|
|
[3.0, 2.8, 2.4, float("inf")],
|
|
Path("report.json"),
|
|
)
|
|
|
|
def test_save_report_path_must_stay_under_cwd(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.lr_finder import save_lr_finder_report
|
|
|
|
# tmp_path is the inner; we chdir to a subdir so tmp_path itself
|
|
# is outside cwd and thus cannot be the target.
|
|
inner = tmp_path / "inner"
|
|
inner.mkdir()
|
|
monkeypatch.chdir(inner)
|
|
outside = (tmp_path / "lr_report.json").resolve()
|
|
with pytest.raises(ValueError, match="under cwd"):
|
|
save_lr_finder_report([1e-6, 1e-5], [3.0, 2.8], outside)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# Cross-cutting: SoupConfig top-level still serializes #
|
|
# --------------------------------------------------------------------------- #
|
|
|
|
class TestNewFieldsRoundTrip:
|
|
def test_roundtrip_yaml(self):
|
|
from soup_cli.config.loader import load_config_from_string
|
|
|
|
yaml_text = """
|
|
base: meta-llama/Llama-3-8B
|
|
task: sft
|
|
data:
|
|
train: data.jsonl
|
|
format: alpaca
|
|
training:
|
|
epochs: 3
|
|
lr: 2e-4
|
|
batch_size: 4
|
|
warmup_auto: true
|
|
loss_watchdog: true
|
|
loss_spike_recovery: true
|
|
loss_spike_recovery_max_attempts: 2
|
|
convergence_detection: true
|
|
output: ./out
|
|
"""
|
|
cfg = load_config_from_string(yaml_text)
|
|
assert cfg.training.warmup_auto is True
|
|
assert cfg.training.loss_spike_recovery is True
|
|
assert cfg.training.loss_spike_recovery_max_attempts == 2
|
|
assert cfg.training.convergence_detection is True
|