mirror of https://github.com/razor-ai/soup.git
735 lines
23 KiB
Python
735 lines
23 KiB
Python
"""Tests for v0.48.0 Part A — Curriculum-Aware Trainer (dynamic re-weighting).
|
|
|
|
BETA feature. Covers:
|
|
- ``DynamicCurriculumPolicy`` frozen + bounds.
|
|
- ``compute_bucket_weights`` math properties + degenerate inputs.
|
|
- ``validate_distributed_curriculum`` multi-rank coordination gate.
|
|
- ``render_curve`` ASCII renderer.
|
|
- ``parse_history_jsonl`` schema check.
|
|
- Schema: ``curriculum_dynamic`` requires ``curriculum=True``; mlx + non-SFT
|
|
rejection at SoupConfig level.
|
|
- ``soup runs curriculum-curve`` CLI smoke.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import DataConfig, SoupConfig, TrainingConfig
|
|
from soup_cli.utils.curriculum_dynamic import (
|
|
BucketStats,
|
|
DynamicCurriculumPolicy,
|
|
compute_bucket_weights,
|
|
parse_history_jsonl,
|
|
render_curve,
|
|
validate_distributed_curriculum,
|
|
)
|
|
|
|
# ---------- DynamicCurriculumPolicy ---------------------------------------
|
|
|
|
|
|
def test_policy_defaults_ok():
|
|
p = DynamicCurriculumPolicy(num_buckets=4)
|
|
assert p.recompute_every_n_steps == 50
|
|
assert p.floor == 0.05
|
|
assert p.temperature == 1.0
|
|
|
|
|
|
def test_policy_frozen():
|
|
p = DynamicCurriculumPolicy(num_buckets=4)
|
|
with pytest.raises(Exception):
|
|
p.num_buckets = 99 # type: ignore[misc]
|
|
|
|
|
|
@pytest.mark.parametrize("nb", [0, -1, 21, 100])
|
|
def test_policy_rejects_invalid_buckets(nb):
|
|
with pytest.raises(ValueError):
|
|
DynamicCurriculumPolicy(num_buckets=nb)
|
|
|
|
|
|
def test_policy_rejects_bool_buckets():
|
|
with pytest.raises(ValueError, match="num_buckets must be int, not bool"):
|
|
DynamicCurriculumPolicy(num_buckets=True) # type: ignore[arg-type]
|
|
|
|
|
|
@pytest.mark.parametrize("rs", [0, -5, 1_000_001])
|
|
def test_policy_rejects_invalid_recompute(rs):
|
|
with pytest.raises(ValueError):
|
|
DynamicCurriculumPolicy(num_buckets=4, recompute_every_n_steps=rs)
|
|
|
|
|
|
def test_policy_floor_must_leave_room():
|
|
# num_buckets=4 → ceiling=0.25.
|
|
DynamicCurriculumPolicy(num_buckets=4, floor=0.25)
|
|
with pytest.raises(ValueError, match="floor must be in"):
|
|
DynamicCurriculumPolicy(num_buckets=4, floor=0.5)
|
|
|
|
|
|
def test_policy_floor_rejects_non_finite():
|
|
with pytest.raises(ValueError, match="must be finite"):
|
|
DynamicCurriculumPolicy(num_buckets=4, floor=float("nan"))
|
|
|
|
|
|
def test_policy_rejects_zero_temperature():
|
|
with pytest.raises(ValueError, match="temperature must be > 0"):
|
|
DynamicCurriculumPolicy(num_buckets=4, temperature=0.0)
|
|
|
|
|
|
def test_policy_should_recompute():
|
|
p = DynamicCurriculumPolicy(num_buckets=4, recompute_every_n_steps=10)
|
|
assert p.should_recompute(0) is False
|
|
assert p.should_recompute(5) is False
|
|
assert p.should_recompute(10) is True
|
|
assert p.should_recompute(20) is True
|
|
|
|
|
|
def test_policy_should_recompute_rejects_bool():
|
|
p = DynamicCurriculumPolicy(num_buckets=4)
|
|
with pytest.raises(ValueError):
|
|
p.should_recompute(True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_policy_should_recompute_rejects_negative():
|
|
p = DynamicCurriculumPolicy(num_buckets=4)
|
|
with pytest.raises(ValueError):
|
|
p.should_recompute(-1)
|
|
|
|
|
|
# ---------- compute_bucket_weights ----------------------------------------
|
|
|
|
|
|
def test_compute_uniform_when_no_data():
|
|
p = DynamicCurriculumPolicy(num_buckets=4)
|
|
weights = compute_bucket_weights({}, p)
|
|
assert len(weights) == 4
|
|
assert all(abs(w - 0.25) < 1e-9 for w in weights)
|
|
|
|
|
|
def test_compute_uniform_when_no_samples():
|
|
p = DynamicCurriculumPolicy(num_buckets=3)
|
|
stats = {
|
|
0: {"num_samples": 0, "mean_loss": 0.0, "mean_grad_norm": 0.0},
|
|
}
|
|
weights = compute_bucket_weights(stats, p)
|
|
assert len(weights) == 3
|
|
assert abs(sum(weights) - 1.0) < 1e-6
|
|
|
|
|
|
def test_compute_high_loss_gets_more_weight():
|
|
p = DynamicCurriculumPolicy(num_buckets=2, floor=0.05, temperature=1.0)
|
|
stats = {
|
|
0: {"num_samples": 10, "mean_loss": 0.5, "mean_grad_norm": 0.1},
|
|
1: {"num_samples": 10, "mean_loss": 5.0, "mean_grad_norm": 1.0},
|
|
}
|
|
w0, w1 = compute_bucket_weights(stats, p)
|
|
assert w1 > w0
|
|
assert abs(w0 + w1 - 1.0) < 1e-6
|
|
|
|
|
|
def test_compute_floor_respected():
|
|
p = DynamicCurriculumPolicy(num_buckets=2, floor=0.2)
|
|
stats = {
|
|
0: {"num_samples": 10, "mean_loss": 0.0, "mean_grad_norm": 0.0},
|
|
1: {"num_samples": 10, "mean_loss": 100.0, "mean_grad_norm": 100.0},
|
|
}
|
|
weights = compute_bucket_weights(stats, p)
|
|
assert min(weights) >= 0.2 - 1e-9
|
|
|
|
|
|
def test_compute_sums_to_one():
|
|
p = DynamicCurriculumPolicy(num_buckets=5)
|
|
stats = {
|
|
i: {"num_samples": 5, "mean_loss": float(i), "mean_grad_norm": 0.1}
|
|
for i in range(5)
|
|
}
|
|
weights = compute_bucket_weights(stats, p)
|
|
assert abs(sum(weights) - 1.0) < 1e-6
|
|
|
|
|
|
def test_compute_rejects_non_mapping_stats():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(TypeError, match="stats must be Mapping"):
|
|
compute_bucket_weights([], p) # type: ignore[arg-type]
|
|
|
|
|
|
def test_compute_rejects_non_policy():
|
|
with pytest.raises(TypeError, match="policy must be"):
|
|
compute_bucket_weights({}, "policy") # type: ignore[arg-type]
|
|
|
|
|
|
def test_compute_rejects_negative_loss():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(ValueError, match="must be >= 0"):
|
|
compute_bucket_weights(
|
|
{0: {"num_samples": 1, "mean_loss": -1.0, "mean_grad_norm": 0.0}},
|
|
p,
|
|
)
|
|
|
|
|
|
def test_compute_rejects_oversize_num_samples():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(ValueError, match="num_samples"):
|
|
compute_bucket_weights(
|
|
{0: {"num_samples": 10_000_001, "mean_loss": 1.0,
|
|
"mean_grad_norm": 1.0}}, p,
|
|
)
|
|
|
|
|
|
def test_compute_rejects_bool_bucket_id():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(TypeError, match="bucket id must be int"):
|
|
compute_bucket_weights(
|
|
{True: {"num_samples": 1, "mean_loss": 1.0,
|
|
"mean_grad_norm": 1.0}}, p,
|
|
)
|
|
|
|
|
|
def test_compute_rejects_negative_bucket_id():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(ValueError, match="bucket id must be >= 0"):
|
|
compute_bucket_weights(
|
|
{-1: {"num_samples": 1, "mean_loss": 1.0,
|
|
"mean_grad_norm": 1.0}}, p,
|
|
)
|
|
|
|
|
|
def test_compute_rejects_non_mapping_payload():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(TypeError, match="bucket payload must be Mapping"):
|
|
compute_bucket_weights({0: "bad"}, p) # type: ignore[dict-item]
|
|
|
|
|
|
def test_bucket_stats_frozen():
|
|
bs = BucketStats(bucket_id=0, num_samples=1, mean_loss=0.5,
|
|
mean_grad_norm=0.1)
|
|
with pytest.raises(Exception):
|
|
bs.num_samples = 99 # type: ignore[misc]
|
|
|
|
|
|
# ---------- validate_distributed_curriculum -------------------------------
|
|
|
|
|
|
def test_distributed_single_rank_ok():
|
|
validate_distributed_curriculum(True, world_size=1, rank_coordinated=False)
|
|
|
|
|
|
def test_distributed_multi_rank_uncoordinated_rejected():
|
|
with pytest.raises(ValueError, match="all_reduce hook"):
|
|
validate_distributed_curriculum(
|
|
True, world_size=4, rank_coordinated=False
|
|
)
|
|
|
|
|
|
def test_distributed_multi_rank_coordinated_ok():
|
|
validate_distributed_curriculum(True, world_size=4, rank_coordinated=True)
|
|
|
|
|
|
def test_distributed_disabled_short_circuit():
|
|
validate_distributed_curriculum(
|
|
False, world_size=4, rank_coordinated=False
|
|
)
|
|
|
|
|
|
def test_distributed_rejects_bool_world_size():
|
|
with pytest.raises(ValueError, match="world_size must be int"):
|
|
validate_distributed_curriculum(
|
|
True, world_size=True, rank_coordinated=True # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_distributed_rejects_non_bool_enabled():
|
|
with pytest.raises(TypeError, match="enabled must be bool"):
|
|
validate_distributed_curriculum(
|
|
"yes", world_size=1, rank_coordinated=False # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_distributed_rejects_non_bool_coordinated():
|
|
with pytest.raises(TypeError, match="rank_coordinated must be bool"):
|
|
validate_distributed_curriculum(
|
|
True, world_size=2, rank_coordinated="yes" # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_distributed_rejects_zero_world_size():
|
|
with pytest.raises(ValueError, match="world_size must be >= 1"):
|
|
validate_distributed_curriculum(
|
|
True, world_size=0, rank_coordinated=True
|
|
)
|
|
|
|
|
|
# ---------- render_curve --------------------------------------------------
|
|
|
|
|
|
def test_render_curve_empty_history():
|
|
text = render_curve([], num_buckets=4)
|
|
assert "no curriculum history" in text
|
|
|
|
|
|
def test_render_curve_basic():
|
|
history = [
|
|
{"step": 100, "weights": [0.25, 0.25, 0.25, 0.25]},
|
|
{"step": 200, "weights": [0.1, 0.2, 0.3, 0.4]},
|
|
]
|
|
text = render_curve(history, num_buckets=4)
|
|
assert "step" in text
|
|
assert "B0" in text and "B3" in text
|
|
assert "100" in text and "200" in text
|
|
|
|
|
|
def test_render_curve_rejects_wrong_arity():
|
|
history = [{"step": 1, "weights": [0.5, 0.5]}]
|
|
with pytest.raises(ValueError, match="weights length"):
|
|
render_curve(history, num_buckets=4)
|
|
|
|
|
|
def test_render_curve_rejects_non_sequence():
|
|
with pytest.raises(TypeError, match="history must be"):
|
|
render_curve("bogus", num_buckets=4)
|
|
|
|
|
|
def test_render_curve_rejects_bool_width():
|
|
with pytest.raises(ValueError, match="width must be int"):
|
|
render_curve([], num_buckets=4, width=True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_render_curve_rejects_invalid_num_buckets():
|
|
with pytest.raises(ValueError, match="num_buckets must be in"):
|
|
render_curve([], num_buckets=999)
|
|
|
|
|
|
# ---------- parse_history_jsonl -------------------------------------------
|
|
|
|
|
|
def test_parse_history_jsonl_happy():
|
|
rows = [
|
|
{"step": 50, "weights": [0.25, 0.25, 0.5]},
|
|
{"step": 100, "weights": [0.3, 0.3, 0.4]},
|
|
]
|
|
out = parse_history_jsonl(rows)
|
|
assert len(out) == 2
|
|
assert isinstance(out[0]["weights"], tuple)
|
|
assert out[0]["step"] == 50
|
|
|
|
|
|
def test_parse_history_jsonl_rejects_non_summing():
|
|
rows = [{"step": 1, "weights": [0.1, 0.2]}]
|
|
with pytest.raises(ValueError, match="weights at step"):
|
|
parse_history_jsonl(rows)
|
|
|
|
|
|
def test_parse_history_jsonl_rejects_non_sequence_weights():
|
|
rows = [{"step": 1, "weights": "abc"}]
|
|
with pytest.raises(TypeError, match="weights must be"):
|
|
parse_history_jsonl(rows)
|
|
|
|
|
|
def test_parse_history_jsonl_rejects_non_mapping_row():
|
|
with pytest.raises(TypeError, match="history row must be Mapping"):
|
|
parse_history_jsonl([42])
|
|
|
|
|
|
def test_parse_history_jsonl_rejects_non_sequence():
|
|
with pytest.raises(TypeError, match="rows must be"):
|
|
parse_history_jsonl("bogus") # type: ignore[arg-type]
|
|
|
|
|
|
# ---------- Schema integration --------------------------------------------
|
|
|
|
|
|
def _base_cfg(**training_overrides):
|
|
base = {
|
|
"curriculum": True,
|
|
"curriculum_dynamic": True,
|
|
"curriculum_buckets": 4,
|
|
}
|
|
base.update(training_overrides)
|
|
return SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="sft",
|
|
data=DataConfig(train="data.jsonl"),
|
|
training=TrainingConfig(**base),
|
|
)
|
|
|
|
|
|
def test_schema_accepts_curriculum_dynamic_with_curriculum():
|
|
cfg = _base_cfg()
|
|
assert cfg.training.curriculum_dynamic is True
|
|
assert cfg.training.curriculum_dynamic_recompute_steps == 50
|
|
|
|
|
|
def test_schema_rejects_dynamic_without_static():
|
|
with pytest.raises(ValidationError, match="curriculum_dynamic requires"):
|
|
SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="sft",
|
|
data=DataConfig(train="data.jsonl"),
|
|
training=TrainingConfig(
|
|
curriculum=False, curriculum_dynamic=True
|
|
),
|
|
)
|
|
|
|
|
|
def test_schema_rejects_dynamic_on_mlx():
|
|
with pytest.raises(ValidationError, match="mlx backend"):
|
|
SoupConfig(
|
|
base="mlx-community/Llama-3.2-1B-Instruct-4bit",
|
|
task="sft",
|
|
backend="mlx",
|
|
data=DataConfig(train="data.jsonl"),
|
|
training=TrainingConfig(
|
|
curriculum=True,
|
|
curriculum_dynamic=True,
|
|
curriculum_buckets=4,
|
|
),
|
|
)
|
|
|
|
|
|
def test_schema_accepts_dynamic_on_dpo():
|
|
"""v0.53.5 #115: multi-trainer expansion accepts every transformer task."""
|
|
cfg = SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="dpo",
|
|
data=DataConfig(train="data.jsonl", format="dpo"),
|
|
training=TrainingConfig(
|
|
curriculum=True,
|
|
curriculum_dynamic=True,
|
|
curriculum_buckets=4,
|
|
),
|
|
)
|
|
assert cfg.training.curriculum_dynamic is True
|
|
|
|
|
|
def test_schema_pretrain_accepted():
|
|
cfg = SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="pretrain",
|
|
data=DataConfig(train="data.txt", format="plaintext"),
|
|
training=TrainingConfig(
|
|
curriculum=True, curriculum_dynamic=True, curriculum_buckets=4
|
|
),
|
|
)
|
|
assert cfg.training.curriculum_dynamic is True
|
|
|
|
|
|
def test_schema_floor_capped_by_uniform():
|
|
# curriculum_buckets=4 → ceiling=0.25.
|
|
with pytest.raises(ValidationError, match="must be <= 1/curriculum_buckets"):
|
|
SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="sft",
|
|
data=DataConfig(train="data.jsonl"),
|
|
training=TrainingConfig(
|
|
curriculum=True,
|
|
curriculum_dynamic=True,
|
|
curriculum_buckets=4,
|
|
curriculum_dynamic_floor=0.5,
|
|
),
|
|
)
|
|
|
|
|
|
def test_schema_recompute_steps_bounded():
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(curriculum_dynamic_recompute_steps=0)
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(curriculum_dynamic_recompute_steps=1_000_001)
|
|
|
|
|
|
def test_schema_temperature_bounded():
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(curriculum_dynamic_temperature=0.0)
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(curriculum_dynamic_temperature=101.0)
|
|
|
|
|
|
def test_schema_disabled_when_dynamic_off():
|
|
cfg = SoupConfig(
|
|
base="meta-llama/Llama-3.2-1B",
|
|
task="dpo",
|
|
data=DataConfig(train="data.jsonl", format="dpo"),
|
|
training=TrainingConfig(curriculum_dynamic=False),
|
|
)
|
|
assert cfg.training.curriculum_dynamic is False
|
|
|
|
|
|
# ---------- CLI smoke ------------------------------------------------------
|
|
|
|
|
|
def test_curriculum_curve_cli_help():
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["runs", "curriculum-curve", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "curriculum" in result.output.lower()
|
|
|
|
|
|
def test_curriculum_curve_run_not_found(tmp_path, monkeypatch):
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
monkeypatch.chdir(tmp_path)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app, ["runs", "curriculum-curve", "nonexistent-run-xyz"]
|
|
)
|
|
assert result.exit_code == 1
|
|
assert "not found" in result.output.lower()
|
|
|
|
|
|
def test_curriculum_curve_missing_history_file(tmp_path, monkeypatch):
|
|
"""When history file does not exist, exits 1 with friendly message."""
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
monkeypatch.chdir(tmp_path)
|
|
runner = CliRunner()
|
|
# Use --history pointing at non-existent file under cwd; the run lookup
|
|
# will fail first so this exercises the run-not-found branch.
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"runs", "curriculum-curve", "anything",
|
|
"--history", str(tmp_path / "missing.jsonl"),
|
|
],
|
|
)
|
|
# Either run-not-found (1) or invalid args; not crash.
|
|
assert result.exit_code in (1, 2)
|
|
|
|
|
|
def test_curriculum_curve_history_outside_cwd(tmp_path, monkeypatch):
|
|
"""--history outside cwd is rejected."""
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
work = tmp_path / "work"
|
|
work.mkdir()
|
|
elsewhere = tmp_path / "elsewhere.jsonl"
|
|
elsewhere.write_text(json.dumps({"step": 1, "weights": [1.0]}) + "\n")
|
|
monkeypatch.chdir(work)
|
|
|
|
# Need a "run" to exist; patch get_run to return a stub.
|
|
import soup_cli.experiment.tracker as et
|
|
|
|
class FakeTracker:
|
|
def get_run(self, run_id):
|
|
return {"run_id": run_id, "output_dir": "."}
|
|
|
|
monkeypatch.setattr(et, "ExperimentTracker", FakeTracker)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"runs", "curriculum-curve", "stub",
|
|
"--history", str(elsewhere),
|
|
],
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "outside cwd" in result.output.lower()
|
|
|
|
|
|
def test_curriculum_curve_render_jsonl(tmp_path, monkeypatch):
|
|
from typer.testing import CliRunner
|
|
|
|
import soup_cli.experiment.tracker as et
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
history = tmp_path / "history.jsonl"
|
|
history.write_text(
|
|
json.dumps({"step": 100, "weights": [0.25, 0.25, 0.25, 0.25]}) + "\n"
|
|
+ json.dumps({"step": 200, "weights": [0.1, 0.2, 0.3, 0.4]}) + "\n"
|
|
)
|
|
|
|
class FakeTracker:
|
|
def get_run(self, run_id):
|
|
return {"run_id": run_id, "output_dir": "."}
|
|
|
|
monkeypatch.setattr(et, "ExperimentTracker", FakeTracker)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"runs", "curriculum-curve", "stub",
|
|
"--history", str(history),
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
assert "B0" in result.output and "B3" in result.output
|
|
assert "100" in result.output
|
|
|
|
|
|
# ---------- Review-fix coverage -------------------------------------------
|
|
|
|
|
|
def test_parse_history_jsonl_rejects_bool_step():
|
|
rows = [{"step": True, "weights": [0.5, 0.5]}]
|
|
with pytest.raises(ValueError, match="step must be int, not bool"):
|
|
parse_history_jsonl(rows)
|
|
|
|
|
|
def test_render_curve_rejects_bool_num_buckets():
|
|
with pytest.raises(ValueError, match="num_buckets must be int, not bool"):
|
|
render_curve([], num_buckets=True) # type: ignore[arg-type]
|
|
|
|
|
|
@pytest.mark.parametrize("w", [3, 201])
|
|
def test_render_curve_rejects_out_of_range_width(w):
|
|
with pytest.raises(ValueError, match="width must be in"):
|
|
render_curve([], num_buckets=4, width=w)
|
|
|
|
|
|
def test_render_curve_rejects_non_mapping_entry():
|
|
with pytest.raises(TypeError, match="history entry must be Mapping"):
|
|
render_curve([42], num_buckets=4)
|
|
|
|
|
|
def test_render_curve_rejects_bool_step_in_entry():
|
|
history = [{"step": True, "weights": [0.5, 0.5]}]
|
|
with pytest.raises(ValueError, match="step must be int, not bool"):
|
|
render_curve(history, num_buckets=2)
|
|
|
|
|
|
def test_render_curve_caps_history_rows():
|
|
huge = [{"step": i, "weights": [0.5, 0.5]} for i in range(101)]
|
|
# Patch the cap to exercise the branch without 100k rows.
|
|
import soup_cli.utils.curriculum_dynamic as cd
|
|
|
|
original = cd._MAX_HISTORY_ROWS
|
|
try:
|
|
cd._MAX_HISTORY_ROWS = 100
|
|
with pytest.raises(ValueError, match="cap is"):
|
|
render_curve(huge, num_buckets=2)
|
|
finally:
|
|
cd._MAX_HISTORY_ROWS = original
|
|
|
|
|
|
def test_parse_history_jsonl_caps_rows():
|
|
huge = [{"step": i, "weights": [0.5, 0.5]} for i in range(101)]
|
|
import soup_cli.utils.curriculum_dynamic as cd
|
|
|
|
original = cd._MAX_HISTORY_ROWS
|
|
try:
|
|
cd._MAX_HISTORY_ROWS = 100
|
|
with pytest.raises(ValueError, match="cap is"):
|
|
parse_history_jsonl(huge)
|
|
finally:
|
|
cd._MAX_HISTORY_ROWS = original
|
|
|
|
|
|
def test_compute_rejects_negative_grad_norm():
|
|
p = DynamicCurriculumPolicy(num_buckets=2)
|
|
with pytest.raises(ValueError, match="must be >= 0"):
|
|
compute_bucket_weights(
|
|
{0: {"num_samples": 1, "mean_loss": 0.5, "mean_grad_norm": -1.0}},
|
|
p,
|
|
)
|
|
|
|
|
|
def test_compute_floor_strict_invariant():
|
|
"""All weights must be >= floor (no floor-violating renorm)."""
|
|
p = DynamicCurriculumPolicy(num_buckets=4, floor=0.1, temperature=0.01)
|
|
stats = {
|
|
0: {"num_samples": 10, "mean_loss": 0.0, "mean_grad_norm": 0.0},
|
|
1: {"num_samples": 10, "mean_loss": 100.0, "mean_grad_norm": 0.0},
|
|
2: {"num_samples": 10, "mean_loss": 100.0, "mean_grad_norm": 0.0},
|
|
3: {"num_samples": 10, "mean_loss": 100.0, "mean_grad_norm": 0.0},
|
|
}
|
|
weights = compute_bucket_weights(stats, p)
|
|
assert min(weights) >= 0.1 - 1e-12
|
|
assert abs(sum(weights) - 1.0) < 1e-9
|
|
|
|
|
|
def test_should_recompute_boundary_minus_one():
|
|
p = DynamicCurriculumPolicy(num_buckets=4, recompute_every_n_steps=50)
|
|
assert p.should_recompute(49) is False
|
|
assert p.should_recompute(50) is True
|
|
|
|
|
|
def test_curriculum_curve_render_includes_exception_info(tmp_path, monkeypatch):
|
|
"""Use the recommended `(result.output, repr(result.exception))` assert form."""
|
|
from typer.testing import CliRunner
|
|
|
|
import soup_cli.experiment.tracker as et
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
history = tmp_path / "history.jsonl"
|
|
history.write_text(
|
|
json.dumps({"step": 100, "weights": [0.5, 0.5]}) + "\n"
|
|
)
|
|
|
|
class FakeTracker:
|
|
def get_run(self, run_id):
|
|
return {"run_id": run_id, "output_dir": "."}
|
|
|
|
monkeypatch.setattr(et, "ExperimentTracker", FakeTracker)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app, ["runs", "curriculum-curve", "stub", "--history", str(history)]
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
|
|
|
|
def test_curriculum_curve_corrupt_history(tmp_path, monkeypatch):
|
|
"""Malformed JSONL (non-summing weights) exits 2 with 'history malformed'."""
|
|
from typer.testing import CliRunner
|
|
|
|
import soup_cli.experiment.tracker as et
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
history = tmp_path / "h.jsonl"
|
|
# Weights don't sum to 1.
|
|
history.write_text(json.dumps({"step": 1, "weights": [0.1, 0.2]}) + "\n")
|
|
|
|
class FakeTracker:
|
|
def get_run(self, run_id):
|
|
return {"run_id": run_id, "output_dir": "."}
|
|
|
|
monkeypatch.setattr(et, "ExperimentTracker", FakeTracker)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app, ["runs", "curriculum-curve", "stub", "--history", str(history)]
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "malformed" in result.output.lower()
|
|
|
|
|
|
def test_curriculum_curve_rejects_oversize_file(tmp_path, monkeypatch):
|
|
"""50 MB cap on curriculum history file."""
|
|
from typer.testing import CliRunner
|
|
|
|
import soup_cli.experiment.tracker as et
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
history = tmp_path / "big.jsonl"
|
|
history.write_text(json.dumps({"step": 1, "weights": [0.5, 0.5]}) + "\n")
|
|
|
|
class FakeTracker:
|
|
def get_run(self, run_id):
|
|
return {"run_id": run_id, "output_dir": "."}
|
|
|
|
# Monkey-patch getsize to simulate large file.
|
|
monkeypatch.setattr(et, "ExperimentTracker", FakeTracker)
|
|
real_getsize = os.path.getsize
|
|
|
|
def fake_getsize(p):
|
|
if str(p).endswith("big.jsonl"):
|
|
return 100 * 1024 * 1024
|
|
return real_getsize(p)
|
|
|
|
monkeypatch.setattr(os.path, "getsize", fake_getsize)
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app, ["runs", "curriculum-curve", "stub", "--history", str(history)]
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "50 mb" in result.output.lower()
|