mirror of https://github.com/razor-ai/soup.git
867 lines
26 KiB
Python
867 lines
26 KiB
Python
"""Tests for v0.48.0 Part B — Data Mixing Optimizer.
|
|
|
|
BETA feature. Covers:
|
|
- ``parse_budget`` digits + suffix matrix.
|
|
- ``validate_datasets`` containment, dedup, symlink, bounds.
|
|
- ``build_optimization_plan`` happy path + rejection.
|
|
- ``BudgetTracker`` start/elapsed/exceeded semantics.
|
|
- ``MixCandidate`` schema, simplex constraint, finite checks.
|
|
- ``run_mix_optimizer`` happy / partial / NaN-loss skip / proxy crash.
|
|
- ``render_mix_recipe_yaml`` shape + injection defence.
|
|
- ``write_mix_recipe`` atomic + TOCTOU symlink reject.
|
|
- ``load_mix_recipe`` round-trip.
|
|
- ``soup data mix`` CLI smoke (--help, --optimize, --apply, error paths).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
|
|
import pytest
|
|
|
|
from soup_cli.utils.data_mix import (
|
|
BudgetTracker,
|
|
MixCandidate,
|
|
MixOptimizationReport,
|
|
build_optimization_plan,
|
|
load_mix_recipe,
|
|
parse_budget,
|
|
render_mix_recipe_yaml,
|
|
run_mix_optimizer,
|
|
validate_datasets,
|
|
write_mix_recipe,
|
|
)
|
|
|
|
# ---------- parse_budget --------------------------------------------------
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"raw,expected",
|
|
[
|
|
("60", 60),
|
|
("60s", 60),
|
|
("5m", 300),
|
|
("1h", 3600),
|
|
("24h", 86400),
|
|
],
|
|
)
|
|
def test_parse_budget_happy(raw, expected):
|
|
assert parse_budget(raw) == expected
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["", "59", "0", "25h", "abc", "10x", "-5m"])
|
|
def test_parse_budget_rejects(raw):
|
|
with pytest.raises(ValueError):
|
|
parse_budget(raw)
|
|
|
|
|
|
def test_parse_budget_rejects_null_byte():
|
|
with pytest.raises(ValueError, match="null bytes"):
|
|
parse_budget("1h\x00")
|
|
|
|
|
|
def test_parse_budget_rejects_non_str():
|
|
with pytest.raises(TypeError):
|
|
parse_budget(60) # type: ignore[arg-type]
|
|
|
|
|
|
# ---------- validate_datasets --------------------------------------------
|
|
|
|
|
|
def _make_files(tmp_path, names):
|
|
for n in names:
|
|
(tmp_path / n).write_text("{}\n")
|
|
|
|
|
|
def test_validate_datasets_happy(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
out = validate_datasets(["a.jsonl", "b.jsonl"])
|
|
assert len(out) == 2
|
|
assert all(os.path.isabs(p) for p in out)
|
|
|
|
|
|
def test_validate_datasets_requires_two(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="at least 2"):
|
|
validate_datasets(["a.jsonl"])
|
|
|
|
|
|
def test_validate_datasets_rejects_empty():
|
|
with pytest.raises(ValueError, match="at least 2"):
|
|
validate_datasets([])
|
|
|
|
|
|
def test_validate_datasets_rejects_non_sequence():
|
|
with pytest.raises(TypeError, match="non-string Sequence"):
|
|
validate_datasets("a.jsonl,b.jsonl") # type: ignore[arg-type]
|
|
|
|
|
|
def test_validate_datasets_rejects_dedup(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="duplicate"):
|
|
validate_datasets(["a.jsonl", "a.jsonl"])
|
|
|
|
|
|
def test_validate_datasets_rejects_oversize_list(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
raw = [f"d{i}.jsonl" for i in range(33)]
|
|
with pytest.raises(ValueError, match="cap is 32"):
|
|
validate_datasets(raw)
|
|
|
|
|
|
def test_validate_datasets_rejects_outside_cwd(tmp_path, monkeypatch):
|
|
work = tmp_path / "work"
|
|
work.mkdir()
|
|
elsewhere = tmp_path / "outside.jsonl"
|
|
elsewhere.write_text("{}")
|
|
monkeypatch.chdir(work)
|
|
with pytest.raises(ValueError, match="outside cwd"):
|
|
validate_datasets([str(elsewhere), "a.jsonl"])
|
|
|
|
|
|
def test_validate_datasets_rejects_null_byte(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null bytes"):
|
|
validate_datasets(["a\x00.jsonl", "b.jsonl"])
|
|
|
|
|
|
def test_validate_datasets_rejects_non_string(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(TypeError, match="dataset must be str"):
|
|
validate_datasets([42, "b.jsonl"]) # type: ignore[list-item]
|
|
|
|
|
|
def test_validate_datasets_rejects_empty_string(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="non-empty"):
|
|
validate_datasets(["", "b.jsonl"])
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX symlink semantics")
|
|
def test_validate_datasets_rejects_symlink(tmp_path, monkeypatch):
|
|
real = tmp_path / "real.jsonl"
|
|
real.write_text("{}")
|
|
link = tmp_path / "link.jsonl"
|
|
link.symlink_to(real)
|
|
other = tmp_path / "other.jsonl"
|
|
other.write_text("{}")
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
validate_datasets(["link.jsonl", "other.jsonl"])
|
|
|
|
|
|
# ---------- build_optimization_plan --------------------------------------
|
|
|
|
|
|
def test_build_plan_happy(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
plan = build_optimization_plan(["a.jsonl", "b.jsonl"], budget="60s")
|
|
assert plan.num_probes == 8
|
|
assert plan.budget_seconds == 60
|
|
assert plan.seed == 42
|
|
assert len(plan.datasets) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("nb", [0, -1, 257])
|
|
def test_build_plan_rejects_invalid_num_probes(tmp_path, monkeypatch, nb):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="num_probes"):
|
|
build_optimization_plan(
|
|
["a.jsonl", "b.jsonl"], budget="60s", num_probes=nb
|
|
)
|
|
|
|
|
|
def test_build_plan_rejects_bool_num_probes(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="num_probes must be int, not bool"):
|
|
build_optimization_plan(
|
|
["a.jsonl", "b.jsonl"], budget="60s", num_probes=True # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_build_plan_rejects_invalid_seed(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="seed"):
|
|
build_optimization_plan(
|
|
["a.jsonl", "b.jsonl"], budget="60s", seed=-1
|
|
)
|
|
|
|
|
|
# ---------- BudgetTracker -------------------------------------------------
|
|
|
|
|
|
def test_budget_tracker_basic():
|
|
fake_time = [0.0]
|
|
|
|
def clock():
|
|
return fake_time[0]
|
|
|
|
tracker = BudgetTracker(60, clock=clock)
|
|
tracker.start()
|
|
assert tracker.elapsed == 0.0
|
|
fake_time[0] = 30.0
|
|
assert tracker.elapsed == 30.0
|
|
assert not tracker.exceeded()
|
|
fake_time[0] = 60.0
|
|
assert tracker.exceeded()
|
|
|
|
|
|
def test_budget_tracker_double_start_raises():
|
|
tracker = BudgetTracker(60)
|
|
tracker.start()
|
|
with pytest.raises(RuntimeError, match="start called twice"):
|
|
tracker.start()
|
|
|
|
|
|
def test_budget_tracker_remaining():
|
|
fake_time = [0.0]
|
|
tracker = BudgetTracker(60, clock=lambda: fake_time[0])
|
|
tracker.start()
|
|
fake_time[0] = 20.0
|
|
assert tracker.remaining == 40.0
|
|
fake_time[0] = 100.0
|
|
assert tracker.remaining == 0.0
|
|
|
|
|
|
def test_budget_tracker_rejects_invalid_budget():
|
|
with pytest.raises(ValueError):
|
|
BudgetTracker(0)
|
|
with pytest.raises(ValueError):
|
|
BudgetTracker(10**10)
|
|
|
|
|
|
def test_budget_tracker_rejects_bool_budget():
|
|
with pytest.raises(ValueError, match="budget_seconds must be int, not bool"):
|
|
BudgetTracker(True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_budget_tracker_elapsed_before_start_zero():
|
|
tracker = BudgetTracker(60)
|
|
assert tracker.elapsed == 0.0
|
|
|
|
|
|
# ---------- MixCandidate --------------------------------------------------
|
|
|
|
|
|
def test_mix_candidate_happy():
|
|
c = MixCandidate(
|
|
weights=(0.5, 0.5), eval_loss=1.0, wall_clock_seconds=5.0
|
|
)
|
|
assert c.eval_loss == 1.0
|
|
|
|
|
|
def test_mix_candidate_frozen():
|
|
c = MixCandidate(
|
|
weights=(0.5, 0.5), eval_loss=1.0, wall_clock_seconds=5.0
|
|
)
|
|
with pytest.raises(Exception):
|
|
c.eval_loss = 2.0 # type: ignore[misc]
|
|
|
|
|
|
def test_mix_candidate_rejects_non_simplex():
|
|
with pytest.raises(ValueError, match="sum to 1"):
|
|
MixCandidate(
|
|
weights=(0.5, 0.3), eval_loss=1.0, wall_clock_seconds=1.0
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_negative_weight():
|
|
with pytest.raises(ValueError, match="weight must be in"):
|
|
MixCandidate(
|
|
weights=(-0.1, 1.1), eval_loss=1.0, wall_clock_seconds=1.0
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_bool_weight():
|
|
with pytest.raises(ValueError, match="weight must be float, not bool"):
|
|
MixCandidate(
|
|
weights=(True, False), eval_loss=1.0, wall_clock_seconds=1.0 # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_empty_weights():
|
|
with pytest.raises(ValueError, match="weights must be non-empty"):
|
|
MixCandidate(weights=(), eval_loss=1.0, wall_clock_seconds=1.0)
|
|
|
|
|
|
def test_mix_candidate_rejects_non_tuple_weights():
|
|
with pytest.raises(TypeError, match="weights must be tuple"):
|
|
MixCandidate(
|
|
weights=[0.5, 0.5], eval_loss=1.0, wall_clock_seconds=1.0 # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_nan_loss():
|
|
with pytest.raises(ValueError, match="eval_loss must be finite"):
|
|
MixCandidate(
|
|
weights=(0.5, 0.5),
|
|
eval_loss=float("nan"),
|
|
wall_clock_seconds=1.0,
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_negative_wall_clock():
|
|
with pytest.raises(ValueError, match="wall_clock_seconds must be >= 0"):
|
|
MixCandidate(
|
|
weights=(0.5, 0.5), eval_loss=1.0, wall_clock_seconds=-1.0
|
|
)
|
|
|
|
|
|
def test_mix_candidate_rejects_oversize_loss():
|
|
with pytest.raises(ValueError, match="eval_loss exceeds"):
|
|
MixCandidate(
|
|
weights=(0.5, 0.5),
|
|
eval_loss=1e7,
|
|
wall_clock_seconds=1.0,
|
|
)
|
|
|
|
|
|
# ---------- run_mix_optimizer ---------------------------------------------
|
|
|
|
|
|
def _make_plan(tmp_path, monkeypatch, num_probes=4, budget="60s"):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
return build_optimization_plan(
|
|
["a.jsonl", "b.jsonl"], budget=budget, num_probes=num_probes
|
|
)
|
|
|
|
|
|
def test_run_mix_optimizer_happy(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=4)
|
|
calls = []
|
|
|
|
def proxy(weights):
|
|
calls.append(weights)
|
|
return float(weights[0]) # lower loss when first dataset dominant
|
|
|
|
report = run_mix_optimizer(plan, proxy)
|
|
assert isinstance(report, MixOptimizationReport)
|
|
assert len(calls) == 4
|
|
assert report.partial is False
|
|
assert len(report.best_weights) == 2
|
|
assert abs(sum(report.best_weights) - 1.0) < 1e-6
|
|
|
|
|
|
def test_run_mix_optimizer_budget_exceeded(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=20)
|
|
fake_time = [0.0]
|
|
|
|
def clock():
|
|
# Each probe consumes 30s; budget is 60s → 2 probes max.
|
|
fake_time[0] += 30.0
|
|
return fake_time[0]
|
|
|
|
def proxy(weights):
|
|
return 0.5
|
|
|
|
report = run_mix_optimizer(plan, proxy, clock=clock)
|
|
assert report.partial is True
|
|
assert len(report.candidates) <= 2
|
|
|
|
|
|
def test_run_mix_optimizer_proxy_crash_skipped(tmp_path, monkeypatch):
|
|
"""Proxy exceptions are isolated per-candidate (project pattern)."""
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
def proxy(weights):
|
|
raise RuntimeError("boom")
|
|
|
|
report = run_mix_optimizer(plan, proxy)
|
|
# Every candidate raised → no valid observations.
|
|
assert len(report.candidates) == 0
|
|
# Uniform fallback used.
|
|
assert abs(sum(report.best_weights) - 1.0) < 1e-6
|
|
|
|
|
|
def test_run_mix_optimizer_propagates_keyboard_interrupt(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
def proxy(weights):
|
|
raise KeyboardInterrupt
|
|
|
|
with pytest.raises(KeyboardInterrupt):
|
|
run_mix_optimizer(plan, proxy)
|
|
|
|
|
|
def test_run_mix_optimizer_skips_nan(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=3)
|
|
seq = iter([float("nan"), 0.5, 0.3])
|
|
|
|
def proxy(weights):
|
|
return next(seq)
|
|
|
|
report = run_mix_optimizer(plan, proxy)
|
|
# NaN observation skipped → 2 valid candidates recorded.
|
|
assert len(report.candidates) == 2
|
|
assert report.best_eval_loss == 0.3
|
|
|
|
|
|
def test_run_mix_optimizer_rejects_non_float_loss(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
def proxy(weights):
|
|
return "loss"
|
|
|
|
with pytest.raises(TypeError, match="proxy_run must return float"):
|
|
run_mix_optimizer(plan, proxy)
|
|
|
|
|
|
def test_run_mix_optimizer_rejects_bool_loss(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
def proxy(weights):
|
|
return True
|
|
|
|
with pytest.raises(TypeError, match="proxy_run must return float"):
|
|
run_mix_optimizer(plan, proxy)
|
|
|
|
|
|
def test_run_mix_optimizer_rejects_non_plan():
|
|
with pytest.raises(TypeError, match="plan must be MixOptimizationPlan"):
|
|
run_mix_optimizer({}, lambda w: 0.0) # type: ignore[arg-type]
|
|
|
|
|
|
def test_run_mix_optimizer_rejects_non_callable_proxy(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch)
|
|
with pytest.raises(TypeError, match="proxy_run must be callable"):
|
|
run_mix_optimizer(plan, "not-callable") # type: ignore[arg-type]
|
|
|
|
|
|
def test_run_mix_optimizer_custom_optimizer(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
class StubOpt:
|
|
def __init__(self):
|
|
self.told = []
|
|
|
|
def ask(self):
|
|
return (0.7, 0.3)
|
|
|
|
def tell(self, weights, loss):
|
|
self.told.append((weights, loss))
|
|
|
|
opt = StubOpt()
|
|
|
|
def proxy(w):
|
|
return 1.0
|
|
|
|
report = run_mix_optimizer(plan, proxy, optimizer=opt)
|
|
assert len(opt.told) == 2
|
|
assert report.best_weights == (0.7, 0.3)
|
|
|
|
|
|
def test_run_mix_optimizer_rejects_bad_optimizer(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch)
|
|
with pytest.raises(TypeError, match="OptimizerProtocol"):
|
|
run_mix_optimizer(plan, lambda w: 0.0, optimizer="not-optimizer") # type: ignore[arg-type]
|
|
|
|
|
|
def test_run_mix_optimizer_all_nan_uniform_fallback(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch, num_probes=2)
|
|
|
|
def proxy(w):
|
|
return float("inf")
|
|
|
|
report = run_mix_optimizer(plan, proxy)
|
|
assert len(report.best_weights) == 2
|
|
assert abs(sum(report.best_weights) - 1.0) < 1e-6
|
|
|
|
|
|
# ---------- render_mix_recipe_yaml ---------------------------------------
|
|
|
|
|
|
def _report(tmp_path):
|
|
return MixOptimizationReport(
|
|
datasets=(str(tmp_path / "a.jsonl"), str(tmp_path / "b.jsonl")),
|
|
candidates=(),
|
|
best_weights=(0.6, 0.4),
|
|
best_eval_loss=0.123,
|
|
partial=False,
|
|
elapsed_seconds=10.0,
|
|
)
|
|
|
|
|
|
def test_render_recipe_shape(tmp_path):
|
|
text = render_mix_recipe_yaml(_report(tmp_path))
|
|
assert "data:" in text
|
|
assert "interleave:" in text
|
|
assert "strategy: probs" in text
|
|
assert "0.600000" in text
|
|
assert "0.400000" in text
|
|
assert "a.jsonl" in text
|
|
|
|
|
|
def test_render_recipe_rejects_non_report():
|
|
with pytest.raises(TypeError, match="MixOptimizationReport"):
|
|
render_mix_recipe_yaml({}) # type: ignore[arg-type]
|
|
|
|
|
|
def test_render_recipe_rejects_newline_in_path():
|
|
bad = MixOptimizationReport(
|
|
datasets=("a\n.jsonl", "b.jsonl"),
|
|
candidates=(),
|
|
best_weights=(0.5, 0.5),
|
|
best_eval_loss=0.0,
|
|
partial=False,
|
|
elapsed_seconds=0.0,
|
|
)
|
|
with pytest.raises(ValueError, match="control characters"):
|
|
render_mix_recipe_yaml(bad)
|
|
|
|
|
|
def test_render_recipe_rejects_null_byte_path():
|
|
bad = MixOptimizationReport(
|
|
datasets=("a\x00.jsonl", "b.jsonl"),
|
|
candidates=(),
|
|
best_weights=(0.5, 0.5),
|
|
best_eval_loss=0.0,
|
|
partial=False,
|
|
elapsed_seconds=0.0,
|
|
)
|
|
with pytest.raises(ValueError, match="control characters"):
|
|
render_mix_recipe_yaml(bad)
|
|
|
|
|
|
def test_render_recipe_partial_note():
|
|
rep = MixOptimizationReport(
|
|
datasets=("a.jsonl", "b.jsonl"),
|
|
candidates=(),
|
|
best_weights=(0.5, 0.5),
|
|
best_eval_loss=0.5,
|
|
partial=True,
|
|
elapsed_seconds=10.0,
|
|
)
|
|
text = render_mix_recipe_yaml(rep)
|
|
assert "Budget exceeded" in text
|
|
|
|
|
|
def test_render_recipe_handles_inf_loss():
|
|
rep = MixOptimizationReport(
|
|
datasets=("a.jsonl", "b.jsonl"),
|
|
candidates=(),
|
|
best_weights=(0.5, 0.5),
|
|
best_eval_loss=float("inf"),
|
|
partial=False,
|
|
elapsed_seconds=10.0,
|
|
)
|
|
text = render_mix_recipe_yaml(rep)
|
|
assert "no valid observation" in text
|
|
|
|
|
|
# ---------- write_mix_recipe + load_mix_recipe ---------------------------
|
|
|
|
|
|
def test_write_recipe_round_trip(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
report = _report(tmp_path)
|
|
out = write_mix_recipe(report, "recipe.yaml")
|
|
assert os.path.isfile(out)
|
|
loaded = load_mix_recipe("recipe.yaml")
|
|
assert "interleave" in loaded
|
|
assert "train" in loaded
|
|
|
|
|
|
def test_write_recipe_outside_cwd(tmp_path, monkeypatch):
|
|
work = tmp_path / "work"
|
|
work.mkdir()
|
|
monkeypatch.chdir(work)
|
|
report = _report(tmp_path)
|
|
with pytest.raises(ValueError, match="outside cwd"):
|
|
write_mix_recipe(report, str(tmp_path / "out.yaml"))
|
|
|
|
|
|
def test_write_recipe_refuses_overwrite(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
target = tmp_path / "exists.yaml"
|
|
target.write_text("# pre")
|
|
with pytest.raises(ValueError, match="already exists"):
|
|
write_mix_recipe(_report(tmp_path), "exists.yaml")
|
|
|
|
|
|
def test_write_recipe_overwrite_ok(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
target = tmp_path / "exists.yaml"
|
|
target.write_text("# pre")
|
|
out = write_mix_recipe(_report(tmp_path), "exists.yaml", overwrite=True)
|
|
text = open(out).read()
|
|
assert "data:" in text
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX symlinks")
|
|
def test_write_recipe_symlink_rejected(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.yaml"
|
|
real.write_text("# real")
|
|
link = tmp_path / "link.yaml"
|
|
link.symlink_to(real)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
write_mix_recipe(_report(tmp_path), "link.yaml", overwrite=True)
|
|
|
|
|
|
def test_write_recipe_null_byte_path(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="null bytes"):
|
|
write_mix_recipe(_report(tmp_path), "out\x00.yaml")
|
|
|
|
|
|
def test_write_recipe_non_str(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(TypeError):
|
|
write_mix_recipe(_report(tmp_path), 42) # type: ignore[arg-type]
|
|
|
|
|
|
def test_load_recipe_missing(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(FileNotFoundError):
|
|
load_mix_recipe("missing.yaml")
|
|
|
|
|
|
def test_load_recipe_outside_cwd(tmp_path, monkeypatch):
|
|
work = tmp_path / "work"
|
|
work.mkdir()
|
|
outside = tmp_path / "out.yaml"
|
|
outside.write_text("data: {interleave: {strategy: probs}}")
|
|
monkeypatch.chdir(work)
|
|
with pytest.raises(ValueError, match="outside cwd"):
|
|
load_mix_recipe(str(outside))
|
|
|
|
|
|
def test_load_recipe_not_yaml_mapping(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
p = tmp_path / "bad.yaml"
|
|
p.write_text("- just a list\n")
|
|
with pytest.raises(ValueError, match="YAML mapping"):
|
|
load_mix_recipe("bad.yaml")
|
|
|
|
|
|
def test_load_recipe_missing_data_block(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
p = tmp_path / "missing.yaml"
|
|
p.write_text("other: value\n")
|
|
with pytest.raises(ValueError, match="missing required"):
|
|
load_mix_recipe("missing.yaml")
|
|
|
|
|
|
# ---------- CLI smoke ----------------------------------------------------
|
|
|
|
|
|
def test_mix_cli_help():
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["data", "mix", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "optimiz" in result.output.lower() or "optimize" in result.output.lower()
|
|
|
|
|
|
def test_mix_cli_requires_mode():
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["data", "mix"])
|
|
assert result.exit_code == 2
|
|
|
|
|
|
def test_mix_cli_mutual_exclusion(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
(tmp_path / "rec.yaml").write_text("data: {}\n")
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
["data", "mix", "--optimize", "--apply", "rec.yaml"],
|
|
)
|
|
assert result.exit_code == 2
|
|
|
|
|
|
def test_mix_cli_optimize_requires_datasets(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["data", "mix", "--optimize"])
|
|
assert result.exit_code == 2
|
|
|
|
|
|
def test_mix_cli_optimize_happy(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"data", "mix",
|
|
"--optimize",
|
|
"--datasets", "a.jsonl,b.jsonl",
|
|
"--budget", "60s",
|
|
"--num-probes", "2",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert os.path.isfile(tmp_path / "mix_recipe.yaml")
|
|
|
|
|
|
def test_mix_cli_optimize_invalid_datasets(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"data", "mix", "--optimize",
|
|
"--datasets", "missing-and-only-one.jsonl",
|
|
"--budget", "60s",
|
|
],
|
|
)
|
|
assert result.exit_code == 2
|
|
assert "validation failed" in result.output.lower()
|
|
|
|
|
|
def test_mix_cli_apply_happy(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
# First generate a recipe.
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"data", "mix",
|
|
"--optimize",
|
|
"--datasets", "a.jsonl,b.jsonl",
|
|
"--budget", "60s",
|
|
"--num-probes", "2",
|
|
"--output", "rec.yaml",
|
|
],
|
|
)
|
|
assert result.exit_code == 0
|
|
# Now apply it.
|
|
result = runner.invoke(app, ["data", "mix", "--apply", "rec.yaml"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert "interleave" in result.output
|
|
|
|
|
|
def test_mix_cli_apply_outside_cwd(tmp_path, monkeypatch):
|
|
work = tmp_path / "work"
|
|
work.mkdir()
|
|
outside = tmp_path / "rec.yaml"
|
|
outside.write_text("data: {interleave: {strategy: probs}}\n")
|
|
monkeypatch.chdir(work)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app, ["data", "mix", "--apply", str(outside)]
|
|
)
|
|
assert result.exit_code == 2
|
|
|
|
|
|
def test_optimization_plan_frozen(tmp_path, monkeypatch):
|
|
plan = _make_plan(tmp_path, monkeypatch)
|
|
with pytest.raises(Exception):
|
|
plan.num_probes = 99 # type: ignore[misc]
|
|
|
|
|
|
# ---------- Review-fix coverage -------------------------------------------
|
|
|
|
|
|
def test_optimization_report_frozen(tmp_path):
|
|
rep = MixOptimizationReport(
|
|
datasets=("a.jsonl", "b.jsonl"),
|
|
candidates=(),
|
|
best_weights=(0.5, 0.5),
|
|
best_eval_loss=0.0,
|
|
partial=False,
|
|
elapsed_seconds=0.0,
|
|
)
|
|
with pytest.raises(Exception):
|
|
rep.partial = True # type: ignore[misc]
|
|
|
|
|
|
def test_build_plan_rejects_seed_at_upper_boundary(tmp_path, monkeypatch):
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="seed must be in"):
|
|
build_optimization_plan(
|
|
["a.jsonl", "b.jsonl"], budget="60s", seed=2**31
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("raw", ["59s", "0s", "0m", "0h"])
|
|
def test_parse_budget_rejects_below_min_with_suffix(raw):
|
|
with pytest.raises(ValueError):
|
|
parse_budget(raw)
|
|
|
|
|
|
@pytest.mark.skipif(os.name == "nt", reason="POSIX symlinks")
|
|
def test_load_mix_recipe_rejects_symlink(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
real = tmp_path / "real.yaml"
|
|
real.write_text("data: {interleave: {strategy: probs}}\n")
|
|
link = tmp_path / "link.yaml"
|
|
link.symlink_to(real)
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
load_mix_recipe("link.yaml")
|
|
|
|
|
|
def test_load_mix_recipe_rejects_oversize(tmp_path, monkeypatch):
|
|
monkeypatch.chdir(tmp_path)
|
|
big = tmp_path / "big.yaml"
|
|
big.write_bytes(b"data: {x: y}\n" + b"# pad\n" * 200_000)
|
|
with pytest.raises(ValueError, match="bytes cap"):
|
|
load_mix_recipe("big.yaml")
|
|
|
|
|
|
def test_mix_cli_optimize_recipe_content(tmp_path, monkeypatch):
|
|
"""Recipe file is non-empty and contains the canonical interleave block."""
|
|
_make_files(tmp_path, ["a.jsonl", "b.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"data", "mix", "--optimize",
|
|
"--datasets", "a.jsonl,b.jsonl",
|
|
"--budget", "60s",
|
|
"--num-probes", "2",
|
|
"--output", "rec.yaml",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
text = (tmp_path / "rec.yaml").read_text()
|
|
assert "interleave:" in text
|
|
assert "strategy: probs" in text
|
|
assert "a.jsonl" in text
|
|
|
|
|
|
def test_validate_datasets_single_entry_message(tmp_path, monkeypatch):
|
|
"""Single-entry input should fail with 'at least 2' immediately."""
|
|
_make_files(tmp_path, ["a.jsonl"])
|
|
monkeypatch.chdir(tmp_path)
|
|
with pytest.raises(ValueError, match="at least 2"):
|
|
validate_datasets(["a.jsonl"])
|