"""Tests for v0.58.0 — soup loop CLI-first data flywheel. Coverage: - Part A: LoopState validation, atomic state file I/O, init/read/write - Part B: canary_router hash-bucket determinism, sticky rollback, BucketStats - Part C: BudgetTracker math, daily counter reset, parse_budget_string - Part D: IterationRecord, write/read/list iterations - Watch daemon: run_once + watch end-to-end with stub callbacks - CLI: init / status / pause / resume / watch --max-iterations / canary / replay """ from __future__ import annotations import dataclasses import json import os import re from datetime import datetime, timezone from pathlib import Path import pytest from typer.testing import CliRunner from soup_cli.cli import app from soup_cli.utils.canary_router import ( BucketStats, CanaryPolicy, rollback, route, ) from soup_cli.utils.loop_budget import ( check_budget, parse_budget_string, reset_daily_counter_if_new_day, ) from soup_cli.utils.loop_daemon import ( WatchConfig, evaluate_canary_verdict, maybe_rollback, run_once, watch, ) from soup_cli.utils.loop_iteration import ( IterationRecord, list_iterations, new_iteration_id, read_iteration, write_iteration, ) from soup_cli.utils.loop_state import ( LOOP_STATUSES, LoopState, default_state_path, init_state, read_state, write_state, ) runner = CliRunner() # --------------------------------------------------------------------------- # Part A — LoopState validation # --------------------------------------------------------------------------- class TestLoopState: def test_default_status_is_stopped(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") assert s.status == "stopped" def test_status_allowlist(self): assert LOOP_STATUSES == frozenset({"running", "paused", "stopped"}) def test_invalid_status_rejected(self): with pytest.raises(ValueError, match="status must be"): LoopState(served_model="m", eval_suite="e", baseline="b", status="weird") @pytest.mark.parametrize("field_name", ["served_model", "eval_suite", "baseline"]) def test_empty_required_field_rejected(self, field_name): kwargs = {"served_model": "m", "eval_suite": "e", "baseline": "b"} kwargs[field_name] = "" with pytest.raises(ValueError, match="must not be empty"): LoopState(**kwargs) def test_null_byte_in_field_rejected(self): with pytest.raises(ValueError, match="NUL"): LoopState(served_model="m\x00", eval_suite="e", baseline="b") def test_oversize_string_rejected(self): with pytest.raises(ValueError, match="exceeds"): LoopState(served_model="m" * 1000, eval_suite="e", baseline="b") def test_non_string_rejected(self): with pytest.raises(TypeError): LoopState(served_model=123, eval_suite="e", baseline="b") # type: ignore @pytest.mark.parametrize( "counter", [ "traces_collected", "pairs_distilled", "runs_gated", "adapters_shipped", "iteration_count", "runs_today", ], ) def test_counter_rejects_negative(self, counter): kwargs = {"served_model": "m", "eval_suite": "e", "baseline": "b", counter: -1} with pytest.raises(ValueError): LoopState(**kwargs) @pytest.mark.parametrize( "counter", [ "traces_collected", "pairs_distilled", "runs_gated", "adapters_shipped", ], ) def test_counter_rejects_bool(self, counter): kwargs = {"served_model": "m", "eval_suite": "e", "baseline": "b", counter: True} with pytest.raises(ValueError): LoopState(**kwargs) def test_canary_traffic_pct_bounds(self): with pytest.raises(ValueError, match=r"\[0, 100\]"): LoopState( served_model="m", eval_suite="e", baseline="b", canary_traffic_pct=150, ) def test_canary_traffic_pct_bool_rejected(self): with pytest.raises(ValueError, match="numeric"): LoopState( served_model="m", eval_suite="e", baseline="b", canary_traffic_pct=True, ) def test_monthly_budget_negative_rejected(self): with pytest.raises(ValueError): LoopState( served_model="m", eval_suite="e", baseline="b", monthly_budget_usd=-1, ) def test_max_runs_per_day_zero_rejected(self): with pytest.raises(ValueError): LoopState( served_model="m", eval_suite="e", baseline="b", max_runs_per_day=0, ) def test_frozen(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(dataclasses.FrozenInstanceError): s.status = "running" # type: ignore def test_to_dict_is_mapping_proxy(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") d = s.to_dict() with pytest.raises(TypeError): d["status"] = "running" # type: ignore def test_with_status_returns_new_instance(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") s2 = s.with_status("running") assert s.status == "stopped" assert s2.status == "running" assert s2.updated_at != "" def test_with_status_rejects_unknown(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError): s.with_status("nope") def test_bumped_increments_counter(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") s2 = s.bumped(traces_collected=3, pairs_distilled=2) assert s2.traces_collected == 3 assert s2.pairs_distilled == 2 def test_bumped_rejects_unknown_counter(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="unknown counter"): s.bumped(nonexistent=1) def test_bumped_rejects_negative_delta(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError): s.bumped(traces_collected=-1) def test_bumped_rejects_bool_delta(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError): s.bumped(traces_collected=True) # --------------------------------------------------------------------------- # Part A — state file I/O # --------------------------------------------------------------------------- class TestStateIO: def test_default_state_path(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) assert default_state_path().endswith(os.path.join(".soup", "loop.yaml")) def test_init_state_creates_file(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) state, path = init_state("m", "e", "b") assert os.path.exists(path) assert state.served_model == "m" assert state.status == "stopped" def test_init_state_refuses_overwrite(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) init_state("m", "e", "b") with pytest.raises(FileExistsError): init_state("m2", "e2", "b2") def test_init_state_force_overwrites(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) init_state("m", "e", "b") state, _ = init_state("m2", "e2", "b2", force=True) assert state.served_model == "m2" def test_write_read_roundtrip(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) s = LoopState( served_model="model-a", eval_suite="suite.yaml", baseline="registry://abc", status="running", traces_collected=42, ) write_state(s) reloaded = read_state() assert reloaded.served_model == "model-a" assert reloaded.status == "running" assert reloaded.traces_collected == 42 def test_write_state_non_loopstate_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) with pytest.raises(TypeError): write_state({"foo": "bar"}) # type: ignore def test_write_state_outside_cwd_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) s = LoopState(served_model="m", eval_suite="e", baseline="b") outside = str(tmp_path.parent / "escape.yaml") with pytest.raises(ValueError, match="cwd"): write_state(s, outside) def test_read_state_missing_raises(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) with pytest.raises(FileNotFoundError): read_state() def test_read_state_invalid_json_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() target.write_text("not json") with pytest.raises(ValueError, match="JSON"): read_state() def test_read_state_non_dict_root_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() target.write_text("[]") with pytest.raises(ValueError, match="object"): read_state() def test_read_state_oversize_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() target.write_text("x" * (2 * 1024 * 1024)) with pytest.raises(ValueError, match="1 MiB"): read_state() def test_read_state_drops_unknown_fields(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() target.write_text( json.dumps( { "served_model": "m", "eval_suite": "e", "baseline": "b", "future_field_v59": "ignored", } ) ) s = read_state() assert s.served_model == "m" @pytest.mark.skipif(os.name == "nt", reason="POSIX symlink test") def test_write_state_rejects_symlink(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) sd = tmp_path / ".soup" sd.mkdir() target = sd / "loop.yaml" target.symlink_to(tmp_path / "elsewhere") s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="symlink"): write_state(s, str(target)) def test_null_byte_path_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="NUL"): write_state(s, "foo\x00bar") # --------------------------------------------------------------------------- # Part B — canary router # --------------------------------------------------------------------------- class TestCanaryPolicy: def test_stable_only_default(self): p = CanaryPolicy(stable="adapter-a") assert p.canary is None assert p.traffic_pct == 0.0 def test_empty_stable_rejected(self): with pytest.raises(ValueError): CanaryPolicy(stable="") def test_null_byte_rejected(self): with pytest.raises(ValueError): CanaryPolicy(stable="a\x00b") def test_canary_same_as_stable_rejected(self): with pytest.raises(ValueError, match="differ"): CanaryPolicy(stable="a", canary="a") def test_traffic_pct_out_of_range(self): with pytest.raises(ValueError): CanaryPolicy(stable="a", canary="b", traffic_pct=150) def test_traffic_pct_bool_rejected(self): with pytest.raises(ValueError): CanaryPolicy(stable="a", canary="b", traffic_pct=True) def test_traffic_pct_nan_rejected(self): with pytest.raises(ValueError): CanaryPolicy(stable="a", canary="b", traffic_pct=float("nan")) def test_traffic_without_canary_rejected(self): with pytest.raises(ValueError, match="cannot route"): CanaryPolicy(stable="a", traffic_pct=5) def test_sticky_bool_required(self): with pytest.raises(ValueError): CanaryPolicy(stable="a", sticky_on_rollback=1) # type: ignore def test_frozen(self): p = CanaryPolicy(stable="a") with pytest.raises(dataclasses.FrozenInstanceError): p.stable = "x" # type: ignore class TestRouting: def test_stable_only_policy_routes_to_stable(self): p = CanaryPolicy(stable="A") for key in ("k1", "k2", "k3"): d = route(p, key) assert d.adapter == "A" assert d.bucket == "stable" def test_zero_pct_canary_still_routes_stable(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=0.0) d = route(p, "anything") assert d.bucket == "stable" def test_full_100_pct_routes_all_canary(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=100.0) for key in ("k1", "k2", "k3"): d = route(p, key) assert d.adapter == "B" assert d.bucket == "canary" def test_deterministic_routing(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=25.0) assert route(p, "abc").adapter == route(p, "abc").adapter assert route(p, "xyz").adapter == route(p, "xyz").adapter def test_split_approximates_traffic_pct(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=25.0) canary_count = sum( 1 for i in range(2000) if route(p, f"key-{i}").bucket == "canary" ) # 25% of 2000 = 500; tolerate ±15% relative drift on a uniform hash. assert 350 <= canary_count <= 650 def test_empty_key_rejected(self): p = CanaryPolicy(stable="A") with pytest.raises(ValueError): route(p, "") def test_non_policy_rejected(self): with pytest.raises(TypeError): route("not-policy", "k") # type: ignore class TestRollback: def test_rollback_clears_canary(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=10.0) cleared = rollback(p) assert cleared.canary is None assert cleared.traffic_pct == 0.0 assert cleared.stable == "A" def test_rollback_after_clears_route_to_stable(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=100.0) cleared = rollback(p) assert route(cleared, "anykey").adapter == "A" def test_rollback_reason_required(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=5.0) with pytest.raises(ValueError): rollback(p, reason="") def test_rollback_non_policy_rejected(self): with pytest.raises(TypeError): rollback("notpolicy") # type: ignore class TestBucketStats: def test_record_and_verdict_ok(self): s = BucketStats() for _ in range(50): s.record("stable", True) s.record("canary", True) assert s.verdict() == "OK" def test_verdict_unknown_below_min_samples(self): s = BucketStats() for _ in range(5): s.record("canary", True) assert s.verdict(min_samples=30) == "UNKNOWN" def test_verdict_major_on_regression(self): s = BucketStats() for _ in range(50): s.record("stable", True) # canary fails most of the time for _ in range(50): s.record("canary", False) assert s.verdict() == "MAJOR" def test_record_invalid_bucket(self): s = BucketStats() with pytest.raises(ValueError): s.record("middle", True) def test_record_non_bool_ok(self): s = BucketStats() with pytest.raises(ValueError): s.record("stable", 1) # type: ignore def test_verdict_invalid_min_samples(self): s = BucketStats() with pytest.raises(ValueError): s.verdict(min_samples=0) def test_verdict_invalid_threshold(self): s = BucketStats() for _ in range(40): s.record("canary", True) with pytest.raises(ValueError): s.verdict(regression_threshold=1.5) def test_snapshot_mapping_proxy(self): s = BucketStats() s.record("stable", True) snap = s.snapshot() with pytest.raises(TypeError): snap["stable_ok"] = 999 # type: ignore # --------------------------------------------------------------------------- # Part C — budget guardrails # --------------------------------------------------------------------------- class TestParseBudget: @pytest.mark.parametrize( "raw,expected", [("50", 50.0), ("50usd", 50.0), ("100 USD", 100.0), ("0", 0.0), ("0.5", 0.5)], ) def test_happy(self, raw, expected): assert parse_budget_string(raw) == expected def test_empty_rejected(self): with pytest.raises(ValueError): parse_budget_string("") def test_garbage_rejected(self): with pytest.raises(ValueError): parse_budget_string("abc") def test_null_byte_rejected(self): with pytest.raises(ValueError): parse_budget_string("50\x00usd") def test_negative_rejected(self): with pytest.raises(ValueError): parse_budget_string("-5") def test_overflow_rejected(self): with pytest.raises(ValueError): parse_budget_string("10000000") def test_non_string_rejected(self): with pytest.raises(TypeError): parse_budget_string(50) # type: ignore class TestCheckBudget: def test_happy_within_budget(self): d = check_budget( estimated_run_usd=1.0, spent_so_far_usd=5.0, monthly_budget_usd=50.0, runs_today=0, max_runs_per_day=10, ) assert d.proceed is True assert d.projected_total_usd == 6.0 def test_blocked_by_budget(self): d = check_budget( estimated_run_usd=10.0, spent_so_far_usd=45.0, monthly_budget_usd=50.0, runs_today=0, max_runs_per_day=10, ) assert d.proceed is False assert "budget" in d.reason def test_blocked_by_daily_cap(self): d = check_budget( estimated_run_usd=1.0, spent_so_far_usd=0.0, monthly_budget_usd=50.0, runs_today=3, max_runs_per_day=3, ) assert d.proceed is False assert "daily cap" in d.reason def test_none_budget_means_unlimited(self): d = check_budget( estimated_run_usd=1e6, spent_so_far_usd=0, monthly_budget_usd=None, runs_today=0, max_runs_per_day=None, ) assert d.proceed is True def test_negative_estimate_rejected(self): with pytest.raises(ValueError): check_budget( estimated_run_usd=-1, spent_so_far_usd=0, monthly_budget_usd=None, runs_today=0, max_runs_per_day=None, ) def test_nan_estimate_rejected(self): with pytest.raises(ValueError): check_budget( estimated_run_usd=float("nan"), spent_so_far_usd=0, monthly_budget_usd=None, runs_today=0, max_runs_per_day=None, ) def test_bool_estimate_rejected(self): with pytest.raises(ValueError): check_budget( estimated_run_usd=True, spent_so_far_usd=0, monthly_budget_usd=None, runs_today=0, max_runs_per_day=None, ) def test_negative_runs_today_rejected(self): with pytest.raises(ValueError): check_budget( estimated_run_usd=0, spent_so_far_usd=0, monthly_budget_usd=None, runs_today=-1, max_runs_per_day=None, ) def test_zero_max_runs_per_day_rejected(self): with pytest.raises(ValueError): check_budget( estimated_run_usd=0, spent_so_far_usd=0, monthly_budget_usd=None, runs_today=0, max_runs_per_day=0, ) class TestDailyCounter: def test_same_day_keeps_count(self): today = datetime(2026, 5, 15, tzinfo=timezone.utc) out, date = reset_daily_counter_if_new_day(3, "2026-05-15", now=today) assert out == 3 assert date == "2026-05-15" def test_new_day_resets(self): today = datetime(2026, 5, 16, tzinfo=timezone.utc) out, date = reset_daily_counter_if_new_day(3, "2026-05-15", now=today) assert out == 0 assert date == "2026-05-16" def test_none_prior_date_treated_as_new_day(self): today = datetime(2026, 5, 15, tzinfo=timezone.utc) out, _ = reset_daily_counter_if_new_day(5, None, now=today) assert out == 0 def test_negative_runs_today_rejected(self): with pytest.raises(ValueError): reset_daily_counter_if_new_day(-1, "2026-05-15") # --------------------------------------------------------------------------- # Part D — iteration artifact # --------------------------------------------------------------------------- def _make_record(**overrides): base = dict( iteration_id="iter-20260515T000000-abcdef01", started_at="2026-05-15T00:00:00+00:00", finished_at="2026-05-15T00:05:00+00:00", pairs_harvested=10, run_id="run-abc", gate_verdict="OK", canary_verdict=None, shipped=True, rolled_back=False, estimated_cost_usd=0.50, ) base.update(overrides) return IterationRecord(**base) class TestIterationRecord: def test_happy(self): r = _make_record() assert r.shipped is True assert r.gate_verdict == "OK" def test_frozen(self): r = _make_record() with pytest.raises(dataclasses.FrozenInstanceError): r.shipped = False # type: ignore def test_invalid_gate_verdict_rejected(self): with pytest.raises(ValueError, match="gate_verdict"): _make_record(gate_verdict="WEIRD") def test_invalid_canary_verdict_rejected(self): with pytest.raises(ValueError, match="canary_verdict"): _make_record(canary_verdict="WEIRD") def test_shipped_must_be_bool(self): with pytest.raises(ValueError): _make_record(shipped=1) # type: ignore def test_negative_pairs_rejected(self): with pytest.raises(ValueError): _make_record(pairs_harvested=-1) def test_negative_cost_rejected(self): with pytest.raises(ValueError): _make_record(estimated_cost_usd=-0.5) def test_iteration_id_path_separator_rejected(self): with pytest.raises(ValueError): _make_record(iteration_id="iter/escape") def test_iteration_id_empty_rejected(self): with pytest.raises(ValueError): _make_record(iteration_id="") def test_iteration_id_null_rejected(self): with pytest.raises(ValueError): _make_record(iteration_id="iter\x00x") def test_oversize_notes_rejected(self): with pytest.raises(ValueError): _make_record(notes="x" * 5000) class TestIterationIO: def test_write_read_roundtrip(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) r = _make_record() path = write_iteration(r) assert os.path.exists(path) loaded = read_iteration(r.iteration_id) assert loaded.iteration_id == r.iteration_id assert loaded.shipped is True def test_write_outside_cwd_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) r = _make_record() with pytest.raises(ValueError): write_iteration(r, base_dir=str(tmp_path.parent / "escape")) def test_read_missing_raises(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) with pytest.raises(FileNotFoundError): read_iteration("iter-missing") def test_list_empty_dir(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) assert list_iterations() == () def test_list_sorted(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) write_iteration(_make_record(iteration_id="iter-001")) write_iteration(_make_record(iteration_id="iter-003")) write_iteration(_make_record(iteration_id="iter-002")) assert list_iterations() == ("iter-001", "iter-002", "iter-003") def test_new_iteration_id_unique(self): ids = {new_iteration_id() for _ in range(20)} assert len(ids) == 20 def test_new_iteration_id_passes_validation(self): # Round-trips through IterationRecord without raising r = _make_record(iteration_id=new_iteration_id()) assert r.iteration_id.startswith("iter-") def test_write_non_record_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) with pytest.raises(TypeError): write_iteration({"foo": "bar"}) # type: ignore def test_read_invalid_manifest_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) d = tmp_path / ".soup-loops" / "iter-bad" d.mkdir(parents=True) (d / "iteration.json").write_text("[]") with pytest.raises(ValueError, match="object"): read_iteration("iter-bad") def test_list_skips_invalid_id_directories(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) # Legitimate write_iteration(_make_record(iteration_id="iter-ok")) # Junk directory with iteration.json but bad id bad = tmp_path / ".soup-loops" / "weird\x00" # OSes may reject NUL in path; if so, just skip try: bad.mkdir(parents=True) (bad / "iteration.json").write_text("{}") except (OSError, ValueError): pass out = list_iterations() assert "iter-ok" in out # --------------------------------------------------------------------------- # Watch daemon # --------------------------------------------------------------------------- class TestRunOnce: def test_default_callbacks_record_iteration(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) state, _ = init_state("m", "e", "b") state = state.with_status("running") write_state(state) cfg = WatchConfig() new_state, record, decision = run_once(state, cfg) assert decision.proceed is True assert record.gate_verdict == "SKIPPED" # default train stub sets skipped assert new_state.iteration_count == 1 def test_budget_skip_records_iteration_with_zero_counters(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) state = LoopState( served_model="m", eval_suite="e", baseline="b", status="running", monthly_budget_usd=10.0, spent_this_month_usd=10.0, ) cfg = WatchConfig(cost_fn=lambda s: 5.0) new_state, record, decision = run_once(state, cfg) assert decision.proceed is False assert record.gate_verdict == "SKIPPED" assert "budget" in record.notes.lower() assert new_state.iteration_count == 0 # not bumped on skip def test_custom_callbacks_invoked(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) state, _ = init_state("m", "e", "b") state = state.with_status("running") write_state(state) cfg = WatchConfig( harvest_fn=lambda s: {"pairs_harvested": 5, "traces_collected": 100}, train_fn=lambda s, c: {"run_id": "run-X", "skipped": False}, gate_fn=lambda s, c: {"gate_verdict": "OK"}, deploy_fn=lambda s, c: {"deployed": True, "canary_verdict": "OK"}, cost_fn=lambda s: 0.10, ) new_state, record, decision = run_once(state, cfg) assert decision.proceed is True assert record.pairs_harvested == 5 assert record.run_id == "run-X" assert record.gate_verdict == "OK" assert record.canary_verdict == "OK" assert record.shipped is True assert new_state.adapters_shipped == 1 assert new_state.pairs_distilled == 5 assert new_state.spent_this_month_usd == pytest.approx(0.10) def test_run_once_rejects_non_state(self): cfg = WatchConfig() with pytest.raises(TypeError): run_once("notstate", cfg) # type: ignore def test_run_once_rejects_non_config(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(TypeError): run_once(s, "notcfg") # type: ignore def test_gate_verdict_normalised(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) state, _ = init_state("m", "e", "b") state = state.with_status("running") write_state(state) cfg = WatchConfig(gate_fn=lambda s, c: {"gate_verdict": "GIBBERISH"}) _, record, _ = run_once(state, cfg) assert record.gate_verdict == "SKIPPED" class TestWatchConfig: def test_default_construct(self): cfg = WatchConfig() assert cfg.poll_interval_sec == 60.0 def test_invalid_poll_interval(self): with pytest.raises(ValueError): WatchConfig(poll_interval_sec=0.5) with pytest.raises(ValueError): WatchConfig(poll_interval_sec=10000) with pytest.raises(ValueError): WatchConfig(poll_interval_sec=float("nan")) def test_non_callable_harvest_rejected(self): with pytest.raises(ValueError): WatchConfig(harvest_fn="not callable") # type: ignore def test_negative_max_iterations_rejected(self): with pytest.raises(ValueError): WatchConfig(max_iterations=-1) class TestWatchDaemon: def test_watch_finite_runs_then_stops(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) init_state("m", "e", "b") cfg = WatchConfig( poll_interval_sec=1.0, max_iterations=3, ) final_state, ran = watch(cfg) assert ran == 3 assert final_state.iteration_count == 3 assert final_state.status == "stopped" def test_watch_respects_pause(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) init_state("m", "e", "b") # Iteration 1 paths through and then we flip to paused via callback. def _pause_after_first(record): s = read_state() write_state(s.with_status("stopped")) cfg = WatchConfig( poll_interval_sec=1.0, max_iterations=10, on_iteration=_pause_after_first, ) final_state, ran = watch(cfg) # Either 1 or 2 — the on_iteration fires before the next read. assert ran >= 1 assert final_state.status == "stopped" class TestRollbackOrchestration: def test_maybe_rollback_no_op_on_ok(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=5.0) assert maybe_rollback(p, "OK").canary == "B" def test_maybe_rollback_clears_on_major(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=5.0) assert maybe_rollback(p, "MAJOR").canary is None def test_maybe_rollback_unknown_no_op(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=5.0) assert maybe_rollback(p, "UNKNOWN").canary == "B" def test_evaluate_canary_verdict_wraps_stats(self): stats = BucketStats() for _ in range(40): stats.record("canary", True) assert evaluate_canary_verdict(stats) == "OK" def test_maybe_rollback_non_policy_rejected(self): with pytest.raises(TypeError): maybe_rollback("notpolicy", "MAJOR") # type: ignore def test_maybe_rollback_non_str_verdict_rejected(self): p = CanaryPolicy(stable="A") with pytest.raises(TypeError): maybe_rollback(p, 5) # type: ignore # --------------------------------------------------------------------------- # CLI smoke # --------------------------------------------------------------------------- class TestCLI: def test_loop_help(self): result = runner.invoke(app, ["loop", "--help"]) assert result.exit_code == 0, result.output assert "init" in result.output assert "watch" in result.output def test_init_command(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) result = runner.invoke( app, ["loop", "init", "model-a", "--eval", "suite.yaml", "--baseline", "ref"], ) assert result.exit_code == 0, result.output assert (tmp_path / ".soup" / "loop.yaml").exists() def test_init_refuses_overwrite(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) result = runner.invoke( app, ["loop", "init", "m2", "--eval", "e", "--baseline", "b"] ) assert result.exit_code == 2, result.output assert "already exists" in result.output def test_init_force_overwrites(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) result = runner.invoke( app, ["loop", "init", "m2", "--eval", "e", "--baseline", "b", "--force"], ) assert result.exit_code == 0, result.output def test_init_invalid_budget(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) result = runner.invoke( app, [ "loop", "init", "m", "--eval", "e", "--baseline", "b", "--monthly-budget", "garbage", ], ) assert result.exit_code == 2 def test_status_without_init(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) result = runner.invoke(app, ["loop", "status"]) assert result.exit_code == 2 assert "init" in result.output.lower() def test_status_after_init(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) result = runner.invoke(app, ["loop", "status"]) assert result.exit_code == 0, result.output assert "stopped" in result.output def test_pause_resume_cycle(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) # Mark running manually so pause has something to flip. s = read_state() write_state(s.with_status("running")) r = runner.invoke(app, ["loop", "pause"]) assert r.exit_code == 0 assert read_state().status == "paused" r = runner.invoke(app, ["loop", "resume"]) assert r.exit_code == 0 assert read_state().status == "running" def test_pause_when_stopped(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke(app, ["loop", "pause"]) assert r.exit_code == 0 assert "already stopped" in r.output def test_resume_when_not_paused(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke(app, ["loop", "resume"]) assert r.exit_code == 0 assert "not paused" in r.output def test_watch_max_iterations(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, [ "loop", "watch", "--foreground", "--max-iterations", "2", "--poll-interval", "1", ], ) assert r.exit_code == 0, r.output assert "iterations=2" in r.output def test_watch_detach_and_foreground_mutex(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, ["loop", "watch", "--foreground", "--detach"] ) assert r.exit_code == 2 def test_canary_command(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "model-a", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, ["loop", "canary", "model-b", "--traffic", "10%"] ) assert r.exit_code == 0, r.output s = read_state() assert s.canary_active == "model-b" assert s.canary_traffic_pct == 10.0 def test_canary_invalid_traffic(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "model-a", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, ["loop", "canary", "model-b", "--traffic", "150"] ) assert r.exit_code == 2 def test_canary_same_as_stable_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "same", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, ["loop", "canary", "same", "--traffic", "5%"] ) assert r.exit_code == 2 def test_replay_list_empty(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke(app, ["loop", "replay"]) assert r.exit_code == 0 assert "no iterations" in r.output def test_replay_show_iteration(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) rec = _make_record() write_iteration(rec) r = runner.invoke(app, ["loop", "replay", rec.iteration_id]) assert r.exit_code == 0 assert rec.iteration_id in r.output def test_replay_unknown_iteration(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "m", "--eval", "e", "--baseline", "b"] ) r = runner.invoke(app, ["loop", "replay", "iter-nonexistent"]) assert r.exit_code == 2 # --------------------------------------------------------------------------- # Source-grep regression guards # --------------------------------------------------------------------------- _REPO_ROOT = Path(__file__).resolve().parent.parent class TestSourceWiring: def test_cli_registers_loop_typer(self): cli_src = (_REPO_ROOT / "src" / "soup_cli" / "cli.py").read_text(encoding="utf-8") assert "from soup_cli.commands import loop as _loop_cmd" in cli_src assert 'name="loop"' in cli_src def test_version_bumped_to_0_58_0(self): # Widened from exact-match to floor-check to match the v0.51.0 / v0.54.0 # / v0.56.0 idiom — v0.58.0 was the floor when these tests landed. init = (_REPO_ROOT / "src" / "soup_cli" / "__init__.py").read_text(encoding="utf-8") match = re.search(r'__version__ = "(\d+)\.(\d+)\.(\d+)"', init) assert match is not None, init major, minor, patch = (int(g) for g in match.groups()) assert (major, minor, patch) >= (0, 58, 0) def test_no_top_level_torch_import_in_loop_modules(self): for name in [ "loop_state.py", "loop_budget.py", "loop_iteration.py", "canary_router.py", "loop_daemon.py", ]: src = (_REPO_ROOT / "src" / "soup_cli" / "utils" / name).read_text(encoding="utf-8") # Check module-level imports only (skip indented imports inside funcs) for line in src.splitlines(): if line.startswith(("import torch", "from torch")): raise AssertionError(f"{name} imports torch at module level") def test_command_module_uses_typer_app(self): src = (_REPO_ROOT / "src" / "soup_cli" / "commands" / "loop.py").read_text(encoding="utf-8") assert "app = typer.Typer(" in src assert 'name="loop"' in src # --------------------------------------------------------------------------- # version sanity # --------------------------------------------------------------------------- def test_version_string(): from soup_cli import __version__ # Widened from exact-match to floor-check (v0.58.0 was the floor here). match = re.match(r"^(\d+)\.(\d+)\.(\d+)$", __version__) assert match is not None, __version__ major, minor, patch = (int(g) for g in match.groups()) assert (major, minor, patch) >= (0, 58, 0) # --------------------------------------------------------------------------- # Code-review wave 2 follow-ups (HIGH #2-#4 + MEDIUM #5-#8 review fixes) # --------------------------------------------------------------------------- class TestReviewFixWave2: def test_watch_preserves_paused_status_on_exit(self, tmp_path, monkeypatch): """HIGH #2: SIGTERM-while-paused must NOT silently promote to stopped.""" monkeypatch.chdir(tmp_path) init_state("m", "e", "b") s = read_state() write_state(s.with_status("paused")) cfg = WatchConfig(poll_interval_sec=1.0, max_iterations=0) final_state, _ = watch(cfg) # Was paused before watch; daemon must not have flipped to stopped. assert final_state.status == "paused", "watch destroyed paused state" def test_watch_flips_running_to_stopped_at_exit(self, tmp_path, monkeypatch): """The legit case still works — running → stopped on max_iterations.""" monkeypatch.chdir(tmp_path) init_state("m", "e", "b") cfg = WatchConfig(poll_interval_sec=1.0, max_iterations=1) final_state, _ = watch(cfg) assert final_state.status == "stopped" def test_budget_skip_does_not_write_iteration_manifest(self, tmp_path, monkeypatch): """HIGH #3: budget-skipped runs do not produce iteration manifests.""" monkeypatch.chdir(tmp_path) init_state("m", "e", "b") s = read_state() write_state( dataclasses.replace( s, status="running", monthly_budget_usd=10.0, spent_this_month_usd=10.0, ) ) cfg = WatchConfig( poll_interval_sec=1.0, max_iterations=1, cost_fn=lambda _s: 5.0, # forces budget rejection ) watch(cfg) # No manifest should exist because the iteration was budget-skipped. assert list_iterations() == () def test_canary_autoroll_persisted(self, tmp_path, monkeypatch): """HIGH #4: --autoroll-on-regress flag must survive into LoopState.""" monkeypatch.chdir(tmp_path) runner.invoke( app, ["loop", "init", "model-a", "--eval", "e", "--baseline", "b"] ) r = runner.invoke( app, [ "loop", "canary", "model-b", "--traffic", "5%", "--no-autoroll-on-regress", ], ) assert r.exit_code == 0, r.output s = read_state() assert s.canary_autoroll_on_regress is False # Default True is also exercised by other canary tests above. def test_canary_autoroll_bool_validator(self): """LoopState rejects non-bool canary_autoroll_on_regress.""" with pytest.raises(ValueError, match="canary_autoroll_on_regress"): LoopState( served_model="m", eval_suite="e", baseline="b", canary_autoroll_on_regress=1, # type: ignore ) def test_route_ceil_at_sub_bucket_fraction(self): """MEDIUM #5: 0.005 % must allocate ≥1 bucket, not round to 0.""" p = CanaryPolicy(stable="A", canary="B", traffic_pct=0.005) # 10_000 buckets * 0.00005 = 0.5 → ceil = 1 bucket reserved for canary. canary_hits = sum( 1 for i in range(10_000) if route(p, f"k-{i}").bucket == "canary" ) assert canary_hits >= 1, "ceil rounding lost the sub-bucket fraction" def test_parse_budget_usd_only(self): """MEDIUM #6: bare 'usd' / ' usd ' raises friendly explicit error.""" with pytest.raises(ValueError, match="numeric value"): parse_budget_string("usd") with pytest.raises(ValueError, match="numeric value"): parse_budget_string(" USD ") def test_list_iterations_swallows_oserror_on_listdir(self, tmp_path, monkeypatch): """MEDIUM #8: list_iterations returns () instead of raising on unreadable dir.""" monkeypatch.chdir(tmp_path) # Create the dir then monkeypatch os.listdir to raise — simulates a # permission flap mid-iteration that would otherwise kill the daemon. d = tmp_path / ".soup-loops" d.mkdir() import soup_cli.utils.loop_iteration as li def _raise(_p): raise PermissionError("simulated permission flap") monkeypatch.setattr(li.os, "listdir", _raise) assert li.list_iterations() == () # --------------------------------------------------------------------------- # TDD-review wave 3: exact-boundary + match= tightening + coverage gaps # --------------------------------------------------------------------------- class TestReviewFixWave3: # --- HIGH #1: exact-boundary tests for _MAX_STR_FIELD = 512 ---------- def test_str_field_accepts_exactly_512(self): s = LoopState(served_model="m" * 512, eval_suite="e", baseline="b") assert len(s.served_model) == 512 def test_str_field_rejects_513(self): with pytest.raises(ValueError, match="exceeds 512"): LoopState(served_model="m" * 513, eval_suite="e", baseline="b") # --- HIGH #2: read_state size cap exact boundary --------------------- def test_read_state_at_one_mib_minus_one_accepted(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() # Pad a valid JSON document to just under 1 MiB. notes_pad = " " * (1024 * 1024 - 200) target.write_text( json.dumps( { "served_model": "m", "eval_suite": "e", "baseline": "b", "last_iteration_id": "iter-pad" + notes_pad[:480], } ) ) size = os.path.getsize(target) assert size < 1024 * 1024 # under the cap s = read_state() assert s.served_model == "m" def test_read_state_at_one_mib_plus_one_rejected(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) target = tmp_path / ".soup" / "loop.yaml" target.parent.mkdir() target.write_text("x" * (1024 * 1024 + 1)) with pytest.raises(ValueError, match="1 MiB"): read_state() # --- HIGH #3: match= tightening on critical reject paths ------------- def test_monthly_budget_negative_match(self): with pytest.raises(ValueError, match="monthly_budget_usd must be"): LoopState( served_model="m", eval_suite="e", baseline="b", monthly_budget_usd=-1.0, ) def test_max_runs_per_day_zero_match(self): with pytest.raises(ValueError, match="max_runs_per_day"): LoopState( served_model="m", eval_suite="e", baseline="b", max_runs_per_day=0, ) def test_with_status_unknown_match(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="status must be"): s.with_status("not-a-real-status") def test_bumped_unknown_counter_match(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="unknown counter"): s.bumped(banana=1) def test_canary_traffic_out_of_range_match(self): with pytest.raises(ValueError, match=r"\[0, 100\]"): CanaryPolicy(stable="A", canary="B", traffic_pct=150.0) # --- MEDIUM: bool rejection on iteration_count / runs_today --------- @pytest.mark.parametrize("counter", ["iteration_count", "runs_today"]) def test_counter_rejects_bool_iter_runs(self, counter): kwargs = {"served_model": "m", "eval_suite": "e", "baseline": "b", counter: True} with pytest.raises(ValueError, match=counter): LoopState(**kwargs) # --- MEDIUM: bool rejection on float budget fields ------------------- def test_monthly_budget_bool_rejected(self): with pytest.raises(ValueError, match="monthly_budget_usd"): LoopState( served_model="m", eval_suite="e", baseline="b", monthly_budget_usd=True, # type: ignore ) def test_spent_this_month_bool_rejected(self): with pytest.raises(ValueError, match="spent_this_month_usd"): LoopState( served_model="m", eval_suite="e", baseline="b", spent_this_month_usd=True, # type: ignore ) # --- MEDIUM: optional-string empty-string rejection ------------------ @pytest.mark.parametrize( "field_name", ["canary_active", "last_iteration_id", "last_run_date"] ) def test_optional_str_field_rejects_empty(self, field_name): kwargs = {"served_model": "m", "eval_suite": "e", "baseline": "b", field_name: ""} with pytest.raises(ValueError, match="must not be empty"): LoopState(**kwargs) # --- MEDIUM: _check_path empty string at write boundary -------------- def test_write_state_rejects_empty_path(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) s = LoopState(served_model="m", eval_suite="e", baseline="b") with pytest.raises(ValueError, match="must not be empty"): write_state(s, "") # --- LOW: canary traffic_pct lower-bound exact 0 + upper-bound 100 --- def test_canary_traffic_pct_zero_accepted(self): p = CanaryPolicy(stable="A") # traffic_pct=0 default assert p.traffic_pct == 0.0 def test_canary_traffic_pct_negative_rejected(self): with pytest.raises(ValueError, match=r"\[0, 100\]"): CanaryPolicy(stable="A", canary="B", traffic_pct=-0.001) def test_canary_traffic_pct_exactly_100_accepted(self): p = CanaryPolicy(stable="A", canary="B", traffic_pct=100.0) assert p.traffic_pct == 100.0 # --- LOW: to_dict keys match dataclass fields (forward-compat lock) -- def test_to_dict_keys_match_dataclass_fields(self): s = LoopState(served_model="m", eval_suite="e", baseline="b") assert set(s.to_dict().keys()) == set(LoopState.__dataclass_fields__.keys()) # --- LOW: read_state preserves created_at/updated_at when present ---- def test_read_state_preserves_created_at(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) s = LoopState( served_model="m", eval_suite="e", baseline="b", created_at="2026-05-15T00:00:00+00:00", ) write_state(s) reloaded = read_state() # created_at survives the roundtrip; updated_at gets refreshed by write_state. assert reloaded.created_at == "2026-05-15T00:00:00+00:00"