"""v0.64.0 Part A — `soup tunability` probe across candidate bases. Tests cover: - CandidateBase frozen dataclass + validation - TunabilityResult frozen dataclass + validation - TunabilityReport frozen dataclass + immutable candidates tuple - validate_probe_steps bounds + bool reject - validate_holdout_size bounds + bool reject - score_candidate happy + delta math (lower-better loss) - pareto_frontier identifies non-dominated points - run_tunability orchestrator (with mocked probe callable) - write_report + load_report atomic + cwd containment + symlink rejection - CLI smoke (--help / outside-cwd reject / unknown candidate) """ from __future__ import annotations import dataclasses import json import os import sys import pytest from typer.testing import CliRunner runner = CliRunner() # --------------------------------------------------------------------------- # Module imports # --------------------------------------------------------------------------- def test_module_imports(): from soup_cli.utils import tunability assert hasattr(tunability, "CandidateBase") assert hasattr(tunability, "TunabilityResult") assert hasattr(tunability, "TunabilityReport") assert hasattr(tunability, "validate_probe_steps") assert hasattr(tunability, "validate_holdout_size") assert hasattr(tunability, "score_candidate") assert hasattr(tunability, "pareto_frontier") assert hasattr(tunability, "run_tunability") assert hasattr(tunability, "write_report") assert hasattr(tunability, "load_report") assert hasattr(tunability, "DEFAULT_CANDIDATES") # --------------------------------------------------------------------------- # DEFAULT_CANDIDATES catalog # --------------------------------------------------------------------------- def test_default_candidates_nonempty(): from soup_cli.utils.tunability import DEFAULT_CANDIDATES assert len(DEFAULT_CANDIDATES) >= 6 # Each entry must be CandidateBase from soup_cli.utils.tunability import CandidateBase for c in DEFAULT_CANDIDATES: assert isinstance(c, CandidateBase) def test_default_candidates_immutable(): from soup_cli.utils.tunability import DEFAULT_CANDIDATES # Tuple, not list assert isinstance(DEFAULT_CANDIDATES, tuple) # --------------------------------------------------------------------------- # CandidateBase # --------------------------------------------------------------------------- def test_candidate_base_frozen(): from soup_cli.utils.tunability import CandidateBase c = CandidateBase( name="qwen3-0.6b", repo_id="Qwen/Qwen3-0.6B", params_b=0.6, license_id="apache-2.0", ) with pytest.raises(dataclasses.FrozenInstanceError): c.name = "other" # type: ignore[misc] def test_candidate_base_rejects_empty_name(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(ValueError, match="name"): CandidateBase(name="", repo_id="x/y", params_b=1.0, license_id="apache-2.0") def test_candidate_base_rejects_null_byte(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(ValueError, match="null"): CandidateBase(name="bad\x00", repo_id="x/y", params_b=1.0, license_id="apache-2.0") def test_candidate_base_rejects_negative_params(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(ValueError, match="params_b"): CandidateBase(name="x", repo_id="x/y", params_b=-1.0, license_id="apache-2.0") def test_candidate_base_rejects_bool_params(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(TypeError, match="bool"): CandidateBase(name="x", repo_id="x/y", params_b=True, license_id="apache-2.0") # type: ignore[arg-type] def test_candidate_base_rejects_non_finite_params(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(ValueError, match="finite"): CandidateBase(name="x", repo_id="x/y", params_b=float("nan"), license_id="apache-2.0") def test_candidate_base_rejects_oversize_name(): from soup_cli.utils.tunability import CandidateBase with pytest.raises(ValueError, match="too long"): CandidateBase(name="x" * 513, repo_id="x/y", params_b=1.0, license_id="apache-2.0") # --------------------------------------------------------------------------- # validate_probe_steps # --------------------------------------------------------------------------- @pytest.mark.parametrize("value", [10, 100, 1000]) def test_validate_probe_steps_happy(value): from soup_cli.utils.tunability import validate_probe_steps assert validate_probe_steps(value) == value def test_validate_probe_steps_boundary_min(): from soup_cli.utils.tunability import validate_probe_steps assert validate_probe_steps(10) == 10 with pytest.raises(ValueError): validate_probe_steps(9) def test_validate_probe_steps_boundary_max(): from soup_cli.utils.tunability import validate_probe_steps assert validate_probe_steps(10_000) == 10_000 with pytest.raises(ValueError): validate_probe_steps(10_001) @pytest.mark.parametrize("bad", [True, False, None, "100", -1, 0, 9, 10_001, 1.5]) def test_validate_probe_steps_rejects(bad): from soup_cli.utils.tunability import validate_probe_steps with pytest.raises((TypeError, ValueError)): validate_probe_steps(bad) # --------------------------------------------------------------------------- # validate_holdout_size # --------------------------------------------------------------------------- def test_validate_holdout_size_happy(): from soup_cli.utils.tunability import validate_holdout_size assert validate_holdout_size(100) == 100 def test_validate_holdout_size_boundary(): from soup_cli.utils.tunability import validate_holdout_size assert validate_holdout_size(10) == 10 with pytest.raises(ValueError): validate_holdout_size(9) assert validate_holdout_size(100_000) == 100_000 with pytest.raises(ValueError): validate_holdout_size(100_001) @pytest.mark.parametrize("bad", [True, False, "100", -1, 0]) def test_validate_holdout_size_rejects(bad): from soup_cli.utils.tunability import validate_holdout_size with pytest.raises((TypeError, ValueError)): validate_holdout_size(bad) # --------------------------------------------------------------------------- # TunabilityResult # --------------------------------------------------------------------------- def test_tunability_result_happy(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase( name="qwen3-0.6b", repo_id="x/y", params_b=0.6, license_id="apache-2.0" ) r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=120.0, estimated_cost_usd=0.05, ) assert r.delta == 0.5 def test_tunability_result_frozen(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=120.0, estimated_cost_usd=0.05, ) with pytest.raises(dataclasses.FrozenInstanceError): r.delta = 0.0 # type: ignore[misc] def test_tunability_result_rejects_non_finite(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") with pytest.raises(ValueError, match="finite"): TunabilityResult( candidate=cand, base_loss=float("nan"), probe_loss=2.0, delta=0.5, wall_clock_seconds=120.0, estimated_cost_usd=0.05, ) def test_tunability_result_rejects_negative_wall_clock(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") with pytest.raises(ValueError, match="wall_clock"): TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=-1.0, estimated_cost_usd=0.05, ) def test_tunability_result_rejects_negative_cost(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") with pytest.raises(ValueError, match="cost"): TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=120.0, estimated_cost_usd=-0.01, ) def test_tunability_result_rejects_bool_loss(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") with pytest.raises(TypeError, match="bool"): TunabilityResult( candidate=cand, base_loss=True, # type: ignore[arg-type] probe_loss=2.0, delta=0.5, wall_clock_seconds=120.0, estimated_cost_usd=0.05, ) # --------------------------------------------------------------------------- # score_candidate # --------------------------------------------------------------------------- def test_score_candidate_delta_math(): """delta = base_loss - probe_loss (positive = improvement).""" from soup_cli.utils.tunability import score_candidate base_loss, probe_loss = 2.5, 2.0 delta = score_candidate(base_loss=base_loss, probe_loss=probe_loss) assert delta == pytest.approx(0.5) def test_score_candidate_zero_when_no_change(): from soup_cli.utils.tunability import score_candidate assert score_candidate(base_loss=2.0, probe_loss=2.0) == 0.0 def test_score_candidate_negative_when_worse(): from soup_cli.utils.tunability import score_candidate assert score_candidate(base_loss=2.0, probe_loss=2.5) == pytest.approx(-0.5) def test_score_candidate_rejects_non_finite(): from soup_cli.utils.tunability import score_candidate with pytest.raises(ValueError): score_candidate(base_loss=float("nan"), probe_loss=2.0) with pytest.raises(ValueError): score_candidate(base_loss=2.0, probe_loss=float("inf")) def test_score_candidate_rejects_bool(): from soup_cli.utils.tunability import score_candidate with pytest.raises(TypeError): score_candidate(base_loss=True, probe_loss=2.0) # type: ignore[arg-type] # --------------------------------------------------------------------------- # pareto_frontier # --------------------------------------------------------------------------- def test_pareto_frontier_simple(): """Maximise delta, minimise cost. Strictly dominated entries get dropped.""" from soup_cli.utils.tunability import CandidateBase, TunabilityResult, pareto_frontier def _mk(name: str, delta: float, cost: float) -> TunabilityResult: cand = CandidateBase(name=name, repo_id="x/y", params_b=1.0, license_id="apache-2.0") return TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.5 - delta, delta=delta, wall_clock_seconds=60.0, estimated_cost_usd=cost, ) # B dominates A: B has higher delta AND lower cost. # C is on the frontier: lower delta but lower cost than B. a = _mk("a", delta=0.1, cost=0.20) b = _mk("b", delta=0.5, cost=0.10) c = _mk("c", delta=0.05, cost=0.05) frontier = pareto_frontier([a, b, c]) names = {r.candidate.name for r in frontier} # A is strictly dominated by B (lower delta, higher cost) assert "a" not in names assert "b" in names assert "c" in names def test_pareto_frontier_empty(): from soup_cli.utils.tunability import pareto_frontier assert pareto_frontier([]) == () def test_pareto_frontier_single(): from soup_cli.utils.tunability import CandidateBase, TunabilityResult, pareto_frontier cand = CandidateBase(name="a", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) frontier = pareto_frontier([r]) assert frontier == (r,) def test_pareto_frontier_returns_tuple(): from soup_cli.utils.tunability import pareto_frontier assert isinstance(pareto_frontier([]), tuple) def test_pareto_frontier_rejects_non_sequence(): from soup_cli.utils.tunability import pareto_frontier with pytest.raises(TypeError): pareto_frontier("not a list") # type: ignore[arg-type] # --------------------------------------------------------------------------- # TunabilityReport # --------------------------------------------------------------------------- def test_tunability_report_frozen(): from soup_cli.utils.tunability import CandidateBase, TunabilityReport, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) report = TunabilityReport(results=(r,), frontier=(r,), probe_steps=100, holdout_size=64) with pytest.raises(dataclasses.FrozenInstanceError): report.probe_steps = 0 # type: ignore[misc] def test_tunability_report_results_tuple(): from soup_cli.utils.tunability import CandidateBase, TunabilityReport, TunabilityResult cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) # Lists rejected — frozen=True doesn't make lists immutable with pytest.raises(TypeError, match="tuple"): TunabilityReport(results=[r], frontier=(r,), probe_steps=100, holdout_size=64) # type: ignore[arg-type] # --------------------------------------------------------------------------- # run_tunability # --------------------------------------------------------------------------- def test_run_tunability_with_mocked_probe(tmp_path): """Inject a deterministic probe to exercise the orchestrator.""" from soup_cli.utils.tunability import ( CandidateBase, TunabilityResult, run_tunability, ) candidates = ( CandidateBase(name="cand-a", repo_id="x/y", params_b=0.5, license_id="apache-2.0"), CandidateBase(name="cand-b", repo_id="x/z", params_b=1.0, license_id="mit"), ) def fake_probe(cand: CandidateBase, dataset_path: str, *, probe_steps: int, holdout_size: int) -> TunabilityResult: # Synthetic: larger param count → bigger delta, longer wall-clock return TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.5 - cand.params_b * 0.3, delta=cand.params_b * 0.3, wall_clock_seconds=cand.params_b * 60.0, estimated_cost_usd=cand.params_b * 0.05, ) dataset = tmp_path / "data.jsonl" dataset.write_text('{"prompt": "x", "completion": "y"}\n') report = run_tunability( candidates=candidates, dataset_path=str(dataset.relative_to(tmp_path)) if False else str(dataset), probe_steps=50, holdout_size=16, probe_fn=fake_probe, ) assert len(report.results) == 2 assert len(report.frontier) >= 1 assert report.probe_steps == 50 def test_run_tunability_rejects_empty_candidates(tmp_path): from soup_cli.utils.tunability import run_tunability dataset = tmp_path / "data.jsonl" dataset.write_text("{}\n") with pytest.raises(ValueError, match="candidates"): run_tunability( candidates=(), dataset_path=str(dataset), probe_steps=50, holdout_size=16, ) def test_run_tunability_rejects_invalid_probe_steps(tmp_path): from soup_cli.utils.tunability import CandidateBase, run_tunability dataset = tmp_path / "data.jsonl" dataset.write_text("{}\n") cands = (CandidateBase(name="a", repo_id="x/y", params_b=1.0, license_id="apache-2.0"),) with pytest.raises(ValueError): run_tunability( candidates=cands, dataset_path=str(dataset), probe_steps=9, # below min holdout_size=16, ) # --------------------------------------------------------------------------- # write_report / load_report # --------------------------------------------------------------------------- def test_write_report_atomic_roundtrip(tmp_path, monkeypatch): from soup_cli.utils.tunability import ( CandidateBase, TunabilityReport, TunabilityResult, load_report, write_report, ) monkeypatch.chdir(tmp_path) cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) report = TunabilityReport(results=(r,), frontier=(r,), probe_steps=100, holdout_size=64) out = tmp_path / "tunability.json" write_report(report, str(out)) loaded = load_report(str(out)) assert loaded.probe_steps == report.probe_steps assert len(loaded.results) == 1 assert loaded.results[0].candidate.name == "x" def test_write_report_outside_cwd_rejected(tmp_path, monkeypatch): from soup_cli.utils.tunability import ( CandidateBase, TunabilityReport, TunabilityResult, write_report, ) monkeypatch.chdir(tmp_path) cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) report = TunabilityReport(results=(r,), frontier=(r,), probe_steps=100, holdout_size=64) outside = tmp_path.parent / "evil.json" with pytest.raises(ValueError, match="cwd"): write_report(report, str(outside)) @pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlinks") def test_write_report_symlink_rejected(tmp_path, monkeypatch): from soup_cli.utils.tunability import ( CandidateBase, TunabilityReport, TunabilityResult, write_report, ) monkeypatch.chdir(tmp_path) cand = CandidateBase(name="x", repo_id="x/y", params_b=1.0, license_id="apache-2.0") r = TunabilityResult( candidate=cand, base_loss=2.5, probe_loss=2.0, delta=0.5, wall_clock_seconds=60.0, estimated_cost_usd=0.05, ) report = TunabilityReport(results=(r,), frontier=(r,), probe_steps=100, holdout_size=64) target = tmp_path / "real.json" target.write_text("{}") link = tmp_path / "link.json" os.symlink(target, link) with pytest.raises(ValueError, match="symlink"): write_report(report, str(link)) def test_write_report_non_report_rejected(tmp_path): from soup_cli.utils.tunability import write_report with pytest.raises(TypeError): write_report("not a report", str(tmp_path / "out.json")) # type: ignore[arg-type] def test_load_report_missing_file(tmp_path, monkeypatch): from soup_cli.utils.tunability import load_report monkeypatch.chdir(tmp_path) with pytest.raises(FileNotFoundError): load_report(str(tmp_path / "nope.json")) def test_load_report_invalid_json(tmp_path): from soup_cli.utils.tunability import load_report p = tmp_path / "bad.json" p.write_text("not json") with pytest.raises((ValueError, json.JSONDecodeError)): load_report(str(p)) # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- def test_cli_tunability_help(): from soup_cli.cli import app result = runner.invoke(app, ["tunability", "--help"]) assert result.exit_code == 0, (result.output, repr(result.exception)) assert "tunability" in result.output.lower() def test_cli_tunability_list_default_candidates(): from soup_cli.cli import app result = runner.invoke(app, ["tunability", "--list"]) assert result.exit_code == 0, (result.output, repr(result.exception)) # Should list at least one default candidate name. assert "qwen" in result.output.lower() or "llama" in result.output.lower() or \ "phi" in result.output.lower() or "gemma" in result.output.lower() or \ "smol" in result.output.lower() def test_cli_tunability_requires_dataset(tmp_path, monkeypatch): """Without --list and without --dataset, exits with usage error.""" from soup_cli.cli import app monkeypatch.chdir(tmp_path) result = runner.invoke(app, ["tunability"]) assert result.exit_code != 0 def test_cli_tunability_outside_cwd(tmp_path, monkeypatch): from soup_cli.cli import app monkeypatch.chdir(tmp_path) outside = tmp_path.parent / "data.jsonl" outside.write_text("{}\n") result = runner.invoke(app, ["tunability", "--dataset", str(outside)]) assert result.exit_code != 0 def test_cli_tunability_plan_only(tmp_path, monkeypatch): """--plan-only enumerates candidates without running probes.""" from soup_cli.cli import app monkeypatch.chdir(tmp_path) dataset = tmp_path / "data.jsonl" dataset.write_text('{"prompt": "x"}\n') result = runner.invoke(app, ["tunability", "--dataset", str(dataset), "--plan-only"]) assert result.exit_code == 0, (result.output, repr(result.exception)) # --------------------------------------------------------------------------- # Source-wiring regression guards # --------------------------------------------------------------------------- def test_cli_registers_tunability(): """cli.py registers the tunability command.""" from pathlib import Path src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "cli.py" text = src.read_text(encoding="utf-8") assert "tunability" in text def test_version_bumped_to_0640(): import soup_cli # Floor check so future minor releases don't regress this test (matches # v0.51.0 / v0.54.0 / v0.57.0 / v0.60.0 floor-check idiom). Exact-equality # at "0.64.0" broke on the v0.65.0 bump — the test name preserves the # intent (≥0.64.0 means v0.64.0 shipped). parts = tuple(int(p) for p in soup_cli.__version__.split(".")[:3]) assert parts >= (0, 64, 0), soup_cli.__version__ def test_no_top_level_heavy_imports(): """tunability module should not import torch/transformers/peft at top-level.""" from pathlib import Path src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "utils" / "tunability.py" text = src.read_text(encoding="utf-8") # Heavy deps must be lazy-imported inside functions for bad in ["^import torch", "^from torch", "^import transformers", "^from transformers"]: # Strict line-start match import re assert not re.search(bad, text, re.MULTILINE), f"top-level {bad} found"