diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4abfa30..df38004 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,7 +116,7 @@ src/soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (271 files, 12134 tests) +tests/ - Test suite (271 files, 12153 tests) examples/ - Real-world config examples and datasets ``` diff --git a/tests/test_serve.py b/tests/test_serve.py index ce37faf..e8fb5cd 100644 --- a/tests/test_serve.py +++ b/tests/test_serve.py @@ -279,13 +279,18 @@ class TestRecordThumbs: return TestClient(app), db def test_flag_in_help(self): + import re + from typer.testing import CliRunner from soup_cli.cli import app result = CliRunner().invoke(app, ["serve", "--help"]) assert result.exit_code == 0, result.output - assert "--record-thumbs" in result.output + # Under color (CI sets FORCE_COLOR), Rich inserts ANSI codes between the + # two dashes of an option name, so strip them before the substring check. + clean = re.sub(r"\x1b\[[0-9;]*m", "", result.output) + assert "--record-thumbs" in clean def test_thumbs_endpoint_registered_when_db_set(self, tmp_path, monkeypatch): try: diff --git a/tests/test_v0610_part_b.py b/tests/test_v0610_part_b.py index 64f5ce1..068a0fc 100644 --- a/tests/test_v0610_part_b.py +++ b/tests/test_v0610_part_b.py @@ -13,6 +13,8 @@ from __future__ import annotations import dataclasses import json +import os +import sys from pathlib import Path import pytest @@ -507,6 +509,80 @@ class TestCli: # ---------- Fixtures ---------- +class TestLoadEvidenceFile: + """v0.71.1 — cover the `soup eval unlearning --evidence` loader.""" + + def test_happy_returns_dict(self, tmp_path, monkeypatch): + from soup_cli.utils.unlearning_eval import load_evidence_file + + monkeypatch.chdir(tmp_path) + (tmp_path / "ev.json").write_text( + json.dumps({"forget_quality": {"pre_loss": 1.0, "post_loss": 2.0}}), + encoding="utf-8", + ) + data = load_evidence_file("ev.json") + assert isinstance(data, dict) + assert "forget_quality" in data + + def test_missing_file_raises(self, tmp_path, monkeypatch): + from soup_cli.utils.unlearning_eval import load_evidence_file + + monkeypatch.chdir(tmp_path) + with pytest.raises(FileNotFoundError): + load_evidence_file("nope.json") + + def test_non_string_path_rejected(self): + from soup_cli.utils.unlearning_eval import load_evidence_file + + with pytest.raises(ValueError): + load_evidence_file(123) # type: ignore[arg-type] + + def test_empty_path_rejected(self): + from soup_cli.utils.unlearning_eval import load_evidence_file + + with pytest.raises(ValueError): + load_evidence_file("") + + def test_null_byte_rejected(self): + from soup_cli.utils.unlearning_eval import load_evidence_file + + with pytest.raises(ValueError, match="null"): + load_evidence_file("a\x00b.json") + + def test_outside_cwd_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.unlearning_eval import load_evidence_file + + outside = tmp_path / "outside" + outside.mkdir() + (outside / "ev.json").write_text("{}", encoding="utf-8") + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError, match="cwd"): + load_evidence_file(str(outside / "ev.json")) + + def test_non_dict_root_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.unlearning_eval import load_evidence_file + + monkeypatch.chdir(tmp_path) + (tmp_path / "arr.json").write_text("[]", encoding="utf-8") + with pytest.raises(ValueError, match="JSON object"): + load_evidence_file("arr.json") + + @pytest.mark.skipif( + sys.platform == "win32", reason="symlink creation needs admin on Windows" + ) + def test_symlink_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.unlearning_eval import load_evidence_file + + monkeypatch.chdir(tmp_path) + (tmp_path / "real.json").write_text("{}", encoding="utf-8") + link = tmp_path / "link.json" + os.symlink(tmp_path / "real.json", link) + with pytest.raises(ValueError, match="symlink"): + load_evidence_file("link.json") + + class TestFixtures: def test_tofu_fixture_exists(self): from soup_cli.utils.unlearning_eval import get_fixture_path diff --git a/tests/test_v0640_part_c.py b/tests/test_v0640_part_c.py index 3e164e4..c00ea51 100644 --- a/tests/test_v0640_part_c.py +++ b/tests/test_v0640_part_c.py @@ -662,3 +662,62 @@ def test_render_install_plan_requirements_conda_comment(): # line rather than a bare `name==version` pip pin (v0.71.1 #209). plan = render_install_plan(_lock_with_conda(), fmt="requirements") assert "# conda: mkl==2023.1" in plan + + +def test_cli_env_fix_corrupt_lock(tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + (tmp_path / "soup-env.lock").write_text("{ this is not valid json", encoding="utf-8") + result = runner.invoke(app, ["env", "fix"]) + assert result.exit_code == 2, result.output + + +def test_cli_env_fix_bad_format(tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + assert runner.invoke(app, ["env", "lock"]).exit_code == 0 + result = runner.invoke(app, ["env", "fix", "--format", "bogus-format"]) + assert result.exit_code == 2, result.output + + +def test_cli_env_check_no_drift(tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + assert runner.invoke(app, ["env", "lock"]).exit_code == 0 + # Nothing changed between snapshots, so the env is ABI-clean. + result = runner.invoke(app, ["env", "check"]) + assert result.exit_code == 0, result.output + assert "ABI-clean" in result.output + + +def test_cli_env_check_missing_lock(tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["env", "check"]) + assert result.exit_code == 1 + assert "soup env lock" in result.output + + +def test_cli_env_check_drift_exits_3(tmp_path, monkeypatch): + from soup_cli.cli import app + from soup_cli.utils.env_lock import snapshot_env, write_lock + + monkeypatch.chdir(tmp_path) + # Write a lock that claims a different Python version → ABI drift on check. + env = snapshot_env() + drifted = dataclasses.replace(env, python_version="2.0.0") + write_lock(drifted, "soup-env.lock") + result = runner.invoke(app, ["env", "check"]) + assert result.exit_code == 3, result.output + + +def test_cli_env_lock_null_byte_output_rejected(tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["env", "lock", "--output", "a\x00b"]) + assert result.exit_code == 2, result.output diff --git a/tests/test_v0670_part_e.py b/tests/test_v0670_part_e.py index e2824b7..ff74bb1 100644 --- a/tests/test_v0670_part_e.py +++ b/tests/test_v0670_part_e.py @@ -400,6 +400,104 @@ class TestCliSmoke: written = json.loads((tmp_path / "soup.lock").read_text(encoding="utf-8")) assert written["env_hash"] == "c" * 64 + def _write_lock(self, runner, tmp_path) -> None: + from soup_cli.commands.lock import app + + result = runner.invoke( + app, + [ + "write", + "--base-model", "test-model", + "--base-sha", "a" * 64, + "--dataset-sha", "b" * 64, + "--env-hash", "c" * 64, + "--output", "soup.lock", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_lock_show_command(self, tmp_path, monkeypatch) -> None: + from typer.testing import CliRunner + + from soup_cli.commands.lock import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + self._write_lock(runner, tmp_path) + result = runner.invoke(app, ["show", "soup.lock"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "test-model" in result.output + + def test_lock_show_missing_file(self, tmp_path, monkeypatch) -> None: + from typer.testing import CliRunner + + from soup_cli.commands.lock import app + + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke(app, ["show", "nonexistent.lock"]) + assert result.exit_code == 2 + + def test_lock_check_no_drift(self, tmp_path, monkeypatch) -> None: + from typer.testing import CliRunner + + from soup_cli.commands.lock import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + self._write_lock(runner, tmp_path) + result = runner.invoke( + app, + [ + "check", "soup.lock", + "--base-model", "test-model", + "--base-sha", "a" * 64, + "--dataset-sha", "b" * 64, + "--env-hash", "c" * 64, + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "OK" in result.output + + def test_lock_check_drift_exits_3(self, tmp_path, monkeypatch) -> None: + from typer.testing import CliRunner + + from soup_cli.commands.lock import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + self._write_lock(runner, tmp_path) + # A changed base-sha drifts both base_model_sha and the closure. + result = runner.invoke( + app, + [ + "check", "soup.lock", + "--base-model", "test-model", + "--base-sha", "d" * 64, + "--dataset-sha", "b" * 64, + "--env-hash", "c" * 64, + ], + ) + assert result.exit_code == 3 + assert "DRIFT" in result.output + + def test_lock_check_missing_file(self, tmp_path, monkeypatch) -> None: + from typer.testing import CliRunner + + from soup_cli.commands.lock import app + + monkeypatch.chdir(tmp_path) + result = CliRunner().invoke( + app, + [ + "check", "nonexistent.lock", + "--base-model", "test-model", + "--base-sha", "a" * 64, + "--dataset-sha", "b" * 64, + "--env-hash", "c" * 64, + ], + ) + assert result.exit_code == 2 + class TestSourceWiring: def test_no_top_level_heavy_imports(self) -> None: