mirror of https://github.com/razor-ai/soup.git
122 lines
3.8 KiB
Python
122 lines
3.8 KiB
Python
"""Tests for soup runs CLI commands."""
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
from soup_cli.experiment.tracker import ExperimentTracker
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _use_temp_db(tmp_path: Path, monkeypatch):
|
|
"""Redirect experiment DB to a temp directory for all tests."""
|
|
db_path = str(tmp_path / "test_experiments.db")
|
|
monkeypatch.setenv("SOUP_DB_PATH", db_path)
|
|
|
|
|
|
@pytest.fixture
|
|
def tracker(tmp_path: Path) -> ExperimentTracker:
|
|
"""Create tracker using the same temp DB."""
|
|
import os
|
|
|
|
db_path = Path(os.environ["SOUP_DB_PATH"])
|
|
return ExperimentTracker(db_path=db_path)
|
|
|
|
|
|
def test_runs_empty():
|
|
"""soup runs with no runs should show a message, not crash."""
|
|
result = runner.invoke(app, ["runs"])
|
|
assert result.exit_code == 0
|
|
assert "No runs found" in result.output
|
|
|
|
|
|
def test_runs_list_with_data(tracker):
|
|
"""soup runs should show a table when runs exist."""
|
|
run_id = tracker.start_run(
|
|
config_dict={"base": "test-model", "task": "sft"},
|
|
device="cpu",
|
|
device_name="CPU",
|
|
gpu_info={"memory_total": "N/A"},
|
|
experiment_name="test-exp",
|
|
)
|
|
result = runner.invoke(app, ["runs"])
|
|
assert result.exit_code == 0
|
|
assert "Training Runs" in result.output
|
|
# Run ID column has no_wrap so it should always be visible
|
|
assert run_id in result.output
|
|
|
|
|
|
def test_runs_show_not_found():
|
|
"""soup runs show nonexistent should fail."""
|
|
result = runner.invoke(app, ["runs", "show", "nonexistent_run_id"])
|
|
assert result.exit_code == 1
|
|
assert "not found" in result.output.lower()
|
|
|
|
|
|
def test_runs_show_with_data(tracker):
|
|
"""soup runs show should display run details."""
|
|
run_id = tracker.start_run(
|
|
config_dict={"base": "test-model", "task": "sft"},
|
|
device="cpu",
|
|
device_name="CPU",
|
|
gpu_info={"memory_total": "16 GB"},
|
|
)
|
|
tracker.finish_run(
|
|
run_id=run_id,
|
|
initial_loss=2.5,
|
|
final_loss=0.8,
|
|
total_steps=100,
|
|
duration_secs=600.0,
|
|
output_dir="/tmp/output",
|
|
)
|
|
result = runner.invoke(app, ["runs", "show", run_id, "--no-plot"])
|
|
assert result.exit_code == 0
|
|
assert run_id in result.output
|
|
assert "test-model" in result.output
|
|
|
|
|
|
def test_runs_compare_not_found():
|
|
"""soup runs compare with bad IDs should fail."""
|
|
result = runner.invoke(app, ["runs", "compare", "run_1", "run_2"])
|
|
assert result.exit_code == 1
|
|
|
|
|
|
def test_runs_compare_with_data(tracker):
|
|
"""soup runs compare should show side-by-side table."""
|
|
id1 = tracker.start_run(
|
|
config_dict={"base": "model-a", "task": "sft", "training": {"epochs": 3, "lr": 2e-5}},
|
|
device="cpu", device_name="CPU", gpu_info={},
|
|
experiment_name="exp-a",
|
|
)
|
|
id2 = tracker.start_run(
|
|
config_dict={"base": "model-b", "task": "dpo", "training": {"epochs": 5, "lr": 1e-5}},
|
|
device="cuda", device_name="RTX 4090", gpu_info={},
|
|
experiment_name="exp-b",
|
|
)
|
|
result = runner.invoke(app, ["runs", "compare", id1, id2])
|
|
assert result.exit_code == 0
|
|
assert "model-a" in result.output
|
|
assert "model-b" in result.output
|
|
|
|
|
|
def test_runs_delete_not_found():
|
|
"""soup runs delete nonexistent should fail."""
|
|
result = runner.invoke(app, ["runs", "delete", "nonexistent", "--force"])
|
|
assert result.exit_code == 1
|
|
|
|
|
|
def test_runs_delete_with_data(tracker):
|
|
"""soup runs delete should remove the run."""
|
|
run_id = tracker.start_run(
|
|
config_dict={}, device="cpu", device_name="CPU", gpu_info={},
|
|
)
|
|
result = runner.invoke(app, ["runs", "delete", run_id, "--force"])
|
|
assert result.exit_code == 0
|
|
assert "Deleted" in result.output
|
|
# Verify it's gone
|
|
assert tracker.get_run(run_id) is None
|