mirror of https://github.com/razor-ai/soup.git
371 lines
13 KiB
Python
371 lines
13 KiB
Python
"""Tests for v0.40.2 Part A — originally scheduled issues (#36, #50, #51)."""
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from io import StringIO
|
|
|
|
import pytest
|
|
from rich.console import Console
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
|
|
|
|
def _plain(s: str) -> str:
|
|
"""Strip ANSI escape sequences AND any whitespace inserted by Rich's
|
|
width-aware option-name wrapping (e.g. ``-\\n-template-dir`` on narrow
|
|
CI terminals).
|
|
"""
|
|
no_ansi = re.sub(r"\x1b\[[0-9;]*[a-zA-Z]", "", s)
|
|
return re.sub(r"\s+", "", no_ansi)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# #50 — `--hf-resume` prefer local newer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestHFResumePreferLocal:
|
|
def test_find_highest_local_checkpoint_empty(self, tmp_path):
|
|
from soup_cli.monitoring.hf_push import _find_highest_local_checkpoint
|
|
|
|
assert _find_highest_local_checkpoint(str(tmp_path)) is None
|
|
|
|
def test_find_highest_local_checkpoint_picks_max(self, tmp_path):
|
|
from soup_cli.monitoring.hf_push import _find_highest_local_checkpoint
|
|
|
|
(tmp_path / "checkpoint-100").mkdir()
|
|
(tmp_path / "checkpoint-50").mkdir()
|
|
(tmp_path / "checkpoint-200").mkdir()
|
|
# Spurious dirs ignored
|
|
(tmp_path / "garbage").mkdir()
|
|
(tmp_path / "checkpoint-bad").mkdir()
|
|
# Files (not dirs) ignored
|
|
(tmp_path / "checkpoint-999").write_text("not a dir")
|
|
|
|
assert _find_highest_local_checkpoint(str(tmp_path)) == 200
|
|
|
|
def test_find_highest_local_checkpoint_missing_dir(self, tmp_path):
|
|
from soup_cli.monitoring.hf_push import _find_highest_local_checkpoint
|
|
|
|
assert _find_highest_local_checkpoint(str(tmp_path / "nope")) is None
|
|
|
|
def test_prepare_hf_resume_skips_download_when_local_equal(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
"""If local checkpoint-N == remote checkpoint-N, skip download."""
|
|
from soup_cli.monitoring.hf_push import prepare_hf_resume
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
out_dir = tmp_path / "runs"
|
|
out_dir.mkdir()
|
|
(out_dir / "checkpoint-300").mkdir()
|
|
|
|
download_called = {"called": False}
|
|
|
|
def fake_download(*args, **kwargs):
|
|
download_called["called"] = True
|
|
return kwargs.get("local_dir", "")
|
|
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push.resolve_latest_checkpoint_revision",
|
|
lambda repo_id, token=None, endpoint=None: "checkpoint-300",
|
|
)
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push._download_checkpoint", fake_download
|
|
)
|
|
|
|
result = prepare_hf_resume(
|
|
repo_id="user/my-model", output_dir=str(out_dir), token="t1"
|
|
)
|
|
assert download_called["called"] is False
|
|
assert result is not None
|
|
assert "checkpoint-300" in result
|
|
|
|
def test_prepare_hf_resume_skips_download_when_local_newer(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
from soup_cli.monitoring.hf_push import prepare_hf_resume
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
out_dir = tmp_path / "runs"
|
|
out_dir.mkdir()
|
|
(out_dir / "checkpoint-500").mkdir()
|
|
|
|
download_called = {"called": False}
|
|
|
|
def fake_download(*args, **kwargs):
|
|
download_called["called"] = True
|
|
return kwargs.get("local_dir", "")
|
|
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push.resolve_latest_checkpoint_revision",
|
|
lambda repo_id, token=None, endpoint=None: "checkpoint-300",
|
|
)
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push._download_checkpoint", fake_download
|
|
)
|
|
|
|
result = prepare_hf_resume(
|
|
repo_id="user/my-model", output_dir=str(out_dir), token="t1"
|
|
)
|
|
assert download_called["called"] is False
|
|
assert result is not None
|
|
assert "checkpoint-500" in result
|
|
|
|
def test_prepare_hf_resume_downloads_when_remote_newer(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
from soup_cli.monitoring.hf_push import prepare_hf_resume
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
out_dir = tmp_path / "runs"
|
|
out_dir.mkdir()
|
|
(out_dir / "checkpoint-100").mkdir()
|
|
|
|
download_called = {"called": False, "revision": None}
|
|
|
|
def fake_download(repo_id, revision, local_dir, token, endpoint):
|
|
download_called["called"] = True
|
|
download_called["revision"] = revision
|
|
return local_dir
|
|
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push.resolve_latest_checkpoint_revision",
|
|
lambda repo_id, token=None, endpoint=None: "checkpoint-500",
|
|
)
|
|
monkeypatch.setattr(
|
|
"soup_cli.monitoring.hf_push._download_checkpoint", fake_download
|
|
)
|
|
|
|
result = prepare_hf_resume(
|
|
repo_id="user/my-model", output_dir=str(out_dir), token="t1"
|
|
)
|
|
assert download_called["called"] is True
|
|
assert download_called["revision"] == "checkpoint-500"
|
|
assert result is not None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# #51 — `soup deploy hf-space --template-dir`
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestHfSpaceCustomTemplate:
|
|
def test_render_custom_template_dir(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "mytpl"
|
|
tdir.mkdir()
|
|
(tdir / "app.py").write_text("MODEL = '{MODEL_REPO}'\n")
|
|
(tdir / "README.md").write_text("# Space for {MODEL_REPO}\n")
|
|
(tdir / "requirements.txt").write_text("gradio\n")
|
|
|
|
rendered = render_custom_template_dir(
|
|
str(tdir), model_repo="user/my-model"
|
|
)
|
|
assert rendered["app.py"] == "MODEL = 'user/my-model'\n"
|
|
assert rendered["README.md"] == "# Space for user/my-model\n"
|
|
assert rendered["requirements.txt"] == "gradio\n"
|
|
|
|
def test_render_custom_template_rejects_outside_cwd(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
cwd = tmp_path / "project"
|
|
cwd.mkdir()
|
|
monkeypatch.chdir(cwd)
|
|
outside = tmp_path / "elsewhere"
|
|
outside.mkdir()
|
|
(outside / "app.py").write_text("x")
|
|
(outside / "README.md").write_text("x")
|
|
|
|
with pytest.raises(ValueError, match="under the current"):
|
|
render_custom_template_dir(str(outside), model_repo="user/my-model")
|
|
|
|
def test_render_custom_template_rejects_invalid_repo_id(
|
|
self, tmp_path, monkeypatch
|
|
):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "tpl"
|
|
tdir.mkdir()
|
|
(tdir / "app.py").write_text("x")
|
|
(tdir / "README.md").write_text("x")
|
|
|
|
with pytest.raises(ValueError):
|
|
render_custom_template_dir(str(tdir), model_repo="bad..repo")
|
|
|
|
def test_render_custom_template_requires_app_py(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "tpl"
|
|
tdir.mkdir()
|
|
(tdir / "README.md").write_text("x")
|
|
|
|
with pytest.raises(FileNotFoundError, match="app.py"):
|
|
render_custom_template_dir(str(tdir), model_repo="user/my-model")
|
|
|
|
def test_render_custom_template_requires_readme(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "tpl"
|
|
tdir.mkdir()
|
|
(tdir / "app.py").write_text("x")
|
|
|
|
with pytest.raises(FileNotFoundError, match="README.md"):
|
|
render_custom_template_dir(str(tdir), model_repo="user/my-model")
|
|
|
|
def test_render_custom_template_size_cap(self, tmp_path, monkeypatch):
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "tpl"
|
|
tdir.mkdir()
|
|
# 256KB+1 byte
|
|
big = "a" * (256 * 1024 + 1)
|
|
(tdir / "app.py").write_text(big)
|
|
(tdir / "README.md").write_text("x")
|
|
|
|
with pytest.raises(ValueError, match="256"):
|
|
render_custom_template_dir(str(tdir), model_repo="user/my-model")
|
|
|
|
def test_render_custom_template_rejects_symlink(self, tmp_path, monkeypatch):
|
|
"""Symlinked app.py must be rejected — TOCTOU defence."""
|
|
import os
|
|
import sys
|
|
|
|
if sys.platform == "win32":
|
|
pytest.skip("symlinks require admin on Windows")
|
|
|
|
from soup_cli.utils.hf_space import render_custom_template_dir
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
tdir = tmp_path / "tpl"
|
|
tdir.mkdir()
|
|
# Real app.py outside, symlink inside
|
|
evil_target = tmp_path / "evil_app.py"
|
|
evil_target.write_text("evil\n")
|
|
os.symlink(str(evil_target), str(tdir / "app.py"))
|
|
(tdir / "README.md").write_text("x")
|
|
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
render_custom_template_dir(str(tdir), model_repo="user/my-model")
|
|
|
|
def test_deploy_hf_space_help_shows_template_dir(self):
|
|
result = runner.invoke(app, ["deploy", "hf-space", "--help"])
|
|
# Rich wraps long option names across lines on narrow CI terminals;
|
|
# strip ANSI + whitespace to match the option name regardless.
|
|
assert "--template-dir" in _plain(result.output)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# #36 — Eval-gate dashboard row
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestEvalGateDashboardRow:
|
|
def test_format_gate_row_disabled(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
# No state -> empty string
|
|
assert format_gate_row(None) == ""
|
|
|
|
def test_format_gate_row_pass(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
state = {
|
|
"tasks": [
|
|
{"name": "helpfulness", "score": 7.8, "passed": True},
|
|
],
|
|
"overall_passed": True,
|
|
"action": None,
|
|
}
|
|
out = format_gate_row(state)
|
|
assert "Gate" in out
|
|
assert "helpfulness" in out
|
|
assert "7.8" in out
|
|
|
|
def test_format_gate_row_regression_stop(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
state = {
|
|
"tasks": [
|
|
{
|
|
"name": "math",
|
|
"score": 0.82,
|
|
"passed": False,
|
|
"delta": -0.06,
|
|
"baseline": 0.88,
|
|
},
|
|
],
|
|
"overall_passed": False,
|
|
"action": "stop",
|
|
}
|
|
out = format_gate_row(state)
|
|
assert "math" in out
|
|
assert "0.82" in out
|
|
assert "STOP" in out.upper() or "stop" in out
|
|
|
|
def test_format_gate_row_warn_action(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
state = {
|
|
"tasks": [{"name": "t", "score": 0.5, "passed": False, "delta": -0.1}],
|
|
"overall_passed": False,
|
|
"action": "warn",
|
|
}
|
|
out = format_gate_row(state)
|
|
assert "WARN" in out
|
|
|
|
def test_format_gate_row_multi_task(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
state = {
|
|
"tasks": [
|
|
{"name": "helpfulness", "score": 7.8, "passed": True},
|
|
{"name": "math", "score": 0.82, "passed": False, "delta": -0.06},
|
|
],
|
|
"overall_passed": False,
|
|
"action": "stop",
|
|
}
|
|
out = format_gate_row(state)
|
|
assert "helpfulness" in out
|
|
assert "math" in out
|
|
assert "|" in out # separator
|
|
assert "STOP" in out
|
|
|
|
def test_format_gate_row_passed_missing_field_renders_neutral(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
# `passed` absent — should not silently render as red ✗
|
|
state = {
|
|
"tasks": [{"name": "t", "score": 0.5}],
|
|
"overall_passed": True,
|
|
"action": None,
|
|
}
|
|
out = format_gate_row(state)
|
|
# No action suffix (None), and the task row renders without
|
|
# falsely claiming pass.
|
|
assert "STOP" not in out
|
|
assert "WARN" not in out
|
|
|
|
def test_format_gate_row_renders_via_console(self):
|
|
from soup_cli.monitoring.display import format_gate_row
|
|
|
|
state = {
|
|
"tasks": [{"name": "t", "score": 1.0, "passed": True}],
|
|
"overall_passed": True,
|
|
"action": None,
|
|
}
|
|
buf = StringIO()
|
|
Console(file=buf, force_terminal=False, no_color=True, width=120).print(
|
|
format_gate_row(state)
|
|
)
|
|
assert "t" in buf.getvalue()
|