mirror of https://github.com/razor-ai/soup.git
390 lines
13 KiB
Python
390 lines
13 KiB
Python
"""v0.67.0 Part D — ``soup adapters pr`` (GitHub-shaped PR rendering).
|
|
|
|
Tests for ``soup_cli/utils/adapter_pr.py``:
|
|
|
|
- Frozen ``EvalDelta`` / ``SampleDiff`` / ``AdapterPR`` dataclasses
|
|
- ``build_adapter_pr`` factory with validation
|
|
- ``render_pr_markdown`` (eval-delta tables + sample diffs)
|
|
- ``render_pr_json`` round-trip
|
|
- Rich-markup escape on every operator-controlled field
|
|
- CLI smoke (`soup adapters pr`)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import math
|
|
|
|
import pytest
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Public surface
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestPublicSurface:
|
|
def test_module_importable(self) -> None:
|
|
from soup_cli.utils import adapter_pr
|
|
|
|
assert hasattr(adapter_pr, "AdapterPR")
|
|
assert hasattr(adapter_pr, "EvalDelta")
|
|
assert hasattr(adapter_pr, "SampleDiff")
|
|
assert hasattr(adapter_pr, "build_adapter_pr")
|
|
assert hasattr(adapter_pr, "render_pr_markdown")
|
|
assert hasattr(adapter_pr, "render_pr_json")
|
|
assert hasattr(adapter_pr, "write_pr_markdown")
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# EvalDelta
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestEvalDelta:
|
|
def test_construct(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
delta = EvalDelta(metric="accuracy", baseline=0.7, candidate=0.85)
|
|
assert math.isclose(delta.delta, 0.15)
|
|
|
|
def test_frozen(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
d = EvalDelta(metric="m", baseline=0.5, candidate=0.6)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
d.candidate = 0.99 # type: ignore[misc]
|
|
|
|
def test_non_finite_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
with pytest.raises(ValueError):
|
|
EvalDelta(metric="m", baseline=math.nan, candidate=0.5)
|
|
with pytest.raises(ValueError):
|
|
EvalDelta(metric="m", baseline=0.5, candidate=math.inf)
|
|
|
|
def test_bool_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
with pytest.raises(TypeError):
|
|
EvalDelta(metric="m", baseline=True, candidate=0.5) # type: ignore[arg-type]
|
|
|
|
def test_metric_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
with pytest.raises(ValueError):
|
|
EvalDelta(metric="m\x00", baseline=0.5, candidate=0.6)
|
|
|
|
def test_metric_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import EvalDelta
|
|
|
|
with pytest.raises(ValueError):
|
|
EvalDelta(metric="a" * 300, baseline=0.5, candidate=0.6)
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# SampleDiff
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestSampleDiff:
|
|
def test_construct(self) -> None:
|
|
from soup_cli.utils.adapter_pr import SampleDiff
|
|
|
|
diff = SampleDiff(
|
|
prompt="What is 2+2?",
|
|
baseline_output="four",
|
|
candidate_output="4",
|
|
)
|
|
assert diff.prompt.startswith("What")
|
|
|
|
def test_frozen(self) -> None:
|
|
from soup_cli.utils.adapter_pr import SampleDiff
|
|
|
|
d = SampleDiff(prompt="p", baseline_output="b", candidate_output="c")
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
d.prompt = "new" # type: ignore[misc]
|
|
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import SampleDiff
|
|
|
|
with pytest.raises(ValueError):
|
|
SampleDiff(prompt="p\x00", baseline_output="b", candidate_output="c")
|
|
|
|
def test_oversize_truncated(self) -> None:
|
|
from soup_cli.utils.adapter_pr import MAX_OUTPUT_LEN, SampleDiff
|
|
|
|
# >MAX_OUTPUT_LEN should be rejected to keep PRs reviewable
|
|
with pytest.raises(ValueError):
|
|
SampleDiff(
|
|
prompt="p" * 10,
|
|
baseline_output="x" * (MAX_OUTPUT_LEN + 1),
|
|
candidate_output="y",
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# AdapterPR
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestAdapterPR:
|
|
def test_construct(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR, EvalDelta, SampleDiff
|
|
|
|
pr = AdapterPR(
|
|
title="add-customer-support-tone",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="+ 100 rows of support data\n",
|
|
deltas=(EvalDelta(metric="accuracy", baseline=0.7, candidate=0.85),),
|
|
samples=(
|
|
SampleDiff(prompt="hi", baseline_output="hello", candidate_output="hey"),
|
|
),
|
|
)
|
|
assert pr.title == "add-customer-support-tone"
|
|
|
|
def test_frozen(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR
|
|
|
|
pr = AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
pr.title = "new" # type: ignore[misc]
|
|
|
|
def test_base_sha_64_hex(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR
|
|
|
|
# Not 64 hex chars
|
|
with pytest.raises(ValueError):
|
|
AdapterPR(
|
|
title="t",
|
|
base_sha="abc",
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
# Non-hex characters
|
|
with pytest.raises(ValueError):
|
|
AdapterPR(
|
|
title="t",
|
|
base_sha="z" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
|
|
def test_title_validation(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR
|
|
|
|
with pytest.raises(ValueError):
|
|
AdapterPR(
|
|
title="",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
|
|
def test_deltas_must_be_tuple(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR
|
|
|
|
with pytest.raises(TypeError):
|
|
AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=[], # type: ignore[arg-type]
|
|
samples=(),
|
|
)
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# build_adapter_pr factory
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestBuildAdapterPR:
|
|
def test_happy(self) -> None:
|
|
from soup_cli.utils.adapter_pr import build_adapter_pr
|
|
|
|
pr = build_adapter_pr(
|
|
title="my-pr",
|
|
base_sha="b" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="diff text",
|
|
deltas=[
|
|
{"metric": "accuracy", "baseline": 0.7, "candidate": 0.85},
|
|
],
|
|
samples=[
|
|
{"prompt": "p", "baseline_output": "a", "candidate_output": "b"},
|
|
],
|
|
)
|
|
assert pr.title == "my-pr"
|
|
assert len(pr.deltas) == 1
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# render_pr_markdown / render_pr_json
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestRenderPR:
|
|
def test_markdown_structure(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR, EvalDelta, SampleDiff, render_pr_markdown
|
|
|
|
pr = AdapterPR(
|
|
title="my-pr",
|
|
base_sha="c" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="+row1\n+row2\n",
|
|
deltas=(EvalDelta(metric="accuracy", baseline=0.7, candidate=0.85),),
|
|
samples=(
|
|
SampleDiff(prompt="hi", baseline_output="a", candidate_output="b"),
|
|
),
|
|
)
|
|
md = render_pr_markdown(pr)
|
|
assert "my-pr" in md
|
|
assert "accuracy" in md
|
|
assert "0.70" in md or "0.7" in md
|
|
# Should include a sample diff section
|
|
assert "hi" in md
|
|
# Should include the dataset diff
|
|
assert "+row1" in md
|
|
|
|
def test_markdown_escapes_markdown_metacharacters(self) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR, EvalDelta, render_pr_markdown
|
|
|
|
# Embed pipe character in metric name; rendered table cell must escape it
|
|
pr = AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(EvalDelta(metric="acc|injection", baseline=0.5, candidate=0.6),),
|
|
samples=(),
|
|
)
|
|
md = render_pr_markdown(pr)
|
|
# The pipe must be escaped to avoid breaking the table
|
|
assert "acc\\|injection" in md or "acc|injection" not in md.split("|")
|
|
|
|
def test_json_roundtrip(self) -> None:
|
|
import json
|
|
|
|
from soup_cli.utils.adapter_pr import (
|
|
AdapterPR,
|
|
EvalDelta,
|
|
render_pr_json,
|
|
)
|
|
|
|
pr = AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(EvalDelta(metric="m", baseline=0.5, candidate=0.6),),
|
|
samples=(),
|
|
)
|
|
text = render_pr_json(pr)
|
|
data = json.loads(text)
|
|
assert data["title"] == "t"
|
|
assert data["deltas"][0]["metric"] == "m"
|
|
|
|
def test_non_pr_rejected(self) -> None:
|
|
from soup_cli.utils.adapter_pr import render_pr_json, render_pr_markdown
|
|
|
|
with pytest.raises(TypeError):
|
|
render_pr_markdown("not-a-pr") # type: ignore[arg-type]
|
|
with pytest.raises(TypeError):
|
|
render_pr_json("not-a-pr") # type: ignore[arg-type]
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Atomic write
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestWritePR:
|
|
def test_write_markdown(self, tmp_path, monkeypatch) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR, write_pr_markdown
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
pr = AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
out = tmp_path / "pr.md"
|
|
write_pr_markdown(pr, str(out))
|
|
assert out.exists()
|
|
assert "t" in out.read_text(encoding="utf-8")
|
|
|
|
def test_write_outside_cwd_rejected(self, tmp_path, monkeypatch) -> None:
|
|
from soup_cli.utils.adapter_pr import AdapterPR, write_pr_markdown
|
|
|
|
cwd = tmp_path / "work"
|
|
cwd.mkdir()
|
|
monkeypatch.chdir(cwd)
|
|
pr = AdapterPR(
|
|
title="t",
|
|
base_sha="a" * 64,
|
|
adapter_path="adapter/",
|
|
dataset_diff="",
|
|
deltas=(),
|
|
samples=(),
|
|
)
|
|
with pytest.raises(ValueError):
|
|
write_pr_markdown(pr, str(tmp_path / "outside.md"))
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# CLI smoke
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestCliSmoke:
|
|
def test_pr_help(self) -> None:
|
|
from typer.testing import CliRunner
|
|
|
|
from soup_cli.commands.adapters import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["pr", "--help"])
|
|
assert result.exit_code == 0
|
|
assert "pr" in result.output.lower()
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Source-grep regression
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class TestSourceWiring:
|
|
def test_no_top_level_heavy_imports(self) -> None:
|
|
from pathlib import Path
|
|
|
|
root = Path(__file__).resolve().parent.parent
|
|
src = (root / "src" / "soup_cli" / "utils" / "adapter_pr.py").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
head_lines = [
|
|
line
|
|
for line in src.splitlines()[:50]
|
|
if line.strip() and not line.strip().startswith("#")
|
|
]
|
|
head = "\n".join(head_lines)
|
|
for forbidden in ("import torch", "import transformers", "import peft"):
|
|
assert forbidden not in head, f"top-level {forbidden!r}"
|