mirror of https://github.com/razor-ai/soup.git
550 lines
19 KiB
Python
550 lines
19 KiB
Python
"""v0.68.0 Part A — ``soup compile`` (DSPy/GEPA prompt-program compiler).
|
|
|
|
Tests for ``soup_cli/utils/prompt_compile.py`` + ``soup_cli/commands/compile_cmd.py``.
|
|
|
|
Coverage:
|
|
- Closed ``SUPPORTED_PROMPT_OPTIMIZERS`` allowlist (frozenset, immutable)
|
|
- ``validate_prompt_optimizer`` happy + bool/null-byte/oversize/empty/non-string/unknown
|
|
- ``validate_max_iters`` bounds + bool rejection
|
|
- ``validate_program_path`` + ``validate_eval_suite_path`` containment + symlink reject
|
|
- Frozen ``CompilePlan`` + ``CompileResult`` + FrozenInstanceError
|
|
- ``build_compile_plan`` factory
|
|
- ``run_compile`` deferred-live stub raises NotImplementedError w/ v0.68.1 marker
|
|
- CLI smoke: help + plan-only + unknown optimizer + missing program
|
|
- Source-grep: no heavy top-level imports (torch / transformers / dspy)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Public surface
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestPublicSurface:
|
|
def test_module_importable(self) -> None:
|
|
from soup_cli.utils import prompt_compile
|
|
|
|
assert hasattr(prompt_compile, "SUPPORTED_PROMPT_OPTIMIZERS")
|
|
assert hasattr(prompt_compile, "validate_prompt_optimizer")
|
|
assert hasattr(prompt_compile, "validate_max_iters")
|
|
assert hasattr(prompt_compile, "validate_program_path")
|
|
assert hasattr(prompt_compile, "validate_eval_suite_path")
|
|
assert hasattr(prompt_compile, "CompilePlan")
|
|
assert hasattr(prompt_compile, "CompileResult")
|
|
assert hasattr(prompt_compile, "build_compile_plan")
|
|
assert hasattr(prompt_compile, "run_compile")
|
|
|
|
def test_allowlist_is_frozenset(self) -> None:
|
|
from soup_cli.utils.prompt_compile import SUPPORTED_PROMPT_OPTIMIZERS
|
|
|
|
assert isinstance(SUPPORTED_PROMPT_OPTIMIZERS, frozenset)
|
|
# Allowlist should cover the canonical DSPy / GEPA / TextGrad set.
|
|
assert "bootstrap_fewshot" in SUPPORTED_PROMPT_OPTIMIZERS
|
|
assert "mipro" in SUPPORTED_PROMPT_OPTIMIZERS
|
|
assert "copro" in SUPPORTED_PROMPT_OPTIMIZERS
|
|
assert "gepa" in SUPPORTED_PROMPT_OPTIMIZERS
|
|
assert "textgrad" in SUPPORTED_PROMPT_OPTIMIZERS
|
|
|
|
def test_allowlist_immutable(self) -> None:
|
|
from soup_cli.utils.prompt_compile import SUPPORTED_PROMPT_OPTIMIZERS
|
|
|
|
with pytest.raises(AttributeError):
|
|
SUPPORTED_PROMPT_OPTIMIZERS.add("evil") # type: ignore[attr-defined]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_prompt_optimizer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidatePromptOptimizer:
|
|
def test_happy(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
assert validate_prompt_optimizer("mipro") == "mipro"
|
|
|
|
def test_case_insensitive(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
assert validate_prompt_optimizer("MIPRO") == "mipro"
|
|
assert validate_prompt_optimizer("Gepa") == "gepa"
|
|
|
|
def test_bool_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_prompt_optimizer(True) # type: ignore[arg-type]
|
|
|
|
def test_non_string_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_prompt_optimizer(42) # type: ignore[arg-type]
|
|
|
|
def test_empty_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_prompt_optimizer("")
|
|
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_prompt_optimizer("mipro\x00")
|
|
|
|
def test_oversize_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_prompt_optimizer("a" * 33)
|
|
|
|
def test_unknown_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_prompt_optimizer
|
|
|
|
with pytest.raises(ValueError, match="unknown"):
|
|
validate_prompt_optimizer("evil-optimizer")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_max_iters
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateMaxIters:
|
|
def test_happy(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_max_iters
|
|
|
|
assert validate_max_iters(10) == 10
|
|
|
|
def test_bool_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_max_iters
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_max_iters(True) # type: ignore[arg-type]
|
|
|
|
def test_non_int_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_max_iters
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_max_iters(3.14) # type: ignore[arg-type]
|
|
|
|
def test_zero_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_max_iters
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_max_iters(0)
|
|
|
|
def test_negative_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_max_iters
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_max_iters(-1)
|
|
|
|
def test_overcap_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import (
|
|
MAX_COMPILE_ITERS,
|
|
validate_max_iters,
|
|
)
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_max_iters(MAX_COMPILE_ITERS + 1)
|
|
|
|
def test_max_boundary_accepted(self) -> None:
|
|
from soup_cli.utils.prompt_compile import (
|
|
MAX_COMPILE_ITERS,
|
|
validate_max_iters,
|
|
)
|
|
|
|
assert validate_max_iters(MAX_COMPILE_ITERS) == MAX_COMPILE_ITERS
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# validate_program_path + validate_eval_suite_path
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestValidateProgramPath:
|
|
def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_program_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "program.py"
|
|
prog.write_text("# dspy program\n", encoding="utf-8")
|
|
assert validate_program_path(str(prog)).endswith("program.py")
|
|
|
|
def test_outside_cwd_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_program_path
|
|
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
prog = outside / "program.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
sub = tmp_path / "sub"
|
|
sub.mkdir()
|
|
monkeypatch.chdir(sub)
|
|
with pytest.raises(ValueError):
|
|
validate_program_path(str(prog))
|
|
|
|
def test_extension_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_program_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
bad = tmp_path / "program.txt"
|
|
bad.write_text("# wrong\n", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="\\.py"):
|
|
validate_program_path(str(bad))
|
|
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_program_path
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_program_path("a\x00b.py")
|
|
|
|
@pytest.mark.skipif(
|
|
not hasattr(__import__("os"), "symlink"), reason="POSIX symlink only"
|
|
)
|
|
def test_symlink_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
import os
|
|
|
|
from soup_cli.utils.prompt_compile import validate_program_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
target = tmp_path / "real.py"
|
|
target.write_text("# real\n", encoding="utf-8")
|
|
link = tmp_path / "link.py"
|
|
try:
|
|
os.symlink(target, link)
|
|
except (OSError, NotImplementedError):
|
|
pytest.skip("symlinks not creatable")
|
|
with pytest.raises(ValueError, match="symlink"):
|
|
validate_program_path(str(link))
|
|
|
|
|
|
class TestValidateEvalSuitePath:
|
|
def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_eval_suite_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
suite = tmp_path / "eval.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
assert validate_eval_suite_path(str(suite)).endswith("eval.json")
|
|
|
|
def test_outside_cwd_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.prompt_compile import validate_eval_suite_path
|
|
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
bad = outside / "suite.json"
|
|
bad.write_text("[]", encoding="utf-8")
|
|
sub = tmp_path / "sub"
|
|
sub.mkdir()
|
|
monkeypatch.chdir(sub)
|
|
with pytest.raises(ValueError):
|
|
validate_eval_suite_path(str(bad))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CompilePlan / CompileResult
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCompilePlan:
|
|
def test_frozen(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.prompt_compile import CompilePlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
|
|
plan = CompilePlan(
|
|
program_path=str(prog),
|
|
eval_suite_path=str(suite),
|
|
optimizer="mipro",
|
|
max_iters=8,
|
|
output_path="out.py",
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
plan.optimizer = "gepa" # type: ignore[misc]
|
|
|
|
def test_invalid_optimizer(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.prompt_compile import CompilePlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
|
|
with pytest.raises(ValueError):
|
|
CompilePlan(
|
|
program_path=str(prog),
|
|
eval_suite_path=str(suite),
|
|
optimizer="evil",
|
|
max_iters=8,
|
|
output_path="out.py",
|
|
)
|
|
|
|
|
|
class TestCompileResult:
|
|
def test_frozen(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
result = CompileResult(
|
|
program_text="# compiled",
|
|
score=0.85,
|
|
iterations=5,
|
|
converged=True,
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
result.score = 0.9 # type: ignore[misc]
|
|
|
|
def test_invalid_score(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(ValueError):
|
|
CompileResult(
|
|
program_text="# x", score=float("nan"), iterations=1, converged=True
|
|
)
|
|
|
|
def test_negative_iterations_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(ValueError):
|
|
CompileResult(
|
|
program_text="# x", score=0.5, iterations=-1, converged=True
|
|
)
|
|
|
|
def test_bool_iterations_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(TypeError):
|
|
CompileResult(
|
|
program_text="# x",
|
|
score=0.5,
|
|
iterations=True, # type: ignore[arg-type]
|
|
converged=True,
|
|
)
|
|
|
|
def test_non_bool_converged_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import CompileResult
|
|
|
|
with pytest.raises(TypeError):
|
|
CompileResult(
|
|
program_text="# x",
|
|
score=0.5,
|
|
iterations=1,
|
|
converged="yes", # type: ignore[arg-type]
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# build_compile_plan
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestBuildCompilePlan:
|
|
def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.utils.prompt_compile import build_compile_plan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
|
|
plan = build_compile_plan(
|
|
program_path=str(prog),
|
|
eval_suite_path=str(suite),
|
|
optimizer="mipro",
|
|
max_iters=4,
|
|
output_path="out.py",
|
|
)
|
|
assert plan.optimizer == "mipro"
|
|
assert plan.max_iters == 4
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# run_compile deferred stub
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestRunCompileDeferred:
|
|
def test_live_missing_dep_friendly_importerror(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
# v0.71.13 #225: the runner is live; with dspy absent the real
|
|
# branch raises a friendly ImportError naming the [compile] extra.
|
|
from soup_cli.utils.prompt_compile import build_compile_plan, run_compile
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("program = 1\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
plan = build_compile_plan(
|
|
program_path=str(prog),
|
|
eval_suite_path=str(suite),
|
|
optimizer="mipro",
|
|
max_iters=4,
|
|
output_path="out.py",
|
|
)
|
|
with pytest.raises(ImportError, match=r"soup-cli\[compile\]"):
|
|
run_compile(plan)
|
|
|
|
def test_non_plan_rejected(self) -> None:
|
|
from soup_cli.utils.prompt_compile import run_compile
|
|
|
|
with pytest.raises(TypeError):
|
|
run_compile("not-a-plan") # type: ignore[arg-type]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# CLI smoke
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestCli:
|
|
def test_help(self) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["compile", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
assert "compile" in result.output.lower()
|
|
|
|
def test_plan_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"compile",
|
|
str(prog),
|
|
"--eval",
|
|
str(suite),
|
|
"--optimizer",
|
|
"mipro",
|
|
"--plan-only",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
|
|
def test_unknown_optimizer_exits_2(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("pass\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"compile",
|
|
str(prog),
|
|
"--eval",
|
|
str(suite),
|
|
"--optimizer",
|
|
"evil",
|
|
"--plan-only",
|
|
],
|
|
)
|
|
assert result.exit_code == 2
|
|
|
|
def test_live_missing_dep_exits_2(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
"""v0.71.13 #225: live runner with dspy absent -> friendly exit 2."""
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
prog = tmp_path / "p.py"
|
|
prog.write_text("program = 1\n", encoding="utf-8")
|
|
suite = tmp_path / "s.json"
|
|
suite.write_text("[]", encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"compile",
|
|
str(prog),
|
|
"--eval",
|
|
str(suite),
|
|
"--optimizer",
|
|
"mipro",
|
|
],
|
|
)
|
|
assert result.exit_code == 2, (result.output, repr(result.exception))
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Source-grep regression guards
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class TestSourceWiring:
|
|
def test_no_top_level_heavy_imports(self) -> None:
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "utils" / "prompt_compile.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
# Heavy / optional deps must be lazy-imported.
|
|
for token in (
|
|
"\nimport torch",
|
|
"\nimport transformers",
|
|
"\nimport dspy",
|
|
):
|
|
assert token not in text
|
|
|
|
def test_cli_registered(self) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
names = [c.name for c in app.registered_commands]
|
|
assert "compile" in names
|
|
|
|
def test_uses_atomic_write_helper(self) -> None:
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "commands" / "compile_cmd.py"
|
|
)
|
|
# If this file is missing the test must fail loudly — Part A CLI is a
|
|
# shipped artefact, not optional. Skipping would hide a regression.
|
|
assert path.exists(), f"missing CLI command module: {path}"
|
|
text = path.read_text(encoding="utf-8")
|
|
util = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "utils" / "prompt_compile.py"
|
|
)
|
|
combined = text + util.read_text(encoding="utf-8")
|
|
assert "atomic_write_text" in combined
|