mirror of https://github.com/razor-ai/soup.git
276 lines
9.0 KiB
Python
276 lines
9.0 KiB
Python
"""v0.68.0 Part C — ``soup compile-tools``.
|
|
|
|
Generate tool schemas + descriptions optimized via TextGrad-style textual
|
|
gradients. CI runs on OpenAPI / MCP schema changes. Composes with v0.46
|
|
Agent Forge.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
from typer.testing import CliRunner
|
|
|
|
|
|
class TestPublicSurface:
|
|
def test_module_importable(self) -> None:
|
|
from soup_cli.utils import compile_tools
|
|
|
|
assert hasattr(compile_tools, "SUPPORTED_TOOL_OPTIMIZERS")
|
|
assert hasattr(compile_tools, "validate_tool_optimizer")
|
|
assert hasattr(compile_tools, "validate_spec_path")
|
|
assert hasattr(compile_tools, "ToolCompilePlan")
|
|
assert hasattr(compile_tools, "build_tool_compile_plan")
|
|
assert hasattr(compile_tools, "run_tool_compile")
|
|
|
|
|
|
class TestAllowlist:
|
|
def test_frozenset(self) -> None:
|
|
from soup_cli.utils.compile_tools import SUPPORTED_TOOL_OPTIMIZERS
|
|
|
|
assert isinstance(SUPPORTED_TOOL_OPTIMIZERS, frozenset)
|
|
assert "textgrad" in SUPPORTED_TOOL_OPTIMIZERS
|
|
assert "gepa" in SUPPORTED_TOOL_OPTIMIZERS
|
|
|
|
def test_immutable(self) -> None:
|
|
from soup_cli.utils.compile_tools import SUPPORTED_TOOL_OPTIMIZERS
|
|
|
|
with pytest.raises(AttributeError):
|
|
SUPPORTED_TOOL_OPTIMIZERS.add("x") # type: ignore[attr-defined]
|
|
|
|
|
|
class TestValidateOptimizer:
|
|
def test_happy(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
assert validate_tool_optimizer("textgrad") == "textgrad"
|
|
|
|
def test_case_insensitive(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
assert validate_tool_optimizer("GEPA") == "gepa"
|
|
|
|
def test_bool_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_tool_optimizer(True) # type: ignore[arg-type]
|
|
|
|
def test_unknown_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
with pytest.raises(ValueError, match="unknown"):
|
|
validate_tool_optimizer("evil")
|
|
|
|
def test_null_byte_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import validate_tool_optimizer
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_tool_optimizer("textgrad\x00")
|
|
|
|
|
|
class TestValidateSpecPath:
|
|
def test_json_happy(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text("{}", encoding="utf-8")
|
|
assert validate_spec_path(str(spec)).endswith("spec.json")
|
|
|
|
def test_yaml_happy(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.yaml"
|
|
spec.write_text("openapi: 3.0\n", encoding="utf-8")
|
|
assert validate_spec_path(str(spec)).endswith("spec.yaml")
|
|
|
|
def test_invalid_extension_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.txt"
|
|
spec.write_text("garbage", encoding="utf-8")
|
|
with pytest.raises(ValueError, match="extension"):
|
|
validate_spec_path(str(spec))
|
|
|
|
def test_outside_cwd_rejected(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import validate_spec_path
|
|
|
|
outside = tmp_path / "outside"
|
|
outside.mkdir()
|
|
bad = outside / "spec.json"
|
|
bad.write_text("{}", encoding="utf-8")
|
|
sub = tmp_path / "sub"
|
|
sub.mkdir()
|
|
monkeypatch.chdir(sub)
|
|
with pytest.raises(ValueError):
|
|
validate_spec_path(str(bad))
|
|
|
|
|
|
class TestToolCompilePlan:
|
|
def test_frozen(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import ToolCompilePlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text("{}", encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text("[]", encoding="utf-8")
|
|
|
|
plan = ToolCompilePlan(
|
|
spec_path=str(spec),
|
|
eval_suite_path=str(eval_suite),
|
|
optimizer="textgrad",
|
|
output_path="tools.json",
|
|
)
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
plan.optimizer = "gepa" # type: ignore[misc]
|
|
|
|
def test_invalid_optimizer(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.utils.compile_tools import ToolCompilePlan
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text("{}", encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text("[]", encoding="utf-8")
|
|
|
|
with pytest.raises(ValueError):
|
|
ToolCompilePlan(
|
|
spec_path=str(spec),
|
|
eval_suite_path=str(eval_suite),
|
|
optimizer="evil",
|
|
output_path="tools.json",
|
|
)
|
|
|
|
|
|
_VALID_OPENAPI = {
|
|
"openapi": "3.0.0",
|
|
"info": {"title": "t", "version": "1"},
|
|
"paths": {
|
|
"/w": {"get": {"operationId": "listW", "description": "List widgets"}}
|
|
},
|
|
}
|
|
|
|
|
|
class TestRunToolCompileDeferred:
|
|
def test_live_missing_dep_importerror(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
# v0.71.13 #227: live runner; textgrad absent -> friendly ImportError.
|
|
from soup_cli.utils.compile_tools import (
|
|
build_tool_compile_plan,
|
|
run_tool_compile,
|
|
)
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text('{"q":"1"}\n', encoding="utf-8")
|
|
plan = build_tool_compile_plan(
|
|
spec_path=str(spec),
|
|
eval_suite_path=str(eval_suite),
|
|
optimizer="textgrad",
|
|
output_path="tools.json",
|
|
)
|
|
with pytest.raises(ImportError, match=r"soup-cli\[compile\]"):
|
|
run_tool_compile(plan)
|
|
|
|
def test_non_plan_rejected(self) -> None:
|
|
from soup_cli.utils.compile_tools import run_tool_compile
|
|
|
|
with pytest.raises(TypeError):
|
|
run_tool_compile("not-a-plan") # type: ignore[arg-type]
|
|
|
|
|
|
class TestCli:
|
|
def test_help(self) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(app, ["compile-tools", "--help"])
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
|
|
def test_plan_only(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text(json.dumps({"openapi": "3.0"}), encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text("[]", encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
[
|
|
"compile-tools",
|
|
str(spec),
|
|
"--eval",
|
|
str(eval_suite),
|
|
"--plan-only",
|
|
],
|
|
)
|
|
assert result.exit_code == 0, (result.output, repr(result.exception))
|
|
|
|
def test_live_missing_dep_exits_2(
|
|
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
# v0.71.13 #227: live runner; textgrad absent -> friendly exit 2.
|
|
from soup_cli.cli import app
|
|
|
|
monkeypatch.chdir(tmp_path)
|
|
spec = tmp_path / "spec.json"
|
|
spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8")
|
|
eval_suite = tmp_path / "eval.jsonl"
|
|
eval_suite.write_text('{"q":"1"}\n', encoding="utf-8")
|
|
runner = CliRunner()
|
|
result = runner.invoke(
|
|
app,
|
|
["compile-tools", str(spec), "--eval", str(eval_suite)],
|
|
)
|
|
assert result.exit_code == 2, (result.output, repr(result.exception))
|
|
|
|
|
|
class TestSourceWiring:
|
|
def test_no_top_level_heavy_imports(self) -> None:
|
|
path = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli"
|
|
/ "utils"
|
|
/ "compile_tools.py"
|
|
)
|
|
text = path.read_text(encoding="utf-8")
|
|
for token in (
|
|
"\nimport torch",
|
|
"\nimport transformers",
|
|
"\nimport dspy",
|
|
"\nimport textgrad",
|
|
):
|
|
assert token not in text
|
|
|
|
def test_cli_registered(self) -> None:
|
|
from soup_cli.cli import app
|
|
|
|
names = [c.name for c in app.registered_commands]
|
|
assert "compile-tools" in names
|