mirror of https://github.com/razor-ai/soup.git
feat(mcp): tool registry + read-only handlers (v0.71.28 Part A)
This commit is contained in:
Alpamys
parent
d034b23d19
commit
3d797afcae
|
|
@ -0,0 +1,9 @@
|
|||
"""Soup MCP server (v0.71.28).
|
||||
|
||||
``soup mcp serve`` exposes Soup's read-only commands (plus two plan-only
|
||||
mutating tools) to any Model Context Protocol client over stdio.
|
||||
|
||||
The package is split so the tool *table* (:mod:`registry`) is pure Python
|
||||
with NO dependency on the ``mcp`` SDK — it is fully unit-testable on the
|
||||
light core install. Only :mod:`server` imports the SDK, lazily.
|
||||
"""
|
||||
|
|
@ -0,0 +1,528 @@
|
|||
"""Pure MCP tool registry for ``soup mcp serve`` (v0.71.28).
|
||||
|
||||
This module has **no** dependency on the ``mcp`` SDK: it defines the tool
|
||||
table (:class:`ToolSpec`), the handler functions (each a pure
|
||||
``(dict) -> dict``), and the shared security guards. :mod:`soup_cli.mcp_server.server`
|
||||
is the only file that imports the SDK, and it consumes this registry.
|
||||
|
||||
Every handler lazy-imports its light core inside the function body so that
|
||||
importing this module stays cheap and torch-free.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Mapping
|
||||
|
||||
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
|
||||
|
||||
# Default read cap for JSON tool arguments (mirrors ship/diagnose evidence).
|
||||
_MAX_JSON_BYTES = 16 * 1024 * 1024
|
||||
|
||||
# C0 control bytes (keep tab / newline / CR) + DEL, stripped from every string
|
||||
# in a handler result before it reaches the MCP client. ``rich.markup.escape``
|
||||
# only neutralises ``[...]`` markup, not raw ESC/OSC sequences a malicious
|
||||
# dataset string could smuggle into a client's terminal. Mirrors
|
||||
# ``commands/data_doctor.py::_CONTROL_STRIP_TABLE``.
|
||||
_CONTROL_STRIP_TABLE = {i: None for i in range(0x20) if i not in (0x09, 0x0A, 0x0D)}
|
||||
_CONTROL_STRIP_TABLE[0x7F] = None
|
||||
|
||||
|
||||
class McpToolError(Exception):
|
||||
"""A tool-level failure with a pre-sanitized, path-free message.
|
||||
|
||||
The MCP SDK stringifies a raised exception verbatim into an ``isError``
|
||||
result, so handlers must raise THIS (never a bare ``OSError`` whose text
|
||||
could leak a filesystem path).
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolSpec:
|
||||
"""One entry in the MCP tool table."""
|
||||
|
||||
name: str
|
||||
title: str
|
||||
description: str
|
||||
input_schema: dict
|
||||
handler: Callable[[dict], dict]
|
||||
mutating: bool = False
|
||||
|
||||
|
||||
def _sanitize(obj: Any) -> Any:
|
||||
"""Recursively strip C0/ESC/DEL bytes from every string in ``obj``.
|
||||
|
||||
Leaves non-string scalars (int/float/bool/None) untouched; recurses into
|
||||
dicts and lists. Applied to every handler result as defence-in-depth.
|
||||
"""
|
||||
if isinstance(obj, str):
|
||||
return obj.translate(_CONTROL_STRIP_TABLE)
|
||||
if isinstance(obj, Mapping):
|
||||
return {_sanitize(k): _sanitize(v) for k, v in obj.items()}
|
||||
if isinstance(obj, (list, tuple)):
|
||||
return [_sanitize(v) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
def _read_json_under_cwd(path: str, field: str, *, max_bytes: int = _MAX_JSON_BYTES) -> dict:
|
||||
"""Load a JSON object argument (cwd-contained, symlink-rejected, size-capped).
|
||||
|
||||
Opens with ``O_NOFOLLOW`` (where available) and fstats the open fd so a
|
||||
symlink swapped in after the containment check cannot redirect the read
|
||||
(TOCTOU defence, mirrors ``commands/ship.py::_load_evidence``). Raises
|
||||
:class:`McpToolError` with a path-free message on any failure.
|
||||
"""
|
||||
try:
|
||||
enforce_under_cwd_and_no_symlink(path, field)
|
||||
except Exception as exc: # ValueError / OSError from the guard
|
||||
raise McpToolError(f"{field} must be a readable file under the working directory") from exc
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
try:
|
||||
handle_fd = os.open(path, flags)
|
||||
except OSError as exc:
|
||||
raise McpToolError(f"{field} is unreadable ({type(exc).__name__})") from exc
|
||||
try:
|
||||
with os.fdopen(handle_fd, "r", encoding="utf-8") as handle:
|
||||
if os.fstat(handle.fileno()).st_size > max_bytes:
|
||||
raise McpToolError(f"{field} exceeds {max_bytes} bytes")
|
||||
payload = json.load(handle)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise McpToolError(f"{field} is not valid JSON") from exc
|
||||
except OSError as exc:
|
||||
raise McpToolError(f"{field} is unreadable ({type(exc).__name__})") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise McpToolError(f"{field} must contain a JSON object")
|
||||
return payload
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Argument helpers — every handler validates its own args (the SDK also
|
||||
# jsonschema-validates inputSchema, but handlers must not trust that alone).
|
||||
# Error messages NEVER echo raw user input (avoids ANSI/path injection).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _require_str(args: dict, key: str) -> str:
|
||||
val = args.get(key)
|
||||
if not isinstance(val, str) or not val:
|
||||
raise McpToolError(f"'{key}' must be a non-empty string")
|
||||
return val
|
||||
|
||||
|
||||
def _opt_str(args: dict, key: str) -> "str | None":
|
||||
val = args.get(key)
|
||||
if val is None:
|
||||
return None
|
||||
if not isinstance(val, str):
|
||||
raise McpToolError(f"'{key}' must be a string")
|
||||
return val
|
||||
|
||||
|
||||
def _opt_int(args: dict, key: str, default: int, *, lo: int, hi: int) -> int:
|
||||
val = args.get(key, default)
|
||||
if isinstance(val, bool) or not isinstance(val, int):
|
||||
raise McpToolError(f"'{key}' must be an integer")
|
||||
return max(lo, min(hi, val))
|
||||
|
||||
|
||||
def _enforce_data_path(path: str, field: str = "data") -> None:
|
||||
try:
|
||||
enforce_under_cwd_and_no_symlink(path, field)
|
||||
except Exception as exc: # ValueError / OSError from the guard
|
||||
raise McpToolError(
|
||||
f"'{field}' must be a readable file under the working directory"
|
||||
) from exc
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read-only tool handlers (each a pure ``(dict) -> dict``)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def tool_advise(args: dict) -> dict:
|
||||
"""`soup advise` — pre-flight PROMPT_ENG / RAG / SFT / DPO / GRPO verdict."""
|
||||
import dataclasses
|
||||
|
||||
from soup_cli.utils import advise as _advise
|
||||
|
||||
data = _require_str(args, "data")
|
||||
goal = _opt_str(args, "goal")
|
||||
_enforce_data_path(data)
|
||||
try:
|
||||
rows = _advise.load_advise_dataset(data)
|
||||
task = _advise.classify_task(rows, goal)
|
||||
profile = _advise.compute_dataset_profile(rows)
|
||||
verdict = _advise.build_verdict(profile, task, goal=goal)
|
||||
except (OSError, ValueError, TypeError) as exc:
|
||||
raise McpToolError(f"advise failed ({type(exc).__name__})") from exc
|
||||
return dataclasses.asdict(verdict)
|
||||
|
||||
|
||||
def _load_data_rows(path: str) -> list:
|
||||
from pathlib import Path
|
||||
|
||||
from soup_cli.data.loader import load_raw_data
|
||||
|
||||
_enforce_data_path(path)
|
||||
try:
|
||||
return load_raw_data(Path(path))
|
||||
except (OSError, ValueError) as exc:
|
||||
raise McpToolError(f"cannot load data ({type(exc).__name__})") from exc
|
||||
|
||||
|
||||
def tool_data_inspect(args: dict) -> dict:
|
||||
"""`soup data inspect` — dataset stats."""
|
||||
from soup_cli.data.validator import validate_and_stats
|
||||
|
||||
rows = _load_data_rows(_require_str(args, "data"))
|
||||
return validate_and_stats(rows)
|
||||
|
||||
|
||||
def tool_data_validate(args: dict) -> dict:
|
||||
"""`soup data validate` — format-compliance report."""
|
||||
from soup_cli.data.validator import validate_and_stats
|
||||
|
||||
rows = _load_data_rows(_require_str(args, "data"))
|
||||
fmt = _opt_str(args, "format")
|
||||
return validate_and_stats(rows, expected_format=fmt)
|
||||
|
||||
|
||||
def tool_data_score(args: dict) -> dict:
|
||||
"""`soup data score` — PII / toxicity / language / educational scorecard."""
|
||||
from soup_cli.utils.data_score import compute_scorecard
|
||||
|
||||
rows = _load_data_rows(_require_str(args, "data"))
|
||||
rep = compute_scorecard(rows)
|
||||
return {
|
||||
"total": rep.total,
|
||||
"pii_flagged": rep.pii_flagged,
|
||||
"toxic_flagged": rep.toxic_flagged,
|
||||
"decontaminated_removed": rep.decontaminated_removed,
|
||||
"languages": dict(rep.languages),
|
||||
"educational_mean": rep.educational_mean,
|
||||
}
|
||||
|
||||
|
||||
def tool_data_doctor(args: dict) -> dict:
|
||||
"""`soup data doctor` — chat-template compat report (needs the tokenizer stack)."""
|
||||
from soup_cli.data import formats as _formats
|
||||
from soup_cli.utils import data_doctor as _dd
|
||||
|
||||
rows = _load_data_rows(_require_str(args, "data"))
|
||||
model = _require_str(args, "model")
|
||||
fmt = _opt_str(args, "format") or "auto"
|
||||
max_length = _opt_int(args, "max_length", 2048, lo=64, hi=1_048_576)
|
||||
sample_size = _opt_int(args, "sample_size", 200, lo=1, hi=2000)
|
||||
if fmt == "auto":
|
||||
try:
|
||||
fmt = _formats.detect_format(rows)
|
||||
except ValueError as exc:
|
||||
raise McpToolError("could not auto-detect data format; pass 'format'") from exc
|
||||
try:
|
||||
tok = _dd.resolve_tokenizer(model, trust_remote_code=False)
|
||||
except ImportError as exc:
|
||||
raise McpToolError(
|
||||
"data_doctor needs the tokenizer stack: pip install 'soup-cli[train]'"
|
||||
) from exc
|
||||
except (ValueError, TypeError, OSError) as exc:
|
||||
raise McpToolError(f"could not load tokenizer ({type(exc).__name__})") from exc
|
||||
try:
|
||||
report = _dd.run_doctor(
|
||||
rows, tok, fmt=fmt, max_length=max_length, sample_size=sample_size
|
||||
)
|
||||
except (ValueError, TypeError) as exc:
|
||||
raise McpToolError(f"data doctor failed ({type(exc).__name__})") from exc
|
||||
return report.to_dict()
|
||||
|
||||
|
||||
def tool_recipes_search(args: dict) -> dict:
|
||||
"""`soup recipes search` — compact recipe list (no yaml body)."""
|
||||
from soup_cli.recipes.catalog import RECIPES, search_recipes
|
||||
|
||||
results = search_recipes(
|
||||
_opt_str(args, "query"), _opt_str(args, "task"), _opt_str(args, "size")
|
||||
)
|
||||
name_by_id = {id(meta): name for name, meta in RECIPES.items()}
|
||||
out = [
|
||||
{
|
||||
"name": name_by_id.get(id(meta), "?"),
|
||||
"model": meta.model,
|
||||
"task": meta.task,
|
||||
"size": meta.size,
|
||||
"tags": list(meta.tags),
|
||||
"description": meta.description,
|
||||
}
|
||||
for meta in results
|
||||
]
|
||||
return {"results": out, "count": len(out)}
|
||||
|
||||
|
||||
def tool_recipes_show(args: dict) -> dict:
|
||||
"""`soup recipes show` — full recipe incl. the YAML body."""
|
||||
from soup_cli.recipes.catalog import get_recipe
|
||||
|
||||
name = _require_str(args, "name")
|
||||
meta = get_recipe(name)
|
||||
if meta is None:
|
||||
raise McpToolError("unknown recipe (try recipes_search)")
|
||||
return {
|
||||
"name": name,
|
||||
"model": meta.model,
|
||||
"task": meta.task,
|
||||
"size": meta.size,
|
||||
"tags": list(meta.tags),
|
||||
"description": meta.description,
|
||||
"yaml_str": meta.yaml_str,
|
||||
}
|
||||
|
||||
|
||||
def tool_runs_list(args: dict) -> dict:
|
||||
"""`soup runs` — recent experiment runs."""
|
||||
from soup_cli.experiment.tracker import ExperimentTracker
|
||||
|
||||
limit = _opt_int(args, "limit", 50, lo=1, hi=500)
|
||||
runs = ExperimentTracker().list_runs(limit=limit)
|
||||
return {"runs": runs, "count": len(runs)}
|
||||
|
||||
|
||||
def tool_runs_show(args: dict) -> dict:
|
||||
"""`soup runs show` — one run's full record."""
|
||||
from soup_cli.experiment.tracker import ExperimentTracker
|
||||
|
||||
run = ExperimentTracker().get_run(_require_str(args, "run_id"))
|
||||
if run is None:
|
||||
raise McpToolError("run not found")
|
||||
return run
|
||||
|
||||
|
||||
def tool_registry_list(args: dict) -> dict:
|
||||
"""`soup registry list` — model registry entries."""
|
||||
from soup_cli.registry.store import RegistryStore
|
||||
|
||||
limit = _opt_int(args, "limit", 100, lo=1, hi=500)
|
||||
with RegistryStore() as store:
|
||||
entries = store.list(
|
||||
name=_opt_str(args, "name"),
|
||||
tag=_opt_str(args, "tag"),
|
||||
base=_opt_str(args, "base"),
|
||||
task=_opt_str(args, "task"),
|
||||
limit=limit,
|
||||
)
|
||||
return {"entries": entries, "count": len(entries)}
|
||||
|
||||
|
||||
def tool_registry_show(args: dict) -> dict:
|
||||
"""`soup registry show` — one registry entry (id / prefix / name:tag / registry://)."""
|
||||
from soup_cli.registry.store import AmbiguousRefError, RegistryStore
|
||||
|
||||
ref = _require_str(args, "ref")
|
||||
with RegistryStore() as store:
|
||||
try:
|
||||
entry_id = store.resolve(ref)
|
||||
except AmbiguousRefError as exc:
|
||||
raise McpToolError("ambiguous registry ref") from exc
|
||||
if entry_id is None:
|
||||
raise McpToolError("registry entry not found")
|
||||
entry = store.get(entry_id)
|
||||
if entry is None:
|
||||
raise McpToolError("registry entry not found")
|
||||
return entry
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool table
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DATA_ARG = {
|
||||
"type": "string",
|
||||
"description": "Path to a JSONL/JSON dataset under the working directory.",
|
||||
}
|
||||
|
||||
|
||||
def _readonly_specs() -> "list[ToolSpec]":
|
||||
return [
|
||||
ToolSpec(
|
||||
name="advise",
|
||||
title="Advise",
|
||||
description=(
|
||||
"Pre-flight recommendation (PROMPT_ENG / RAG / SFT / DPO / GRPO) "
|
||||
"for a dataset + goal."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": _DATA_ARG,
|
||||
"goal": {
|
||||
"type": "string",
|
||||
"description": "Optional stated goal, e.g. 'improve summaries'.",
|
||||
},
|
||||
},
|
||||
"required": ["data"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_advise,
|
||||
),
|
||||
ToolSpec(
|
||||
name="data_inspect",
|
||||
title="Inspect dataset",
|
||||
description="Dataset stats: row count, columns, length distribution, duplicates.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"data": _DATA_ARG},
|
||||
"required": ["data"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_data_inspect,
|
||||
),
|
||||
ToolSpec(
|
||||
name="data_validate",
|
||||
title="Validate dataset",
|
||||
description=(
|
||||
"Format-compliance report: issues + valid-row count "
|
||||
"(alpaca/sharegpt/chatml/dpo/...)."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": _DATA_ARG,
|
||||
"format": {
|
||||
"type": "string",
|
||||
"description": "Expected format; omit to auto-detect.",
|
||||
},
|
||||
},
|
||||
"required": ["data"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_data_validate,
|
||||
),
|
||||
ToolSpec(
|
||||
name="data_score",
|
||||
title="Score dataset",
|
||||
description="Data-quality scorecard: PII, toxicity, language mix, educational value.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"data": _DATA_ARG},
|
||||
"required": ["data"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_data_score,
|
||||
),
|
||||
ToolSpec(
|
||||
name="data_doctor",
|
||||
title="Chat-template doctor",
|
||||
description=(
|
||||
"Chat-template compatibility report vs a tokenizer (EOS-in-labels, "
|
||||
"BOS dup, truncation risk). Needs the soup-cli[train] tokenizer stack."
|
||||
),
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": _DATA_ARG,
|
||||
"model": {
|
||||
"type": "string",
|
||||
"description": "Tokenizer model id or local path.",
|
||||
},
|
||||
"format": {
|
||||
"type": "string",
|
||||
"description": "Data format; omit to auto-detect.",
|
||||
},
|
||||
"max_length": {"type": "integer", "minimum": 64, "maximum": 1048576},
|
||||
"sample_size": {"type": "integer", "minimum": 1, "maximum": 2000},
|
||||
},
|
||||
"required": ["data", "model"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_data_doctor,
|
||||
),
|
||||
ToolSpec(
|
||||
name="recipes_search",
|
||||
title="Search recipes",
|
||||
description="Search the ready-made recipe catalog by keyword / task / model size.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {"type": "string"},
|
||||
"task": {"type": "string"},
|
||||
"size": {"type": "string"},
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_recipes_search,
|
||||
),
|
||||
ToolSpec(
|
||||
name="recipes_show",
|
||||
title="Show recipe",
|
||||
description="Full recipe details incl. the ready-to-use soup.yaml body.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"name": {"type": "string", "description": "Recipe name."}},
|
||||
"required": ["name"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_recipes_show,
|
||||
),
|
||||
ToolSpec(
|
||||
name="runs_list",
|
||||
title="List runs",
|
||||
description="Recent experiment runs from the local tracker.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"limit": {"type": "integer", "minimum": 1, "maximum": 500}},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_runs_list,
|
||||
),
|
||||
ToolSpec(
|
||||
name="runs_show",
|
||||
title="Show run",
|
||||
description="One run's full record (config, metrics summary). Accepts an id prefix.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"run_id": {"type": "string"}},
|
||||
"required": ["run_id"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_runs_show,
|
||||
),
|
||||
ToolSpec(
|
||||
name="registry_list",
|
||||
title="List registry",
|
||||
description="Model-registry entries, filterable by name/tag/base/task.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {"type": "string"},
|
||||
"tag": {"type": "string"},
|
||||
"base": {"type": "string"},
|
||||
"task": {"type": "string"},
|
||||
"limit": {"type": "integer", "minimum": 1, "maximum": 500},
|
||||
},
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_registry_list,
|
||||
),
|
||||
ToolSpec(
|
||||
name="registry_show",
|
||||
title="Show registry entry",
|
||||
description="One registry entry by id / prefix / name:tag / registry:// ref.",
|
||||
input_schema={
|
||||
"type": "object",
|
||||
"properties": {"ref": {"type": "string"}},
|
||||
"required": ["ref"],
|
||||
"additionalProperties": False,
|
||||
},
|
||||
handler=tool_registry_show,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def build_registry(*, allow_mutating: bool) -> "list[ToolSpec]":
|
||||
"""Assemble the MCP tool table.
|
||||
|
||||
The read-only tools are always present. ``allow_mutating`` gates the
|
||||
plan-only mutating tools (added in a later part).
|
||||
"""
|
||||
specs = _readonly_specs()
|
||||
return specs
|
||||
|
|
@ -0,0 +1,318 @@
|
|||
"""v0.71.28 — `soup mcp serve` MCP server.
|
||||
|
||||
Covers the pure tool registry (handlers + guards), the SDK server wiring
|
||||
(via the in-memory transport), and the `soup mcp` Typer command.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from soup_cli.mcp_server import registry as reg
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _sanitize — recursive C0/ESC strip on handler output
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestSanitize:
|
||||
def test_strips_c0_and_esc_keeps_tab_newline_cr(self):
|
||||
raw = "a\x1b[31mred\x1b[0m\tb\nc\rd\x07\x7f"
|
||||
cleaned = reg._sanitize(raw)
|
||||
assert "\x1b" not in cleaned
|
||||
assert "\x07" not in cleaned
|
||||
assert "\x7f" not in cleaned
|
||||
assert "\t" in cleaned and "\n" in cleaned and "\r" in cleaned
|
||||
assert "red" in cleaned
|
||||
|
||||
def test_recurses_dict_and_list(self):
|
||||
obj = {"k": ["x\x1by", {"n": "z\x00w"}], "keep": 5, "b": True, "none": None}
|
||||
out = reg._sanitize(obj)
|
||||
assert out["k"][0] == "xy"
|
||||
assert out["k"][1]["n"] == "zw"
|
||||
assert out["keep"] == 5
|
||||
assert out["b"] is True
|
||||
assert out["none"] is None
|
||||
|
||||
def test_leaves_non_str_scalars_untouched(self):
|
||||
assert reg._sanitize(3.14) == 3.14
|
||||
assert reg._sanitize(42) == 42
|
||||
assert reg._sanitize(False) is False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _read_json_under_cwd — cwd-contained, symlink-rejected, size-capped loader
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestReadJsonUnderCwd:
|
||||
def test_reads_dict_under_cwd(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "ev.json"
|
||||
p.write_text(json.dumps({"a": 1}), encoding="utf-8")
|
||||
assert reg._read_json_under_cwd("ev.json", "evidence") == {"a": 1}
|
||||
|
||||
def test_rejects_outside_cwd(self, tmp_path, monkeypatch):
|
||||
work = tmp_path / "work"
|
||||
work.mkdir()
|
||||
(tmp_path / "evil.json").write_text("{}", encoding="utf-8")
|
||||
monkeypatch.chdir(work)
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg._read_json_under_cwd("../evil.json", "evidence")
|
||||
|
||||
def test_missing_file_raises_tool_error(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg._read_json_under_cwd("nope.json", "evidence")
|
||||
|
||||
def test_non_dict_json_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "arr.json"
|
||||
p.write_text("[1,2,3]", encoding="utf-8")
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg._read_json_under_cwd("arr.json", "evidence")
|
||||
|
||||
def test_oversize_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "big.json"
|
||||
p.write_text("{}", encoding="utf-8")
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg._read_json_under_cwd("big.json", "evidence", max_bytes=1)
|
||||
|
||||
def test_error_message_has_no_raw_path(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(reg.McpToolError) as exc:
|
||||
reg._read_json_under_cwd("secret-name.json", "evidence")
|
||||
assert "secret-name.json" not in str(exc.value)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ToolSpec / McpToolError basics
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TestToolSpec:
|
||||
def test_toolspec_is_frozen(self):
|
||||
spec = reg.ToolSpec(
|
||||
name="x",
|
||||
title="X",
|
||||
description="does x",
|
||||
input_schema={"type": "object"},
|
||||
handler=lambda args: {},
|
||||
mutating=False,
|
||||
)
|
||||
with pytest.raises((AttributeError, TypeError)):
|
||||
spec.name = "y" # type: ignore[misc]
|
||||
|
||||
def test_tool_error_is_exception(self):
|
||||
assert issubclass(reg.McpToolError, Exception)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Read-only handlers (Part A)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _write_jsonl(path, rows):
|
||||
path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8")
|
||||
|
||||
|
||||
_ADVISE_ROWS = [
|
||||
{"instruction": "Summarize this article", "output": "A short summary."},
|
||||
{"instruction": "Translate to French", "output": "Bonjour le monde."},
|
||||
{"instruction": "Write a poem about spring", "output": "Petals fall softly."},
|
||||
{"instruction": "Explain gravity", "output": "Mass attracts mass."},
|
||||
{"instruction": "List three fruits", "output": "Apple, pear, plum."},
|
||||
]
|
||||
|
||||
|
||||
class TestAdviseHandler:
|
||||
def test_returns_verdict_dict(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "d.jsonl"
|
||||
_write_jsonl(p, _ADVISE_ROWS)
|
||||
out = reg.tool_advise({"data": "d.jsonl", "goal": "improve summaries"})
|
||||
assert "choice" in out and "task_category" in out
|
||||
assert 0.0 <= out["confidence"] <= 1.0
|
||||
assert "estimated_roi" in out
|
||||
|
||||
def test_bad_path_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_advise({"data": "missing.jsonl"})
|
||||
|
||||
def test_missing_data_arg_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_advise({})
|
||||
|
||||
|
||||
class TestDataInspectValidateHandlers:
|
||||
def test_inspect_returns_stats(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "d.jsonl"
|
||||
_write_jsonl(p, _ADVISE_ROWS)
|
||||
out = reg.tool_data_inspect({"data": "d.jsonl"})
|
||||
assert out["total"] == 5
|
||||
assert "columns" in out
|
||||
|
||||
def test_validate_returns_issues_and_valid_rows(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "d.jsonl"
|
||||
_write_jsonl(p, _ADVISE_ROWS)
|
||||
out = reg.tool_data_validate({"data": "d.jsonl"})
|
||||
assert "issues" in out and "valid_rows" in out
|
||||
|
||||
def test_inspect_bad_path_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_data_inspect({"data": "nope.jsonl"})
|
||||
|
||||
|
||||
class TestDataScoreHandler:
|
||||
def test_returns_scorecard(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
p = tmp_path / "d.jsonl"
|
||||
_write_jsonl(p, _ADVISE_ROWS)
|
||||
out = reg.tool_data_score({"data": "d.jsonl"})
|
||||
assert out["total"] == 5
|
||||
assert "pii_flagged" in out and "educational_mean" in out
|
||||
assert isinstance(out["languages"], dict)
|
||||
|
||||
|
||||
class TestDataDoctorHandler:
|
||||
def test_returns_report_dict(self, tmp_path, monkeypatch):
|
||||
from tests.test_v07127 import _FakeTokenizer
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(
|
||||
"soup_cli.utils.data_doctor.resolve_tokenizer",
|
||||
lambda model, **kw: _FakeTokenizer(),
|
||||
)
|
||||
p = tmp_path / "chat.jsonl"
|
||||
_write_jsonl(
|
||||
p,
|
||||
[
|
||||
{
|
||||
"messages": [
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "assistant", "content": "hello"},
|
||||
]
|
||||
}
|
||||
for _ in range(4)
|
||||
],
|
||||
)
|
||||
out = reg.tool_data_doctor(
|
||||
{"data": "chat.jsonl", "model": "fake/model", "format": "chatml"}
|
||||
)
|
||||
assert out["overall"] in ("OK", "MINOR", "MAJOR")
|
||||
assert "checks" in out and isinstance(out["checks"], list)
|
||||
|
||||
def test_missing_transformers_friendly_error(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
||||
def _boom(model, **kw):
|
||||
raise ImportError("no transformers")
|
||||
|
||||
monkeypatch.setattr("soup_cli.utils.data_doctor.resolve_tokenizer", _boom)
|
||||
p = tmp_path / "chat.jsonl"
|
||||
_write_jsonl(p, [{"messages": [{"role": "user", "content": "hi"}]}])
|
||||
with pytest.raises(reg.McpToolError) as exc:
|
||||
reg.tool_data_doctor({"data": "chat.jsonl", "model": "x", "format": "chatml"})
|
||||
assert "soup-cli[train]" in str(exc.value) or "install" in str(exc.value).lower()
|
||||
|
||||
|
||||
class TestRecipesHandlers:
|
||||
def test_search_returns_results(self):
|
||||
out = reg.tool_recipes_search({"query": "qwen"})
|
||||
assert out["count"] >= 1
|
||||
assert all("name" in r and "model" in r for r in out["results"])
|
||||
# search results stay compact — no full yaml body
|
||||
assert all("yaml_str" not in r for r in out["results"])
|
||||
|
||||
def test_show_returns_full_recipe(self):
|
||||
# pick a known recipe from the search
|
||||
name = reg.tool_recipes_search({"query": "qwen"})["results"][0]["name"]
|
||||
out = reg.tool_recipes_show({"name": name})
|
||||
assert out["name"] == name
|
||||
assert "yaml_str" in out and out["yaml_str"]
|
||||
|
||||
def test_show_unknown_raises(self):
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_recipes_show({"name": "definitely-not-a-recipe-xyz"})
|
||||
|
||||
|
||||
class TestRunsHandlers:
|
||||
def test_list_returns_runs_key(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SOUP_DB_PATH", str(tmp_path / "exp.db"))
|
||||
out = reg.tool_runs_list({})
|
||||
assert "runs" in out and isinstance(out["runs"], list)
|
||||
assert "count" in out
|
||||
|
||||
def test_show_unknown_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SOUP_DB_PATH", str(tmp_path / "exp.db"))
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_runs_show({"run_id": "nope"})
|
||||
|
||||
|
||||
class TestRegistryHandlers:
|
||||
def test_list_returns_entries_key(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db"))
|
||||
out = reg.tool_registry_list({})
|
||||
assert "entries" in out and isinstance(out["entries"], list)
|
||||
assert "count" in out
|
||||
|
||||
def test_show_unknown_raises(self, tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db"))
|
||||
with pytest.raises(reg.McpToolError):
|
||||
reg.tool_registry_show({"ref": "nonexistent-id"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# build_registry — the tool table
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_EXPECTED_READONLY = {
|
||||
"advise",
|
||||
"data_inspect",
|
||||
"data_validate",
|
||||
"data_score",
|
||||
"data_doctor",
|
||||
"recipes_search",
|
||||
"recipes_show",
|
||||
"runs_list",
|
||||
"runs_show",
|
||||
"registry_list",
|
||||
"registry_show",
|
||||
}
|
||||
|
||||
|
||||
class TestBuildRegistry:
|
||||
def test_readonly_tools_present(self):
|
||||
names = {s.name for s in reg.build_registry(allow_mutating=False)}
|
||||
assert _EXPECTED_READONLY <= names
|
||||
|
||||
def test_names_unique(self):
|
||||
specs = reg.build_registry(allow_mutating=True)
|
||||
names = [s.name for s in specs]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
def test_every_schema_is_valid_json_schema(self):
|
||||
import jsonschema
|
||||
|
||||
for spec in reg.build_registry(allow_mutating=True):
|
||||
jsonschema.Draft202012Validator.check_schema(spec.input_schema)
|
||||
assert spec.input_schema.get("type") == "object"
|
||||
|
||||
def test_every_spec_well_formed(self):
|
||||
for spec in reg.build_registry(allow_mutating=True):
|
||||
assert spec.name and spec.description
|
||||
assert callable(spec.handler)
|
||||
assert isinstance(spec.mutating, bool)
|
||||
|
||||
def test_no_mutating_in_readonly_registry_is_executable(self):
|
||||
# read-only build has no non-mutating gap: every listed tool is callable
|
||||
for spec in reg.build_registry(allow_mutating=False):
|
||||
assert callable(spec.handler)
|
||||
Loading…
Reference in New Issue