diff --git a/src/soup_cli/mcp_server/__init__.py b/src/soup_cli/mcp_server/__init__.py new file mode 100644 index 0000000..838cb35 --- /dev/null +++ b/src/soup_cli/mcp_server/__init__.py @@ -0,0 +1,9 @@ +"""Soup MCP server (v0.71.28). + +``soup mcp serve`` exposes Soup's read-only commands (plus two plan-only +mutating tools) to any Model Context Protocol client over stdio. + +The package is split so the tool *table* (:mod:`registry`) is pure Python +with NO dependency on the ``mcp`` SDK — it is fully unit-testable on the +light core install. Only :mod:`server` imports the SDK, lazily. +""" diff --git a/src/soup_cli/mcp_server/registry.py b/src/soup_cli/mcp_server/registry.py new file mode 100644 index 0000000..b843063 --- /dev/null +++ b/src/soup_cli/mcp_server/registry.py @@ -0,0 +1,528 @@ +"""Pure MCP tool registry for ``soup mcp serve`` (v0.71.28). + +This module has **no** dependency on the ``mcp`` SDK: it defines the tool +table (:class:`ToolSpec`), the handler functions (each a pure +``(dict) -> dict``), and the shared security guards. :mod:`soup_cli.mcp_server.server` +is the only file that imports the SDK, and it consumes this registry. + +Every handler lazy-imports its light core inside the function body so that +importing this module stays cheap and torch-free. +""" + +from __future__ import annotations + +import json +import os +from dataclasses import dataclass +from typing import Any, Callable, Mapping + +from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + +# Default read cap for JSON tool arguments (mirrors ship/diagnose evidence). +_MAX_JSON_BYTES = 16 * 1024 * 1024 + +# C0 control bytes (keep tab / newline / CR) + DEL, stripped from every string +# in a handler result before it reaches the MCP client. ``rich.markup.escape`` +# only neutralises ``[...]`` markup, not raw ESC/OSC sequences a malicious +# dataset string could smuggle into a client's terminal. Mirrors +# ``commands/data_doctor.py::_CONTROL_STRIP_TABLE``. +_CONTROL_STRIP_TABLE = {i: None for i in range(0x20) if i not in (0x09, 0x0A, 0x0D)} +_CONTROL_STRIP_TABLE[0x7F] = None + + +class McpToolError(Exception): + """A tool-level failure with a pre-sanitized, path-free message. + + The MCP SDK stringifies a raised exception verbatim into an ``isError`` + result, so handlers must raise THIS (never a bare ``OSError`` whose text + could leak a filesystem path). + """ + + +@dataclass(frozen=True) +class ToolSpec: + """One entry in the MCP tool table.""" + + name: str + title: str + description: str + input_schema: dict + handler: Callable[[dict], dict] + mutating: bool = False + + +def _sanitize(obj: Any) -> Any: + """Recursively strip C0/ESC/DEL bytes from every string in ``obj``. + + Leaves non-string scalars (int/float/bool/None) untouched; recurses into + dicts and lists. Applied to every handler result as defence-in-depth. + """ + if isinstance(obj, str): + return obj.translate(_CONTROL_STRIP_TABLE) + if isinstance(obj, Mapping): + return {_sanitize(k): _sanitize(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [_sanitize(v) for v in obj] + return obj + + +def _read_json_under_cwd(path: str, field: str, *, max_bytes: int = _MAX_JSON_BYTES) -> dict: + """Load a JSON object argument (cwd-contained, symlink-rejected, size-capped). + + Opens with ``O_NOFOLLOW`` (where available) and fstats the open fd so a + symlink swapped in after the containment check cannot redirect the read + (TOCTOU defence, mirrors ``commands/ship.py::_load_evidence``). Raises + :class:`McpToolError` with a path-free message on any failure. + """ + try: + enforce_under_cwd_and_no_symlink(path, field) + except Exception as exc: # ValueError / OSError from the guard + raise McpToolError(f"{field} must be a readable file under the working directory") from exc + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + handle_fd = os.open(path, flags) + except OSError as exc: + raise McpToolError(f"{field} is unreadable ({type(exc).__name__})") from exc + try: + with os.fdopen(handle_fd, "r", encoding="utf-8") as handle: + if os.fstat(handle.fileno()).st_size > max_bytes: + raise McpToolError(f"{field} exceeds {max_bytes} bytes") + payload = json.load(handle) + except json.JSONDecodeError as exc: + raise McpToolError(f"{field} is not valid JSON") from exc + except OSError as exc: + raise McpToolError(f"{field} is unreadable ({type(exc).__name__})") from exc + if not isinstance(payload, dict): + raise McpToolError(f"{field} must contain a JSON object") + return payload + + +# --------------------------------------------------------------------------- +# Argument helpers — every handler validates its own args (the SDK also +# jsonschema-validates inputSchema, but handlers must not trust that alone). +# Error messages NEVER echo raw user input (avoids ANSI/path injection). +# --------------------------------------------------------------------------- + + +def _require_str(args: dict, key: str) -> str: + val = args.get(key) + if not isinstance(val, str) or not val: + raise McpToolError(f"'{key}' must be a non-empty string") + return val + + +def _opt_str(args: dict, key: str) -> "str | None": + val = args.get(key) + if val is None: + return None + if not isinstance(val, str): + raise McpToolError(f"'{key}' must be a string") + return val + + +def _opt_int(args: dict, key: str, default: int, *, lo: int, hi: int) -> int: + val = args.get(key, default) + if isinstance(val, bool) or not isinstance(val, int): + raise McpToolError(f"'{key}' must be an integer") + return max(lo, min(hi, val)) + + +def _enforce_data_path(path: str, field: str = "data") -> None: + try: + enforce_under_cwd_and_no_symlink(path, field) + except Exception as exc: # ValueError / OSError from the guard + raise McpToolError( + f"'{field}' must be a readable file under the working directory" + ) from exc + + +# --------------------------------------------------------------------------- +# Read-only tool handlers (each a pure ``(dict) -> dict``) +# --------------------------------------------------------------------------- + + +def tool_advise(args: dict) -> dict: + """`soup advise` — pre-flight PROMPT_ENG / RAG / SFT / DPO / GRPO verdict.""" + import dataclasses + + from soup_cli.utils import advise as _advise + + data = _require_str(args, "data") + goal = _opt_str(args, "goal") + _enforce_data_path(data) + try: + rows = _advise.load_advise_dataset(data) + task = _advise.classify_task(rows, goal) + profile = _advise.compute_dataset_profile(rows) + verdict = _advise.build_verdict(profile, task, goal=goal) + except (OSError, ValueError, TypeError) as exc: + raise McpToolError(f"advise failed ({type(exc).__name__})") from exc + return dataclasses.asdict(verdict) + + +def _load_data_rows(path: str) -> list: + from pathlib import Path + + from soup_cli.data.loader import load_raw_data + + _enforce_data_path(path) + try: + return load_raw_data(Path(path)) + except (OSError, ValueError) as exc: + raise McpToolError(f"cannot load data ({type(exc).__name__})") from exc + + +def tool_data_inspect(args: dict) -> dict: + """`soup data inspect` — dataset stats.""" + from soup_cli.data.validator import validate_and_stats + + rows = _load_data_rows(_require_str(args, "data")) + return validate_and_stats(rows) + + +def tool_data_validate(args: dict) -> dict: + """`soup data validate` — format-compliance report.""" + from soup_cli.data.validator import validate_and_stats + + rows = _load_data_rows(_require_str(args, "data")) + fmt = _opt_str(args, "format") + return validate_and_stats(rows, expected_format=fmt) + + +def tool_data_score(args: dict) -> dict: + """`soup data score` — PII / toxicity / language / educational scorecard.""" + from soup_cli.utils.data_score import compute_scorecard + + rows = _load_data_rows(_require_str(args, "data")) + rep = compute_scorecard(rows) + return { + "total": rep.total, + "pii_flagged": rep.pii_flagged, + "toxic_flagged": rep.toxic_flagged, + "decontaminated_removed": rep.decontaminated_removed, + "languages": dict(rep.languages), + "educational_mean": rep.educational_mean, + } + + +def tool_data_doctor(args: dict) -> dict: + """`soup data doctor` — chat-template compat report (needs the tokenizer stack).""" + from soup_cli.data import formats as _formats + from soup_cli.utils import data_doctor as _dd + + rows = _load_data_rows(_require_str(args, "data")) + model = _require_str(args, "model") + fmt = _opt_str(args, "format") or "auto" + max_length = _opt_int(args, "max_length", 2048, lo=64, hi=1_048_576) + sample_size = _opt_int(args, "sample_size", 200, lo=1, hi=2000) + if fmt == "auto": + try: + fmt = _formats.detect_format(rows) + except ValueError as exc: + raise McpToolError("could not auto-detect data format; pass 'format'") from exc + try: + tok = _dd.resolve_tokenizer(model, trust_remote_code=False) + except ImportError as exc: + raise McpToolError( + "data_doctor needs the tokenizer stack: pip install 'soup-cli[train]'" + ) from exc + except (ValueError, TypeError, OSError) as exc: + raise McpToolError(f"could not load tokenizer ({type(exc).__name__})") from exc + try: + report = _dd.run_doctor( + rows, tok, fmt=fmt, max_length=max_length, sample_size=sample_size + ) + except (ValueError, TypeError) as exc: + raise McpToolError(f"data doctor failed ({type(exc).__name__})") from exc + return report.to_dict() + + +def tool_recipes_search(args: dict) -> dict: + """`soup recipes search` — compact recipe list (no yaml body).""" + from soup_cli.recipes.catalog import RECIPES, search_recipes + + results = search_recipes( + _opt_str(args, "query"), _opt_str(args, "task"), _opt_str(args, "size") + ) + name_by_id = {id(meta): name for name, meta in RECIPES.items()} + out = [ + { + "name": name_by_id.get(id(meta), "?"), + "model": meta.model, + "task": meta.task, + "size": meta.size, + "tags": list(meta.tags), + "description": meta.description, + } + for meta in results + ] + return {"results": out, "count": len(out)} + + +def tool_recipes_show(args: dict) -> dict: + """`soup recipes show` — full recipe incl. the YAML body.""" + from soup_cli.recipes.catalog import get_recipe + + name = _require_str(args, "name") + meta = get_recipe(name) + if meta is None: + raise McpToolError("unknown recipe (try recipes_search)") + return { + "name": name, + "model": meta.model, + "task": meta.task, + "size": meta.size, + "tags": list(meta.tags), + "description": meta.description, + "yaml_str": meta.yaml_str, + } + + +def tool_runs_list(args: dict) -> dict: + """`soup runs` — recent experiment runs.""" + from soup_cli.experiment.tracker import ExperimentTracker + + limit = _opt_int(args, "limit", 50, lo=1, hi=500) + runs = ExperimentTracker().list_runs(limit=limit) + return {"runs": runs, "count": len(runs)} + + +def tool_runs_show(args: dict) -> dict: + """`soup runs show` — one run's full record.""" + from soup_cli.experiment.tracker import ExperimentTracker + + run = ExperimentTracker().get_run(_require_str(args, "run_id")) + if run is None: + raise McpToolError("run not found") + return run + + +def tool_registry_list(args: dict) -> dict: + """`soup registry list` — model registry entries.""" + from soup_cli.registry.store import RegistryStore + + limit = _opt_int(args, "limit", 100, lo=1, hi=500) + with RegistryStore() as store: + entries = store.list( + name=_opt_str(args, "name"), + tag=_opt_str(args, "tag"), + base=_opt_str(args, "base"), + task=_opt_str(args, "task"), + limit=limit, + ) + return {"entries": entries, "count": len(entries)} + + +def tool_registry_show(args: dict) -> dict: + """`soup registry show` — one registry entry (id / prefix / name:tag / registry://).""" + from soup_cli.registry.store import AmbiguousRefError, RegistryStore + + ref = _require_str(args, "ref") + with RegistryStore() as store: + try: + entry_id = store.resolve(ref) + except AmbiguousRefError as exc: + raise McpToolError("ambiguous registry ref") from exc + if entry_id is None: + raise McpToolError("registry entry not found") + entry = store.get(entry_id) + if entry is None: + raise McpToolError("registry entry not found") + return entry + + +# --------------------------------------------------------------------------- +# Tool table +# --------------------------------------------------------------------------- + +_DATA_ARG = { + "type": "string", + "description": "Path to a JSONL/JSON dataset under the working directory.", +} + + +def _readonly_specs() -> "list[ToolSpec]": + return [ + ToolSpec( + name="advise", + title="Advise", + description=( + "Pre-flight recommendation (PROMPT_ENG / RAG / SFT / DPO / GRPO) " + "for a dataset + goal." + ), + input_schema={ + "type": "object", + "properties": { + "data": _DATA_ARG, + "goal": { + "type": "string", + "description": "Optional stated goal, e.g. 'improve summaries'.", + }, + }, + "required": ["data"], + "additionalProperties": False, + }, + handler=tool_advise, + ), + ToolSpec( + name="data_inspect", + title="Inspect dataset", + description="Dataset stats: row count, columns, length distribution, duplicates.", + input_schema={ + "type": "object", + "properties": {"data": _DATA_ARG}, + "required": ["data"], + "additionalProperties": False, + }, + handler=tool_data_inspect, + ), + ToolSpec( + name="data_validate", + title="Validate dataset", + description=( + "Format-compliance report: issues + valid-row count " + "(alpaca/sharegpt/chatml/dpo/...)." + ), + input_schema={ + "type": "object", + "properties": { + "data": _DATA_ARG, + "format": { + "type": "string", + "description": "Expected format; omit to auto-detect.", + }, + }, + "required": ["data"], + "additionalProperties": False, + }, + handler=tool_data_validate, + ), + ToolSpec( + name="data_score", + title="Score dataset", + description="Data-quality scorecard: PII, toxicity, language mix, educational value.", + input_schema={ + "type": "object", + "properties": {"data": _DATA_ARG}, + "required": ["data"], + "additionalProperties": False, + }, + handler=tool_data_score, + ), + ToolSpec( + name="data_doctor", + title="Chat-template doctor", + description=( + "Chat-template compatibility report vs a tokenizer (EOS-in-labels, " + "BOS dup, truncation risk). Needs the soup-cli[train] tokenizer stack." + ), + input_schema={ + "type": "object", + "properties": { + "data": _DATA_ARG, + "model": { + "type": "string", + "description": "Tokenizer model id or local path.", + }, + "format": { + "type": "string", + "description": "Data format; omit to auto-detect.", + }, + "max_length": {"type": "integer", "minimum": 64, "maximum": 1048576}, + "sample_size": {"type": "integer", "minimum": 1, "maximum": 2000}, + }, + "required": ["data", "model"], + "additionalProperties": False, + }, + handler=tool_data_doctor, + ), + ToolSpec( + name="recipes_search", + title="Search recipes", + description="Search the ready-made recipe catalog by keyword / task / model size.", + input_schema={ + "type": "object", + "properties": { + "query": {"type": "string"}, + "task": {"type": "string"}, + "size": {"type": "string"}, + }, + "additionalProperties": False, + }, + handler=tool_recipes_search, + ), + ToolSpec( + name="recipes_show", + title="Show recipe", + description="Full recipe details incl. the ready-to-use soup.yaml body.", + input_schema={ + "type": "object", + "properties": {"name": {"type": "string", "description": "Recipe name."}}, + "required": ["name"], + "additionalProperties": False, + }, + handler=tool_recipes_show, + ), + ToolSpec( + name="runs_list", + title="List runs", + description="Recent experiment runs from the local tracker.", + input_schema={ + "type": "object", + "properties": {"limit": {"type": "integer", "minimum": 1, "maximum": 500}}, + "additionalProperties": False, + }, + handler=tool_runs_list, + ), + ToolSpec( + name="runs_show", + title="Show run", + description="One run's full record (config, metrics summary). Accepts an id prefix.", + input_schema={ + "type": "object", + "properties": {"run_id": {"type": "string"}}, + "required": ["run_id"], + "additionalProperties": False, + }, + handler=tool_runs_show, + ), + ToolSpec( + name="registry_list", + title="List registry", + description="Model-registry entries, filterable by name/tag/base/task.", + input_schema={ + "type": "object", + "properties": { + "name": {"type": "string"}, + "tag": {"type": "string"}, + "base": {"type": "string"}, + "task": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 500}, + }, + "additionalProperties": False, + }, + handler=tool_registry_list, + ), + ToolSpec( + name="registry_show", + title="Show registry entry", + description="One registry entry by id / prefix / name:tag / registry:// ref.", + input_schema={ + "type": "object", + "properties": {"ref": {"type": "string"}}, + "required": ["ref"], + "additionalProperties": False, + }, + handler=tool_registry_show, + ), + ] + + +def build_registry(*, allow_mutating: bool) -> "list[ToolSpec]": + """Assemble the MCP tool table. + + The read-only tools are always present. ``allow_mutating`` gates the + plan-only mutating tools (added in a later part). + """ + specs = _readonly_specs() + return specs diff --git a/tests/test_v07128.py b/tests/test_v07128.py new file mode 100644 index 0000000..e4cbf55 --- /dev/null +++ b/tests/test_v07128.py @@ -0,0 +1,318 @@ +"""v0.71.28 — `soup mcp serve` MCP server. + +Covers the pure tool registry (handlers + guards), the SDK server wiring +(via the in-memory transport), and the `soup mcp` Typer command. +""" + +from __future__ import annotations + +import json + +import pytest + +from soup_cli.mcp_server import registry as reg + +# --------------------------------------------------------------------------- +# _sanitize — recursive C0/ESC strip on handler output +# --------------------------------------------------------------------------- + + +class TestSanitize: + def test_strips_c0_and_esc_keeps_tab_newline_cr(self): + raw = "a\x1b[31mred\x1b[0m\tb\nc\rd\x07\x7f" + cleaned = reg._sanitize(raw) + assert "\x1b" not in cleaned + assert "\x07" not in cleaned + assert "\x7f" not in cleaned + assert "\t" in cleaned and "\n" in cleaned and "\r" in cleaned + assert "red" in cleaned + + def test_recurses_dict_and_list(self): + obj = {"k": ["x\x1by", {"n": "z\x00w"}], "keep": 5, "b": True, "none": None} + out = reg._sanitize(obj) + assert out["k"][0] == "xy" + assert out["k"][1]["n"] == "zw" + assert out["keep"] == 5 + assert out["b"] is True + assert out["none"] is None + + def test_leaves_non_str_scalars_untouched(self): + assert reg._sanitize(3.14) == 3.14 + assert reg._sanitize(42) == 42 + assert reg._sanitize(False) is False + + +# --------------------------------------------------------------------------- +# _read_json_under_cwd — cwd-contained, symlink-rejected, size-capped loader +# --------------------------------------------------------------------------- + + +class TestReadJsonUnderCwd: + def test_reads_dict_under_cwd(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "ev.json" + p.write_text(json.dumps({"a": 1}), encoding="utf-8") + assert reg._read_json_under_cwd("ev.json", "evidence") == {"a": 1} + + def test_rejects_outside_cwd(self, tmp_path, monkeypatch): + work = tmp_path / "work" + work.mkdir() + (tmp_path / "evil.json").write_text("{}", encoding="utf-8") + monkeypatch.chdir(work) + with pytest.raises(reg.McpToolError): + reg._read_json_under_cwd("../evil.json", "evidence") + + def test_missing_file_raises_tool_error(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with pytest.raises(reg.McpToolError): + reg._read_json_under_cwd("nope.json", "evidence") + + def test_non_dict_json_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "arr.json" + p.write_text("[1,2,3]", encoding="utf-8") + with pytest.raises(reg.McpToolError): + reg._read_json_under_cwd("arr.json", "evidence") + + def test_oversize_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "big.json" + p.write_text("{}", encoding="utf-8") + with pytest.raises(reg.McpToolError): + reg._read_json_under_cwd("big.json", "evidence", max_bytes=1) + + def test_error_message_has_no_raw_path(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with pytest.raises(reg.McpToolError) as exc: + reg._read_json_under_cwd("secret-name.json", "evidence") + assert "secret-name.json" not in str(exc.value) + + +# --------------------------------------------------------------------------- +# ToolSpec / McpToolError basics +# --------------------------------------------------------------------------- + + +class TestToolSpec: + def test_toolspec_is_frozen(self): + spec = reg.ToolSpec( + name="x", + title="X", + description="does x", + input_schema={"type": "object"}, + handler=lambda args: {}, + mutating=False, + ) + with pytest.raises((AttributeError, TypeError)): + spec.name = "y" # type: ignore[misc] + + def test_tool_error_is_exception(self): + assert issubclass(reg.McpToolError, Exception) + + +# --------------------------------------------------------------------------- +# Read-only handlers (Part A) +# --------------------------------------------------------------------------- + + +def _write_jsonl(path, rows): + path.write_text("\n".join(json.dumps(r) for r in rows), encoding="utf-8") + + +_ADVISE_ROWS = [ + {"instruction": "Summarize this article", "output": "A short summary."}, + {"instruction": "Translate to French", "output": "Bonjour le monde."}, + {"instruction": "Write a poem about spring", "output": "Petals fall softly."}, + {"instruction": "Explain gravity", "output": "Mass attracts mass."}, + {"instruction": "List three fruits", "output": "Apple, pear, plum."}, +] + + +class TestAdviseHandler: + def test_returns_verdict_dict(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "d.jsonl" + _write_jsonl(p, _ADVISE_ROWS) + out = reg.tool_advise({"data": "d.jsonl", "goal": "improve summaries"}) + assert "choice" in out and "task_category" in out + assert 0.0 <= out["confidence"] <= 1.0 + assert "estimated_roi" in out + + def test_bad_path_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with pytest.raises(reg.McpToolError): + reg.tool_advise({"data": "missing.jsonl"}) + + def test_missing_data_arg_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with pytest.raises(reg.McpToolError): + reg.tool_advise({}) + + +class TestDataInspectValidateHandlers: + def test_inspect_returns_stats(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "d.jsonl" + _write_jsonl(p, _ADVISE_ROWS) + out = reg.tool_data_inspect({"data": "d.jsonl"}) + assert out["total"] == 5 + assert "columns" in out + + def test_validate_returns_issues_and_valid_rows(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "d.jsonl" + _write_jsonl(p, _ADVISE_ROWS) + out = reg.tool_data_validate({"data": "d.jsonl"}) + assert "issues" in out and "valid_rows" in out + + def test_inspect_bad_path_raises(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + with pytest.raises(reg.McpToolError): + reg.tool_data_inspect({"data": "nope.jsonl"}) + + +class TestDataScoreHandler: + def test_returns_scorecard(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + p = tmp_path / "d.jsonl" + _write_jsonl(p, _ADVISE_ROWS) + out = reg.tool_data_score({"data": "d.jsonl"}) + assert out["total"] == 5 + assert "pii_flagged" in out and "educational_mean" in out + assert isinstance(out["languages"], dict) + + +class TestDataDoctorHandler: + def test_returns_report_dict(self, tmp_path, monkeypatch): + from tests.test_v07127 import _FakeTokenizer + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + "soup_cli.utils.data_doctor.resolve_tokenizer", + lambda model, **kw: _FakeTokenizer(), + ) + p = tmp_path / "chat.jsonl" + _write_jsonl( + p, + [ + { + "messages": [ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "hello"}, + ] + } + for _ in range(4) + ], + ) + out = reg.tool_data_doctor( + {"data": "chat.jsonl", "model": "fake/model", "format": "chatml"} + ) + assert out["overall"] in ("OK", "MINOR", "MAJOR") + assert "checks" in out and isinstance(out["checks"], list) + + def test_missing_transformers_friendly_error(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + def _boom(model, **kw): + raise ImportError("no transformers") + + monkeypatch.setattr("soup_cli.utils.data_doctor.resolve_tokenizer", _boom) + p = tmp_path / "chat.jsonl" + _write_jsonl(p, [{"messages": [{"role": "user", "content": "hi"}]}]) + with pytest.raises(reg.McpToolError) as exc: + reg.tool_data_doctor({"data": "chat.jsonl", "model": "x", "format": "chatml"}) + assert "soup-cli[train]" in str(exc.value) or "install" in str(exc.value).lower() + + +class TestRecipesHandlers: + def test_search_returns_results(self): + out = reg.tool_recipes_search({"query": "qwen"}) + assert out["count"] >= 1 + assert all("name" in r and "model" in r for r in out["results"]) + # search results stay compact — no full yaml body + assert all("yaml_str" not in r for r in out["results"]) + + def test_show_returns_full_recipe(self): + # pick a known recipe from the search + name = reg.tool_recipes_search({"query": "qwen"})["results"][0]["name"] + out = reg.tool_recipes_show({"name": name}) + assert out["name"] == name + assert "yaml_str" in out and out["yaml_str"] + + def test_show_unknown_raises(self): + with pytest.raises(reg.McpToolError): + reg.tool_recipes_show({"name": "definitely-not-a-recipe-xyz"}) + + +class TestRunsHandlers: + def test_list_returns_runs_key(self, tmp_path, monkeypatch): + monkeypatch.setenv("SOUP_DB_PATH", str(tmp_path / "exp.db")) + out = reg.tool_runs_list({}) + assert "runs" in out and isinstance(out["runs"], list) + assert "count" in out + + def test_show_unknown_raises(self, tmp_path, monkeypatch): + monkeypatch.setenv("SOUP_DB_PATH", str(tmp_path / "exp.db")) + with pytest.raises(reg.McpToolError): + reg.tool_runs_show({"run_id": "nope"}) + + +class TestRegistryHandlers: + def test_list_returns_entries_key(self, tmp_path, monkeypatch): + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db")) + out = reg.tool_registry_list({}) + assert "entries" in out and isinstance(out["entries"], list) + assert "count" in out + + def test_show_unknown_raises(self, tmp_path, monkeypatch): + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(tmp_path / "reg.db")) + with pytest.raises(reg.McpToolError): + reg.tool_registry_show({"ref": "nonexistent-id"}) + + +# --------------------------------------------------------------------------- +# build_registry — the tool table +# --------------------------------------------------------------------------- + +_EXPECTED_READONLY = { + "advise", + "data_inspect", + "data_validate", + "data_score", + "data_doctor", + "recipes_search", + "recipes_show", + "runs_list", + "runs_show", + "registry_list", + "registry_show", +} + + +class TestBuildRegistry: + def test_readonly_tools_present(self): + names = {s.name for s in reg.build_registry(allow_mutating=False)} + assert _EXPECTED_READONLY <= names + + def test_names_unique(self): + specs = reg.build_registry(allow_mutating=True) + names = [s.name for s in specs] + assert len(names) == len(set(names)) + + def test_every_schema_is_valid_json_schema(self): + import jsonschema + + for spec in reg.build_registry(allow_mutating=True): + jsonschema.Draft202012Validator.check_schema(spec.input_schema) + assert spec.input_schema.get("type") == "object" + + def test_every_spec_well_formed(self): + for spec in reg.build_registry(allow_mutating=True): + assert spec.name and spec.description + assert callable(spec.handler) + assert isinstance(spec.mutating, bool) + + def test_no_mutating_in_readonly_registry_is_executable(self): + # read-only build has no non-mutating gap: every listed tool is callable + for spec in reg.build_registry(allow_mutating=False): + assert callable(spec.handler)