From aa71658f50cbc583e6a0b7268e688626c45a84be Mon Sep 17 00:00:00 2001 From: Alpamys Date: Sun, 24 May 2026 20:24:39 +0500 Subject: [PATCH] =?UTF-8?q?feat(v0.68.0):=20Anti-trend=20Insurance=20?= =?UTF-8?q?=E2=80=94=20compile=20(DSPy/GEPA)=20+=20distill-prompt=20+=20co?= =?UTF-8?q?mpile-tools=20+=20apple-adapter=20+=20local-rl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 5 commands that hedge Soup against paradigm shifts. If 1M-context kills FT, `soup compile` (DSPy + GEPA + TextGrad prompt-program compilation) takes its place. If teams hit prompt-cost walls, `soup distill-prompt` bridges to small FT. If only Apple Foundation Models win on-device, `soup apple-adapter` ships the converter+signing surface. If personal-LLM flywheels become the shape, `soup local-rl` captures thumbs into SQLite and emits DPO pairs. - Part A: `soup compile --eval [--optimizer mipro|gepa|...]` - Part B: `soup distill-prompt --traces --teacher --student --strategy` - Part C: `soup compile-tools --eval ` - Part D: `soup apple-adapter --direction hf-to-mlx|... --output` - Part E: `soup local-rl init/status/record/harvest/train` (LIVE except train) Schema + path containment + symlink rejection + atomic-write surface ship now; live runners for Parts A/B/C/D + Part E nightly scheduler deferred to v0.68.1 (stub-then-live, mirrors v0.50.0 / v0.61.0 / v0.62.0 / v0.67.0 cadence). Test count: 11021 -> 11225 (+204). Review-fix: 0 CRIT + 4 HIGH + 10 MED + 4 LOW. Co-Authored-By: Claude Opus 4.7 (1M context) --- CONTRIBUTING.md | 2 +- README.md | 45 +- SECURITY.md | 3 +- pyproject.toml | 2 +- soup_cli/__init__.py | 2 +- soup_cli/cli.py | 34 ++ soup_cli/commands/apple_adapter.py | 73 +++ soup_cli/commands/compile_cmd.py | 98 ++++ soup_cli/commands/compile_tools.py | 73 +++ soup_cli/commands/distill_prompt.py | 75 +++ soup_cli/commands/local_rl.py | 197 ++++++++ soup_cli/utils/apple_adapter.py | 144 ++++++ soup_cli/utils/compile_tools.py | 129 +++++ soup_cli/utils/local_rl.py | 326 ++++++++++++ soup_cli/utils/prompt_compile.py | 252 ++++++++++ soup_cli/utils/prompt_distill.py | 161 ++++++ tests/test_v0680_followups.py | 734 ++++++++++++++++++++++++++++ tests/test_v0680_part_a.py | 536 ++++++++++++++++++++ tests/test_v0680_part_b.py | 351 +++++++++++++ tests/test_v0680_part_c.py | 264 ++++++++++ tests/test_v0680_part_d.py | 311 ++++++++++++ tests/test_v0680_part_e.py | 471 ++++++++++++++++++ 22 files changed, 4271 insertions(+), 12 deletions(-) create mode 100644 soup_cli/commands/apple_adapter.py create mode 100644 soup_cli/commands/compile_cmd.py create mode 100644 soup_cli/commands/compile_tools.py create mode 100644 soup_cli/commands/distill_prompt.py create mode 100644 soup_cli/commands/local_rl.py create mode 100644 soup_cli/utils/apple_adapter.py create mode 100644 soup_cli/utils/compile_tools.py create mode 100644 soup_cli/utils/local_rl.py create mode 100644 soup_cli/utils/prompt_compile.py create mode 100644 soup_cli/utils/prompt_distill.py create mode 100644 tests/test_v0680_followups.py create mode 100644 tests/test_v0680_part_a.py create mode 100644 tests/test_v0680_part_b.py create mode 100644 tests/test_v0680_part_c.py create mode 100644 tests/test_v0680_part_d.py create mode 100644 tests/test_v0680_part_e.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 87bb15d..bfa264c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,7 @@ soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (251 files, 11021 tests) +tests/ - Test suite (257 files, 11225 tests) examples/ - Real-world config examples and datasets ``` diff --git a/README.md b/README.md index 419dc4b..a77393a 100644 --- a/README.md +++ b/README.md @@ -42,15 +42,44 @@ soup train Latest highlights only. Full history: [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). -**v0.67.0 — Adapter Lifecycle Finish: CMA-ES evolutionary merge + VeRA/VB-LoRA bank storage + MoLE per-token routing + GitHub-shaped adapter PRs + `soup.lock` + `soup adapters bisect`.** Six surfaces finish what v0.57 started — turning adapter management into a real version-control + collaboration workflow. None of these exist in hosted vendors: Sakana-style evolutionary merge is research demo only, VeRA storage hurts hosted unit economics (price by GPU-hour not adapter count), MoLE routing requires both training + serving stacks, adapter PRs need weights + eval + history together. +**v0.68.0 — Anti-trend Insurance: `soup compile` (DSPy/GEPA) + `soup distill-prompt` + `soup compile-tools` + `soup apple-adapter` + `soup local-rl` daemon.** Five bets that hedge Soup against paradigm shifts. If 1M-token contexts kill fine-tuning, `soup compile` (DSPy + GEPA prompt-program compilation) takes its place. If teams hit prompt-cost walls, `soup distill-prompt` bridges to small FT. If only Apple Foundation Models win on-device, `soup apple-adapter` ships the converter+signing surface. If personal-LLM flywheels become the shape, `soup local-rl` daemon captures thumbs into SQLite and emits DPO pairs nightly. None of these exist as CLI workflows in the OSS or hosted ecosystem — DSPy/GEPA are libraries, prompt distillation is unowned, Apple FoundationModels adapter format is undocumented territory, personal-LLM flywheels are research-stage. Schema-only release for Parts A/B/C/D — live wiring deferred to v0.68.1. Part E ships LIVE except the nightly train scheduler. -- **`soup adapters merge --strategy cmaes --eval --budget 1h`** — Sakana-style evolutionary search over LoRA merge weights. Pure-Python rank-mu CMA-ES (no `cma` dependency). Softmaxes N-1 logits onto the simplex, samples a population, keeps elite half, plateau-detects (3 generations without improvement → `converged=True`). Operator-supplied `eval_fn` closure; failures swallowed with sentinel score so one broken eval doesn't crash the run. Budget bounds reused from v0.57 `blame.parse_budget` (60s..24h). Live auto-wiring of the eval suite is deferred to v0.67.1; the CLI prints the validated plan today. -- **`soup_cli.utils.vector_bank`** — VeRA / VB-LoRA storage format. `VectorBank = {shared random projection matrix P, per-user scaling vector v_u}` — thousands of per-user adapters at MB-each instead of hundreds-of-MB per LoRA. Atomic JSON I/O via shared `paths.atomic_write_text` + cwd containment + symlink rejection + 16 MiB cap. `estimate_bank_size(num_users, vector_dim)` for sizing. Live multi-tenant serving via v0.22 multi-adapter surface lands in v0.67.1. -- **`task='moe_lora_routing'`** — MoLE per-token gating over N task LoRAs (Mixture of LoRA Experts). New `MoleGatingConfig` (num_task_adapters [2,64] / hidden_dim / temperature / top_k); cross-validator rejects mlx backend. Live gating-kernel training + per-token softmax routing lands in v0.67.1. -- **`soup adapters pr --base-sha <hex> --adapter <path> --eval <json>`** — GitHub-shaped PR rendering. PR = {base SHA, dataset diff, adapter weights, eval-delta report} → review-friendly Markdown with eval-delta tables + per-sample baseline/candidate diffs. `_md_table_escape` neutralises `\\` `|` `\n` `\r` `\t` in cells. JSON output for downstream consumers (v0.68 GitHub Action). Composes with `soup adapters diff`. -- **`soup lock write / show / check`** — Shared run lockfile. Closure of `(base_model_sha, dataset_sha, env_hash)` → committed to git so teams coordinate on reproducible training runs. `soup lock check` exits 3 on drift; `soup_version` + `created_at` are advisory-only (legitimate operator upgrades don't trigger drift). Composes with v0.64 `soup env lock`. -- **`soup adapters bisect <ckpt1> <ckpt2> ... --eval-command "..."`** — Binary search over training history. Operator supplies a shell template with `{ckpt}` placeholder (argv-list mode via `shlex.split` after `shlex.quote(ckpt)` — no `shell=True`). Probes both endpoints first (short-circuits all-OK / all-broken), then ~log₂(n) midpoint probes. Exit 3 on BROKEN_AT. Composes with v0.66 influence-blame for fine-grained attribution. -- **+165 new tests** (10836 → 11021) across 7 new test files. Review-fix coverage across 2 sequential waves: 0 CRITICAL + 0 HIGH (project policies already enforced by source-grep guards inherited from v0.66) + ~5 MEDIUM (plateau-convergence test; vector-bank forward-compat for unknown JSON fields; PR empty-sections renders without `None` leakage; soup.lock `soup_version` drift is advisory-only; bisect non-monotonic eval_fn does not crash) + ~3 LOW (exact MAX-boundary acceptance tests; `validate_user_id` bool rejection; source-grep regression for `atomic_write_text` usage + no `shell=True` + `shlex.quote` in bisect subprocess call). Manual CPU smokes for `soup lock write/show/check` (incl. drift exit 3), `soup adapters pr` (markdown + JSON output), `soup adapters bisect --plan-only`, `soup adapters merge --strategy cmaes` (plan-only output), plus 4 failure-mode rejection paths (cmaes-without-eval / unknown-strategy / mlx + moe_lora_routing / lock drift). +- **`soup compile <program.py> --eval <suite> [--optimizer mipro|gepa|textgrad|copro|bootstrap_fewshot] [--plan-only]`** — DSPy / GEPA prompt-program compiler. Closed allowlist over the canonical 5 optimisers. `CompilePlan` is frozen and re-validated on construction — direct-construction bugs that bypass `build_compile_plan` still produce loud failures. `CompileResult` rejects NaN / ±Inf scores via `math.isfinite`, bool-as-int, and negative iterations. Live orchestrator (DSPy / GEPA / TextGrad) deferred to v0.68.1; the CLI renders the validated plan today and exits 3 on the deferred path (mirrors v0.50.0 / v0.61.0 stub-then-live cadence). +- **`soup distill-prompt --traces <jsonl> --teacher <m> --student <m> --strategy sft|preference|kl`** — Take prompt-heavy traces from GPT-5 / Claude calls and prepare a distillation dataset targeting a small student model. Bridge between prompt-engineering and FT worlds. Composes with v0.70 cross-tokenizer KD when that ships. Schema + path containment + symlink rejection ship now; the teacher/student tokeniser-bridge runner lands in v0.68.1. +- **`soup compile-tools <spec.json|yaml> --eval <jsonl> [--optimizer textgrad|gepa]`** — TextGrad / GEPA tool-schema optimiser. Reads OpenAPI / MCP / GraphQL specs and prepares to optimise tool descriptions via textual gradients. Composes with v0.46 Agent Forge — Agent Forge parses the spec, `soup compile-tools` optimises the descriptions. `validate_spec_path` enforces JSON / YAML extension allowlist + cwd containment + symlink rejection. +- **`soup apple-adapter <source-dir> --direction hf-to-mlx|mlx-to-hf|hf-to-apple|mlx-to-apple --output <dir> [--sign]`** — Adapter format conversion + optional v0.60 Merkle-root signing. Extends v0.25 MLX backend. `validate_source_adapter` enforces cwd containment + `stat.S_ISDIR` (must be a directory) + symlink rejection. The `--sign` flag flows the converted adapter through v0.60 signing for distribution / sigstore. +- **`soup local-rl init / status / record / harvest / train`** — Personal-LLM flywheel daemon. Smaller-scope cousin of v0.58 `soup loop` — runs locally on a single workstation, trains your personal model from your own feedback. **LIVE today**: `init` creates the SQLite schema (`interactions` + `thumbs` with CHECK constraints + `0o600` perms on POSIX); `record` appends a thumbs-up/down row via parameterised query; `status` renders a Rich table of counters; `harvest` walks the thumbs table and emits `{prompt, chosen, rejected}` JSONL pairs (last-writes-win dedup so duplicate thumbs collapse to one pair per prompt). The `train` subcommand (Ollama / MLX × DPO/KTO/ORPO) is deferred to v0.68.1 — operators today can harvest into `dpo_pairs.jsonl` and feed it to existing `soup train --task dpo`. +- **+204 new tests** (11021 → 11225) across 6 new test files (5 part files + 1 followups covering every TDD-review finding). Review-fix coverage: 0 CRITICAL + 4 HIGH (validate_student_id rejection-matrix parity / Part C CLI unknown-optimizer exit 2 / harvest_dpo_pairs edge cases / POSIX symlink skip predicate uses `sys.platform`) + 10 MEDIUM (CompileResult ±Inf rejection + bool-score / validate_eval_suite_path symlink test / Part C empty + oversize on tool-optimizer / Part C null-byte + symlink on spec_path / TestBuildToolCompilePlan factory / Part D non-string + oversize on direction / TestBuildAppleAdapterPlan factory / Part E backend + train_method full rejection matrix / record_thumb response null-byte + oversize + empty / SUPPORTED_LOCAL_RL_TRAIN_METHODS immutability) + 4 LOW (TestInitDb column-level schema via PRAGMA / Part D output_dir validation / validate_db_path public + in __all__ / DpoPair frozen). Manual CPU smokes (Step 6): every command's `--help` + happy-path `--plan-only` + deferred-live exit 3 + validation rejection exit 2; full `soup local-rl init → record up → record down → status → harvest` end-to-end producing valid DPO JSONL. + +## Anti-trend Insurance + +The v0.68.0 release ships 5 commands that hedge Soup against paradigm shifts. Each is one of the 5 "if X kills FT, Soup still survives" bets. + +```bash +# DSPy / GEPA prompt-program compilation — replaces FT if 1M-context wins +soup compile my_program.py --eval evals.json --optimizer mipro --max-iters 10 + +# Distill prompt-heavy traces into small FT — bridges prompt-eng to FT worlds +soup distill-prompt --traces traces.jsonl --teacher anthropic/claude-3-5-sonnet \ + --student meta-llama/Llama-3.2-1B --strategy sft + +# Tool-schema textual-gradient optimization — composes with `soup agent` (v0.46) +soup compile-tools api_spec.yaml --eval tool_eval.jsonl --optimizer textgrad + +# HF / MLX / Apple FoundationModels adapter conversion + optional signing +soup apple-adapter ./my_adapter --direction hf-to-mlx --output ./mlx_adapter +soup apple-adapter ./my_adapter --direction hf-to-apple --output ./apple_adapter --sign + +# Personal-LLM flywheel — capture thumbs into SQLite, harvest DPO pairs nightly +soup local-rl init --db local_rl.db +soup local-rl record --db local_rl.db --prompt "What is X?" --response "..." --thumb up +soup local-rl record --db local_rl.db --prompt "What is X?" --response "wrong" --thumb down +soup local-rl status --db local_rl.db +soup local-rl harvest --db local_rl.db -o dpo_pairs.jsonl +# soup local-rl train (nightly DPO/KTO/ORPO) — v0.68.1 +``` + +Live runners for Parts A/B/C/D + the local-rl nightly scheduler land in v0.68.1. All schema + validation + path containment + symlink-rejection + atomic-write surfaces ship today, so misconfigured commands fail loudly at config load. ## Why Soup? diff --git a/SECURITY.md b/SECURITY.md index 7e55455..bb0feb1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,7 +9,8 @@ We provide security updates for the following versions: - **Versions older than 3 minor versions:** No support Example: -- v0.67.0 -- Full support (latest) +- v0.68.0 -- Full support (latest) +- v0.67.0 -- Full support - v0.66.0 -- Full support - v0.65.0 -- Full support - v0.64.0 -- Full support diff --git a/pyproject.toml b/pyproject.toml index fb402bd..281b47e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.67.0" +version = "0.68.0" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" diff --git a/soup_cli/__init__.py b/soup_cli/__init__.py index 79846c6..5518907 100644 --- a/soup_cli/__init__.py +++ b/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.67.0" +__version__ = "0.68.0" diff --git a/soup_cli/cli.py b/soup_cli/cli.py index e1181ff..d4ee733 100644 --- a/soup_cli/cli.py +++ b/soup_cli/cli.py @@ -417,6 +417,40 @@ app.add_typer( help="Shared run lockfile (write / show / check) - v0.67.0 Part E.", ) +# v0.68.0 — Anti-trend insurance (compile / distill-prompt / compile-tools / +# apple-adapter / local-rl). +from soup_cli.commands import apple_adapter as _apple_cmd # noqa: E402 +from soup_cli.commands import compile_cmd as _compile_cmd # noqa: E402 +from soup_cli.commands import compile_tools as _compile_tools_cmd # noqa: E402 +from soup_cli.commands import distill_prompt as _distill_prompt_cmd # noqa: E402 +from soup_cli.commands import local_rl as _local_rl_cmd # noqa: E402 + +app.command( + name="compile", + help="Compile a DSPy / GEPA prompt program against an eval suite (v0.68.0).", +)(_compile_cmd.compile_cmd) + +app.command( + name="distill-prompt", + help="Distill prompt-heavy traces into a small FT plan (v0.68.0).", +)(_distill_prompt_cmd.distill_prompt_cmd) + +app.command( + name="compile-tools", + help="Compile / optimize tool schemas + descriptions (v0.68.0).", +)(_compile_tools_cmd.compile_tools_cmd) + +app.command( + name="apple-adapter", + help="Convert / sign HF / MLX / Apple FoundationModels adapters (v0.68.0).", +)(_apple_cmd.apple_adapter_cmd) + +app.add_typer( + _local_rl_cmd.app, + name="local-rl", + help="Personal-LLM flywheel daemon (init / status / record / harvest / train) (v0.68.0).", +) + def _rewrite_advise_argv(argv: list) -> list: """Inject `run` between `advise` and a non-subcommand first argument. diff --git a/soup_cli/commands/apple_adapter.py b/soup_cli/commands/apple_adapter.py new file mode 100644 index 0000000..c31ae00 --- /dev/null +++ b/soup_cli/commands/apple_adapter.py @@ -0,0 +1,73 @@ +"""``soup apple-adapter`` CLI — v0.68.0 Part D. + +HF / PEFT <-> MLX <-> Apple Foundation Models adapter conversion + signing. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel + +from soup_cli.utils.apple_adapter import ( + SUPPORTED_ADAPTER_DIRECTIONS, + build_apple_adapter_plan, + convert_apple_adapter, +) + +console = Console() + + +def apple_adapter_cmd( + source: str = typer.Argument(..., help="Source adapter directory"), + direction: str = typer.Option( + "hf-to-mlx", + "--direction", + help="Allowed: " + ", ".join(sorted(SUPPORTED_ADAPTER_DIRECTIONS)), + ), + output: str = typer.Option(..., "--output", "-o", help="Output directory"), + sign: bool = typer.Option( + False, + "--sign/--no-sign", + help="Sign converted adapter via v0.60 Merkle-root signing.", + ), + plan_only: bool = typer.Option( + False, "--plan-only", help="Render plan + exit 0 (no live conversion)." + ), +) -> None: + """Convert / sign HF / MLX / Apple FoundationModels adapters.""" + try: + plan = build_apple_adapter_plan( + source_dir=source, + output_dir=output, + direction=direction, + sign=sign, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Source: [bold]{escape(plan.source_dir)}[/]\n" + f"Output: [bold]{escape(plan.output_dir)}[/]\n" + f"Direction: [bold]{escape(plan.direction)}[/]\n" + f"Sign: [bold]{plan.sign}[/]", + title="soup apple-adapter — plan", + ) + ) + + if plan_only: + return + + try: + convert_apple_adapter(plan) + except NotImplementedError as exc: + console.print( + Panel( + f"[yellow]{escape(str(exc))}[/]", + title="Live apple-adapter deferred", + ) + ) + raise typer.Exit(3) from exc diff --git a/soup_cli/commands/compile_cmd.py b/soup_cli/commands/compile_cmd.py new file mode 100644 index 0000000..980971c --- /dev/null +++ b/soup_cli/commands/compile_cmd.py @@ -0,0 +1,98 @@ +"""``soup compile`` — DSPy / GEPA prompt-program compiler CLI (v0.68.0 Part A). + +Renders a ``CompilePlan`` panel and, when ``--plan-only`` is omitted, +invokes the deferred-live ``run_compile`` (raises NotImplementedError +with a v0.68.1 marker). Mirrors v0.61.0 / v0.62.0 stub-then-live CLI policy. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel + +from soup_cli.utils.paths import atomic_write_text +from soup_cli.utils.prompt_compile import ( + SUPPORTED_PROMPT_OPTIMIZERS, + build_compile_plan, + run_compile, +) + +console = Console() + + +def compile_cmd( + program: str = typer.Argument(..., help="Path to DSPy / GEPA program (.py)"), + eval_suite: str = typer.Option( + ..., "--eval", help="Path to eval-suite JSON / JSONL file" + ), + optimizer: str = typer.Option( + "mipro", + "--optimizer", + help=( + "Optimizer name. Allowed: " + + ", ".join(sorted(SUPPORTED_PROMPT_OPTIMIZERS)) + ), + ), + max_iters: int = typer.Option(10, "--max-iters", help="Maximum optimizer iterations"), + output: str = typer.Option( + "compiled_program.py", "--output", "-o", help="Output program path" + ), + plan_only: bool = typer.Option( + False, + "--plan-only", + help="Render the resolved plan + exit 0 (no live compile).", + ), +) -> None: + """Compile a DSPy / GEPA prompt program against an eval suite.""" + try: + plan = build_compile_plan( + program_path=program, + eval_suite_path=eval_suite, + optimizer=optimizer, + max_iters=max_iters, + output_path=output, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Program: [bold]{escape(plan.program_path)}[/]\n" + f"Eval suite: [bold]{escape(plan.eval_suite_path)}[/]\n" + f"Optimizer: [bold]{escape(plan.optimizer)}[/]\n" + f"Max iters: [bold]{plan.max_iters}[/]\n" + f"Output: [bold]{escape(plan.output_path)}[/]", + title="soup compile — plan", + ) + ) + + if plan_only: + return + + try: + result = run_compile(plan) + except NotImplementedError as exc: + console.print( + Panel( + f"[yellow]{escape(str(exc))}[/]", + title="Live compile deferred", + ) + ) + raise typer.Exit(3) from exc + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + atomic_write_text(result.program_text, plan.output_path, field="output_path") + console.print( + Panel( + f"Output: [bold]{escape(plan.output_path)}[/]\n" + f"Score: [bold]{result.score:.3f}[/]\n" + f"Iterations: [bold]{result.iterations}[/]\n" + f"Converged: [bold]{result.converged}[/]", + title="soup compile — done", + ) + ) diff --git a/soup_cli/commands/compile_tools.py b/soup_cli/commands/compile_tools.py new file mode 100644 index 0000000..8666630 --- /dev/null +++ b/soup_cli/commands/compile_tools.py @@ -0,0 +1,73 @@ +"""``soup compile-tools`` CLI — v0.68.0 Part C. + +Generate tool schemas + descriptions optimized via textual gradients. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel + +from soup_cli.utils.compile_tools import ( + SUPPORTED_TOOL_OPTIMIZERS, + build_tool_compile_plan, + run_tool_compile, +) + +console = Console() + + +def compile_tools_cmd( + spec: str = typer.Argument(..., help="OpenAPI / MCP / GraphQL spec path"), + eval_suite: str = typer.Option( + ..., "--eval", help="Path to tool-call eval JSONL" + ), + optimizer: str = typer.Option( + "textgrad", + "--optimizer", + help="Allowed: " + ", ".join(sorted(SUPPORTED_TOOL_OPTIMIZERS)), + ), + output: str = typer.Option( + "compiled_tools.json", "--output", "-o", help="Output schema path" + ), + plan_only: bool = typer.Option( + False, "--plan-only", help="Render plan + exit 0 (no live optimise)." + ), +) -> None: + """Compile / optimise tool schemas via textual-gradient methods.""" + try: + plan = build_tool_compile_plan( + spec_path=spec, + eval_suite_path=eval_suite, + optimizer=optimizer, + output_path=output, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Spec: [bold]{escape(plan.spec_path)}[/]\n" + f"Eval: [bold]{escape(plan.eval_suite_path)}[/]\n" + f"Optimizer: [bold]{escape(plan.optimizer)}[/]\n" + f"Output: [bold]{escape(plan.output_path)}[/]", + title="soup compile-tools — plan", + ) + ) + + if plan_only: + return + + try: + run_tool_compile(plan) + except NotImplementedError as exc: + console.print( + Panel( + f"[yellow]{escape(str(exc))}[/]", + title="Live compile-tools deferred", + ) + ) + raise typer.Exit(3) from exc diff --git a/soup_cli/commands/distill_prompt.py b/soup_cli/commands/distill_prompt.py new file mode 100644 index 0000000..2d9bb5d --- /dev/null +++ b/soup_cli/commands/distill_prompt.py @@ -0,0 +1,75 @@ +"""``soup distill-prompt`` CLI — v0.68.0 Part B. + +Distill prompt-heavy traces into a small FT plan. +""" + +from __future__ import annotations + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel + +from soup_cli.utils.prompt_distill import ( + SUPPORTED_DISTILL_STRATEGIES, + build_distill_prompt_plan, + prepare_distill_dataset, +) + +console = Console() + + +def distill_prompt_cmd( + traces: str = typer.Option(..., "--traces", help="Path to traces JSONL"), + teacher: str = typer.Option(..., "--teacher", help="Teacher model id"), + student: str = typer.Option(..., "--student", help="Student model id"), + strategy: str = typer.Option( + "sft", + "--strategy", + help="Distill strategy. Allowed: " + + ", ".join(sorted(SUPPORTED_DISTILL_STRATEGIES)), + ), + output: str = typer.Option( + "distilled.jsonl", "--output", "-o", help="Output JSONL path" + ), + plan_only: bool = typer.Option( + False, "--plan-only", help="Render plan + exit 0 (no live preparation)." + ), +) -> None: + """Distill prompt-heavy traces into a small FT plan.""" + try: + plan = build_distill_prompt_plan( + traces_path=traces, + teacher=teacher, + student=student, + strategy=strategy, + output_path=output, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Traces: [bold]{escape(plan.traces_path)}[/]\n" + f"Teacher: [bold]{escape(plan.teacher)}[/]\n" + f"Student: [bold]{escape(plan.student)}[/]\n" + f"Strategy: [bold]{escape(plan.strategy)}[/]\n" + f"Output: [bold]{escape(plan.output_path)}[/]", + title="soup distill-prompt — plan", + ) + ) + + if plan_only: + return + + try: + prepare_distill_dataset(plan) + except NotImplementedError as exc: + console.print( + Panel( + f"[yellow]{escape(str(exc))}[/]", + title="Live distill-prompt deferred", + ) + ) + raise typer.Exit(3) from exc diff --git a/soup_cli/commands/local_rl.py b/soup_cli/commands/local_rl.py new file mode 100644 index 0000000..dbf07fc --- /dev/null +++ b/soup_cli/commands/local_rl.py @@ -0,0 +1,197 @@ +"""``soup local-rl`` — personal-LLM flywheel daemon CLI (v0.68.0 Part E). + +Subcommands: + +- ``init`` — create the SQLite schema +- ``status`` — print counters +- ``record`` — append a thumbs-up/down record +- ``harvest`` — emit DPO pairs as JSONL +- ``train`` — nightly DPO/KTO/ORPO train (deferred to v0.68.1) +""" + +from __future__ import annotations + +import dataclasses +import json +import sqlite3 + +import typer +from rich.console import Console +from rich.markup import escape +from rich.panel import Panel +from rich.table import Table + +from soup_cli.utils.local_rl import ( + SUPPORTED_LOCAL_RL_BACKENDS, + SUPPORTED_LOCAL_RL_TRAIN_METHODS, + LocalRLConfig, + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + run_nightly_train, +) +from soup_cli.utils.paths import atomic_write_text + +console = Console() + +app = typer.Typer(no_args_is_help=True, help="Personal-LLM flywheel daemon (v0.68.0).") + + +@app.command(name="init") +def init_cmd( + db: str = typer.Option( + "local_rl.db", "--db", help="Path to local-RL SQLite database" + ), +) -> None: + """Create the local-RL SQLite schema.""" + try: + init_local_rl_db(db) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"DB: [bold]{escape(db)}[/]\n" + f"Tables: [bold]interactions[/], [bold]thumbs[/]", + title="soup local-rl init", + ) + ) + + +@app.command(name="status") +def status_cmd( + db: str = typer.Option( + "local_rl.db", "--db", help="Path to local-RL SQLite database" + ), +) -> None: + """Print counters from the local-RL database.""" + import os + + from soup_cli.utils.local_rl import validate_db_path + + try: + validate_db_path(db) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + real = os.path.abspath(db) + if not os.path.exists(real): + console.print(f"[red]db not found: {escape(db)}[/]") + raise typer.Exit(2) + + with sqlite3.connect(real) as conn: + up = conn.execute( + "SELECT COUNT(*) FROM thumbs WHERE thumb='up'" + ).fetchone()[0] + down = conn.execute( + "SELECT COUNT(*) FROM thumbs WHERE thumb='down'" + ).fetchone()[0] + interactions = conn.execute( + "SELECT COUNT(*) FROM interactions" + ).fetchone()[0] + + table = Table(title=f"soup local-rl status — {db}") + table.add_column("Metric") + table.add_column("Count", justify="right") + table.add_row("Interactions", str(interactions)) + table.add_row("Thumbs up", str(up)) + table.add_row("Thumbs down", str(down)) + console.print(table) + + +@app.command(name="record") +def record_cmd( + db: str = typer.Option( + "local_rl.db", "--db", help="Path to local-RL SQLite database" + ), + prompt: str = typer.Option(..., "--prompt", help="Prompt text"), + response: str = typer.Option(..., "--response", help="Model response"), + thumb: str = typer.Option(..., "--thumb", help="up / down"), +) -> None: + """Append a thumbs-up / thumbs-down record.""" + try: + record_thumb(db_path=db, prompt=prompt, response=response, thumb=thumb) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + f"[green]Recorded {escape(thumb)} for prompt {escape(prompt[:32])}…[/]" + ) + + +@app.command(name="harvest") +def harvest_cmd( + db: str = typer.Option( + "local_rl.db", "--db", help="Path to local-RL SQLite database" + ), + output: str = typer.Option( + "dpo_pairs.jsonl", "--output", "-o", help="Output JSONL path" + ), +) -> None: + """Harvest DPO pairs from thumbs into a JSONL file.""" + try: + pairs = harvest_dpo_pairs(db) + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + lines = [json.dumps(dataclasses.asdict(p), ensure_ascii=False) for p in pairs] + text = "\n".join(lines) + ("\n" if lines else "") + try: + atomic_write_text(text, output, field="output") + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"DB: [bold]{escape(db)}[/]\n" + f"Output: [bold]{escape(output)}[/]\n" + f"Pairs: [bold]{len(pairs)}[/]", + title="soup local-rl harvest", + ) + ) + + +@app.command(name="train") +def train_cmd( + db: str = typer.Option( + "local_rl.db", "--db", help="Path to local-RL SQLite database" + ), + backend: str = typer.Option( + "ollama", + "--backend", + help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_BACKENDS)), + ), + model: str = typer.Option(..., "--model", help="Model id (Ollama tag or MLX path)"), + train_method: str = typer.Option( + "dpo", + "--train-method", + help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_TRAIN_METHODS)), + ), +) -> None: + """Trigger the nightly DPO/KTO/ORPO train. Deferred to v0.68.1.""" + try: + cfg = LocalRLConfig( + backend=backend, + model=model, + db_path=db, + train_method=train_method, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + try: + run_nightly_train(cfg) + except NotImplementedError as exc: + console.print( + Panel( + f"[yellow]{escape(str(exc))}[/]", + title="Live local-rl train deferred", + ) + ) + raise typer.Exit(3) from exc diff --git a/soup_cli/utils/apple_adapter.py b/soup_cli/utils/apple_adapter.py new file mode 100644 index 0000000..40dd811 --- /dev/null +++ b/soup_cli/utils/apple_adapter.py @@ -0,0 +1,144 @@ +"""``soup apple-adapter`` — HF / PEFT ↔ MLX ↔ Apple FoundationModels (v0.68.0 Part D). + +Schema-only release: live conversion (HF safetensors -> MLX npz / Apple +FoundationModels adapter blob) lands in v0.68.1. Reuses v0.60 Part B +signing for the optional ``--sign`` path. + +Public surface: + +- ``SUPPORTED_ADAPTER_DIRECTIONS`` — closed frozenset +- ``validate_direction(name)`` — bool-first / null-byte / case-insensitive +- ``validate_source_adapter(path)`` — cwd containment + directory check + symlink reject +- ``AppleAdapterPlan`` frozen dataclass +- ``build_apple_adapter_plan(...)`` factory +- ``convert_apple_adapter(plan)`` — NotImplementedError stub w/ v0.68.1 marker +""" + +from __future__ import annotations + +import os +import stat +from dataclasses import dataclass + +from soup_cli.utils.paths import is_under_cwd + +SUPPORTED_ADAPTER_DIRECTIONS: frozenset = frozenset( + {"hf-to-mlx", "mlx-to-hf", "hf-to-apple", "mlx-to-apple"} +) + +_MAX_DIRECTION_LEN = 32 + + +def validate_direction(name: object) -> str: + if isinstance(name, bool): + raise TypeError("direction must not be bool") + if not isinstance(name, str): + raise TypeError("direction must be str") + if not name: + raise ValueError("direction must be non-empty") + if "\x00" in name: + raise ValueError("direction must not contain null bytes") + if len(name) > _MAX_DIRECTION_LEN: + raise ValueError( + f"direction length {len(name)} > {_MAX_DIRECTION_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_ADAPTER_DIRECTIONS: + raise ValueError( + f"unknown direction {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_ADAPTER_DIRECTIONS)) + ) + return canonical + + +def validate_source_adapter(path: object) -> str: + """Validate a source-adapter directory (cwd-contained, no symlink).""" + if isinstance(path, bool): + raise TypeError("source_dir must not be bool") + if not isinstance(path, str): + raise TypeError("source_dir must be str") + if not path: + raise ValueError("source_dir must be non-empty") + if "\x00" in path: + raise ValueError("source_dir must not contain null bytes") + if not is_under_cwd(path): + raise ValueError( + f"source_dir {os.path.basename(path)!r} must stay under cwd" + ) + if os.path.lexists(path): + try: + st = os.lstat(path) + except OSError as exc: + raise ValueError( + f"source_dir unreadable: {type(exc).__name__}" + ) from exc + if stat.S_ISLNK(st.st_mode): + raise ValueError( + "source_dir must not be a symlink (TOCTOU defence)" + ) + if not stat.S_ISDIR(st.st_mode): + raise ValueError("source_dir must be a directory") + return os.path.realpath(path) + + +def _validate_output_dir(path: object) -> str: + if isinstance(path, bool): + raise TypeError("output_dir must not be bool") + if not isinstance(path, str): + raise TypeError("output_dir must be str") + if not path: + raise ValueError("output_dir must be non-empty") + if "\x00" in path: + raise ValueError("output_dir must not contain null bytes") + return path + + +@dataclass(frozen=True) +class AppleAdapterPlan: + source_dir: str + output_dir: str + direction: str + sign: bool + + def __post_init__(self) -> None: + validate_source_adapter(self.source_dir) + _validate_output_dir(self.output_dir) + object.__setattr__( + self, "direction", validate_direction(self.direction) + ) + if not isinstance(self.sign, bool): + raise TypeError("sign must be bool") + + +def build_apple_adapter_plan( + *, + source_dir: str, + output_dir: str, + direction: str, + sign: bool = False, +) -> AppleAdapterPlan: + return AppleAdapterPlan( + source_dir=source_dir, + output_dir=output_dir, + direction=validate_direction(direction), + sign=sign, + ) + + +def convert_apple_adapter(plan: AppleAdapterPlan) -> None: + """Live conversion. Deferred to v0.68.1.""" + if not isinstance(plan, AppleAdapterPlan): + raise TypeError("plan must be AppleAdapterPlan") + raise NotImplementedError( + "apple-adapter live conversion is deferred to v0.68.1" + ) + + +__all__ = [ + "SUPPORTED_ADAPTER_DIRECTIONS", + "validate_direction", + "validate_source_adapter", + "AppleAdapterPlan", + "build_apple_adapter_plan", + "convert_apple_adapter", +] diff --git a/soup_cli/utils/compile_tools.py b/soup_cli/utils/compile_tools.py new file mode 100644 index 0000000..1ea82a4 --- /dev/null +++ b/soup_cli/utils/compile_tools.py @@ -0,0 +1,129 @@ +"""``soup compile-tools`` — TextGrad / GEPA tool-schema optimizer (v0.68.0 Part C). + +Generate tool schemas + descriptions optimized via textual gradients. +Schema-only release: live optimizer pass lands in v0.68.1. + +Composes with v0.46 Agent Forge (OpenAPI / MCP / GraphQL parser) — Agent +Forge produces the spec, ``compile-tools`` optimises the descriptions. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + +SUPPORTED_TOOL_OPTIMIZERS: frozenset = frozenset({"textgrad", "gepa"}) +_SUPPORTED_SPEC_EXTENSIONS: frozenset = frozenset({".json", ".yaml", ".yml"}) + +_MAX_OPTIMIZER_NAME_LEN = 32 + + +def validate_tool_optimizer(name: object) -> str: + if isinstance(name, bool): + raise TypeError("optimizer must not be bool") + if not isinstance(name, str): + raise TypeError("optimizer must be str") + if not name: + raise ValueError("optimizer must be non-empty") + if "\x00" in name: + raise ValueError("optimizer must not contain null bytes") + if len(name) > _MAX_OPTIMIZER_NAME_LEN: + raise ValueError( + f"optimizer length {len(name)} > {_MAX_OPTIMIZER_NAME_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_TOOL_OPTIMIZERS: + raise ValueError( + f"unknown optimizer {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_TOOL_OPTIMIZERS)) + ) + return canonical + + +def validate_spec_path(path: object) -> str: + """Validate an OpenAPI / MCP / GraphQL spec path (json / yaml / yml only).""" + if isinstance(path, bool): + raise TypeError("spec_path must not be bool") + if not isinstance(path, str): + raise TypeError("spec_path must be str") + lower = path.lower() + if not any(lower.endswith(ext) for ext in _SUPPORTED_SPEC_EXTENSIONS): + raise ValueError( + "spec_path extension must be .json / .yaml / .yml" + ) + enforce_under_cwd_and_no_symlink(path, field="spec_path") + return os.path.realpath(path) + + +def validate_eval_suite_path(path: object) -> str: + if isinstance(path, bool): + raise TypeError("eval_suite_path must not be bool") + if not isinstance(path, str): + raise TypeError("eval_suite_path must be str") + enforce_under_cwd_and_no_symlink(path, field="eval_suite_path") + return os.path.realpath(path) + + +def _validate_output_path(path: object) -> str: + if isinstance(path, bool): + raise TypeError("output_path must not be bool") + if not isinstance(path, str): + raise TypeError("output_path must be str") + if not path: + raise ValueError("output_path must be non-empty") + if "\x00" in path: + raise ValueError("output_path must not contain null bytes") + return path + + +@dataclass(frozen=True) +class ToolCompilePlan: + spec_path: str + eval_suite_path: str + optimizer: str + output_path: str + + def __post_init__(self) -> None: + validate_spec_path(self.spec_path) + validate_eval_suite_path(self.eval_suite_path) + object.__setattr__( + self, "optimizer", validate_tool_optimizer(self.optimizer) + ) + _validate_output_path(self.output_path) + + +def build_tool_compile_plan( + *, + spec_path: str, + eval_suite_path: str, + optimizer: str, + output_path: str, +) -> ToolCompilePlan: + return ToolCompilePlan( + spec_path=spec_path, + eval_suite_path=eval_suite_path, + optimizer=validate_tool_optimizer(optimizer), + output_path=output_path, + ) + + +def run_tool_compile(plan: ToolCompilePlan) -> None: + """Live optimiser pass. Deferred to v0.68.1.""" + if not isinstance(plan, ToolCompilePlan): + raise TypeError("plan must be ToolCompilePlan") + raise NotImplementedError( + "compile-tools live optimisation is deferred to v0.68.1" + ) + + +__all__ = [ + "SUPPORTED_TOOL_OPTIMIZERS", + "validate_tool_optimizer", + "validate_spec_path", + "validate_eval_suite_path", + "ToolCompilePlan", + "build_tool_compile_plan", + "run_tool_compile", +] diff --git a/soup_cli/utils/local_rl.py b/soup_cli/utils/local_rl.py new file mode 100644 index 0000000..b8e378e --- /dev/null +++ b/soup_cli/utils/local_rl.py @@ -0,0 +1,326 @@ +"""``soup local-rl`` — personal-LLM flywheel daemon (v0.68.0 Part E). + +Wrap Ollama / MLX inference, capture thumbs into SQLite, harvest DPO pairs, +and (in v0.68.1) DPO-train nightly via systemd / launchd. Smaller-scope +cousin of v0.58 ``soup loop`` — runs locally on a single workstation, +trains the user's personal model from their own feedback. + +Schema + thumbs recording + DPO-pair harvester are LIVE in v0.68.0; the +nightly train scheduler is the deferred stub. + +Public surface: + +- ``SUPPORTED_LOCAL_RL_BACKENDS`` (``ollama``/``mlx``) +- ``SUPPORTED_LOCAL_RL_TRAIN_METHODS`` (``dpo``/``kto``/``orpo``) +- ``validate_local_rl_backend`` / ``validate_local_rl_train_method`` +- ``LocalRLConfig`` frozen dataclass +- ``init_local_rl_db(db_path)`` — atomic table creation +- ``record_thumb(...)`` — append a thumbs-up/down record +- ``harvest_dpo_pairs(db_path)`` — pair up/down responses to same prompt +- ``run_nightly_train(config)`` — NotImplementedError stub w/ v0.68.1 marker +""" + +from __future__ import annotations + +import os +import sqlite3 +import stat +import time +from dataclasses import dataclass +from typing import Tuple + +from soup_cli.utils.paths import is_under_cwd + +SUPPORTED_LOCAL_RL_BACKENDS: frozenset = frozenset({"ollama", "mlx"}) +SUPPORTED_LOCAL_RL_TRAIN_METHODS: frozenset = frozenset({"dpo", "kto", "orpo"}) +_VALID_THUMBS: frozenset = frozenset({"up", "down"}) + +MAX_PROMPT_LEN = 16_384 +MAX_RESPONSE_LEN = 16_384 +_MAX_BACKEND_LEN = 32 +_MAX_TRAIN_METHOD_LEN = 32 +_MAX_MODEL_LEN = 512 + + +# --------------------------------------------------------------------------- +# Validators +# --------------------------------------------------------------------------- + + +def validate_local_rl_backend(name: object) -> str: + if isinstance(name, bool): + raise TypeError("backend must not be bool") + if not isinstance(name, str): + raise TypeError("backend must be str") + if not name: + raise ValueError("backend must be non-empty") + if "\x00" in name: + raise ValueError("backend must not contain null bytes") + if len(name) > _MAX_BACKEND_LEN: + raise ValueError( + f"backend length {len(name)} > {_MAX_BACKEND_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_LOCAL_RL_BACKENDS: + raise ValueError( + f"unknown backend {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_LOCAL_RL_BACKENDS)) + ) + return canonical + + +def validate_local_rl_train_method(name: object) -> str: + if isinstance(name, bool): + raise TypeError("train_method must not be bool") + if not isinstance(name, str): + raise TypeError("train_method must be str") + if not name: + raise ValueError("train_method must be non-empty") + if "\x00" in name: + raise ValueError("train_method must not contain null bytes") + if len(name) > _MAX_TRAIN_METHOD_LEN: + raise ValueError( + f"train_method length {len(name)} > {_MAX_TRAIN_METHOD_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_LOCAL_RL_TRAIN_METHODS: + raise ValueError( + f"unknown train_method {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_LOCAL_RL_TRAIN_METHODS)) + ) + return canonical + + +def _validate_model(value: object) -> str: + if isinstance(value, bool): + raise TypeError("model must not be bool") + if not isinstance(value, str): + raise TypeError("model must be str") + if not value: + raise ValueError("model must be non-empty") + if "\x00" in value: + raise ValueError("model must not contain null bytes") + if len(value) > _MAX_MODEL_LEN: + raise ValueError(f"model length {len(value)} > {_MAX_MODEL_LEN}") + return value + + +def _validate_thumb(value: object) -> str: + if isinstance(value, bool): + raise TypeError("thumb must not be bool") + if not isinstance(value, str): + raise TypeError("thumb must be str") + if value not in _VALID_THUMBS: + raise ValueError( + f"thumb must be one of {sorted(_VALID_THUMBS)}, got {value!r}" + ) + return value + + +def validate_db_path(path: object) -> str: + if isinstance(path, bool): + raise TypeError("db_path must not be bool") + if not isinstance(path, str): + raise TypeError("db_path must be str") + if not path: + raise ValueError("db_path must be non-empty") + if "\x00" in path: + raise ValueError("db_path must not contain null bytes") + if not is_under_cwd(path): + raise ValueError( + f"db_path {os.path.basename(path)!r} must stay under cwd" + ) + if os.path.lexists(path): + try: + st = os.lstat(path) + except OSError as exc: + raise ValueError( + f"db_path unreadable: {type(exc).__name__}" + ) from exc + if stat.S_ISLNK(st.st_mode): + raise ValueError( + "db_path must not be a symlink (TOCTOU defence)" + ) + return path + + +def _validate_text(value: object, field: str, max_len: int) -> str: + if isinstance(value, bool): + raise TypeError(f"{field} must not be bool") + if not isinstance(value, str): + raise TypeError(f"{field} must be str") + if not value: + raise ValueError(f"{field} must be non-empty") + if "\x00" in value: + raise ValueError(f"{field} must not contain null bytes") + if len(value) > max_len: + raise ValueError(f"{field} length {len(value)} > {max_len}") + return value + + +# --------------------------------------------------------------------------- +# LocalRLConfig +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class LocalRLConfig: + """Runtime config for the local-RL flywheel.""" + + backend: str + model: str + db_path: str + train_method: str + + def __post_init__(self) -> None: + object.__setattr__( + self, "backend", validate_local_rl_backend(self.backend) + ) + _validate_model(self.model) + validate_db_path(self.db_path) + object.__setattr__( + self, + "train_method", + validate_local_rl_train_method(self.train_method), + ) + + +@dataclass(frozen=True) +class DpoPair: + """A harvested DPO training pair (prompt + chosen + rejected).""" + + prompt: str + chosen: str + rejected: str + + +# --------------------------------------------------------------------------- +# DB I/O +# --------------------------------------------------------------------------- + + +_SCHEMA_INTERACTIONS = """ +CREATE TABLE IF NOT EXISTS interactions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, + prompt TEXT NOT NULL, + response TEXT NOT NULL +) +""" + +_SCHEMA_THUMBS = """ +CREATE TABLE IF NOT EXISTS thumbs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + ts REAL NOT NULL, + prompt TEXT NOT NULL, + response TEXT NOT NULL, + thumb TEXT NOT NULL CHECK (thumb IN ('up', 'down')) +) +""" + + +def init_local_rl_db(db_path: str) -> None: + """Atomically create the local-RL SQLite schema. Idempotent.""" + validate_db_path(db_path) + real = os.path.abspath(db_path) + parent = os.path.dirname(real) or "." + os.makedirs(parent, exist_ok=True) + with sqlite3.connect(real) as conn: + conn.execute(_SCHEMA_INTERACTIONS) + conn.execute(_SCHEMA_THUMBS) + conn.commit() + # Best-effort 0o600 perms (matches v0.26.0 registry.db policy on POSIX). + if os.name == "posix": + try: + os.chmod(real, 0o600) + except OSError: + pass + + +def record_thumb( + *, + db_path: str, + prompt: str, + response: str, + thumb: str, +) -> None: + """Append a thumbs-up/down record. cwd-contained, symlink-rejected.""" + validate_db_path(db_path) + _validate_text(prompt, field="prompt", max_len=MAX_PROMPT_LEN) + _validate_text(response, field="response", max_len=MAX_RESPONSE_LEN) + _validate_thumb(thumb) + real = os.path.abspath(db_path) + with sqlite3.connect(real) as conn: + conn.execute( + "INSERT INTO thumbs (ts, prompt, response, thumb) VALUES (?, ?, ?, ?)", + (time.time(), prompt, response, thumb), + ) + conn.commit() + + +def harvest_dpo_pairs(db_path: str) -> Tuple[DpoPair, ...]: + """Pair up/down responses to the same prompt into DPO training rows. + + For each prompt that has at least one ``up`` AND at least one ``down`` + thumb, emit a single (chosen, rejected) tuple using the most recent + up + most recent down. Subsequent up/down pairs for the same prompt + are skipped (one row per prompt, last writes win) — keeps the harvest + deterministic without exploding into a Cartesian product. + """ + validate_db_path(db_path) + real = os.path.abspath(db_path) + if not os.path.exists(real): + raise FileNotFoundError(f"db_path not found: {db_path!r}") + with sqlite3.connect(real) as conn: + rows = conn.execute( + "SELECT prompt, response, thumb, ts FROM thumbs ORDER BY ts ASC" + ).fetchall() + last_up: dict = {} + last_down: dict = {} + for prompt, response, thumb, _ts in rows: + if thumb == "up": + last_up[prompt] = response + elif thumb == "down": + last_down[prompt] = response + pairs = [] + for prompt in sorted(set(last_up) & set(last_down)): + pairs.append( + DpoPair( + prompt=prompt, + chosen=last_up[prompt], + rejected=last_down[prompt], + ) + ) + return tuple(pairs) + + +# --------------------------------------------------------------------------- +# Live train stub (v0.68.1) +# --------------------------------------------------------------------------- + + +def run_nightly_train(config: LocalRLConfig) -> None: + """Run the nightly DPO/KTO/ORPO train. Deferred to v0.68.1.""" + if not isinstance(config, LocalRLConfig): + raise TypeError("config must be LocalRLConfig") + raise NotImplementedError( + "local-rl nightly train is deferred to v0.68.1 — " + "harvest DPO pairs today, train them once the runner lands" + ) + + +__all__ = [ + "SUPPORTED_LOCAL_RL_BACKENDS", + "SUPPORTED_LOCAL_RL_TRAIN_METHODS", + "MAX_PROMPT_LEN", + "MAX_RESPONSE_LEN", + "validate_local_rl_backend", + "validate_local_rl_train_method", + "validate_db_path", + "LocalRLConfig", + "DpoPair", + "init_local_rl_db", + "record_thumb", + "harvest_dpo_pairs", + "run_nightly_train", +] diff --git a/soup_cli/utils/prompt_compile.py b/soup_cli/utils/prompt_compile.py new file mode 100644 index 0000000..ad29c74 --- /dev/null +++ b/soup_cli/utils/prompt_compile.py @@ -0,0 +1,252 @@ +"""``soup compile`` — DSPy / GEPA prompt-program compiler (v0.68.0 Part A). + +Schema-only release: live wiring (DSPy / GEPA / TextGrad orchestrator) lands +in v0.68.1. The validators + frozen dataclasses ship now so operators can +build a plan today, then re-run with the live runner when it lands. + +Public surface: + +- ``SUPPORTED_PROMPT_OPTIMIZERS`` — closed frozenset +- ``MAX_COMPILE_ITERS`` — hard upper bound on iteration count +- ``validate_prompt_optimizer(name)`` — bool-first / null-byte / oversize / case-insensitive +- ``validate_max_iters(n)`` — bool-first / non-int / bounds +- ``validate_program_path(path)`` — cwd containment + symlink rejection + ``.py`` only +- ``validate_eval_suite_path(path)`` — cwd containment + symlink rejection +- ``CompilePlan`` frozen dataclass +- ``CompileResult`` frozen dataclass +- ``build_compile_plan(...)`` — factory +- ``run_compile(plan)`` — NotImplementedError stub w/ v0.68.1 marker + +The CLI command lives in ``soup_cli/commands/compile_cmd.py`` (named +``compile_cmd`` to avoid shadowing the Python builtin ``compile()``). +""" + +from __future__ import annotations + +import math +import os +from dataclasses import dataclass + +from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + +# --------------------------------------------------------------------------- +# Allowlists + bounds +# --------------------------------------------------------------------------- + + +SUPPORTED_PROMPT_OPTIMIZERS: frozenset = frozenset( + { + "bootstrap_fewshot", # DSPy classic + "mipro", # DSPy Multi-stage Instruction Proposal Optimizer + "copro", # DSPy COordinated PROmpt optimizer + "gepa", # Reflective Prompt Evolution (gradient-free) + "textgrad", # Textual-gradient optimizer + } +) + +MAX_COMPILE_ITERS = 1000 +_MIN_COMPILE_ITERS = 1 +_MAX_OPTIMIZER_NAME_LEN = 32 + + +# --------------------------------------------------------------------------- +# Validators +# --------------------------------------------------------------------------- + + +def validate_prompt_optimizer(name: object) -> str: + """Return the canonical lowercase optimizer name. + + Rejects bool / non-string / empty / null-byte / >32-char / unknown. + Mirrors v0.41.0 ``validate_optimizer_name`` policy. + """ + if isinstance(name, bool): + raise TypeError("optimizer must not be bool") + if not isinstance(name, str): + raise TypeError("optimizer must be str") + if not name: + raise ValueError("optimizer must be non-empty") + if "\x00" in name: + raise ValueError("optimizer must not contain null bytes") + if len(name) > _MAX_OPTIMIZER_NAME_LEN: + raise ValueError( + f"optimizer length {len(name)} > {_MAX_OPTIMIZER_NAME_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_PROMPT_OPTIMIZERS: + raise ValueError( + f"unknown optimizer {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_PROMPT_OPTIMIZERS)) + ) + return canonical + + +def validate_max_iters(value: object) -> int: + """Return ``value`` when an in-bounds positive int. + + Rejects bool / non-int / <=0 / > ``MAX_COMPILE_ITERS``. + """ + if isinstance(value, bool): + raise TypeError("max_iters must not be bool") + if not isinstance(value, int): + raise TypeError("max_iters must be int") + if value < _MIN_COMPILE_ITERS: + raise ValueError(f"max_iters must be >= {_MIN_COMPILE_ITERS}") + if value > MAX_COMPILE_ITERS: + raise ValueError(f"max_iters {value} > {MAX_COMPILE_ITERS}") + return value + + +def validate_program_path(path: object) -> str: + """Validate a prompt-program path (cwd-contained, ``.py`` only, no symlink). + + Returns the realpath of the validated path. + """ + if isinstance(path, bool): + raise TypeError("program_path must not be bool") + if not isinstance(path, str): + raise TypeError("program_path must be str") + if not path.endswith(".py"): + raise ValueError("program_path must end in .py") + enforce_under_cwd_and_no_symlink(path, field="program_path") + return os.path.realpath(path) + + +def validate_eval_suite_path(path: object) -> str: + """Validate an eval-suite path (cwd-contained, no symlink).""" + if isinstance(path, bool): + raise TypeError("eval_suite_path must not be bool") + if not isinstance(path, str): + raise TypeError("eval_suite_path must be str") + enforce_under_cwd_and_no_symlink(path, field="eval_suite_path") + return os.path.realpath(path) + + +def _validate_output_path(path: object) -> str: + if isinstance(path, bool): + raise TypeError("output_path must not be bool") + if not isinstance(path, str): + raise TypeError("output_path must be str") + if not path: + raise ValueError("output_path must be non-empty") + if "\x00" in path: + raise ValueError("output_path must not contain null bytes") + # Containment + symlink rejection enforced at write time by + # ``atomic_write_text``; we only shape-check here so a plan can be + # rendered before the output file is materialised. + return path + + +# --------------------------------------------------------------------------- +# Frozen dataclasses +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class CompilePlan: + """A resolved compilation plan, validated at construction time.""" + + program_path: str + eval_suite_path: str + optimizer: str + max_iters: int + output_path: str + + def __post_init__(self) -> None: + # Re-validate so callers bypassing ``build_compile_plan`` cannot + # smuggle in inconsistent fields (mirrors v0.61.0 EditPlan policy). + validate_program_path(self.program_path) + validate_eval_suite_path(self.eval_suite_path) + # ``optimizer`` should already be lowercase canonical from the + # factory; re-running the validator catches direct-construction + # bugs that bypass ``build_compile_plan``. + object.__setattr__( + self, "optimizer", validate_prompt_optimizer(self.optimizer) + ) + validate_max_iters(self.max_iters) + _validate_output_path(self.output_path) + + +@dataclass(frozen=True) +class CompileResult: + """The frozen result of running a compilation.""" + + program_text: str + score: float + iterations: int + converged: bool + + def __post_init__(self) -> None: + if isinstance(self.program_text, bool) or not isinstance(self.program_text, str): + raise TypeError("program_text must be str") + if isinstance(self.score, bool): + raise TypeError("score must not be bool") + if not isinstance(self.score, (int, float)): + raise TypeError("score must be number") + if not math.isfinite(float(self.score)): + raise ValueError("score must be finite") + if isinstance(self.iterations, bool): + raise TypeError("iterations must not be bool") + if not isinstance(self.iterations, int): + raise TypeError("iterations must be int") + if self.iterations < 0: + raise ValueError("iterations must be >= 0") + if not isinstance(self.converged, bool): + raise TypeError("converged must be bool") + + +# --------------------------------------------------------------------------- +# Factory +# --------------------------------------------------------------------------- + + +def build_compile_plan( + *, + program_path: str, + eval_suite_path: str, + optimizer: str, + max_iters: int, + output_path: str, +) -> CompilePlan: + """Validate every input and build a frozen ``CompilePlan``.""" + canonical_opt = validate_prompt_optimizer(optimizer) + return CompilePlan( + program_path=program_path, + eval_suite_path=eval_suite_path, + optimizer=canonical_opt, + max_iters=max_iters, + output_path=output_path, + ) + + +# --------------------------------------------------------------------------- +# Live runner stub (v0.68.1) +# --------------------------------------------------------------------------- + + +def run_compile(plan: CompilePlan) -> CompileResult: + """Run the live compilation. Deferred to v0.68.1. + + Validates the plan type at the boundary so callers passing a bare dict + get a clean ``TypeError`` rather than a confusing AttributeError when + the v0.68.1 runner finally lands. + """ + if not isinstance(plan, CompilePlan): + raise TypeError("plan must be CompilePlan") + raise NotImplementedError( + "soup compile live runner is deferred to v0.68.1 — re-run after upgrading" + ) + + +__all__ = [ + "SUPPORTED_PROMPT_OPTIMIZERS", + "MAX_COMPILE_ITERS", + "validate_prompt_optimizer", + "validate_max_iters", + "validate_program_path", + "validate_eval_suite_path", + "CompilePlan", + "CompileResult", + "build_compile_plan", + "run_compile", +] diff --git a/soup_cli/utils/prompt_distill.py b/soup_cli/utils/prompt_distill.py new file mode 100644 index 0000000..7547efb --- /dev/null +++ b/soup_cli/utils/prompt_distill.py @@ -0,0 +1,161 @@ +"""``soup distill-prompt`` — distill prompt-heavy traces into a small FT plan (v0.68.0 Part B). + +Bridge between prompt-engineering and FT worlds: take a JSONL of +large-prompt teacher calls (GPT-5 / Claude / etc.) and prepare a +distillation dataset targeting a small student model. Schema-only release: +live dataset preparation lands in v0.68.1 (composes with v0.70 Part B +cross-tokenizer KD when that ships). + +Public surface: + +- ``SUPPORTED_DISTILL_STRATEGIES`` — closed frozenset {sft, preference, kl} +- ``validate_distill_strategy(name)`` — bool-first / null-byte / case-insensitive +- ``validate_teacher_id`` / ``validate_student_id`` — null-byte / oversize / bool +- ``validate_traces_path(path)`` — cwd containment + symlink rejection +- ``DistillPromptPlan`` frozen dataclass +- ``build_distill_prompt_plan(...)`` factory +- ``prepare_distill_dataset(plan)`` — NotImplementedError stub w/ v0.68.1 marker +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass + +from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + +SUPPORTED_DISTILL_STRATEGIES: frozenset = frozenset({"sft", "preference", "kl"}) + +_MAX_STRATEGY_LEN = 32 +_MAX_MODEL_ID_LEN = 512 + + +def validate_distill_strategy(name: object) -> str: + """Return canonical lowercase strategy name.""" + if isinstance(name, bool): + raise TypeError("strategy must not be bool") + if not isinstance(name, str): + raise TypeError("strategy must be str") + if not name: + raise ValueError("strategy must be non-empty") + if "\x00" in name: + raise ValueError("strategy must not contain null bytes") + if len(name) > _MAX_STRATEGY_LEN: + raise ValueError( + f"strategy length {len(name)} > {_MAX_STRATEGY_LEN}" + ) + canonical = name.lower() + if canonical not in SUPPORTED_DISTILL_STRATEGIES: + raise ValueError( + f"unknown strategy {name!r}; supported: " + + ", ".join(sorted(SUPPORTED_DISTILL_STRATEGIES)) + ) + return canonical + + +def _validate_model_id(value: object, field: str) -> str: + if isinstance(value, bool): + raise TypeError(f"{field} must not be bool") + if not isinstance(value, str): + raise TypeError(f"{field} must be str") + if not value: + raise ValueError(f"{field} must be non-empty") + if "\x00" in value: + raise ValueError(f"{field} must not contain null bytes") + if len(value) > _MAX_MODEL_ID_LEN: + raise ValueError(f"{field} length {len(value)} > {_MAX_MODEL_ID_LEN}") + return value + + +def validate_teacher_id(value: object) -> str: + """Validate a teacher model id (HF repo id or local path-shape).""" + return _validate_model_id(value, field="teacher") + + +def validate_student_id(value: object) -> str: + """Validate a student model id.""" + return _validate_model_id(value, field="student") + + +def validate_traces_path(path: object) -> str: + """Validate a traces JSONL path (cwd-contained, no symlink).""" + if isinstance(path, bool): + raise TypeError("traces_path must not be bool") + if not isinstance(path, str): + raise TypeError("traces_path must be str") + enforce_under_cwd_and_no_symlink(path, field="traces_path") + return os.path.realpath(path) + + +def _validate_output_path(path: object) -> str: + if isinstance(path, bool): + raise TypeError("output_path must not be bool") + if not isinstance(path, str): + raise TypeError("output_path must be str") + if not path: + raise ValueError("output_path must be non-empty") + if "\x00" in path: + raise ValueError("output_path must not contain null bytes") + return path + + +@dataclass(frozen=True) +class DistillPromptPlan: + """A resolved distill-prompt plan.""" + + traces_path: str + teacher: str + student: str + strategy: str + output_path: str + + def __post_init__(self) -> None: + validate_traces_path(self.traces_path) + validate_teacher_id(self.teacher) + validate_student_id(self.student) + object.__setattr__( + self, "strategy", validate_distill_strategy(self.strategy) + ) + _validate_output_path(self.output_path) + + +def build_distill_prompt_plan( + *, + traces_path: str, + teacher: str, + student: str, + strategy: str, + output_path: str, +) -> DistillPromptPlan: + """Validate inputs and return a frozen ``DistillPromptPlan``.""" + return DistillPromptPlan( + traces_path=traces_path, + teacher=teacher, + student=student, + strategy=validate_distill_strategy(strategy), + output_path=output_path, + ) + + +def prepare_distill_dataset(plan: DistillPromptPlan) -> None: + """Live dataset preparation. Deferred to v0.68.1. + + Validates plan type at the boundary so a bare dict raises cleanly. + """ + if not isinstance(plan, DistillPromptPlan): + raise TypeError("plan must be DistillPromptPlan") + raise NotImplementedError( + "distill-prompt live dataset preparation is deferred to v0.68.1" + ) + + +__all__ = [ + "SUPPORTED_DISTILL_STRATEGIES", + "validate_distill_strategy", + "validate_teacher_id", + "validate_student_id", + "validate_traces_path", + "DistillPromptPlan", + "build_distill_prompt_plan", + "prepare_distill_dataset", +] diff --git a/tests/test_v0680_followups.py b/tests/test_v0680_followups.py new file mode 100644 index 0000000..2130b85 --- /dev/null +++ b/tests/test_v0680_followups.py @@ -0,0 +1,734 @@ +"""v0.68.0 review-fix follow-ups. + +Closes every TDD-review finding plus the additional manual code/security +review surface gaps: + +- HIGH H1: `validate_student_id` rejection-matrix parity with teacher +- HIGH H2: Part C CLI `unknown-optimizer` exit 2 test +- HIGH H3: `harvest_dpo_pairs` edge cases (one-up-no-down / dedup / multi-prompt) +- HIGH H4: POSIX symlink skip predicate uses `sys.platform` not `hasattr(os, "symlink")` +- MED M1: `CompileResult` Inf rejection + bool-score rejection +- MED M2: `validate_eval_suite_path` symlink test +- MED M3: Part C `validate_tool_optimizer` empty + oversize tests +- MED M4: Part C `validate_spec_path` null-byte + symlink tests +- MED M5: `TestBuildToolCompilePlan` factory test (Part C) +- MED M6: Part D `validate_direction` non-string TypeError + oversize tests +- MED M7: `TestBuildAppleAdapterPlan` factory test (Part D) +- MED M8: Part E backend / train_method validators — full rejection matrix +- MED M9: `record_thumb` null-byte + oversize on `response` +- MED M10: `SUPPORTED_LOCAL_RL_TRAIN_METHODS` immutability test +- LOW L1: Part E `TestInitDb` column-level schema assertions +- LOW L2: Part D `output_dir` null-byte / oversize / outside-cwd tests +- LOW L3: `validate_db_path` is a public symbol +- LOW L4: `DpoPair` frozen + invariants +- Sec/code: `validate_db_path` exported in `__all__` +""" + +from __future__ import annotations + +import dataclasses +import os +import sqlite3 +import sys +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +# --------------------------------------------------------------------------- +# Cross-platform symlink test gate (HIGH H4) +# --------------------------------------------------------------------------- + + +def _symlinks_available() -> bool: + """``os.symlink`` exists on Windows but needs elevation. Use platform.""" + return sys.platform != "win32" + + +# --------------------------------------------------------------------------- +# Part B — validate_student_id rejection-matrix parity (HIGH H1) +# --------------------------------------------------------------------------- + + +class TestValidateStudentIdParity: + def test_oversize_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + with pytest.raises(ValueError): + validate_student_id("a" * 513) + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + with pytest.raises(ValueError): + validate_student_id("a\x00b") + + def test_bool_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + with pytest.raises(TypeError): + validate_student_id(True) # type: ignore[arg-type] + + def test_empty_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + with pytest.raises(ValueError): + validate_student_id("") + + def test_non_string_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + with pytest.raises(TypeError): + validate_student_id(42) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Part C — CLI unknown-optimizer exits 2 (HIGH H2) +# --------------------------------------------------------------------------- + + +class TestPartCUnknownOptimizerCli: + def test_exits_2(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "compile-tools", + str(spec), + "--eval", + str(eval_suite), + "--optimizer", + "evil", + "--plan-only", + ], + ) + assert result.exit_code == 2 + + +# --------------------------------------------------------------------------- +# Part E — harvest_dpo_pairs edge cases (HIGH H3) +# --------------------------------------------------------------------------- + + +class TestHarvestEdgeCases: + def test_one_up_no_down_returns_empty( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + record_thumb( + db_path="rl.db", prompt="q", response="r", thumb="up" + ) + assert harvest_dpo_pairs("rl.db") == () + + def test_one_down_no_up_returns_empty( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + record_thumb( + db_path="rl.db", prompt="q", response="r", thumb="down" + ) + assert harvest_dpo_pairs("rl.db") == () + + def test_multiple_prompts_independent( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + for prompt, chosen, rejected in (("q1", "c1", "r1"), ("q2", "c2", "r2")): + record_thumb(db_path="rl.db", prompt=prompt, response=chosen, thumb="up") + record_thumb( + db_path="rl.db", prompt=prompt, response=rejected, thumb="down" + ) + pairs = harvest_dpo_pairs("rl.db") + assert len(pairs) == 2 + prompts = sorted(p.prompt for p in pairs) + assert prompts == ["q1", "q2"] + + def test_dedup_keeps_latest_per_prompt( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """Duplicate thumbs for the same prompt collapse to one DPO pair.""" + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + # Two ups + two downs for the same prompt should still yield exactly one pair. + record_thumb(db_path="rl.db", prompt="q", response="up1", thumb="up") + record_thumb(db_path="rl.db", prompt="q", response="up2", thumb="up") + record_thumb(db_path="rl.db", prompt="q", response="down1", thumb="down") + record_thumb(db_path="rl.db", prompt="q", response="down2", thumb="down") + pairs = harvest_dpo_pairs("rl.db") + assert len(pairs) == 1 + + +# --------------------------------------------------------------------------- +# Part A — CompileResult Inf + bool-score (MED M1) +# --------------------------------------------------------------------------- + + +class TestCompileResultBoundaries: + def test_positive_inf_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(ValueError): + CompileResult( + program_text="# x", + score=float("inf"), + iterations=1, + converged=True, + ) + + def test_negative_inf_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(ValueError): + CompileResult( + program_text="# x", + score=float("-inf"), + iterations=1, + converged=True, + ) + + def test_bool_score_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(TypeError): + CompileResult( + program_text="# x", + score=True, # type: ignore[arg-type] + iterations=1, + converged=True, + ) + + def test_zero_iterations_accepted(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + result = CompileResult( + program_text="# x", score=0.5, iterations=0, converged=False + ) + assert result.iterations == 0 + + +# --------------------------------------------------------------------------- +# Part A — validate_eval_suite_path symlink test (MED M2) +# --------------------------------------------------------------------------- + + +@pytest.mark.skipif(not _symlinks_available(), reason="POSIX symlink only") +class TestEvalSuitePathSymlink: + def test_symlink_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import validate_eval_suite_path + + monkeypatch.chdir(tmp_path) + real = tmp_path / "suite.json" + real.write_text("[]", encoding="utf-8") + link = tmp_path / "link.json" + try: + os.symlink(real, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not creatable on this filesystem") + with pytest.raises(ValueError, match="symlink"): + validate_eval_suite_path(str(link)) + + +# --------------------------------------------------------------------------- +# Part C — validate_tool_optimizer empty + oversize (MED M3) +# --------------------------------------------------------------------------- + + +class TestValidateToolOptimizerExtras: + def test_empty_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + with pytest.raises(ValueError): + validate_tool_optimizer("") + + def test_oversize_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + with pytest.raises(ValueError): + validate_tool_optimizer("a" * 33) + + +# --------------------------------------------------------------------------- +# Part C — validate_spec_path null-byte + symlink (MED M4) +# --------------------------------------------------------------------------- + + +class TestValidateSpecPathExtras: + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + with pytest.raises(ValueError): + validate_spec_path("spec\x00.json") + + @pytest.mark.skipif(not _symlinks_available(), reason="POSIX symlink only") + def test_symlink_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + monkeypatch.chdir(tmp_path) + real = tmp_path / "real.json" + real.write_text("{}", encoding="utf-8") + link = tmp_path / "link.json" + try: + os.symlink(real, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not creatable on this filesystem") + with pytest.raises(ValueError, match="symlink"): + validate_spec_path(str(link)) + + +# --------------------------------------------------------------------------- +# Part C — build_tool_compile_plan factory (MED M5) +# --------------------------------------------------------------------------- + + +class TestBuildToolCompilePlan: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.compile_tools import build_tool_compile_plan + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + plan = build_tool_compile_plan( + spec_path=str(spec), + eval_suite_path=str(eval_suite), + optimizer="GEPA", # case-insensitive + output_path="tools.json", + ) + assert plan.optimizer == "gepa" + + +# --------------------------------------------------------------------------- +# Part D — validate_direction non-string + oversize (MED M6) +# --------------------------------------------------------------------------- + + +class TestValidateDirectionExtras: + def test_non_string_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(TypeError): + validate_direction(42) # type: ignore[arg-type] + + def test_oversize_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(ValueError): + validate_direction("a" * 33) + + +# --------------------------------------------------------------------------- +# Part D — build_apple_adapter_plan factory (MED M7) +# --------------------------------------------------------------------------- + + +class TestBuildAppleAdapterPlan: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.apple_adapter import build_apple_adapter_plan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + plan = build_apple_adapter_plan( + source_dir=str(adapter), + output_dir="out", + direction="HF-TO-MLX", # case-insensitive + sign=False, + ) + assert plan.direction == "hf-to-mlx" + + +# --------------------------------------------------------------------------- +# Part D — output_dir validation (LOW L2) +# --------------------------------------------------------------------------- + + +class TestAppleAdapterOutputDirValidation: + def test_empty_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "a" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError): + AppleAdapterPlan( + source_dir=str(adapter), + output_dir="", + direction="hf-to-mlx", + sign=False, + ) + + def test_null_byte_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "a" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError): + AppleAdapterPlan( + source_dir=str(adapter), + output_dir="out\x00", + direction="hf-to-mlx", + sign=False, + ) + + def test_bool_output_dir_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "a" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + with pytest.raises(TypeError): + AppleAdapterPlan( + source_dir=str(adapter), + output_dir=True, # type: ignore[arg-type] + direction="hf-to-mlx", + sign=False, + ) + + +# --------------------------------------------------------------------------- +# Part E — backend / train_method full rejection matrix (MED M8) +# --------------------------------------------------------------------------- + + +class TestPartEValidatorParity: + def test_backend_null_byte_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(ValueError): + validate_local_rl_backend("ollama\x00") + + def test_backend_empty_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(ValueError): + validate_local_rl_backend("") + + def test_backend_oversize_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(ValueError): + validate_local_rl_backend("a" * 33) + + def test_backend_non_string_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(TypeError): + validate_local_rl_backend(42) # type: ignore[arg-type] + + def test_train_method_null_byte_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(ValueError): + validate_local_rl_train_method("dpo\x00") + + def test_train_method_empty_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(ValueError): + validate_local_rl_train_method("") + + def test_train_method_oversize_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(ValueError): + validate_local_rl_train_method("a" * 33) + + def test_train_method_non_string_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(TypeError): + validate_local_rl_train_method(42) # type: ignore[arg-type] + + def test_train_method_bool_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(TypeError): + validate_local_rl_train_method(True) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Part E — record_thumb response validation (MED M9) +# --------------------------------------------------------------------------- + + +class TestRecordThumbResponseValidation: + def test_response_null_byte_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", prompt="p", response="r\x00", thumb="up" + ) + + def test_response_oversize_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + MAX_RESPONSE_LEN, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", + prompt="p", + response="a" * (MAX_RESPONSE_LEN + 1), + thumb="up", + ) + + def test_response_empty_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", prompt="p", response="", thumb="up" + ) + + +# --------------------------------------------------------------------------- +# Part E — train_method allowlist immutability (MED M10) +# --------------------------------------------------------------------------- + + +class TestTrainMethodAllowlistImmutability: + def test_immutable(self) -> None: + from soup_cli.utils.local_rl import SUPPORTED_LOCAL_RL_TRAIN_METHODS + + with pytest.raises(AttributeError): + SUPPORTED_LOCAL_RL_TRAIN_METHODS.add("ppo") # type: ignore[attr-defined] + + +# --------------------------------------------------------------------------- +# Part E — schema column-level assertions (LOW L1) +# --------------------------------------------------------------------------- + + +class TestInitDbColumnLevel: + def test_thumbs_columns( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with sqlite3.connect("rl.db") as conn: + rows = conn.execute("PRAGMA table_info(thumbs)").fetchall() + names = {r[1] for r in rows} + # Schema regression guard: every named column must exist. + for col in ("id", "ts", "prompt", "response", "thumb"): + assert col in names + + def test_interactions_columns( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with sqlite3.connect("rl.db") as conn: + rows = conn.execute("PRAGMA table_info(interactions)").fetchall() + names = {r[1] for r in rows} + for col in ("id", "ts", "prompt", "response"): + assert col in names + + +# --------------------------------------------------------------------------- +# Part E — validate_db_path is public (LOW L3) +# --------------------------------------------------------------------------- + + +class TestValidateDbPathPublic: + def test_importable(self) -> None: + from soup_cli.utils.local_rl import validate_db_path # noqa: F401 + + def test_in_all(self) -> None: + from soup_cli.utils import local_rl + + assert "validate_db_path" in local_rl.__all__ + + def test_rejects_null_byte(self) -> None: + from soup_cli.utils.local_rl import validate_db_path + + with pytest.raises(ValueError): + validate_db_path("db\x00.db") + + def test_rejects_bool(self) -> None: + from soup_cli.utils.local_rl import validate_db_path + + with pytest.raises(TypeError): + validate_db_path(True) # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# Part E — DpoPair frozen invariant (LOW L4) +# --------------------------------------------------------------------------- + + +class TestDpoPairFrozen: + def test_frozen(self) -> None: + from soup_cli.utils.local_rl import DpoPair + + pair = DpoPair(prompt="p", chosen="c", rejected="r") + with pytest.raises(dataclasses.FrozenInstanceError): + pair.prompt = "evil" # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# Part E — harvest_dpo_pairs missing-file rejection (manual review) +# --------------------------------------------------------------------------- + + +class TestHarvestMissingFile: + def test_missing_db_raises( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import harvest_dpo_pairs + + monkeypatch.chdir(tmp_path) + with pytest.raises(FileNotFoundError): + harvest_dpo_pairs("no_such.db") + + +# --------------------------------------------------------------------------- +# Source-grep regression guards (LOW) +# --------------------------------------------------------------------------- + + +class TestSourceGuards: + def test_no_shell_true_in_v068_modules(self) -> None: + """No subprocess invocation in v0.68.0 utility modules.""" + for stem in ( + "prompt_compile", + "prompt_distill", + "compile_tools", + "apple_adapter", + "local_rl", + ): + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / f"{stem}.py" + ) + text = path.read_text(encoding="utf-8") + assert "shell=True" not in text, f"{stem}.py contains shell=True" + + def test_no_resolve_relative_to(self) -> None: + """Project policy: realpath + commonpath, not Path.resolve()+relative_to.""" + for stem in ( + "prompt_compile", + "prompt_distill", + "compile_tools", + "apple_adapter", + "local_rl", + ): + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / f"{stem}.py" + ) + text = path.read_text(encoding="utf-8") + assert ".resolve()" not in text or ".resolve().parent" in text, ( + f"{stem}.py should not use Path.resolve() for containment" + ) + + def test_local_rl_cli_imports_public_helper(self) -> None: + """CLI must import the public `validate_db_path`, not the private one.""" + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "commands" + / "local_rl.py" + ) + text = path.read_text(encoding="utf-8") + assert "_validate_db_path" not in text, ( + "CLI must use public `validate_db_path`, not the private alias" + ) + assert "validate_db_path" in text + + def test_all_command_modules_use_typer_exit(self) -> None: + """All v0.68 CLI command modules raise typer.Exit on rejection.""" + for stem in ( + "compile_cmd", + "distill_prompt", + "compile_tools", + "apple_adapter", + "local_rl", + ): + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "commands" + / f"{stem}.py" + ) + text = path.read_text(encoding="utf-8") + assert "typer.Exit" in text, f"{stem}.py missing typer.Exit" + + def test_version_bumped(self) -> None: + from soup_cli import __version__ + + # Floor-check: must be at or above the v0.68.0 release. + major, minor, patch = (int(x) for x in __version__.split(".")) + assert (major, minor) >= (0, 68) diff --git a/tests/test_v0680_part_a.py b/tests/test_v0680_part_a.py new file mode 100644 index 0000000..598521c --- /dev/null +++ b/tests/test_v0680_part_a.py @@ -0,0 +1,536 @@ +"""v0.68.0 Part A — ``soup compile`` (DSPy/GEPA prompt-program compiler). + +Tests for ``soup_cli/utils/prompt_compile.py`` + ``soup_cli/commands/compile_cmd.py``. + +Coverage: +- Closed ``SUPPORTED_PROMPT_OPTIMIZERS`` allowlist (frozenset, immutable) +- ``validate_prompt_optimizer`` happy + bool/null-byte/oversize/empty/non-string/unknown +- ``validate_max_iters`` bounds + bool rejection +- ``validate_program_path`` + ``validate_eval_suite_path`` containment + symlink reject +- Frozen ``CompilePlan`` + ``CompileResult`` + FrozenInstanceError +- ``build_compile_plan`` factory +- ``run_compile`` deferred-live stub raises NotImplementedError w/ v0.68.1 marker +- CLI smoke: help + plan-only + unknown optimizer + missing program +- Source-grep: no heavy top-level imports (torch / transformers / dspy) +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +# --------------------------------------------------------------------------- +# Public surface +# --------------------------------------------------------------------------- + + +class TestPublicSurface: + def test_module_importable(self) -> None: + from soup_cli.utils import prompt_compile + + assert hasattr(prompt_compile, "SUPPORTED_PROMPT_OPTIMIZERS") + assert hasattr(prompt_compile, "validate_prompt_optimizer") + assert hasattr(prompt_compile, "validate_max_iters") + assert hasattr(prompt_compile, "validate_program_path") + assert hasattr(prompt_compile, "validate_eval_suite_path") + assert hasattr(prompt_compile, "CompilePlan") + assert hasattr(prompt_compile, "CompileResult") + assert hasattr(prompt_compile, "build_compile_plan") + assert hasattr(prompt_compile, "run_compile") + + def test_allowlist_is_frozenset(self) -> None: + from soup_cli.utils.prompt_compile import SUPPORTED_PROMPT_OPTIMIZERS + + assert isinstance(SUPPORTED_PROMPT_OPTIMIZERS, frozenset) + # Allowlist should cover the canonical DSPy / GEPA / TextGrad set. + assert "bootstrap_fewshot" in SUPPORTED_PROMPT_OPTIMIZERS + assert "mipro" in SUPPORTED_PROMPT_OPTIMIZERS + assert "copro" in SUPPORTED_PROMPT_OPTIMIZERS + assert "gepa" in SUPPORTED_PROMPT_OPTIMIZERS + assert "textgrad" in SUPPORTED_PROMPT_OPTIMIZERS + + def test_allowlist_immutable(self) -> None: + from soup_cli.utils.prompt_compile import SUPPORTED_PROMPT_OPTIMIZERS + + with pytest.raises(AttributeError): + SUPPORTED_PROMPT_OPTIMIZERS.add("evil") # type: ignore[attr-defined] + + +# --------------------------------------------------------------------------- +# validate_prompt_optimizer +# --------------------------------------------------------------------------- + + +class TestValidatePromptOptimizer: + def test_happy(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + assert validate_prompt_optimizer("mipro") == "mipro" + + def test_case_insensitive(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + assert validate_prompt_optimizer("MIPRO") == "mipro" + assert validate_prompt_optimizer("Gepa") == "gepa" + + def test_bool_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(TypeError): + validate_prompt_optimizer(True) # type: ignore[arg-type] + + def test_non_string_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(TypeError): + validate_prompt_optimizer(42) # type: ignore[arg-type] + + def test_empty_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(ValueError): + validate_prompt_optimizer("") + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(ValueError): + validate_prompt_optimizer("mipro\x00") + + def test_oversize_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(ValueError): + validate_prompt_optimizer("a" * 33) + + def test_unknown_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_prompt_optimizer + + with pytest.raises(ValueError, match="unknown"): + validate_prompt_optimizer("evil-optimizer") + + +# --------------------------------------------------------------------------- +# validate_max_iters +# --------------------------------------------------------------------------- + + +class TestValidateMaxIters: + def test_happy(self) -> None: + from soup_cli.utils.prompt_compile import validate_max_iters + + assert validate_max_iters(10) == 10 + + def test_bool_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_max_iters + + with pytest.raises(TypeError): + validate_max_iters(True) # type: ignore[arg-type] + + def test_non_int_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_max_iters + + with pytest.raises(TypeError): + validate_max_iters(3.14) # type: ignore[arg-type] + + def test_zero_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_max_iters + + with pytest.raises(ValueError): + validate_max_iters(0) + + def test_negative_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_max_iters + + with pytest.raises(ValueError): + validate_max_iters(-1) + + def test_overcap_rejected(self) -> None: + from soup_cli.utils.prompt_compile import ( + MAX_COMPILE_ITERS, + validate_max_iters, + ) + + with pytest.raises(ValueError): + validate_max_iters(MAX_COMPILE_ITERS + 1) + + def test_max_boundary_accepted(self) -> None: + from soup_cli.utils.prompt_compile import ( + MAX_COMPILE_ITERS, + validate_max_iters, + ) + + assert validate_max_iters(MAX_COMPILE_ITERS) == MAX_COMPILE_ITERS + + +# --------------------------------------------------------------------------- +# validate_program_path + validate_eval_suite_path +# --------------------------------------------------------------------------- + + +class TestValidateProgramPath: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_compile import validate_program_path + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "program.py" + prog.write_text("# dspy program\n", encoding="utf-8") + assert validate_program_path(str(prog)).endswith("program.py") + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import validate_program_path + + outside = tmp_path / "outside" + outside.mkdir() + prog = outside / "program.py" + prog.write_text("pass\n", encoding="utf-8") + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + validate_program_path(str(prog)) + + def test_extension_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import validate_program_path + + monkeypatch.chdir(tmp_path) + bad = tmp_path / "program.txt" + bad.write_text("# wrong\n", encoding="utf-8") + with pytest.raises(ValueError, match="\\.py"): + validate_program_path(str(bad)) + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_compile import validate_program_path + + with pytest.raises(ValueError): + validate_program_path("a\x00b.py") + + @pytest.mark.skipif( + not hasattr(__import__("os"), "symlink"), reason="POSIX symlink only" + ) + def test_symlink_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import os + + from soup_cli.utils.prompt_compile import validate_program_path + + monkeypatch.chdir(tmp_path) + target = tmp_path / "real.py" + target.write_text("# real\n", encoding="utf-8") + link = tmp_path / "link.py" + try: + os.symlink(target, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not creatable") + with pytest.raises(ValueError, match="symlink"): + validate_program_path(str(link)) + + +class TestValidateEvalSuitePath: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_compile import validate_eval_suite_path + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "eval.json" + suite.write_text("[]", encoding="utf-8") + assert validate_eval_suite_path(str(suite)).endswith("eval.json") + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import validate_eval_suite_path + + outside = tmp_path / "outside" + outside.mkdir() + bad = outside / "suite.json" + bad.write_text("[]", encoding="utf-8") + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + validate_eval_suite_path(str(bad)) + + +# --------------------------------------------------------------------------- +# CompilePlan / CompileResult +# --------------------------------------------------------------------------- + + +class TestCompilePlan: + def test_frozen(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_compile import CompilePlan + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + + plan = CompilePlan( + program_path=str(prog), + eval_suite_path=str(suite), + optimizer="mipro", + max_iters=8, + output_path="out.py", + ) + with pytest.raises(dataclasses.FrozenInstanceError): + plan.optimizer = "gepa" # type: ignore[misc] + + def test_invalid_optimizer( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import CompilePlan + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError): + CompilePlan( + program_path=str(prog), + eval_suite_path=str(suite), + optimizer="evil", + max_iters=8, + output_path="out.py", + ) + + +class TestCompileResult: + def test_frozen(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + result = CompileResult( + program_text="# compiled", + score=0.85, + iterations=5, + converged=True, + ) + with pytest.raises(dataclasses.FrozenInstanceError): + result.score = 0.9 # type: ignore[misc] + + def test_invalid_score(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(ValueError): + CompileResult( + program_text="# x", score=float("nan"), iterations=1, converged=True + ) + + def test_negative_iterations_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(ValueError): + CompileResult( + program_text="# x", score=0.5, iterations=-1, converged=True + ) + + def test_bool_iterations_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(TypeError): + CompileResult( + program_text="# x", + score=0.5, + iterations=True, # type: ignore[arg-type] + converged=True, + ) + + def test_non_bool_converged_rejected(self) -> None: + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(TypeError): + CompileResult( + program_text="# x", + score=0.5, + iterations=1, + converged="yes", # type: ignore[arg-type] + ) + + +# --------------------------------------------------------------------------- +# build_compile_plan +# --------------------------------------------------------------------------- + + +class TestBuildCompilePlan: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_compile import build_compile_plan + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + + plan = build_compile_plan( + program_path=str(prog), + eval_suite_path=str(suite), + optimizer="mipro", + max_iters=4, + output_path="out.py", + ) + assert plan.optimizer == "mipro" + assert plan.max_iters == 4 + + +# --------------------------------------------------------------------------- +# run_compile deferred stub +# --------------------------------------------------------------------------- + + +class TestRunCompileDeferred: + def test_raises_with_v068_1_marker( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_compile import build_compile_plan, run_compile + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + plan = build_compile_plan( + program_path=str(prog), + eval_suite_path=str(suite), + optimizer="mipro", + max_iters=4, + output_path="out.py", + ) + with pytest.raises(NotImplementedError, match="v0.68.1"): + run_compile(plan) + + def test_non_plan_rejected(self) -> None: + from soup_cli.utils.prompt_compile import run_compile + + with pytest.raises(TypeError): + run_compile("not-a-plan") # type: ignore[arg-type] + + +# --------------------------------------------------------------------------- +# CLI smoke +# --------------------------------------------------------------------------- + + +class TestCli: + def test_help(self) -> None: + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["compile", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "compile" in result.output.lower() + + def test_plan_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "compile", + str(prog), + "--eval", + str(suite), + "--optimizer", + "mipro", + "--plan-only", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_unknown_optimizer_exits_2( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "compile", + str(prog), + "--eval", + str(suite), + "--optimizer", + "evil", + "--plan-only", + ], + ) + assert result.exit_code == 2 + + def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Without --plan-only the live runner raises and CLI exits 3.""" + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + prog = tmp_path / "p.py" + prog.write_text("pass\n", encoding="utf-8") + suite = tmp_path / "s.json" + suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "compile", + str(prog), + "--eval", + str(suite), + "--optimizer", + "mipro", + ], + ) + assert result.exit_code == 3, (result.output, repr(result.exception)) + + +# --------------------------------------------------------------------------- +# Source-grep regression guards +# --------------------------------------------------------------------------- + + +class TestSourceWiring: + def test_no_top_level_heavy_imports(self) -> None: + path = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "prompt_compile.py" + text = path.read_text(encoding="utf-8") + # Heavy / optional deps must be lazy-imported. + for token in ( + "\nimport torch", + "\nimport transformers", + "\nimport dspy", + ): + assert token not in text + + def test_cli_registered(self) -> None: + from soup_cli.cli import app + + names = [c.name for c in app.registered_commands] + assert "compile" in names + + def test_uses_atomic_write_helper(self) -> None: + path = Path(__file__).resolve().parent.parent / "soup_cli" / "commands" / "compile_cmd.py" + # If this file is missing the test must fail loudly — Part A CLI is a + # shipped artefact, not optional. Skipping would hide a regression. + assert path.exists(), f"missing CLI command module: {path}" + text = path.read_text(encoding="utf-8") + util = Path(__file__).resolve().parent.parent / "soup_cli" / "utils" / "prompt_compile.py" + combined = text + util.read_text(encoding="utf-8") + assert "atomic_write_text" in combined diff --git a/tests/test_v0680_part_b.py b/tests/test_v0680_part_b.py new file mode 100644 index 0000000..0373c67 --- /dev/null +++ b/tests/test_v0680_part_b.py @@ -0,0 +1,351 @@ +"""v0.68.0 Part B — ``soup distill-prompt``. + +Distill prompt-heavy traces (large-prompt GPT-5 / Claude calls) into a small +FT plan. Bridge between prompt-engineering and FT worlds. Composes with the +v0.70 Part B cross-tokenizer KD when that ships. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +class TestPublicSurface: + def test_module_importable(self) -> None: + from soup_cli.utils import prompt_distill + + assert hasattr(prompt_distill, "SUPPORTED_DISTILL_STRATEGIES") + assert hasattr(prompt_distill, "validate_distill_strategy") + assert hasattr(prompt_distill, "validate_teacher_id") + assert hasattr(prompt_distill, "validate_student_id") + assert hasattr(prompt_distill, "validate_traces_path") + assert hasattr(prompt_distill, "DistillPromptPlan") + assert hasattr(prompt_distill, "build_distill_prompt_plan") + assert hasattr(prompt_distill, "prepare_distill_dataset") + + +class TestAllowlist: + def test_frozenset(self) -> None: + from soup_cli.utils.prompt_distill import SUPPORTED_DISTILL_STRATEGIES + + assert isinstance(SUPPORTED_DISTILL_STRATEGIES, frozenset) + assert "sft" in SUPPORTED_DISTILL_STRATEGIES + assert "preference" in SUPPORTED_DISTILL_STRATEGIES + assert "kl" in SUPPORTED_DISTILL_STRATEGIES + + def test_immutable(self) -> None: + from soup_cli.utils.prompt_distill import SUPPORTED_DISTILL_STRATEGIES + + with pytest.raises(AttributeError): + SUPPORTED_DISTILL_STRATEGIES.add("evil") # type: ignore[attr-defined] + + +class TestValidateStrategy: + def test_happy(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + assert validate_distill_strategy("sft") == "sft" + + def test_case_insensitive(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + assert validate_distill_strategy("SFT") == "sft" + + def test_bool_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + with pytest.raises(TypeError): + validate_distill_strategy(True) # type: ignore[arg-type] + + def test_unknown_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + with pytest.raises(ValueError, match="unknown"): + validate_distill_strategy("evil") + + def test_empty_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + with pytest.raises(ValueError): + validate_distill_strategy("") + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_distill_strategy + + with pytest.raises(ValueError): + validate_distill_strategy("sft\x00") + + +class TestValidateModelId: + def test_teacher_happy(self) -> None: + from soup_cli.utils.prompt_distill import validate_teacher_id + + assert validate_teacher_id("anthropic/claude-3-5-sonnet") == "anthropic/claude-3-5-sonnet" + + def test_teacher_oversize_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_teacher_id + + with pytest.raises(ValueError): + validate_teacher_id("a" * 513) + + def test_teacher_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_teacher_id + + with pytest.raises(ValueError): + validate_teacher_id("model\x00") + + def test_teacher_bool_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_teacher_id + + with pytest.raises(TypeError): + validate_teacher_id(True) # type: ignore[arg-type] + + def test_teacher_empty_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_teacher_id + + with pytest.raises(ValueError): + validate_teacher_id("") + + def test_student_happy(self) -> None: + from soup_cli.utils.prompt_distill import validate_student_id + + assert validate_student_id("meta-llama/Llama-3.2-1B") == "meta-llama/Llama-3.2-1B" + + +class TestValidateTracesPath: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_distill import validate_traces_path + + monkeypatch.chdir(tmp_path) + p = tmp_path / "traces.jsonl" + p.write_text('{"prompt":"x","response":"y"}\n', encoding="utf-8") + assert validate_traces_path(str(p)).endswith("traces.jsonl") + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_distill import validate_traces_path + + outside = tmp_path / "outside" + outside.mkdir() + p = outside / "t.jsonl" + p.write_text("[]", encoding="utf-8") + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + validate_traces_path(str(p)) + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.prompt_distill import validate_traces_path + + with pytest.raises(ValueError): + validate_traces_path("t\x00.jsonl") + + @pytest.mark.skipif( + not hasattr(__import__("os"), "symlink"), reason="POSIX symlink only" + ) + def test_symlink_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import os + + from soup_cli.utils.prompt_distill import validate_traces_path + + monkeypatch.chdir(tmp_path) + real = tmp_path / "r.jsonl" + real.write_text("[]", encoding="utf-8") + link = tmp_path / "link.jsonl" + try: + os.symlink(real, link) + except (OSError, NotImplementedError): + pytest.skip("symlinks not creatable") + with pytest.raises(ValueError, match="symlink"): + validate_traces_path(str(link)) + + +class TestDistillPromptPlan: + def test_frozen(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_distill import DistillPromptPlan + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + + plan = DistillPromptPlan( + traces_path=str(traces), + teacher="anthropic/claude-3-5-sonnet", + student="meta-llama/Llama-3.2-1B", + strategy="sft", + output_path="distilled.jsonl", + ) + with pytest.raises(dataclasses.FrozenInstanceError): + plan.strategy = "preference" # type: ignore[misc] + + def test_invalid_strategy_propagates( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.prompt_distill import DistillPromptPlan + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError): + DistillPromptPlan( + traces_path=str(traces), + teacher="t", + student="s", + strategy="evil", + output_path="o.jsonl", + ) + + +class TestBuildPlan: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_distill import build_distill_prompt_plan + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + plan = build_distill_prompt_plan( + traces_path=str(traces), + teacher="t/x", + student="s/y", + strategy="sft", + output_path="o.jsonl", + ) + assert plan.strategy == "sft" + + +class TestPrepareDataset: + def test_deferred(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.prompt_distill import ( + build_distill_prompt_plan, + prepare_distill_dataset, + ) + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + plan = build_distill_prompt_plan( + traces_path=str(traces), + teacher="t/x", + student="s/y", + strategy="sft", + output_path="o.jsonl", + ) + with pytest.raises(NotImplementedError, match="v0.68.1"): + prepare_distill_dataset(plan) + + def test_non_plan_rejected(self) -> None: + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + with pytest.raises(TypeError): + prepare_distill_dataset({}) # type: ignore[arg-type] + + +class TestCli: + def test_help(self) -> None: + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["distill-prompt", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_plan_only(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text(json.dumps({"prompt": "x"}) + "\n", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "distill-prompt", + "--traces", + str(traces), + "--teacher", + "anthropic/claude-3-5-sonnet", + "--student", + "meta-llama/Llama-3.2-1B", + "--strategy", + "sft", + "--plan-only", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_unknown_strategy_exits_2( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "distill-prompt", + "--traces", + str(traces), + "--teacher", + "t", + "--student", + "s", + "--strategy", + "evil", + "--plan-only", + ], + ) + assert result.exit_code == 2 + + def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + traces = tmp_path / "traces.jsonl" + traces.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "distill-prompt", + "--traces", + str(traces), + "--teacher", + "t", + "--student", + "s", + "--strategy", + "sft", + ], + ) + assert result.exit_code == 3, (result.output, repr(result.exception)) + + +class TestSourceWiring: + def test_no_top_level_heavy_imports(self) -> None: + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / "prompt_distill.py" + ) + text = path.read_text(encoding="utf-8") + for token in ("\nimport torch", "\nimport transformers", "\nimport anthropic"): + assert token not in text + + def test_cli_registered(self) -> None: + from soup_cli.cli import app + + names = [c.name for c in app.registered_commands] + assert "distill-prompt" in names diff --git a/tests/test_v0680_part_c.py b/tests/test_v0680_part_c.py new file mode 100644 index 0000000..f012237 --- /dev/null +++ b/tests/test_v0680_part_c.py @@ -0,0 +1,264 @@ +"""v0.68.0 Part C — ``soup compile-tools``. + +Generate tool schemas + descriptions optimized via TextGrad-style textual +gradients. CI runs on OpenAPI / MCP schema changes. Composes with v0.46 +Agent Forge. +""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +class TestPublicSurface: + def test_module_importable(self) -> None: + from soup_cli.utils import compile_tools + + assert hasattr(compile_tools, "SUPPORTED_TOOL_OPTIMIZERS") + assert hasattr(compile_tools, "validate_tool_optimizer") + assert hasattr(compile_tools, "validate_spec_path") + assert hasattr(compile_tools, "ToolCompilePlan") + assert hasattr(compile_tools, "build_tool_compile_plan") + assert hasattr(compile_tools, "run_tool_compile") + + +class TestAllowlist: + def test_frozenset(self) -> None: + from soup_cli.utils.compile_tools import SUPPORTED_TOOL_OPTIMIZERS + + assert isinstance(SUPPORTED_TOOL_OPTIMIZERS, frozenset) + assert "textgrad" in SUPPORTED_TOOL_OPTIMIZERS + assert "gepa" in SUPPORTED_TOOL_OPTIMIZERS + + def test_immutable(self) -> None: + from soup_cli.utils.compile_tools import SUPPORTED_TOOL_OPTIMIZERS + + with pytest.raises(AttributeError): + SUPPORTED_TOOL_OPTIMIZERS.add("x") # type: ignore[attr-defined] + + +class TestValidateOptimizer: + def test_happy(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + assert validate_tool_optimizer("textgrad") == "textgrad" + + def test_case_insensitive(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + assert validate_tool_optimizer("GEPA") == "gepa" + + def test_bool_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + with pytest.raises(TypeError): + validate_tool_optimizer(True) # type: ignore[arg-type] + + def test_unknown_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + with pytest.raises(ValueError, match="unknown"): + validate_tool_optimizer("evil") + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.compile_tools import validate_tool_optimizer + + with pytest.raises(ValueError): + validate_tool_optimizer("textgrad\x00") + + +class TestValidateSpecPath: + def test_json_happy( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + assert validate_spec_path(str(spec)).endswith("spec.json") + + def test_yaml_happy( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.yaml" + spec.write_text("openapi: 3.0\n", encoding="utf-8") + assert validate_spec_path(str(spec)).endswith("spec.yaml") + + def test_invalid_extension_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.txt" + spec.write_text("garbage", encoding="utf-8") + with pytest.raises(ValueError, match="extension"): + validate_spec_path(str(spec)) + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import validate_spec_path + + outside = tmp_path / "outside" + outside.mkdir() + bad = outside / "spec.json" + bad.write_text("{}", encoding="utf-8") + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + validate_spec_path(str(bad)) + + +class TestToolCompilePlan: + def test_frozen( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import ToolCompilePlan + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + + plan = ToolCompilePlan( + spec_path=str(spec), + eval_suite_path=str(eval_suite), + optimizer="textgrad", + output_path="tools.json", + ) + with pytest.raises(dataclasses.FrozenInstanceError): + plan.optimizer = "gepa" # type: ignore[misc] + + def test_invalid_optimizer( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import ToolCompilePlan + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + + with pytest.raises(ValueError): + ToolCompilePlan( + spec_path=str(spec), + eval_suite_path=str(eval_suite), + optimizer="evil", + output_path="tools.json", + ) + + +class TestRunToolCompileDeferred: + def test_raises_v068_1( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.compile_tools import ( + build_tool_compile_plan, + run_tool_compile, + ) + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + plan = build_tool_compile_plan( + spec_path=str(spec), + eval_suite_path=str(eval_suite), + optimizer="textgrad", + output_path="tools.json", + ) + with pytest.raises(NotImplementedError, match="v0.68.1"): + run_tool_compile(plan) + + def test_non_plan_rejected(self) -> None: + from soup_cli.utils.compile_tools import run_tool_compile + + with pytest.raises(TypeError): + run_tool_compile("not-a-plan") # type: ignore[arg-type] + + +class TestCli: + def test_help(self) -> None: + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["compile-tools", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_plan_only( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text(json.dumps({"openapi": "3.0"}), encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "compile-tools", + str(spec), + "--eval", + str(eval_suite), + "--plan-only", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_live_exits_3( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + spec = tmp_path / "spec.json" + spec.write_text("{}", encoding="utf-8") + eval_suite = tmp_path / "eval.jsonl" + eval_suite.write_text("[]", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + ["compile-tools", str(spec), "--eval", str(eval_suite)], + ) + assert result.exit_code == 3, (result.output, repr(result.exception)) + + +class TestSourceWiring: + def test_no_top_level_heavy_imports(self) -> None: + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / "compile_tools.py" + ) + text = path.read_text(encoding="utf-8") + for token in ( + "\nimport torch", + "\nimport transformers", + "\nimport dspy", + "\nimport textgrad", + ): + assert token not in text + + def test_cli_registered(self) -> None: + from soup_cli.cli import app + + names = [c.name for c in app.registered_commands] + assert "compile-tools" in names diff --git a/tests/test_v0680_part_d.py b/tests/test_v0680_part_d.py new file mode 100644 index 0000000..4d55f86 --- /dev/null +++ b/tests/test_v0680_part_d.py @@ -0,0 +1,311 @@ +"""v0.68.0 Part D — ``soup apple-adapter``. + +HF / PEFT <-> MLX <-> Apple Foundation Models adapter conversion + signing. +Extends v0.25 MLX backend, reuses v0.60 Part B Merkle-root signing. +""" + +from __future__ import annotations + +import dataclasses +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +class TestPublicSurface: + def test_module_importable(self) -> None: + from soup_cli.utils import apple_adapter + + assert hasattr(apple_adapter, "SUPPORTED_ADAPTER_DIRECTIONS") + assert hasattr(apple_adapter, "validate_direction") + assert hasattr(apple_adapter, "validate_source_adapter") + assert hasattr(apple_adapter, "AppleAdapterPlan") + assert hasattr(apple_adapter, "build_apple_adapter_plan") + assert hasattr(apple_adapter, "convert_apple_adapter") + + +class TestAllowlist: + def test_frozenset(self) -> None: + from soup_cli.utils.apple_adapter import SUPPORTED_ADAPTER_DIRECTIONS + + assert isinstance(SUPPORTED_ADAPTER_DIRECTIONS, frozenset) + assert "hf-to-mlx" in SUPPORTED_ADAPTER_DIRECTIONS + assert "mlx-to-hf" in SUPPORTED_ADAPTER_DIRECTIONS + assert "hf-to-apple" in SUPPORTED_ADAPTER_DIRECTIONS + assert "mlx-to-apple" in SUPPORTED_ADAPTER_DIRECTIONS + + def test_immutable(self) -> None: + from soup_cli.utils.apple_adapter import SUPPORTED_ADAPTER_DIRECTIONS + + with pytest.raises(AttributeError): + SUPPORTED_ADAPTER_DIRECTIONS.add("x") # type: ignore[attr-defined] + + +class TestValidateDirection: + def test_happy(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + assert validate_direction("hf-to-mlx") == "hf-to-mlx" + + def test_case_insensitive(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + assert validate_direction("HF-TO-MLX") == "hf-to-mlx" + + def test_bool_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(TypeError): + validate_direction(True) # type: ignore[arg-type] + + def test_unknown_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(ValueError, match="unknown"): + validate_direction("evil") + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(ValueError): + validate_direction("hf-to-mlx\x00") + + def test_empty_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_direction + + with pytest.raises(ValueError): + validate_direction("") + + +class TestValidateSourceAdapter: + def test_happy(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.apple_adapter import validate_source_adapter + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + assert validate_source_adapter(str(adapter)).endswith("adapter") + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import validate_source_adapter + + outside = tmp_path / "outside" + outside.mkdir() + adapter = outside / "ad" + adapter.mkdir() + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + validate_source_adapter(str(adapter)) + + def test_non_directory_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import validate_source_adapter + + monkeypatch.chdir(tmp_path) + f = tmp_path / "not_a_dir.json" + f.write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="directory"): + validate_source_adapter(str(f)) + + def test_null_byte_rejected(self) -> None: + from soup_cli.utils.apple_adapter import validate_source_adapter + + with pytest.raises(ValueError): + validate_source_adapter("source\x00") + + +class TestAppleAdapterPlan: + def test_frozen( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + + plan = AppleAdapterPlan( + source_dir=str(adapter), + output_dir="out", + direction="hf-to-mlx", + sign=False, + ) + with pytest.raises(dataclasses.FrozenInstanceError): + plan.sign = True # type: ignore[misc] + + def test_invalid_direction( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + + with pytest.raises(ValueError): + AppleAdapterPlan( + source_dir=str(adapter), + output_dir="out", + direction="evil", + sign=False, + ) + + def test_sign_must_be_bool( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import AppleAdapterPlan + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + + with pytest.raises(TypeError): + AppleAdapterPlan( + source_dir=str(adapter), + output_dir="out", + direction="hf-to-mlx", + sign="yes", # type: ignore[arg-type] + ) + + +class TestConvertDeferred: + def test_raises_v068_1( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.apple_adapter import ( + build_apple_adapter_plan, + convert_apple_adapter, + ) + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + plan = build_apple_adapter_plan( + source_dir=str(adapter), + output_dir="out", + direction="hf-to-mlx", + sign=False, + ) + with pytest.raises(NotImplementedError, match="v0.68.1"): + convert_apple_adapter(plan) + + def test_non_plan_rejected(self) -> None: + from soup_cli.utils.apple_adapter import convert_apple_adapter + + with pytest.raises(TypeError): + convert_apple_adapter({}) # type: ignore[arg-type] + + +class TestCli: + def test_help(self) -> None: + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["apple-adapter", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_plan_only( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "apple-adapter", + str(adapter), + "--direction", + "hf-to-mlx", + "--output", + "out", + "--plan-only", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_unknown_direction_exits_2( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "apple-adapter", + str(adapter), + "--direction", + "evil", + "--output", + "out", + "--plan-only", + ], + ) + assert result.exit_code == 2 + + def test_live_exits_3( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text("{}", encoding="utf-8") + runner = CliRunner() + result = runner.invoke( + app, + [ + "apple-adapter", + str(adapter), + "--direction", + "hf-to-mlx", + "--output", + "out", + ], + ) + assert result.exit_code == 3, (result.output, repr(result.exception)) + + +class TestSourceWiring: + def test_no_top_level_heavy_imports(self) -> None: + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / "apple_adapter.py" + ) + text = path.read_text(encoding="utf-8") + for token in ( + "\nimport torch", + "\nimport transformers", + "\nimport mlx", + "\nimport safetensors", + ): + assert token not in text + + def test_cli_registered(self) -> None: + from soup_cli.cli import app + + names = [c.name for c in app.registered_commands] + assert "apple-adapter" in names diff --git a/tests/test_v0680_part_e.py b/tests/test_v0680_part_e.py new file mode 100644 index 0000000..5c0c162 --- /dev/null +++ b/tests/test_v0680_part_e.py @@ -0,0 +1,471 @@ +"""v0.68.0 Part E — ``soup local-rl`` personal-LLM flywheel daemon. + +Wrap Ollama / MLX inference, capture thumbs into SQLite, harvest DPO pairs, +and (in v0.68.1) DPO-train nightly from systemd / launchd. v0.68.0 ships +the SQLite schema + thumbs recording + DPO-pair harvester live; nightly +train scheduler is the stub. Smaller-scope cousin of v0.58 ``soup loop``. +""" + +from __future__ import annotations + +import dataclasses +import os +from pathlib import Path + +import pytest +from typer.testing import CliRunner + + +class TestPublicSurface: + def test_module_importable(self) -> None: + from soup_cli.utils import local_rl + + assert hasattr(local_rl, "SUPPORTED_LOCAL_RL_BACKENDS") + assert hasattr(local_rl, "SUPPORTED_LOCAL_RL_TRAIN_METHODS") + assert hasattr(local_rl, "validate_local_rl_backend") + assert hasattr(local_rl, "validate_local_rl_train_method") + assert hasattr(local_rl, "LocalRLConfig") + assert hasattr(local_rl, "init_local_rl_db") + assert hasattr(local_rl, "record_thumb") + assert hasattr(local_rl, "harvest_dpo_pairs") + assert hasattr(local_rl, "run_nightly_train") + + +class TestAllowlists: + def test_backend_frozenset(self) -> None: + from soup_cli.utils.local_rl import SUPPORTED_LOCAL_RL_BACKENDS + + assert isinstance(SUPPORTED_LOCAL_RL_BACKENDS, frozenset) + assert "ollama" in SUPPORTED_LOCAL_RL_BACKENDS + assert "mlx" in SUPPORTED_LOCAL_RL_BACKENDS + + def test_train_method_frozenset(self) -> None: + from soup_cli.utils.local_rl import SUPPORTED_LOCAL_RL_TRAIN_METHODS + + assert isinstance(SUPPORTED_LOCAL_RL_TRAIN_METHODS, frozenset) + assert "dpo" in SUPPORTED_LOCAL_RL_TRAIN_METHODS + assert "kto" in SUPPORTED_LOCAL_RL_TRAIN_METHODS + assert "orpo" in SUPPORTED_LOCAL_RL_TRAIN_METHODS + + def test_backend_immutable(self) -> None: + from soup_cli.utils.local_rl import SUPPORTED_LOCAL_RL_BACKENDS + + with pytest.raises(AttributeError): + SUPPORTED_LOCAL_RL_BACKENDS.add("x") # type: ignore[attr-defined] + + +class TestValidators: + def test_backend_happy(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + assert validate_local_rl_backend("ollama") == "ollama" + + def test_backend_case_insensitive(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + assert validate_local_rl_backend("OLLAMA") == "ollama" + + def test_backend_bool_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(TypeError): + validate_local_rl_backend(True) # type: ignore[arg-type] + + def test_backend_unknown_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_backend + + with pytest.raises(ValueError, match="unknown"): + validate_local_rl_backend("evil") + + def test_train_method_happy(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + assert validate_local_rl_train_method("dpo") == "dpo" + + def test_train_method_unknown_rejected(self) -> None: + from soup_cli.utils.local_rl import validate_local_rl_train_method + + with pytest.raises(ValueError, match="unknown"): + validate_local_rl_train_method("ppo") # PPO not in allowlist + + +class TestLocalRLConfig: + def test_frozen(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.local_rl import LocalRLConfig + + monkeypatch.chdir(tmp_path) + cfg = LocalRLConfig( + backend="ollama", + model="llama3:8b", + db_path="local_rl.db", + train_method="dpo", + ) + with pytest.raises(dataclasses.FrozenInstanceError): + cfg.backend = "mlx" # type: ignore[misc] + + def test_invalid_backend_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import LocalRLConfig + + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + LocalRLConfig( + backend="evil", + model="m", + db_path="db.db", + train_method="dpo", + ) + + def test_invalid_train_method_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import LocalRLConfig + + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + LocalRLConfig( + backend="ollama", + model="m", + db_path="db.db", + train_method="ppo", + ) + + +class TestInitDb: + def test_creates_tables( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import sqlite3 + + from soup_cli.utils.local_rl import init_local_rl_db + + monkeypatch.chdir(tmp_path) + db_path = "rl.db" + init_local_rl_db(db_path) + assert os.path.exists(db_path) + with sqlite3.connect(db_path) as conn: + rows = conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + names = {r[0] for r in rows} + assert "interactions" in names + assert "thumbs" in names + + def test_idempotent( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + # Second call must not raise. + init_local_rl_db("rl.db") + + def test_outside_cwd_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db + + outside = tmp_path / "outside" + outside.mkdir() + sub = tmp_path / "sub" + sub.mkdir() + monkeypatch.chdir(sub) + with pytest.raises(ValueError): + init_local_rl_db(str(outside / "rl.db")) + + +class TestRecordThumb: + def test_happy_up( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + import sqlite3 + + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + record_thumb( + db_path="rl.db", + prompt="capital of france?", + response="paris", + thumb="up", + ) + with sqlite3.connect("rl.db") as conn: + rows = conn.execute("SELECT thumb FROM thumbs").fetchall() + assert rows == [("up",)] + + def test_happy_down( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + record_thumb( + db_path="rl.db", prompt="x", response="y", thumb="down" + ) + + def test_invalid_thumb_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", prompt="x", response="y", thumb="meh" + ) + + def test_null_byte_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", + prompt="x\x00", + response="y", + thumb="up", + ) + + def test_bool_thumb_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(TypeError): + record_thumb( + db_path="rl.db", + prompt="x", + response="y", + thumb=True, # type: ignore[arg-type] + ) + + def test_oversize_prompt_rejected( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + MAX_PROMPT_LEN, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + with pytest.raises(ValueError): + record_thumb( + db_path="rl.db", + prompt="a" * (MAX_PROMPT_LEN + 1), + response="y", + thumb="up", + ) + + +class TestHarvestDpoPairs: + def test_empty(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + assert harvest_dpo_pairs("rl.db") == () + + def test_pairs_from_thumbs( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + record_thumb( + db_path="rl.db", + prompt="q", + response="good response", + thumb="up", + ) + record_thumb( + db_path="rl.db", + prompt="q", + response="bad response", + thumb="down", + ) + pairs = harvest_dpo_pairs("rl.db") + # One prompt with both up + down should yield exactly one DPO pair. + assert len(pairs) == 1 + pair = pairs[0] + assert pair.prompt == "q" + assert pair.chosen == "good response" + assert pair.rejected == "bad response" + + def test_returns_tuple( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + assert isinstance(harvest_dpo_pairs("rl.db"), tuple) + + +class TestNightlyTrainDeferred: + def test_raises_v068_1( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.utils.local_rl import ( + LocalRLConfig, + init_local_rl_db, + run_nightly_train, + ) + + monkeypatch.chdir(tmp_path) + init_local_rl_db("rl.db") + cfg = LocalRLConfig( + backend="ollama", + model="llama3:8b", + db_path="rl.db", + train_method="dpo", + ) + with pytest.raises(NotImplementedError, match="v0.68.1"): + run_nightly_train(cfg) + + def test_non_config_rejected(self) -> None: + from soup_cli.utils.local_rl import run_nightly_train + + with pytest.raises(TypeError): + run_nightly_train({}) # type: ignore[arg-type] + + +class TestCli: + def test_help(self) -> None: + from soup_cli.cli import app + + runner = CliRunner() + result = runner.invoke(app, ["local-rl", "--help"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_init_command( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + result = runner.invoke( + app, + ["local-rl", "init", "--db", "rl.db"], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert (tmp_path / "rl.db").exists() + + def test_record_command( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + runner.invoke(app, ["local-rl", "init", "--db", "rl.db"]) + result = runner.invoke( + app, + [ + "local-rl", + "record", + "--db", + "rl.db", + "--prompt", + "q", + "--response", + "a", + "--thumb", + "up", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_status_command( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + runner.invoke(app, ["local-rl", "init", "--db", "rl.db"]) + result = runner.invoke(app, ["local-rl", "status", "--db", "rl.db"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_harvest_command( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + runner.invoke(app, ["local-rl", "init", "--db", "rl.db"]) + result = runner.invoke( + app, ["local-rl", "harvest", "--db", "rl.db", "--output", "p.jsonl"] + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + + def test_train_command_exits_3( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + runner = CliRunner() + runner.invoke(app, ["local-rl", "init", "--db", "rl.db"]) + result = runner.invoke( + app, + [ + "local-rl", + "train", + "--db", + "rl.db", + "--backend", + "ollama", + "--model", + "llama3:8b", + ], + ) + assert result.exit_code == 3, (result.output, repr(result.exception)) + + +class TestSourceWiring: + def test_no_top_level_heavy_imports(self) -> None: + path = ( + Path(__file__).resolve().parent.parent + / "soup_cli" + / "utils" + / "local_rl.py" + ) + text = path.read_text(encoding="utf-8") + for token in ( + "\nimport torch", + "\nimport transformers", + "\nimport ollama", + "\nimport mlx", + ): + assert token not in text + + def test_cli_registered(self) -> None: + from soup_cli.cli import app + + names = [t.name for t in app.registered_groups] + assert "local-rl" in names