From f528da53289f29afb679b728c2a18aeebb989672 Mon Sep 17 00:00:00 2001 From: Alpamys Date: Thu, 4 Jun 2026 22:14:43 +0500 Subject: [PATCH] feat(prompt-compile): live soup compile / distill-prompt / compile-tools / local-rl train (v0.71.13) Lift the v0.68.0 deferred-stub family to live (closes #225, #226, #227, #229): - #229 local-rl train --once: harvest thumbs -> DPO/KTO/ORPO train via a soup train subprocess (argv list, no shell); state table tracks last_train_at (skip-on-no-new-thumbs + skip-on-insufficient-pairs); no --once renders a systemd/launchd nightly scheduler scaffold. New local_rl_scheduler.py. - #226 distill-prompt: call the teacher once per trace (Ollama/Anthropic/vLLM) and write a real dataset (sft/kl -> messages; preference -> chosen/rejected). - #225 compile / #227 compile-tools: live DSPy/GEPA/TextGrad dispatch behind the new [compile] extra with a friendly ImportError when absent; injectable seams. Security: reject \n/\r in the model id + shell-quote ExecStart args (systemd injection defence). Fix: render train output as a plain string (schema-valid), with a regression test against SoupConfig. Tests 13329 -> 13424. Smoked end-to-end: real DPO train on SmolLM2-135M (RTX 3050) + real Ollama teacher distillation. --- CHANGELOG.md | 38 + CONTRIBUTING.md | 4 +- README.md | 24 +- SECURITY.md | 2 +- docs/commands.md | 9 +- pyproject.toml | 6 +- src/soup_cli/__init__.py | 2 +- src/soup_cli/commands/compile_cmd.py | 14 +- src/soup_cli/commands/compile_tools.py | 19 +- src/soup_cli/commands/distill_prompt.py | 40 +- src/soup_cli/commands/local_rl.py | 102 +- src/soup_cli/utils/compile_tools.py | 133 +- src/soup_cli/utils/local_rl.py | 327 ++++- src/soup_cli/utils/local_rl_scheduler.py | 276 +++++ src/soup_cli/utils/prompt_compile.py | 272 ++++- src/soup_cli/utils/prompt_distill.py | 189 ++- tests/test_v0680_part_a.py | 18 +- tests/test_v0680_part_b.py | 33 +- tests/test_v0680_part_c.py | 27 +- tests/test_v0680_part_e.py | 23 +- tests/test_v07113.py | 1409 ++++++++++++++++++++++ 21 files changed, 2836 insertions(+), 131 deletions(-) create mode 100644 src/soup_cli/utils/local_rl_scheduler.py create mode 100644 tests/test_v07113.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e2ee63a..b633187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,44 @@ reproducing 70+ versions of notes. ## [Unreleased] +## [0.71.13] - 2026-06-04 + +### Added +- **Prompt-compile family — live wiring** (closes #225, #226, #227, #229). Four + `soup` commands that shipped as deferred-stub `NotImplementedError` in v0.68.0 + are now real, validated end-to-end (real DPO train on SmolLM2-135M + real + Ollama teacher distillation on RTX 3050). +- **`soup local-rl train` runs a real nightly DPO/KTO/ORPO train** (#229). + `--once` harvests the latest thumbs-up/down DPO pairs from the local-RL SQLite + and trains them via a `soup train` subprocess (argv list, no shell); a `state` + table tracks `last_train_at` so a re-run with no new feedback skips, and a run + with fewer than `--min-pairs` (default 10) skips. Without `--once` it renders a + systemd `.service`/`.timer` + launchd `.plist` scheduler scaffold into + `--scheduler-dir` for the user to install. New flags: `--once`, `--min-pairs`, + `--output/-o`, `--scheduler-dir`, `--hour`, `--minute`. +- **`soup distill-prompt` prepares a real distillation dataset** (#226). For + each prompt in the traces JSONL the teacher is called once via the v0.20 + provider helpers (Ollama / Anthropic / vLLM); `sft`/`kl` emit + `{messages:[user, assistant=teacher]}` and `preference` emits + `{prompt, chosen=teacher, rejected=student}`. New flags: `--provider`, + `--base-url`, `--temperature`, `--max-rows`. +- **`soup compile` runs DSPy / GEPA / TextGrad prompt-program optimisation** (#225) + and **`soup compile-tools` runs the TextGrad / GEPA tool-schema optimiser** (#227), + both lazy-importing the optimiser libraries behind the new `[compile]` extra + (`pip install 'soup-cli[compile]'`) with a friendly `ImportError` naming the + extra when absent. `--plan-only` still renders the plan and exits 0. + +### Security +- **systemd / launchd injection defence** (#229). `local-rl` and the scheduler + renderers reject `\n` / `\r` in the model id and shell-quote every `ExecStart` + argument, so a crafted model id cannot inject extra unit directives. + +### Fixed +- **`local-rl` train config rendered `output` as a mapping** (#229). The nightly + `soup train` YAML now emits `output: ` (a plain string the schema accepts) + instead of `output: {dir: }`; a regression test validates the rendered + config against `SoupConfig`. + ## [0.71.12] - 2026-06-04 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ee5d41c..e998454 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -116,11 +116,11 @@ src/soup_cli/ cans/ - Shareable .can artifact format + run/publish orchestrator (v0.26.0 + v0.33.0) data/traces/ - Trace-to-Preference harvester (v0.26.0) data/collators.py - CrossDocCollator for sample packing (v0.33.0) - utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats, deploy_measure, advise, advise_history, adapter_diff, adapter_merge, blame, adapter_branch, unlearning, unlearning_eval, knowledge_edit, edit_governor, edit_diff, ra_dit, steering, citation_faithful, grace_codebook, ingest_sources, prune_prompt, active_sampler, ab_test, drift_alarm, tunability, terraform_plan, env_lock, hardware_fit, completions, license_advisor, behavior_battery, capability_suite, checklist_dsl, irt, sae_diff, sleeper_probe, interference, probe_pack, cmaes_merge, vector_bank, mole_routing, adapter_pr, soup_lock, adapter_bisect, prompt_compile, prompt_distill, compile_tools, apple_adapter, local_rl, build_dag, expectations, magpie, persona_hub, brain_rot, reward_hacking, uld, minillm, rl_checkpoint, iterative_dpo, echo_trap, mod + utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats, deploy_measure, advise, advise_history, adapter_diff, adapter_merge, blame, adapter_branch, unlearning, unlearning_eval, knowledge_edit, edit_governor, edit_diff, ra_dit, steering, citation_faithful, grace_codebook, ingest_sources, prune_prompt, active_sampler, ab_test, drift_alarm, tunability, terraform_plan, env_lock, hardware_fit, completions, license_advisor, behavior_battery, capability_suite, checklist_dsl, irt, sae_diff, sleeper_probe, interference, probe_pack, cmaes_merge, vector_bank, mole_routing, adapter_pr, soup_lock, adapter_bisect, prompt_compile, prompt_distill, compile_tools, apple_adapter, local_rl, build_dag, expectations, magpie, persona_hub, brain_rot, reward_hacking, uld, minillm, rl_checkpoint, iterative_dpo, echo_trap, mod, local_rl_scheduler templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (283 files, 13329 tests) +tests/ - Test suite (284 files, 13424 tests) examples/ - Real-world config examples and datasets ``` diff --git a/README.md b/README.md index dca3441..44b0fdc 100644 --- a/README.md +++ b/README.md @@ -49,20 +49,18 @@ infrastructure instead of improving models. Soup fixes that. ## What's New -**v0.71.12 — Architecture, distillation & adapter-training (live).** Seven schema-only surfaces -from earlier releases are now real, validated end-to-end on tiny models: +**v0.71.13 — Prompt-compile family (live).** Four commands that shipped as deferred stubs in +v0.68.0 are now real, validated end-to-end on tiny models: -- **`soup serve --bank `** — multi-tenant VeRA / VB-LoRA serving: load N personas at - KB-per-user (shared projection + per-user vectors) and pick the active one per request via the - `X-User-Id` header. An unknown / absent id is a zero-delta no-op (no cross-request leak). -- **`task: moe_lora_routing`** — MoLE per-token routing trains a small gating network over N frozen - task LoRAs (`mole_task_adapters`, `mole_top_k`, `mole_temperature`); only the router trains. -- **`task: distill` with `distill_mode: sequence`** — sequence-level KD trains the student on the - teacher's generated continuations (cross-tokenizer-friendly), alongside the existing token logit-KL. -- **`task: classifier` with a `lora` section** — train a frozen encoder + LoRA adapter classifier - instead of the full model. -- **`use_mod` (Mixture-of-Depths) · `expand_layers` (LLaMA Pro) · `use_longlora` (S² attention)** — - the architecture knobs that were schema-only are now live for Llama / Qwen / Mistral (+ Phi for LongLoRA). +- **`soup local-rl train --once`** — harvest your latest 👍/👎 feedback into DPO pairs and train a + real DPO/KTO/ORPO round via `soup train` (argv list, no shell). A `state` table tracks the last + train so a re-run with no new feedback skips. Without `--once` it renders a systemd/launchd nightly + scheduler scaffold you can install. +- **`soup distill-prompt`** — call a teacher model (Ollama / Anthropic / vLLM) once per trace and + write a real distillation dataset: `sft`/`kl` → `{messages}`, `preference` → `{prompt, chosen, rejected}`. +- **`soup compile`** — DSPy / GEPA / TextGrad prompt-program optimisation, and **`soup compile-tools`** + — TextGrad / GEPA tool-schema optimisation, both behind the new `[compile]` extra + (`pip install 'soup-cli[compile]'`). Full history: [CHANGELOG.md](CHANGELOG.md) · [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). diff --git a/SECURITY.md b/SECURITY.md index fa85596..2580dc6 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -36,7 +36,7 @@ in-scope issues: dataset, or artifact paths; - SSRF in the synthetic-data providers, inference server, or hub/endpoint validators; -- command, Modelfile, or Jinja chat-template injection; +- command, Modelfile, Jinja chat-template, or systemd/launchd unit injection; - secret leakage in logs, crash bundles, or generated artifacts; - sandbox escape in the RLVR code-execution reward path. diff --git a/docs/commands.md b/docs/commands.md index 004441f..ce3b734 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -216,15 +216,16 @@ soup adapters bisect ... --eval-command "..." Binary search over training soup lock write --base-sha --dataset-sha --env-hash Write soup.lock (v0.67.0) soup lock write --base-sha --dataset-sha --env-lock soup-env.lock Auto-derive --env-hash from soup-env.lock (v0.71.1) soup lock show / soup lock check Show + drift-check (exit 3 on drift) -soup compile --eval [--optimizer mipro|gepa|textgrad|copro|bootstrap_fewshot] DSPy / GEPA prompt-program compiler (v0.68.0) -soup distill-prompt --traces --teacher --student --strategy sft|preference|kl Distill prompt-heavy traces to small FT (v0.68.0) -soup compile-tools --eval [--optimizer textgrad|gepa] TextGrad / GEPA tool-schema optimiser (v0.68.0) +soup compile --eval [--optimizer mipro|gepa|textgrad|copro|bootstrap_fewshot] [--plan-only] DSPy / GEPA / TextGrad prompt-program compiler — live (v0.71.13; pip install 'soup-cli[compile]') +soup distill-prompt --traces --teacher --student --strategy sft|preference|kl [--provider ollama|anthropic|vllm] [--base-url ] [--temperature F] [--max-rows N] Distill prompt-heavy traces via a live teacher (v0.71.13) +soup compile-tools --eval [--optimizer textgrad|gepa] [--plan-only] TextGrad / GEPA tool-schema optimiser — live (v0.71.13; pip install 'soup-cli[compile]') soup apple-adapter --direction hf-to-mlx|mlx-to-hf|hf-to-apple|mlx-to-apple --output [--sign] HF / MLX / Apple FoundationModels adapter conversion (v0.68.0) soup local-rl init --db Create personal-LLM flywheel SQLite schema (v0.68.0) soup local-rl status --db Print interactions / thumbs-up / thumbs-down counters soup local-rl record --db --prompt --response --thumb up|down Append thumbs record soup local-rl harvest --db -o Harvest DPO pairs from thumbs into JSONL -soup local-rl train --db --backend ollama|mlx --model Nightly DPO/KTO/ORPO train (v0.68.1) +soup local-rl train --db --model --once [--train-method dpo|kto|orpo] [--min-pairs N] [-o ] Ad-hoc DPO/KTO/ORPO train from harvested thumbs — live (v0.71.13) +soup local-rl train --db --model [--scheduler-dir ] [--hour H] [--minute M] Render a systemd/launchd nightly-train scaffold (no --once) (v0.71.13) soup build [--dry-run] [--output-dir ] dbt-for-SFT DAG: validate + plan + live materialise (v0.69.0; live v0.71.6) soup expect Expectations suite: PII / token-length / refusal / judge (v0.69.0) soup data gen-magpie --base --provider ollama|vllm --target N --output [--base-url ] [--quality-filter] Magpie synthetic generator — live (v0.69.0; live v0.71.6) diff --git a/pyproject.toml b/pyproject.toml index 9dd2c19..8e2f97c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.71.12" +version = "0.71.13" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" @@ -94,6 +94,10 @@ pdf = ["reportlab>=4.0.0"] # v0.71.3 #180 — codecarbon offline energy/CO2 measurement for # `soup train --track-energy` (offline; no IP-geolocation network call). carbon = ["codecarbon>=2.0.0"] +# v0.71.13 #225/#227 — prompt-program / tool-schema compilers +# (`soup compile` / `soup compile-tools`). Lazy-imported with a friendly +# ImportError; not installed by default (heavy dependency trees). +compile = ["dspy-ai>=2.5.0", "textgrad>=0.1.0", "gepa>=0.0.1"] [project.scripts] soup = "soup_cli.cli:run" diff --git a/src/soup_cli/__init__.py b/src/soup_cli/__init__.py index dcca27f..5dc6ddc 100644 --- a/src/soup_cli/__init__.py +++ b/src/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.71.12" +__version__ = "0.71.13" diff --git a/src/soup_cli/commands/compile_cmd.py b/src/soup_cli/commands/compile_cmd.py index 980971c..1993a9b 100644 --- a/src/soup_cli/commands/compile_cmd.py +++ b/src/soup_cli/commands/compile_cmd.py @@ -1,8 +1,8 @@ -"""``soup compile`` — DSPy / GEPA prompt-program compiler CLI (v0.68.0 Part A). +"""``soup compile`` — DSPy / GEPA / TextGrad prompt-program compiler CLI. -Renders a ``CompilePlan`` panel and, when ``--plan-only`` is omitted, -invokes the deferred-live ``run_compile`` (raises NotImplementedError -with a v0.68.1 marker). Mirrors v0.61.0 / v0.62.0 stub-then-live CLI policy. +Renders a ``CompilePlan`` panel and, when ``--plan-only`` is omitted, runs +the live optimizer (v0.71.13 #225). A missing optimizer library surfaces a +friendly exit-2 advisory naming ``pip install 'soup-cli[compile]'``. """ from __future__ import annotations @@ -74,14 +74,14 @@ def compile_cmd( try: result = run_compile(plan) - except NotImplementedError as exc: + except ImportError as exc: console.print( Panel( f"[yellow]{escape(str(exc))}[/]", - title="Live compile deferred", + title="compile — missing dependency", ) ) - raise typer.Exit(3) from exc + raise typer.Exit(2) from exc except (TypeError, ValueError) as exc: console.print(f"[red]{escape(str(exc))}[/]") raise typer.Exit(2) from exc diff --git a/src/soup_cli/commands/compile_tools.py b/src/soup_cli/commands/compile_tools.py index 8666630..f435931 100644 --- a/src/soup_cli/commands/compile_tools.py +++ b/src/soup_cli/commands/compile_tools.py @@ -62,12 +62,23 @@ def compile_tools_cmd( return try: - run_tool_compile(plan) - except NotImplementedError as exc: + n = run_tool_compile(plan) + except ImportError as exc: console.print( Panel( f"[yellow]{escape(str(exc))}[/]", - title="Live compile-tools deferred", + title="compile-tools — missing dependency", ) ) - raise typer.Exit(3) from exc + raise typer.Exit(2) from exc + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Tools: [bold]{n}[/]\n" + f"Output: [bold]{escape(plan.output_path)}[/]", + title="soup compile-tools — done", + ) + ) diff --git a/src/soup_cli/commands/distill_prompt.py b/src/soup_cli/commands/distill_prompt.py index 2d9bb5d..deca69e 100644 --- a/src/soup_cli/commands/distill_prompt.py +++ b/src/soup_cli/commands/distill_prompt.py @@ -29,6 +29,20 @@ def distill_prompt_cmd( help="Distill strategy. Allowed: " + ", ".join(sorted(SUPPORTED_DISTILL_STRATEGIES)), ), + provider: str = typer.Option( + "ollama", + "--provider", + help="Teacher/student provider: ollama / anthropic / vllm.", + ), + base_url: str = typer.Option( + None, "--base-url", help="Provider base URL (ollama / vllm)." + ), + temperature: float = typer.Option( + 0.0, "--temperature", help="Sampling temperature for teacher/student." + ), + max_rows: int = typer.Option( + None, "--max-rows", help="Cap the number of distilled rows." + ), output: str = typer.Option( "distilled.jsonl", "--output", "-o", help="Output JSONL path" ), @@ -55,6 +69,7 @@ def distill_prompt_cmd( f"Teacher: [bold]{escape(plan.teacher)}[/]\n" f"Student: [bold]{escape(plan.student)}[/]\n" f"Strategy: [bold]{escape(plan.strategy)}[/]\n" + f"Provider: [bold]{escape(provider)}[/]\n" f"Output: [bold]{escape(plan.output_path)}[/]", title="soup distill-prompt — plan", ) @@ -64,12 +79,29 @@ def distill_prompt_cmd( return try: - prepare_distill_dataset(plan) - except NotImplementedError as exc: + n = prepare_distill_dataset( + plan, + provider=provider, + base_url=base_url, + temperature=temperature, + max_rows=max_rows, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + except ImportError as exc: console.print( Panel( f"[yellow]{escape(str(exc))}[/]", - title="Live distill-prompt deferred", + title="distill-prompt — missing dependency", ) ) - raise typer.Exit(3) from exc + raise typer.Exit(2) from exc + + console.print( + Panel( + f"Rows: [bold]{n}[/]\n" + f"Output: [bold]{escape(plan.output_path)}[/]", + title="soup distill-prompt — done", + ) + ) diff --git a/src/soup_cli/commands/local_rl.py b/src/soup_cli/commands/local_rl.py index dbf07fc..13971eb 100644 --- a/src/soup_cli/commands/local_rl.py +++ b/src/soup_cli/commands/local_rl.py @@ -6,7 +6,7 @@ Subcommands: - ``status`` — print counters - ``record`` — append a thumbs-up/down record - ``harvest`` — emit DPO pairs as JSONL -- ``train`` — nightly DPO/KTO/ORPO train (deferred to v0.68.1) +- ``train`` — run (``--once``) or schedule the nightly DPO/KTO/ORPO train (v0.71.13 #229) """ from __future__ import annotations @@ -53,7 +53,7 @@ def init_cmd( console.print( Panel( f"DB: [bold]{escape(db)}[/]\n" - f"Tables: [bold]interactions[/], [bold]thumbs[/]", + f"Tables: [bold]interactions[/], [bold]thumbs[/], [bold]state[/]", title="soup local-rl init", ) ) @@ -166,14 +166,36 @@ def train_cmd( "--backend", help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_BACKENDS)), ), - model: str = typer.Option(..., "--model", help="Model id (Ollama tag or MLX path)"), + model: str = typer.Option( + ..., + "--model", + help="Training base — HF repo id or local path (NOT the Ollama tag)", + ), train_method: str = typer.Option( "dpo", "--train-method", help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_TRAIN_METHODS)), ), + once: bool = typer.Option( + False, + "--once", + help="Run the train now (ad-hoc); skip scheduler install.", + ), + min_pairs: int = typer.Option( + 10, "--min-pairs", help="Skip training when fewer pairs are harvested." + ), + output: str = typer.Option( + "local_rl_adapter", "--output", "-o", help="Adapter output directory" + ), + scheduler_dir: str = typer.Option( + "local-rl-scheduler", + "--scheduler-dir", + help="Directory to render the systemd / launchd scaffold into.", + ), + hour: int = typer.Option(3, "--hour", help="Daily train hour (local time)."), + minute: int = typer.Option(0, "--minute", help="Daily train minute."), ) -> None: - """Trigger the nightly DPO/KTO/ORPO train. Deferred to v0.68.1.""" + """Run (``--once``) or schedule the nightly DPO/KTO/ORPO train.""" try: cfg = LocalRLConfig( backend=backend, @@ -185,13 +207,73 @@ def train_cmd( console.print(f"[red]{escape(str(exc))}[/]") raise typer.Exit(2) from exc - try: - run_nightly_train(cfg) - except NotImplementedError as exc: + if not once: + # Render the scheduler scaffold; never run systemctl / launchctl. + import sys + + from soup_cli.utils.local_rl_scheduler import write_scheduler_files + + try: + written = write_scheduler_files( + scheduler_dir, + soup_python=sys.executable, + db_path=db, + model=model, + train_method=train_method, + hour=hour, + minute=minute, + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + files = "\n".join(f" [bold]{escape(n)}[/]" for n in sorted(written)) console.print( Panel( - f"[yellow]{escape(str(exc))}[/]", - title="Live local-rl train deferred", + f"Rendered the nightly-train scaffold into " + f"[bold]{escape(scheduler_dir)}[/]:\n{files}\n\n" + "[dim]Linux:[/] cp soup-local-rl.{service,timer} ~/.config/systemd/user/ " + "&& systemctl --user enable --now soup-local-rl.timer\n" + "[dim]macOS:[/] cp com.soup.local-rl.plist ~/Library/LaunchAgents/ " + "&& launchctl load ~/Library/LaunchAgents/com.soup.local-rl.plist\n" + "[dim]Now:[/] re-run with [bold]--once[/] for an ad-hoc train.", + title="soup local-rl train — scheduler scaffold", ) ) - raise typer.Exit(3) from exc + return + + import subprocess + + try: + result = run_nightly_train( + cfg, once=True, min_pairs=min_pairs, output_dir=output + ) + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + except subprocess.SubprocessError as exc: + # The underlying `soup train` subprocess failed (e.g. OOM / bad base). + console.print( + Panel( + f"[red]Training subprocess failed:[/] {escape(str(exc))}", + title="soup local-rl train — failed", + ) + ) + raise typer.Exit(1) from exc + + if result.status.startswith("skipped"): + console.print( + Panel( + f"[yellow]Skipped:[/] {escape(result.reason)}", + title="soup local-rl train — skipped", + ) + ) + return + + console.print( + Panel( + f"Trained: [bold]{result.num_pairs}[/] pairs via " + f"[bold]{escape(cfg.train_method)}[/]\n" + f"Output: [bold]{escape(result.output_dir or output)}[/]", + title="soup local-rl train — done", + ) + ) diff --git a/src/soup_cli/utils/compile_tools.py b/src/soup_cli/utils/compile_tools.py index 1ea82a4..414fe08 100644 --- a/src/soup_cli/utils/compile_tools.py +++ b/src/soup_cli/utils/compile_tools.py @@ -1,7 +1,9 @@ -"""``soup compile-tools`` — TextGrad / GEPA tool-schema optimizer (v0.68.0 Part C). +"""``soup compile-tools`` — TextGrad / GEPA tool-schema optimizer. Generate tool schemas + descriptions optimized via textual gradients. -Schema-only release: live optimizer pass lands in v0.68.1. +Schema + validators from v0.68.0 Part C; the live optimizer pass lands in +v0.71.13 (#227), lazy-importing TextGrad / GEPA with a friendly +``ImportError`` (``pip install 'soup-cli[compile]'``). Composes with v0.46 Agent Forge (OpenAPI / MCP / GraphQL parser) — Agent Forge produces the spec, ``compile-tools`` optimises the descriptions. @@ -9,15 +11,29 @@ Forge produces the spec, ``compile-tools`` optimises the descriptions. from __future__ import annotations +import json import os from dataclasses import dataclass +from typing import Callable, List, Optional -from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink +from soup_cli.utils.paths import ( + atomic_write_text, + enforce_under_cwd_and_no_symlink, +) -SUPPORTED_TOOL_OPTIMIZERS: frozenset = frozenset({"textgrad", "gepa"}) -_SUPPORTED_SPEC_EXTENSIONS: frozenset = frozenset({".json", ".yaml", ".yml"}) +SUPPORTED_TOOL_OPTIMIZERS: frozenset[str] = frozenset({"textgrad", "gepa"}) +_SUPPORTED_SPEC_EXTENSIONS: frozenset[str] = frozenset({".json", ".yaml", ".yml"}) _MAX_OPTIMIZER_NAME_LEN = 32 +_INSTALL_HINT = ( + "Run: pip install 'soup-cli[compile]' (installs textgrad / gepa)" +) + +# Injectable seam: tests set this to a ``(description, examples, optimizer) +# -> str`` callable so the parse -> iterate -> write orchestration is +# exercised without the TextGrad / GEPA libraries (mirrors v0.71.13 #225 +# ``prompt_compile._OPTIMIZER_RUN_OVERRIDE``). +_TOOL_OPTIMIZER_OVERRIDE: "Optional[Callable[[str, list, str], str]]" = None def validate_tool_optimizer(name: object) -> str: @@ -109,13 +125,110 @@ def build_tool_compile_plan( ) -def run_tool_compile(plan: ToolCompilePlan) -> None: - """Live optimiser pass. Deferred to v0.68.1.""" +# --------------------------------------------------------------------------- +# Live runner (v0.71.13 #227) — TextGrad / GEPA tool-schema optimiser +# --------------------------------------------------------------------------- + + +def _optimise_description( + description: str, examples: List[dict], optimizer: str +) -> str: + """Optimise one tool description via the chosen textual-gradient method. + + The ``_TOOL_OPTIMIZER_OVERRIDE`` seam short-circuits to a test fake. + Otherwise the real branch lazy-imports the optimiser library and raises a + friendly ``ImportError`` naming the ``[compile]`` extra when absent. + """ + if _TOOL_OPTIMIZER_OVERRIDE is not None: + result = _TOOL_OPTIMIZER_OVERRIDE(description, examples, optimizer) + if not isinstance(result, str): + raise TypeError("tool optimizer override must return a str") + return result + + if optimizer == "textgrad": + try: + import textgrad as tg + except ImportError as exc: + raise ImportError( + f"TextGrad is required for the 'textgrad' optimizer. {_INSTALL_HINT}" + ) from exc + variable = tg.Variable( + description, + requires_grad=True, + role_description="tool description", + ) + opt = tg.TGD(parameters=[variable]) + for _ in range(min(8, max(1, len(examples)))): + opt.zero_grad() + opt.step() + return str(variable.value) + + # gepa + try: + import gepa # noqa: F401 + except ImportError as exc: + raise ImportError( + f"GEPA is required for the 'gepa' optimizer. {_INSTALL_HINT}" + ) from exc + optimised = gepa.optimize( # type: ignore[attr-defined] + seed_candidate=description, + trainset=examples, + max_metric_calls=max(1, len(examples)), + ) + return str(getattr(optimised, "best_candidate", optimised)) + + +def _serialise_tools(out: dict, output_path: str) -> None: + """Write the optimised tool catalog as JSON / YAML per the extension.""" + lower = output_path.lower() + if lower.endswith((".yaml", ".yml")): + import yaml + + text = yaml.safe_dump(out, default_flow_style=False, sort_keys=False) + else: # default JSON + text = json.dumps(out, ensure_ascii=False, indent=2) + "\n" + atomic_write_text(text, output_path, field="output_path") + + +def run_tool_compile(plan: ToolCompilePlan) -> int: + """Optimise every tool description in the spec (v0.71.13 #227). + + Reuses v0.46 ``agent_forge.parse_spec`` to lift OpenAPI / MCP / GraphQL + into ``Endpoint`` objects, runs the textual-gradient optimiser on each + tool's description (scored against the eval suite), and writes a flat + optimised tool catalog (``{"tools": [...]}``) to ``plan.output_path`` as + JSON / YAML matching the output extension. Returns the tool count. + + Validates the plan type so a bare dict raises cleanly. + """ if not isinstance(plan, ToolCompilePlan): raise TypeError("plan must be ToolCompilePlan") - raise NotImplementedError( - "compile-tools live optimisation is deferred to v0.68.1" - ) + + from soup_cli.utils.agent_forge import load_spec_file, parse_spec + from soup_cli.utils.prompt_compile import load_eval_examples + + spec = load_spec_file(plan.spec_path) + endpoints, report = parse_spec(spec) + examples = load_eval_examples(plan.eval_suite_path) + + tools: list[dict] = [] + for ep in endpoints: + new_desc = _optimise_description( + ep.description, examples, plan.optimizer + ) + tools.append( + { + "tool": ep.tool, + "description": new_desc, + "method": ep.method, + "path": ep.path, + "parameters": list(ep.parameters), + } + ) + + out = {"spec_kind": report.spec_kind, "tools": tools} + _serialise_tools(out, plan.output_path) + return len(tools) __all__ = [ diff --git a/src/soup_cli/utils/local_rl.py b/src/soup_cli/utils/local_rl.py index b8e378e..8af8fad 100644 --- a/src/soup_cli/utils/local_rl.py +++ b/src/soup_cli/utils/local_rl.py @@ -5,8 +5,9 @@ and (in v0.68.1) DPO-train nightly via systemd / launchd. Smaller-scope cousin of v0.58 ``soup loop`` — runs locally on a single workstation, trains the user's personal model from their own feedback. -Schema + thumbs recording + DPO-pair harvester are LIVE in v0.68.0; the -nightly train scheduler is the deferred stub. +Schema + thumbs recording + DPO-pair harvester are LIVE from v0.68.0; the +nightly DPO/KTO/ORPO train runner + systemd/launchd scheduler scaffold land +in v0.71.13 (#229). Public surface: @@ -14,10 +15,12 @@ Public surface: - ``SUPPORTED_LOCAL_RL_TRAIN_METHODS`` (``dpo``/``kto``/``orpo``) - ``validate_local_rl_backend`` / ``validate_local_rl_train_method`` - ``LocalRLConfig`` frozen dataclass -- ``init_local_rl_db(db_path)`` — atomic table creation +- ``init_local_rl_db(db_path)`` — atomic table creation (interactions/thumbs/state) - ``record_thumb(...)`` — append a thumbs-up/down record - ``harvest_dpo_pairs(db_path)`` — pair up/down responses to same prompt -- ``run_nightly_train(config)`` — NotImplementedError stub w/ v0.68.1 marker +- ``get_state`` / ``set_state`` / ``count_new_thumbs_since`` — state-table I/O +- ``pairs_to_rows(pairs, method)`` — DPO pairs -> dpo/orpo/kto JSONL rows +- ``run_nightly_train(config)`` — live DPO/KTO/ORPO train runner (v0.71.13 #229) """ from __future__ import annotations @@ -27,13 +30,13 @@ import sqlite3 import stat import time from dataclasses import dataclass -from typing import Tuple +from typing import Callable, Optional, Tuple from soup_cli.utils.paths import is_under_cwd -SUPPORTED_LOCAL_RL_BACKENDS: frozenset = frozenset({"ollama", "mlx"}) -SUPPORTED_LOCAL_RL_TRAIN_METHODS: frozenset = frozenset({"dpo", "kto", "orpo"}) -_VALID_THUMBS: frozenset = frozenset({"up", "down"}) +SUPPORTED_LOCAL_RL_BACKENDS: frozenset[str] = frozenset({"ollama", "mlx"}) +SUPPORTED_LOCAL_RL_TRAIN_METHODS: frozenset[str] = frozenset({"dpo", "kto", "orpo"}) +_VALID_THUMBS: frozenset[str] = frozenset({"up", "down"}) MAX_PROMPT_LEN = 16_384 MAX_RESPONSE_LEN = 16_384 @@ -41,6 +44,14 @@ _MAX_BACKEND_LEN = 32 _MAX_TRAIN_METHOD_LEN = 32 _MAX_MODEL_LEN = 512 +# v0.71.13 #229 — nightly train scheduler defaults. +MIN_PAIRS_DEFAULT = 10 +_MAX_MIN_PAIRS = 1_000_000 +_LAST_TRAIN_KEY = "last_train_at" +# task -> data format the trainer consumes (dpo/orpo read chosen+rejected; +# kto reads prompt+completion+label). +_METHOD_FORMAT = {"dpo": "dpo", "orpo": "dpo", "kto": "kto"} + # --------------------------------------------------------------------------- # Validators @@ -100,6 +111,10 @@ def _validate_model(value: object) -> str: raise ValueError("model must be non-empty") if "\x00" in value: raise ValueError("model must not contain null bytes") + # Reject newlines / CR (systemd unit-file injection defence — the model + # flows into a scheduler ExecStart line via local_rl_scheduler). + if "\n" in value or "\r" in value: + raise ValueError("model must not contain newline / carriage return") if len(value) > _MAX_MODEL_LEN: raise ValueError(f"model length {len(value)} > {_MAX_MODEL_LEN}") return value @@ -126,6 +141,10 @@ def validate_db_path(path: object) -> str: raise ValueError("db_path must be non-empty") if "\x00" in path: raise ValueError("db_path must not contain null bytes") + # Newline / CR rejection: db_path flows into a scheduler ExecStart line + # via local_rl_scheduler (systemd unit-file injection defence). + if "\n" in path or "\r" in path: + raise ValueError("db_path must not contain newline / carriage return") if not is_under_cwd(path): raise ValueError( f"db_path {os.path.basename(path)!r} must stay under cwd" @@ -218,6 +237,14 @@ CREATE TABLE IF NOT EXISTS thumbs ( ) """ +# v0.71.13 #229 — key/value state table (tracks last_train_at across runs). +_SCHEMA_STATE = """ +CREATE TABLE IF NOT EXISTS state ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) +""" + def init_local_rl_db(db_path: str) -> None: """Atomically create the local-RL SQLite schema. Idempotent.""" @@ -228,6 +255,7 @@ def init_local_rl_db(db_path: str) -> None: with sqlite3.connect(real) as conn: conn.execute(_SCHEMA_INTERACTIONS) conn.execute(_SCHEMA_THUMBS) + conn.execute(_SCHEMA_STATE) conn.commit() # Best-effort 0o600 perms (matches v0.26.0 registry.db policy on POSIX). if os.name == "posix": @@ -275,8 +303,8 @@ def harvest_dpo_pairs(db_path: str) -> Tuple[DpoPair, ...]: rows = conn.execute( "SELECT prompt, response, thumb, ts FROM thumbs ORDER BY ts ASC" ).fetchall() - last_up: dict = {} - last_down: dict = {} + last_up: dict[str, str] = {} + last_down: dict[str, str] = {} for prompt, response, thumb, _ts in rows: if thumb == "up": last_up[prompt] = response @@ -295,17 +323,278 @@ def harvest_dpo_pairs(db_path: str) -> Tuple[DpoPair, ...]: # --------------------------------------------------------------------------- -# Live train stub (v0.68.1) +# State table I/O (v0.71.13 #229) # --------------------------------------------------------------------------- -def run_nightly_train(config: LocalRLConfig) -> None: - """Run the nightly DPO/KTO/ORPO train. Deferred to v0.68.1.""" +def _ensure_state_table(conn: sqlite3.Connection) -> None: + """Create the ``state`` table if a pre-v0.71.13 DB lacks it.""" + conn.execute(_SCHEMA_STATE) + + +def get_state(db_path: str, key: str) -> "str | None": + """Return the stored value for ``key`` or ``None`` when absent.""" + validate_db_path(db_path) + if not isinstance(key, str) or not key: + raise ValueError("key must be a non-empty string") + real = os.path.abspath(db_path) + if not os.path.exists(real): + raise FileNotFoundError(f"db_path not found: {db_path!r}") + with sqlite3.connect(real) as conn: + _ensure_state_table(conn) + row = conn.execute( + "SELECT value FROM state WHERE key = ?", (key,) + ).fetchone() + return row[0] if row is not None else None + + +def set_state(db_path: str, key: str, value: str) -> None: + """Upsert ``key`` -> ``value`` in the ``state`` table.""" + validate_db_path(db_path) + if not isinstance(key, str) or not key: + raise ValueError("key must be a non-empty string") + if not isinstance(value, str): + raise TypeError("value must be a string") + real = os.path.abspath(db_path) + with sqlite3.connect(real) as conn: + _ensure_state_table(conn) + conn.execute( + "INSERT INTO state (key, value) VALUES (?, ?) " + "ON CONFLICT(key) DO UPDATE SET value = excluded.value", + (key, value), + ) + conn.commit() + + +def count_new_thumbs_since(db_path: str, since_ts: "float | None") -> int: + """Count thumbs with ``ts > since_ts`` (all thumbs when ``since_ts`` is None).""" + validate_db_path(db_path) + if since_ts is not None and ( + isinstance(since_ts, bool) or not isinstance(since_ts, (int, float)) + ): + raise TypeError("since_ts must be a number or None") + real = os.path.abspath(db_path) + if not os.path.exists(real): + raise FileNotFoundError(f"db_path not found: {db_path!r}") + with sqlite3.connect(real) as conn: + if since_ts is None: + row = conn.execute("SELECT COUNT(*) FROM thumbs").fetchone() + else: + row = conn.execute( + "SELECT COUNT(*) FROM thumbs WHERE ts > ?", (float(since_ts),) + ).fetchone() + return int(row[0]) + + +# --------------------------------------------------------------------------- +# Pair -> training rows (v0.71.13 #229) +# --------------------------------------------------------------------------- + + +def pairs_to_rows( + pairs: "Tuple[DpoPair, ...]", train_method: str +) -> "list[dict]": + """Convert harvested DPO pairs to JSONL rows for ``train_method``. + + - ``dpo`` / ``orpo`` -> one ``{prompt, chosen, rejected}`` row per pair. + - ``kto`` -> two unpaired ``{prompt, completion, label}`` rows per pair + (chosen -> label True, rejected -> label False). + """ + method = validate_local_rl_train_method(train_method) + rows: "list[dict]" = [] + for pair in pairs: + if not isinstance(pair, DpoPair): + raise TypeError("pairs must contain DpoPair objects") + if method == "kto": + rows.append( + {"prompt": pair.prompt, "completion": pair.chosen, "label": True} + ) + rows.append( + {"prompt": pair.prompt, "completion": pair.rejected, "label": False} + ) + else: # dpo / orpo + rows.append( + { + "prompt": pair.prompt, + "chosen": pair.chosen, + "rejected": pair.rejected, + } + ) + return rows + + +# --------------------------------------------------------------------------- +# Nightly train runner (v0.71.13 #229) — lifts the v0.68.1 stub +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class NightlyTrainResult: + """Outcome of a ``run_nightly_train`` invocation.""" + + status: str # "trained" / "skipped_no_new_thumbs" / "skipped_insufficient_pairs" + num_pairs: int + output_dir: "str | None" + reason: str + + +def _default_train_fn( + *, + base_model: str, + pairs_path: str, + output_dir: str, + train_method: str, +) -> None: + """Train one round via a ``soup train`` subprocess (no shell). + + Mirrors v0.71.11 ``iterative_dpo._default_train_fn``: render a YAML via + ``yaml.safe_dump`` (no value can inject extra keys) and invoke + ``python -m soup_cli.cli train --config --yes`` with an argv list. + """ + import subprocess + import sys + import tempfile + + import yaml + + data_format = _METHOD_FORMAT[train_method] + yaml_text = yaml.safe_dump( + { + "base": base_model, + "task": train_method, + "data": {"train": pairs_path, "format": data_format, "max_length": 256}, + "training": {"epochs": 1, "batch_size": 1}, + "output": output_dir, + }, + default_flow_style=False, + sort_keys=False, + ) + fd, tmp_yaml = tempfile.mkstemp( + suffix=".yaml", prefix=".soup_localrl_", dir=os.getcwd() + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(yaml_text) + subprocess.run( # noqa: S603 — argv list, no shell + [ + sys.executable, + "-m", + "soup_cli.cli", + "train", + "--config", + tmp_yaml, + "--yes", + ], + check=True, + ) + finally: + try: + os.remove(tmp_yaml) + except OSError: + pass + + +def run_nightly_train( + config: LocalRLConfig, + *, + once: bool = False, + min_pairs: int = MIN_PAIRS_DEFAULT, + output_dir: str = "local_rl_adapter", + train_fn: Optional[Callable[..., None]] = None, +) -> NightlyTrainResult: + """Harvest the latest DPO pairs and train them (v0.71.13 #229). + + The flow: read ``last_train_at`` from the SQLite ``state`` table; if a + prior train exists and no new thumbs landed since, skip. Otherwise + harvest pairs and — when at least ``min_pairs`` are available — write a + tmp JSONL in the right shape for ``config.train_method`` and invoke the + DPO/KTO/ORPO trainer via ``soup train`` (or an injected ``train_fn`` for + tests). ``last_train_at`` is stamped only after a real train. + + ``config.model`` is the **training base** here (an HF repo id or local + path) — distinct from the Ollama tag used by the record/harvest loop. + + The ``once`` flag is accepted for symmetry with the CLI (the runner is + the same whether invoked ad-hoc or by the scheduler); it is recorded but + does not change the harvest/skip logic. + """ if not isinstance(config, LocalRLConfig): raise TypeError("config must be LocalRLConfig") - raise NotImplementedError( - "local-rl nightly train is deferred to v0.68.1 — " - "harvest DPO pairs today, train them once the runner lands" + if not isinstance(once, bool): + raise TypeError("once must be bool") + if isinstance(min_pairs, bool) or not isinstance(min_pairs, int): + raise TypeError("min_pairs must be int") + if min_pairs < 1 or min_pairs > _MAX_MIN_PAIRS: + raise ValueError(f"min_pairs must be in [1, {_MAX_MIN_PAIRS}]") + if not isinstance(output_dir, str) or not output_dir or "\x00" in output_dir: + raise ValueError("output_dir must be a non-empty NUL-free string") + + runner = train_fn if train_fn is not None else _default_train_fn + if not callable(runner): + raise TypeError("train_fn must be callable") + + db = config.db_path + last_raw = get_state(db, _LAST_TRAIN_KEY) + last_ts: "float | None" = None + if last_raw is not None: + try: + last_ts = float(last_raw) + except (TypeError, ValueError): + last_ts = None + + if last_ts is not None: + new = count_new_thumbs_since(db, last_ts) + if new == 0: + return NightlyTrainResult( + status="skipped_no_new_thumbs", + num_pairs=0, + output_dir=None, + reason=f"no new thumbs since last train at {last_ts}", + ) + + # Stamp the run start (BEFORE harvest), not the post-train wall-clock, so + # any thumbs recorded *during* the train window keep ``ts > last_train_at`` + # and are counted by the next run instead of being silently dropped + # (code-review MEDIUM fix). + run_started = time.time() + pairs = harvest_dpo_pairs(db) + if len(pairs) < min_pairs: + return NightlyTrainResult( + status="skipped_insufficient_pairs", + num_pairs=len(pairs), + output_dir=None, + reason=f"{len(pairs)} pairs < min_pairs {min_pairs}", + ) + + import json + import tempfile + + rows = pairs_to_rows(pairs, config.train_method) + fd, tmp_jsonl = tempfile.mkstemp( + suffix=".jsonl", prefix=".soup_localrl_pairs_", dir=os.getcwd() + ) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, ensure_ascii=False) + "\n") + runner( + base_model=config.model, + pairs_path=tmp_jsonl, + output_dir=output_dir, + train_method=config.train_method, + ) + finally: + try: + os.remove(tmp_jsonl) + except OSError: + pass + + set_state(db, _LAST_TRAIN_KEY, repr(run_started)) + return NightlyTrainResult( + status="trained", + num_pairs=len(pairs), + output_dir=output_dir, + reason=f"trained {len(pairs)} pairs via {config.train_method}", ) @@ -314,13 +603,19 @@ __all__ = [ "SUPPORTED_LOCAL_RL_TRAIN_METHODS", "MAX_PROMPT_LEN", "MAX_RESPONSE_LEN", + "MIN_PAIRS_DEFAULT", "validate_local_rl_backend", "validate_local_rl_train_method", "validate_db_path", "LocalRLConfig", "DpoPair", + "NightlyTrainResult", "init_local_rl_db", "record_thumb", "harvest_dpo_pairs", + "get_state", + "set_state", + "count_new_thumbs_since", + "pairs_to_rows", "run_nightly_train", ] diff --git a/src/soup_cli/utils/local_rl_scheduler.py b/src/soup_cli/utils/local_rl_scheduler.py new file mode 100644 index 0000000..8f84a48 --- /dev/null +++ b/src/soup_cli/utils/local_rl_scheduler.py @@ -0,0 +1,276 @@ +"""``soup local-rl train`` scheduler scaffold (v0.71.13 #229). + +Render a systemd-user ``local-rl.service`` + ``.timer`` (Linux) or a +``com.soup.local-rl.plist`` (macOS launchd) that invokes +``soup local-rl train --once`` daily at a configurable local time. + +This module only **renders** the unit files (and writes them to a chosen +directory) — it never runs ``systemctl`` / ``launchctl`` itself, so it is +fully testable on any OS. The CLI prints the manual install command. This +mirrors v0.46 deploy-autopilot / v0.68 apple-adapter (render-not-execute). + +Public surface: + +- ``build_train_argv(...)`` — argv list the scheduler invokes +- ``render_systemd_service(...)`` / ``render_systemd_timer(...)`` +- ``render_launchd_plist(...)`` +- ``write_scheduler_files(target_dir, ...)`` — atomic write of the units +- ``SYSTEMD_SERVICE_NAME`` / ``SYSTEMD_TIMER_NAME`` / ``LAUNCHD_PLIST_NAME`` +""" + +from __future__ import annotations + +import os +from typing import Dict, List + +from soup_cli.utils.local_rl import ( + validate_db_path, + validate_local_rl_train_method, +) +from soup_cli.utils.paths import atomic_write_text + +SYSTEMD_SERVICE_NAME = "soup-local-rl.service" +SYSTEMD_TIMER_NAME = "soup-local-rl.timer" +LAUNCHD_PLIST_NAME = "com.soup.local-rl.plist" + +_MIN_HOUR = 0 +_MAX_HOUR = 23 +_MIN_MINUTE = 0 +_MAX_MINUTE = 59 +_MAX_MODEL_LEN = 512 + + +def _validate_model(value: object) -> str: + if isinstance(value, bool): + raise TypeError("model must not be bool") + if not isinstance(value, str): + raise TypeError("model must be str") + if not value: + raise ValueError("model must be non-empty") + if "\x00" in value or "\n" in value or "\r" in value: + raise ValueError( + "model must not contain NUL / newline / carriage return" + ) + if len(value) > _MAX_MODEL_LEN: + raise ValueError(f"model length {len(value)} > {_MAX_MODEL_LEN}") + return value + + +def _validate_hm(hour: object, minute: object) -> "tuple[int, int]": + for name, value, lo, hi in ( + ("hour", hour, _MIN_HOUR, _MAX_HOUR), + ("minute", minute, _MIN_MINUTE, _MAX_MINUTE), + ): + if isinstance(value, bool): + raise TypeError(f"{name} must not be bool") + if not isinstance(value, int): + raise TypeError(f"{name} must be int") + if value < lo or value > hi: + raise ValueError(f"{name} must be in [{lo}, {hi}]") + return int(hour), int(minute) # type: ignore[arg-type] + + +def _validate_soup_python(value: object) -> str: + """The python executable that runs ``-m soup_cli.cli``.""" + if isinstance(value, bool): + raise TypeError("soup_python must not be bool") + if not isinstance(value, str): + raise TypeError("soup_python must be str") + if not value: + raise ValueError("soup_python must be non-empty") + if "\x00" in value or "\n" in value or "\r" in value: + raise ValueError("soup_python must not contain NUL / newline") + return value + + +def build_train_argv( + *, + soup_python: str, + db_path: str, + model: str, + train_method: str, +) -> List[str]: + """Return the argv list the scheduler invokes (no shell).""" + soup_python = _validate_soup_python(soup_python) + validate_db_path(db_path) + model = _validate_model(model) + method = validate_local_rl_train_method(train_method) + return [ + soup_python, + "-m", + "soup_cli.cli", + "local-rl", + "train", + "--once", + "--db", + db_path, + "--model", + model, + "--train-method", + method, + ] + + +def _systemd_quote(arg: str) -> str: + """Double-quote an argv element for a systemd ``ExecStart`` line. + + systemd splits ``ExecStart`` on whitespace unless quoted; ``"`` and + ``\\`` inside a quoted token must be escaped. Defence-in-depth: refuse + any newline / CR (a quoted token cannot span lines — this would inject a + new directive). The argv builder already rejects these on db_path/model. + """ + if "\n" in arg or "\r" in arg: + raise ValueError("systemd argument must not contain newline / CR") + escaped = arg.replace("\\", "\\\\").replace('"', '\\"') + return f'"{escaped}"' + + +def render_systemd_service( + *, + soup_python: str, + db_path: str, + model: str, + train_method: str, +) -> str: + """Render the systemd-user ``.service`` unit.""" + argv = build_train_argv( + soup_python=soup_python, + db_path=db_path, + model=model, + train_method=train_method, + ) + exec_start = " ".join(_systemd_quote(a) for a in argv) + return ( + "[Unit]\n" + "Description=Soup local-RL nightly DPO/KTO/ORPO train\n" + "\n" + "[Service]\n" + "Type=oneshot\n" + f"WorkingDirectory={_systemd_quote(os.getcwd())}\n" + f"ExecStart={exec_start}\n" + ) + + +def render_systemd_timer(*, hour: int = 3, minute: int = 0) -> str: + """Render the systemd-user ``.timer`` unit (daily at ``hour:minute``).""" + hour, minute = _validate_hm(hour, minute) + return ( + "[Unit]\n" + "Description=Soup local-RL nightly train timer\n" + "\n" + "[Timer]\n" + f"OnCalendar=*-*-* {hour:02d}:{minute:02d}:00\n" + "Persistent=true\n" + "\n" + "[Install]\n" + "WantedBy=timers.target\n" + ) + + +def render_launchd_plist( + *, + soup_python: str, + db_path: str, + model: str, + train_method: str, + hour: int = 3, + minute: int = 0, +) -> str: + """Render a macOS launchd ``.plist`` (daily ``StartCalendarInterval``).""" + from xml.sax.saxutils import escape as _xml_escape + + hour, minute = _validate_hm(hour, minute) + argv = build_train_argv( + soup_python=soup_python, + db_path=db_path, + model=model, + train_method=train_method, + ) + args_xml = "\n".join( + f" {_xml_escape(a)}" for a in argv + ) + label = LAUNCHD_PLIST_NAME[: -len(".plist")] + return ( + '\n' + '\n' + '\n' + "\n" + " Label\n" + f" {_xml_escape(label)}\n" + " ProgramArguments\n" + " \n" + f"{args_xml}\n" + " \n" + " WorkingDirectory\n" + f" {_xml_escape(os.getcwd())}\n" + " StartCalendarInterval\n" + " \n" + " Hour\n" + f" {hour}\n" + " Minute\n" + f" {minute}\n" + " \n" + "\n" + "\n" + ) + + +def write_scheduler_files( + target_dir: str, + *, + soup_python: str, + db_path: str, + model: str, + train_method: str, + hour: int = 3, + minute: int = 0, +) -> Dict[str, str]: + """Render both the systemd units and the launchd plist into ``target_dir``. + + All three are written so an operator on either OS finds the right scaffold. + Returns ``{filename: realpath}``. Writes are atomic + cwd-contained + + symlink-rejected via ``atomic_write_text``. + """ + if not isinstance(target_dir, str) or not target_dir or "\x00" in target_dir: + raise ValueError("target_dir must be a non-empty NUL-free string") + os.makedirs(target_dir, exist_ok=True) + + service = render_systemd_service( + soup_python=soup_python, + db_path=db_path, + model=model, + train_method=train_method, + ) + timer = render_systemd_timer(hour=hour, minute=minute) + plist = render_launchd_plist( + soup_python=soup_python, + db_path=db_path, + model=model, + train_method=train_method, + hour=hour, + minute=minute, + ) + + written: Dict[str, str] = {} + for name, body in ( + (SYSTEMD_SERVICE_NAME, service), + (SYSTEMD_TIMER_NAME, timer), + (LAUNCHD_PLIST_NAME, plist), + ): + path = os.path.join(target_dir, name) + atomic_write_text(body, path, field="scheduler_file") + written[name] = os.path.realpath(path) + return written + + +__all__ = [ + "SYSTEMD_SERVICE_NAME", + "SYSTEMD_TIMER_NAME", + "LAUNCHD_PLIST_NAME", + "build_train_argv", + "render_systemd_service", + "render_systemd_timer", + "render_launchd_plist", + "write_scheduler_files", +] diff --git a/src/soup_cli/utils/prompt_compile.py b/src/soup_cli/utils/prompt_compile.py index ad29c74..2051cf2 100644 --- a/src/soup_cli/utils/prompt_compile.py +++ b/src/soup_cli/utils/prompt_compile.py @@ -1,8 +1,9 @@ -"""``soup compile`` — DSPy / GEPA prompt-program compiler (v0.68.0 Part A). +"""``soup compile`` — DSPy / GEPA / TextGrad prompt-program compiler. -Schema-only release: live wiring (DSPy / GEPA / TextGrad orchestrator) lands -in v0.68.1. The validators + frozen dataclasses ship now so operators can -build a plan today, then re-run with the live runner when it lands. +Schema + validators ship from v0.68.0 Part A; the live DSPy / GEPA / +TextGrad orchestrator lands in v0.71.13 (#225). The optimizer libraries are +lazy-imported with a friendly ``ImportError`` (naming ``pip install +'soup-cli[compile]'``) so the command works without them installed. Public surface: @@ -12,10 +13,10 @@ Public surface: - ``validate_max_iters(n)`` — bool-first / non-int / bounds - ``validate_program_path(path)`` — cwd containment + symlink rejection + ``.py`` only - ``validate_eval_suite_path(path)`` — cwd containment + symlink rejection -- ``CompilePlan`` frozen dataclass -- ``CompileResult`` frozen dataclass +- ``load_eval_examples(path)`` — JSON-list / JSONL eval loader (cwd-contained) +- ``CompilePlan`` / ``CompileResult`` frozen dataclasses - ``build_compile_plan(...)`` — factory -- ``run_compile(plan)`` — NotImplementedError stub w/ v0.68.1 marker +- ``run_compile(plan)`` — live dispatcher (DSPy / GEPA / TextGrad) The CLI command lives in ``soup_cli/commands/compile_cmd.py`` (named ``compile_cmd`` to avoid shadowing the Python builtin ``compile()``). @@ -23,9 +24,12 @@ The CLI command lives in ``soup_cli/commands/compile_cmd.py`` (named from __future__ import annotations +import importlib.util +import json import math import os from dataclasses import dataclass +from typing import Any, Callable, List, Optional from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink @@ -34,7 +38,7 @@ from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink # --------------------------------------------------------------------------- -SUPPORTED_PROMPT_OPTIMIZERS: frozenset = frozenset( +SUPPORTED_PROMPT_OPTIMIZERS: frozenset[str] = frozenset( { "bootstrap_fewshot", # DSPy classic "mipro", # DSPy Multi-stage Instruction Proposal Optimizer @@ -44,9 +48,21 @@ SUPPORTED_PROMPT_OPTIMIZERS: frozenset = frozenset( } ) +# DSPy-backed optimizers (dispatched to ``dspy``); GEPA + TextGrad have their +# own libraries. +_DSPY_OPTIMIZERS: frozenset[str] = frozenset({"bootstrap_fewshot", "mipro", "copro"}) + MAX_COMPILE_ITERS = 1000 _MIN_COMPILE_ITERS = 1 _MAX_OPTIMIZER_NAME_LEN = 32 +_MAX_EVAL_BYTES = 64 * 1024 * 1024 # 64 MiB cap on the eval-suite file + +# Injectable seam: tests / advanced operators set this to a +# ``(plan) -> CompileResult`` callable so the dispatcher + result-handling + +# atomic-write are exercised without the heavy DSPy / GEPA / TextGrad libs. +# Mirrors v0.67 ``cmaes_merge._CMAES_SCORER_OVERRIDE`` / +# v0.53.1 ``deploy_measure._DEPLOY_MEASURE_BEFORE_GEN`` policy. +_OPTIMIZER_RUN_OVERRIDE: "Optional[Callable[[CompilePlan], CompileResult]]" = None # --------------------------------------------------------------------------- @@ -220,22 +236,243 @@ def build_compile_plan( # --------------------------------------------------------------------------- -# Live runner stub (v0.68.1) +# Live runner (v0.71.13 #225) — DSPy / GEPA / TextGrad dispatcher # --------------------------------------------------------------------------- -def run_compile(plan: CompilePlan) -> CompileResult: - """Run the live compilation. Deferred to v0.68.1. +_INSTALL_HINT = ( + "Run: pip install 'soup-cli[compile]' (installs dspy-ai / textgrad / gepa)" +) - Validates the plan type at the boundary so callers passing a bare dict - get a clean ``TypeError`` rather than a confusing AttributeError when - the v0.68.1 runner finally lands. + +def load_eval_examples(eval_suite_path: str) -> List[dict]: + """Load eval-suite examples (cwd-contained, symlink-safe, O_NOFOLLOW). + + Accepts a JSON array of objects, a single JSON object, or a JSONL file + (one object per line). Each example is a dict — the optimizer decides + which keys are inputs / outputs (DSPy: ``with_inputs``; TextGrad / GEPA: + free-form). + """ + canonical = enforce_under_cwd_and_no_symlink(eval_suite_path, "eval_suite_path") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(canonical, flags) + with os.fdopen(fd, encoding="utf-8") as handle: + raw = handle.read(_MAX_EVAL_BYTES + 1) + if len(raw) > _MAX_EVAL_BYTES: + raise ValueError(f"eval suite exceeds {_MAX_EVAL_BYTES} bytes") + stripped = raw.lstrip() + examples: List[dict] = [] + # Try a whole-file JSON parse first (handles pretty-printed arrays + a + # single top-level object); fall back to JSONL line-by-line. + parsed = None + if stripped[:1] in ("[", "{"): + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, list): + examples = [d for d in parsed if isinstance(d, dict)] + elif isinstance(parsed, dict): + examples = [parsed] + else: + for line in raw.splitlines(): + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, dict): + examples.append(obj) + return examples + + +def _load_program_module(program_path: str) -> Any: + """Import the user's ``.py`` program from a validated path. + + Re-validates the path immediately before ``exec_module`` to close the + plan-build -> run TOCTOU window (security-review LOW). The program itself + is operator-trusted (arbitrary code by design) but must still be a + cwd-contained, non-symlink ``.py``. + """ + validate_program_path(program_path) + spec = importlib.util.spec_from_file_location("soup_compile_program", program_path) + if spec is None or spec.loader is None: + raise ValueError(f"could not import program from {program_path!r}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) # noqa: S102 — operator-trusted program + return module + + +def _resolve_program(module: Any) -> Any: + """Find the DSPy program object exported by the user module. + + Convention: a module-level ``program`` attribute, or a ``get_program()`` + factory. Mirrors the DSPy idiom of building a ``dspy.Module`` instance. + """ + if hasattr(module, "program"): + return module.program + if hasattr(module, "get_program") and callable(module.get_program): + return module.get_program() + raise ValueError( + "program module must expose a `program` attribute or `get_program()`" + ) + + +def _resolve_metric(module: Any) -> "Optional[Callable]": + """Return the user metric (``metric`` attr) or None (caller defaults).""" + metric = getattr(module, "metric", None) + return metric if callable(metric) else None + + +def _run_dspy(plan: CompilePlan) -> CompileResult: + try: + import dspy + except ImportError as exc: + raise ImportError( + f"DSPy is required for the {plan.optimizer!r} optimizer. {_INSTALL_HINT}" + ) from exc + + module = _load_program_module(plan.program_path) + program = _resolve_program(module) + metric = _resolve_metric(module) + trainset = _dspy_examples(dspy, module, plan.eval_suite_path) + + if plan.optimizer == "bootstrap_fewshot": + optimizer = dspy.BootstrapFewShot(metric=metric, max_rounds=plan.max_iters) + elif plan.optimizer == "mipro": + optimizer = dspy.MIPROv2(metric=metric, auto="light") + else: # copro + optimizer = dspy.COPRO(metric=metric, depth=plan.max_iters) + + compiled = optimizer.compile(program, trainset=trainset) + program_text = _serialize_dspy(compiled) + return CompileResult( + program_text=program_text, + score=0.0, + iterations=plan.max_iters, + converged=True, + ) + + +def _dspy_examples(dspy: Any, module: Any, eval_suite_path: str) -> List[Any]: + """Build ``dspy.Example`` objects from the eval suite. + + Honours a module-level ``input_keys`` list (which keys are inputs); else + treats every key except common output names as inputs. + """ + raw = load_eval_examples(eval_suite_path) + input_keys = getattr(module, "input_keys", None) + out: List[Any] = [] + for ex in raw: + example = dspy.Example(**ex) + if isinstance(input_keys, (list, tuple)) and input_keys: + example = example.with_inputs(*input_keys) + else: + inferred = [ + k for k in ex if k not in ("output", "answer", "completion", "label") + ] + if inferred: + example = example.with_inputs(*inferred) + out.append(example) + return out + + +def _serialize_dspy(compiled: Any) -> str: + """Best-effort serialise a compiled DSPy program to text.""" + for attr in ("dump_state", "__repr__"): + try: + value = getattr(compiled, attr)() + return value if isinstance(value, str) else json.dumps(value, default=str) + except Exception: # noqa: BLE001 — fall through to repr + continue + return repr(compiled) + + +def _run_gepa(plan: CompilePlan) -> CompileResult: + try: + import gepa # noqa: F401 + except ImportError as exc: + raise ImportError( + f"GEPA is required for the 'gepa' optimizer. {_INSTALL_HINT}" + ) from exc + module = _load_program_module(plan.program_path) + program = _resolve_program(module) + examples = load_eval_examples(plan.eval_suite_path) + optimised = gepa.optimize( # type: ignore[attr-defined] + seed_candidate=program, + trainset=examples, + max_metric_calls=plan.max_iters, + ) + program_text = str(getattr(optimised, "best_candidate", optimised)) + score = float(getattr(optimised, "best_score", 0.0) or 0.0) + return CompileResult( + program_text=program_text, + score=score if math.isfinite(score) else 0.0, + iterations=plan.max_iters, + converged=True, + ) + + +def _run_textgrad(plan: CompilePlan) -> CompileResult: + try: + import textgrad as tg + except ImportError as exc: + raise ImportError( + f"TextGrad is required for the 'textgrad' optimizer. {_INSTALL_HINT}" + ) from exc + module = _load_program_module(plan.program_path) + program = _resolve_program(module) + # The user program is expected to be (or contain) a tg.Variable to + # optimise. We run a textual-gradient descent loop over the eval suite. + if isinstance(program, tg.Variable): + variable = program + else: + variable = tg.Variable( + str(program), requires_grad=True, role_description="prompt program" + ) + optimizer = tg.TGD(parameters=[variable]) + examples = load_eval_examples(plan.eval_suite_path) + iterations = min(plan.max_iters, max(1, len(examples))) + for _ in range(iterations): + optimizer.zero_grad() + optimizer.step() + return CompileResult( + program_text=str(variable.value), + score=0.0, + iterations=iterations, + converged=True, + ) + + +def run_compile(plan: CompilePlan) -> CompileResult: + """Run the live compilation (v0.71.13 #225). + + Dispatches by ``plan.optimizer``: DSPy (bootstrap_fewshot / mipro / copro), + GEPA, or TextGrad. Each branch lazy-imports its library and raises a + friendly ``ImportError`` (naming ``pip install soup-cli[compile]``) when it + is absent. The ``_OPTIMIZER_RUN_OVERRIDE`` seam lets tests exercise the + dispatcher + result handling without the heavy optimizer libraries. + + Validates the plan type at the boundary so callers passing a bare dict get + a clean ``TypeError``. """ if not isinstance(plan, CompilePlan): raise TypeError("plan must be CompilePlan") - raise NotImplementedError( - "soup compile live runner is deferred to v0.68.1 — re-run after upgrading" - ) + if _OPTIMIZER_RUN_OVERRIDE is not None: + result = _OPTIMIZER_RUN_OVERRIDE(plan) + if not isinstance(result, CompileResult): + raise TypeError("optimizer override must return a CompileResult") + return result + if plan.optimizer in _DSPY_OPTIMIZERS: + return _run_dspy(plan) + if plan.optimizer == "gepa": + return _run_gepa(plan) + if plan.optimizer == "textgrad": + return _run_textgrad(plan) + # Unreachable: the optimizer is validated by the schema. + raise ValueError(f"unhandled optimizer {plan.optimizer!r}") __all__ = [ @@ -245,6 +482,7 @@ __all__ = [ "validate_max_iters", "validate_program_path", "validate_eval_suite_path", + "load_eval_examples", "CompilePlan", "CompileResult", "build_compile_plan", diff --git a/src/soup_cli/utils/prompt_distill.py b/src/soup_cli/utils/prompt_distill.py index 7547efb..828cc08 100644 --- a/src/soup_cli/utils/prompt_distill.py +++ b/src/soup_cli/utils/prompt_distill.py @@ -2,9 +2,9 @@ Bridge between prompt-engineering and FT worlds: take a JSONL of large-prompt teacher calls (GPT-5 / Claude / etc.) and prepare a -distillation dataset targeting a small student model. Schema-only release: -live dataset preparation lands in v0.68.1 (composes with v0.70 Part B -cross-tokenizer KD when that ships). +distillation dataset targeting a small student model. Live dataset +preparation lands in v0.71.13 (#226) via the v0.20.0 provider helpers +(composes with v0.70 Part B cross-tokenizer KD). Public surface: @@ -14,20 +14,30 @@ Public surface: - ``validate_traces_path(path)`` — cwd containment + symlink rejection - ``DistillPromptPlan`` frozen dataclass - ``build_distill_prompt_plan(...)`` factory -- ``prepare_distill_dataset(plan)`` — NotImplementedError stub w/ v0.68.1 marker +- ``prepare_distill_dataset(plan)`` — live teacher/student dataset prep (v0.71.13 #226) """ from __future__ import annotations +import json +import logging import os from dataclasses import dataclass +from typing import Any, Callable, List, Mapping, Optional -from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink +from soup_cli.utils.paths import ( + atomic_write_text, + enforce_under_cwd_and_no_symlink, +) -SUPPORTED_DISTILL_STRATEGIES: frozenset = frozenset({"sft", "preference", "kl"}) +_LOG = logging.getLogger("soup.distill_prompt") + +SUPPORTED_DISTILL_STRATEGIES: frozenset[str] = frozenset({"sft", "preference", "kl"}) _MAX_STRATEGY_LEN = 32 _MAX_MODEL_ID_LEN = 512 +_MAX_TRACE_ROWS = 1_000_000 +_MAX_PROMPT_CHARS = 100_000 def validate_distill_strategy(name: object) -> str: @@ -137,16 +147,170 @@ def build_distill_prompt_plan( ) -def prepare_distill_dataset(plan: DistillPromptPlan) -> None: - """Live dataset preparation. Deferred to v0.68.1. +def extract_prompt(row: Mapping[str, Any]) -> Optional[str]: + """Extract the teacher-facing prompt from a trace row. - Validates plan type at the boundary so a bare dict raises cleanly. + Handles the common trace shapes: an explicit ``prompt`` / ``input`` / + ``instruction`` / ``question`` field, or the last user turn of a + ``messages`` list. Returns ``None`` when no prompt can be found. + """ + if not isinstance(row, Mapping): + return None + for key in ("prompt", "input", "instruction", "question", "query"): + value = row.get(key) + if isinstance(value, str) and value.strip(): + return value[:_MAX_PROMPT_CHARS] + messages = row.get("messages") + if isinstance(messages, list): + for msg in reversed(messages): + if ( + isinstance(msg, Mapping) + and msg.get("role") == "user" + and isinstance(msg.get("content"), str) + and msg["content"].strip() + ): + return str(msg["content"])[:_MAX_PROMPT_CHARS] + return None + + +def _read_traces(traces_path: str) -> List[Mapping[str, Any]]: + """Read trace rows (cwd-contained, symlink-safe, O_NOFOLLOW).""" + canonical = enforce_under_cwd_and_no_symlink(traces_path, "traces_path") + rows: List[Mapping[str, Any]] = [] + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(canonical, flags) + with os.fdopen(fd, encoding="utf-8") as handle: + for line in handle: + line = line.strip() + if not line: + continue + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(obj, Mapping): + rows.append(obj) + if len(rows) >= _MAX_TRACE_ROWS: + break + return rows + + +def _build_provider_fn( + provider: str, model: str, *, base_url: Optional[str], temperature: float +) -> "Callable[[str], Mapping[str, Any]]": + """Lazy-build a ``judge(prompt) -> {'text': str}`` callable. + + Reuses the v0.20.0 provider helpers (Ollama / Anthropic / vLLM) via + ``data_forge.make_judge_provider_fn``. Anthropic reads the key from the + environment; Ollama / vLLM are SSRF-validated loopback by default. + """ + from soup_cli.utils.data_forge import make_judge_provider_fn + + return make_judge_provider_fn( + provider, + model=model, + base_url=base_url, + temperature=temperature, + ) + + +def prepare_distill_dataset( + plan: DistillPromptPlan, + *, + provider: str = "ollama", + base_url: Optional[str] = None, + temperature: float = 0.0, + max_rows: Optional[int] = None, + teacher_fn: "Optional[Callable[[str], Mapping[str, Any]]]" = None, + student_fn: "Optional[Callable[[str], Mapping[str, Any]]]" = None, +) -> int: + """Prepare a distillation dataset from prompt-heavy traces (v0.71.13 #226). + + For each prompt in ``plan.traces_path`` the teacher model is called once; + the output row depends on ``plan.strategy``: + + - ``sft`` -> ``{messages: [user, assistant=teacher_response]}`` (feed to + ``soup train --task sft``). + - ``preference`` -> ``{prompt, chosen=teacher_response, + rejected=student_baseline}`` (feed to ``soup train --task dpo``). The + student model is called once per prompt for the rejected response. + - ``kl`` -> the same ``{messages}`` rows as ``sft`` — ``DistillTrainerWrapper`` + (``soup train --task distill``) computes the per-token logit-KL live, so + no pre-computed logprobs are emitted; cross-tokenizer projection is the + v0.70 Part B deliverable. + + The ``teacher_fn`` / ``student_fn`` seams default to real provider calls + (``make_judge_provider_fn``); tests inject fast fakes. Returns the number + of rows written. ``max_rows`` caps the number of rows *written* (not the + number of teacher calls — a prompt whose teacher reply is empty still + consumes a call without producing a row). """ if not isinstance(plan, DistillPromptPlan): raise TypeError("plan must be DistillPromptPlan") - raise NotImplementedError( - "distill-prompt live dataset preparation is deferred to v0.68.1" - ) + if max_rows is not None and ( + isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1 + ): + raise ValueError("max_rows must be a positive int or None") + + teacher = teacher_fn + if teacher is None: + teacher = _build_provider_fn( + provider, plan.teacher, base_url=base_url, temperature=temperature + ) + student = student_fn + if plan.strategy == "preference" and student is None: + student = _build_provider_fn( + provider, plan.student, base_url=base_url, temperature=temperature + ) + + rows = _read_traces(plan.traces_path) + out_lines: List[str] = [] + for row in rows: + if max_rows is not None and len(out_lines) >= max_rows: + break + prompt = extract_prompt(row) + if prompt is None: + continue + try: + t_reply = teacher(prompt) + except Exception as exc: # noqa: BLE001 — provider error variety + _LOG.debug("teacher call failed: %s", exc) + continue + t_text = t_reply.get("text") if isinstance(t_reply, Mapping) else None + if not isinstance(t_text, str) or not t_text.strip(): + continue + + if plan.strategy in ("sft", "kl"): + out_lines.append( + json.dumps( + { + "messages": [ + {"role": "user", "content": prompt}, + {"role": "assistant", "content": t_text}, + ] + }, + ensure_ascii=False, + ) + ) + else: # preference + try: + s_reply = student(prompt) if student is not None else None + except Exception as exc: # noqa: BLE001 + _LOG.debug("student call failed: %s", exc) + continue + s_text = s_reply.get("text") if isinstance(s_reply, Mapping) else None + if not isinstance(s_text, str) or not s_text.strip(): + continue + out_lines.append( + json.dumps( + {"prompt": prompt, "chosen": t_text, "rejected": s_text}, + ensure_ascii=False, + ) + ) + + text = "\n".join(out_lines) + ("\n" if out_lines else "") + atomic_write_text(text, plan.output_path, field="output_path") + return len(out_lines) __all__ = [ @@ -155,6 +319,7 @@ __all__ = [ "validate_teacher_id", "validate_student_id", "validate_traces_path", + "extract_prompt", "DistillPromptPlan", "build_distill_prompt_plan", "prepare_distill_dataset", diff --git a/tests/test_v0680_part_a.py b/tests/test_v0680_part_a.py index 92aac69..6fa749a 100644 --- a/tests/test_v0680_part_a.py +++ b/tests/test_v0680_part_a.py @@ -389,14 +389,16 @@ class TestBuildCompilePlan: class TestRunCompileDeferred: - def test_raises_with_v068_1_marker( + def test_live_missing_dep_friendly_importerror( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # v0.71.13 #225: the runner is live; with dspy absent the real + # branch raises a friendly ImportError naming the [compile] extra. from soup_cli.utils.prompt_compile import build_compile_plan, run_compile monkeypatch.chdir(tmp_path) prog = tmp_path / "p.py" - prog.write_text("pass\n", encoding="utf-8") + prog.write_text("program = 1\n", encoding="utf-8") suite = tmp_path / "s.json" suite.write_text("[]", encoding="utf-8") plan = build_compile_plan( @@ -406,7 +408,7 @@ class TestRunCompileDeferred: max_iters=4, output_path="out.py", ) - with pytest.raises(NotImplementedError, match="v0.68.1"): + with pytest.raises(ImportError, match=r"soup-cli\[compile\]"): run_compile(plan) def test_non_plan_rejected(self) -> None: @@ -478,13 +480,15 @@ class TestCli: ) assert result.exit_code == 2 - def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: - """Without --plan-only the live runner raises and CLI exits 3.""" + def test_live_missing_dep_exits_2( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + """v0.71.13 #225: live runner with dspy absent -> friendly exit 2.""" from soup_cli.cli import app monkeypatch.chdir(tmp_path) prog = tmp_path / "p.py" - prog.write_text("pass\n", encoding="utf-8") + prog.write_text("program = 1\n", encoding="utf-8") suite = tmp_path / "s.json" suite.write_text("[]", encoding="utf-8") runner = CliRunner() @@ -499,7 +503,7 @@ class TestCli: "mipro", ], ) - assert result.exit_code == 3, (result.output, repr(result.exception)) + assert result.exit_code == 2, (result.output, repr(result.exception)) # --------------------------------------------------------------------------- diff --git a/tests/test_v0680_part_b.py b/tests/test_v0680_part_b.py index 9b03f25..bd5a7b2 100644 --- a/tests/test_v0680_part_b.py +++ b/tests/test_v0680_part_b.py @@ -224,7 +224,10 @@ class TestBuildPlan: class TestPrepareDataset: - def test_deferred(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def test_live_writes_rows( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # v0.71.13 #226: live dataset preparation with an injected teacher. from soup_cli.utils.prompt_distill import ( build_distill_prompt_plan, prepare_distill_dataset, @@ -232,7 +235,9 @@ class TestPrepareDataset: monkeypatch.chdir(tmp_path) traces = tmp_path / "traces.jsonl" - traces.write_text("[]", encoding="utf-8") + traces.write_text( + json.dumps({"prompt": "q"}) + "\n", encoding="utf-8" + ) plan = build_distill_prompt_plan( traces_path=str(traces), teacher="t/x", @@ -240,8 +245,9 @@ class TestPrepareDataset: strategy="sft", output_path="o.jsonl", ) - with pytest.raises(NotImplementedError, match="v0.68.1"): - prepare_distill_dataset(plan) + n = prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": "T"}) + assert n == 1 + assert (tmp_path / "o.jsonl").is_file() def test_non_plan_rejected(self) -> None: from soup_cli.utils.prompt_distill import prepare_distill_dataset @@ -308,12 +314,22 @@ class TestCli: ) assert result.exit_code == 2 - def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + def test_live_writes_output( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # v0.71.13 #226: live runner writes a distilled dataset (provider + # mocked so the test never touches the network). + import soup_cli.utils.prompt_distill as pd from soup_cli.cli import app monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + pd, "_build_provider_fn", lambda *a, **k: (lambda p: {"text": "R"}) + ) traces = tmp_path / "traces.jsonl" - traces.write_text("[]", encoding="utf-8") + traces.write_text( + json.dumps({"prompt": "x"}) + "\n", encoding="utf-8" + ) runner = CliRunner() result = runner.invoke( app, @@ -327,9 +343,12 @@ class TestCli: "s", "--strategy", "sft", + "--output", + "o.jsonl", ], ) - assert result.exit_code == 3, (result.output, repr(result.exception)) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert (tmp_path / "o.jsonl").is_file() class TestSourceWiring: diff --git a/tests/test_v0680_part_c.py b/tests/test_v0680_part_c.py index 9f27f78..179205c 100644 --- a/tests/test_v0680_part_c.py +++ b/tests/test_v0680_part_c.py @@ -161,10 +161,20 @@ class TestToolCompilePlan: ) +_VALID_OPENAPI = { + "openapi": "3.0.0", + "info": {"title": "t", "version": "1"}, + "paths": { + "/w": {"get": {"operationId": "listW", "description": "List widgets"}} + }, +} + + class TestRunToolCompileDeferred: - def test_raises_v068_1( + def test_live_missing_dep_importerror( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # v0.71.13 #227: live runner; textgrad absent -> friendly ImportError. from soup_cli.utils.compile_tools import ( build_tool_compile_plan, run_tool_compile, @@ -172,16 +182,16 @@ class TestRunToolCompileDeferred: monkeypatch.chdir(tmp_path) spec = tmp_path / "spec.json" - spec.write_text("{}", encoding="utf-8") + spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8") eval_suite = tmp_path / "eval.jsonl" - eval_suite.write_text("[]", encoding="utf-8") + eval_suite.write_text('{"q":"1"}\n', encoding="utf-8") plan = build_tool_compile_plan( spec_path=str(spec), eval_suite_path=str(eval_suite), optimizer="textgrad", output_path="tools.json", ) - with pytest.raises(NotImplementedError, match="v0.68.1"): + with pytest.raises(ImportError, match=r"soup-cli\[compile\]"): run_tool_compile(plan) def test_non_plan_rejected(self) -> None: @@ -222,22 +232,23 @@ class TestCli: ) assert result.exit_code == 0, (result.output, repr(result.exception)) - def test_live_exits_3( + def test_live_missing_dep_exits_2( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # v0.71.13 #227: live runner; textgrad absent -> friendly exit 2. from soup_cli.cli import app monkeypatch.chdir(tmp_path) spec = tmp_path / "spec.json" - spec.write_text("{}", encoding="utf-8") + spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8") eval_suite = tmp_path / "eval.jsonl" - eval_suite.write_text("[]", encoding="utf-8") + eval_suite.write_text('{"q":"1"}\n', encoding="utf-8") runner = CliRunner() result = runner.invoke( app, ["compile-tools", str(spec), "--eval", str(eval_suite)], ) - assert result.exit_code == 3, (result.output, repr(result.exception)) + assert result.exit_code == 2, (result.output, repr(result.exception)) class TestSourceWiring: diff --git a/tests/test_v0680_part_e.py b/tests/test_v0680_part_e.py index 36d3d22..c2058d6 100644 --- a/tests/test_v0680_part_e.py +++ b/tests/test_v0680_part_e.py @@ -325,9 +325,11 @@ class TestHarvestDpoPairs: class TestNightlyTrainDeferred: - def test_raises_v068_1( + def test_live_skips_when_no_pairs( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # v0.71.13 #229: live runner; an empty DB harvests 0 pairs and skips + # (no NotImplementedError, no train call). from soup_cli.utils.local_rl import ( LocalRLConfig, init_local_rl_db, @@ -338,12 +340,14 @@ class TestNightlyTrainDeferred: init_local_rl_db("rl.db") cfg = LocalRLConfig( backend="ollama", - model="llama3:8b", + model="org/model", db_path="rl.db", train_method="dpo", ) - with pytest.raises(NotImplementedError, match="v0.68.1"): - run_nightly_train(cfg) + res = run_nightly_train( + cfg, min_pairs=1, train_fn=lambda **kw: pytest.fail("no pairs") + ) + assert res.status == "skipped_insufficient_pairs" def test_non_config_rejected(self) -> None: from soup_cli.utils.local_rl import run_nightly_train @@ -423,9 +427,11 @@ class TestCli: ) assert result.exit_code == 0, (result.output, repr(result.exception)) - def test_train_command_exits_3( + def test_train_no_once_renders_scheduler( self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: + # v0.71.13 #229: `train` (no --once) renders the systemd/launchd + # scaffold and exits 0 (no systemctl call). from soup_cli.cli import app monkeypatch.chdir(tmp_path) @@ -441,10 +447,13 @@ class TestCli: "--backend", "ollama", "--model", - "llama3:8b", + "org/model", + "--scheduler-dir", + "sched", ], ) - assert result.exit_code == 3, (result.output, repr(result.exception)) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert (tmp_path / "sched" / "soup-local-rl.timer").is_file() class TestSourceWiring: diff --git a/tests/test_v07113.py b/tests/test_v07113.py new file mode 100644 index 0000000..259c41d --- /dev/null +++ b/tests/test_v07113.py @@ -0,0 +1,1409 @@ +"""v0.71.13 — "Prompt-compile family" live wiring. + +Closes #225 (soup compile), #226 (soup distill-prompt), #227 (soup +compile-tools), #229 (soup local-rl train). Lifts the v0.68.1 deferred +stubs from ``utils/prompt_compile.py`` / ``prompt_distill.py`` / +``compile_tools.py`` / ``local_rl.py``. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import sys + +import pytest +import typer +from typer.testing import CliRunner + +runner = CliRunner() + + +# =========================================================================== +# #229 — soup local-rl train (live DPO/KTO/ORPO runner + scheduler) +# =========================================================================== + + +def _seed_db(path: str, n_pairs: int) -> None: + from soup_cli.utils.local_rl import init_local_rl_db, record_thumb + + init_local_rl_db(path) + for i in range(n_pairs): + record_thumb( + db_path=path, prompt=f"q{i}", response=f"good{i}", thumb="up" + ) + record_thumb( + db_path=path, prompt=f"q{i}", response=f"bad{i}", thumb="down" + ) + + +class TestStateTable: + def test_init_creates_state_table(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + + init_local_rl_db("db.sqlite") + with sqlite3.connect(str(tmp_path / "db.sqlite")) as conn: + names = { + r[0] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table'" + ).fetchall() + } + assert "state" in names + + def test_get_set_state_roundtrip(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + get_state, + init_local_rl_db, + set_state, + ) + + init_local_rl_db("db.sqlite") + assert get_state("db.sqlite", "k") is None + set_state("db.sqlite", "k", "v1") + assert get_state("db.sqlite", "k") == "v1" + set_state("db.sqlite", "k", "v2") + assert get_state("db.sqlite", "k") == "v2" + + def test_get_state_empty_key_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db, set_state + + init_local_rl_db("db.sqlite") + with pytest.raises(ValueError): + set_state("db.sqlite", "", "v") + + def test_set_state_non_str_value_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db, set_state + + init_local_rl_db("db.sqlite") + with pytest.raises(TypeError): + set_state("db.sqlite", "k", 5) + + def test_state_table_auto_migrates_old_db(self, tmp_path, monkeypatch): + # A pre-v0.71.13 DB without the state table still works. + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import get_state + + with sqlite3.connect(str(tmp_path / "old.sqlite")) as conn: + conn.execute( + "CREATE TABLE thumbs (id INTEGER PRIMARY KEY, ts REAL, " + "prompt TEXT, response TEXT, thumb TEXT)" + ) + conn.commit() + assert get_state("old.sqlite", "k") is None + + +class TestCountNewThumbs: + def test_counts_all_when_since_none(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import count_new_thumbs_since + + _seed_db("db.sqlite", 3) # 6 thumbs + assert count_new_thumbs_since("db.sqlite", None) == 6 + + def test_counts_after_ts(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + count_new_thumbs_since, + init_local_rl_db, + record_thumb, + ) + + init_local_rl_db("db.sqlite") + record_thumb(db_path="db.sqlite", prompt="a", response="x", thumb="up") + with sqlite3.connect(str(tmp_path / "db.sqlite")) as conn: + cutoff = conn.execute("SELECT MAX(ts) FROM thumbs").fetchone()[0] + record_thumb(db_path="db.sqlite", prompt="b", response="y", thumb="down") + assert count_new_thumbs_since("db.sqlite", cutoff) == 1 + + def test_bool_since_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import count_new_thumbs_since + + _seed_db("db.sqlite", 1) + with pytest.raises(TypeError): + count_new_thumbs_since("db.sqlite", True) + + +class TestPairsToRows: + def _pairs(self): + from soup_cli.utils.local_rl import DpoPair + + return (DpoPair(prompt="p", chosen="c", rejected="r"),) + + def test_dpo_shape(self): + from soup_cli.utils.local_rl import pairs_to_rows + + rows = pairs_to_rows(self._pairs(), "dpo") + assert rows == [{"prompt": "p", "chosen": "c", "rejected": "r"}] + + def test_orpo_same_as_dpo(self): + from soup_cli.utils.local_rl import pairs_to_rows + + assert pairs_to_rows(self._pairs(), "orpo") == [ + {"prompt": "p", "chosen": "c", "rejected": "r"} + ] + + def test_kto_two_rows_per_pair(self): + from soup_cli.utils.local_rl import pairs_to_rows + + rows = pairs_to_rows(self._pairs(), "kto") + assert rows == [ + {"prompt": "p", "completion": "c", "label": True}, + {"prompt": "p", "completion": "r", "label": False}, + ] + + def test_unknown_method_rejected(self): + from soup_cli.utils.local_rl import pairs_to_rows + + with pytest.raises(ValueError): + pairs_to_rows(self._pairs(), "ppo") + + def test_non_pair_rejected(self): + from soup_cli.utils.local_rl import pairs_to_rows + + with pytest.raises(TypeError): + pairs_to_rows(({"prompt": "p"},), "dpo") + + +class TestRunNightlyTrain: + def _cfg(self, db, method="dpo"): + from soup_cli.utils.local_rl import LocalRLConfig + + return LocalRLConfig( + backend="ollama", + model="HuggingFaceTB/SmolLM2-135M-Instruct", + db_path=db, + train_method=method, + ) + + def test_first_run_trains_and_stamps(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + get_state, + run_nightly_train, + ) + + _seed_db("db.sqlite", 5) + seen = {} + + def fake_train(*, base_model, pairs_path, output_dir, train_method): + seen["base_model"] = base_model + seen["train_method"] = train_method + seen["output_dir"] = output_dir + with open(pairs_path, encoding="utf-8") as fh: + seen["rows"] = [json.loads(line) for line in fh if line.strip()] + + res = run_nightly_train( + self._cfg("db.sqlite"), + min_pairs=3, + output_dir="adapter", + train_fn=fake_train, + ) + assert res.status == "trained" + assert res.num_pairs == 5 + assert seen["base_model"] == "HuggingFaceTB/SmolLM2-135M-Instruct" + assert seen["train_method"] == "dpo" + assert seen["output_dir"] == "adapter" + assert len(seen["rows"]) == 5 + assert set(seen["rows"][0]) == {"prompt", "chosen", "rejected"} + # last_train_at stamped. + assert get_state("db.sqlite", "last_train_at") is not None + + def test_kto_rows_written(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import run_nightly_train + + _seed_db("db.sqlite", 4) + seen = {} + + def fake_train(*, base_model, pairs_path, output_dir, train_method): + with open(pairs_path, encoding="utf-8") as fh: + seen["rows"] = [json.loads(line) for line in fh if line.strip()] + + run_nightly_train( + self._cfg("db.sqlite", "kto"), + min_pairs=1, + train_fn=fake_train, + ) + # 4 pairs -> 8 KTO rows. + assert len(seen["rows"]) == 8 + assert set(seen["rows"][0]) == {"prompt", "completion", "label"} + + def test_skip_no_new_thumbs(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import run_nightly_train, set_state + + _seed_db("db.sqlite", 5) + # Stamp last_train_at AFTER all thumbs -> no new thumbs. + import time + + set_state("db.sqlite", "last_train_at", repr(time.time() + 100.0)) + calls = [] + res = run_nightly_train( + self._cfg("db.sqlite"), + min_pairs=1, + train_fn=lambda **kw: calls.append(kw), + ) + assert res.status == "skipped_no_new_thumbs" + assert calls == [] + + def test_skip_insufficient_pairs(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import run_nightly_train + + _seed_db("db.sqlite", 2) + calls = [] + res = run_nightly_train( + self._cfg("db.sqlite"), + min_pairs=10, + train_fn=lambda **kw: calls.append(kw), + ) + assert res.status == "skipped_insufficient_pairs" + assert res.num_pairs == 2 + assert calls == [] + + def test_non_config_rejected(self): + from soup_cli.utils.local_rl import run_nightly_train + + with pytest.raises(TypeError): + run_nightly_train({"backend": "ollama"}) + + def test_bad_min_pairs_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import run_nightly_train + + _seed_db("db.sqlite", 1) + with pytest.raises(ValueError): + run_nightly_train(self._cfg("db.sqlite"), min_pairs=0, train_fn=lambda **k: None) + with pytest.raises(TypeError): + run_nightly_train(self._cfg("db.sqlite"), min_pairs=True, train_fn=lambda **k: None) + + def test_non_callable_train_fn_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import run_nightly_train + + _seed_db("db.sqlite", 5) + with pytest.raises(TypeError): + run_nightly_train(self._cfg("db.sqlite"), train_fn=123) + + def test_default_train_fn_renders_schema_valid_config(self, tmp_path, monkeypatch): + """The real ``_default_train_fn`` must render a YAML that the SoupConfig + schema accepts — the injected-``train_fn`` tests never exercise the YAML + rendering, so this guards the ``output:`` shape (regression: it was once + emitted as ``{"dir": ...}`` which the schema rejects as not-a-string).""" + monkeypatch.chdir(tmp_path) + from soup_cli.config.loader import load_config_from_string + from soup_cli.utils import local_rl as lr + + captured = {} + + def fake_run(argv, check): # noqa: ARG001 — capture the rendered config + cfg_idx = argv.index("--config") + 1 + with open(argv[cfg_idx], encoding="utf-8") as fh: + captured["yaml"] = fh.read() + + # ``_default_train_fn`` does a lazy ``import subprocess`` -> the global + # module; patch that module's ``run``. + import subprocess as _sp + + monkeypatch.setattr(_sp, "run", fake_run) + + with open("pairs.jsonl", "w", encoding="utf-8") as fh: + fh.write('{"prompt": "p", "chosen": "c", "rejected": "r"}\n') + + lr._default_train_fn( + base_model="HuggingFaceTB/SmolLM2-135M-Instruct", + pairs_path="pairs.jsonl", + output_dir="adapter_out", + train_method="dpo", + ) + assert "yaml" in captured, "subprocess.run was not invoked" + cfg = load_config_from_string(captured["yaml"]) # must NOT raise + assert cfg.output == "adapter_out" + assert cfg.task == "dpo" + + +class TestScheduler: + def test_build_train_argv(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + from soup_cli.utils.local_rl_scheduler import build_train_argv + + init_local_rl_db("db.sqlite") + argv = build_train_argv( + soup_python=sys.executable, + db_path="db.sqlite", + model="org/model", + train_method="dpo", + ) + assert argv[0] == sys.executable + assert "local-rl" in argv and "train" in argv and "--once" in argv + assert "db.sqlite" in argv and "org/model" in argv + # No shell metacharacters injected — it's a plain list. + assert all(isinstance(a, str) for a in argv) + + def test_systemd_service_has_execstart(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + from soup_cli.utils.local_rl_scheduler import render_systemd_service + + init_local_rl_db("db.sqlite") + out = render_systemd_service( + soup_python=sys.executable, + db_path="db.sqlite", + model="org/model", + train_method="dpo", + ) + assert "ExecStart=" in out + assert "Type=oneshot" in out + assert "org/model" in out + + def test_systemd_timer_oncalendar(self): + from soup_cli.utils.local_rl_scheduler import render_systemd_timer + + out = render_systemd_timer(hour=3, minute=0) + assert "OnCalendar=*-*-* 03:00:00" in out + assert "WantedBy=timers.target" in out + + def test_systemd_timer_bad_hour(self): + from soup_cli.utils.local_rl_scheduler import render_systemd_timer + + with pytest.raises(ValueError): + render_systemd_timer(hour=24, minute=0) + with pytest.raises(TypeError): + render_systemd_timer(hour=True, minute=0) + + def test_launchd_plist(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + from soup_cli.utils.local_rl_scheduler import render_launchd_plist + + init_local_rl_db("db.sqlite") + out = render_launchd_plist( + soup_python=sys.executable, + db_path="db.sqlite", + model="org/model", + train_method="dpo", + hour=3, + minute=30, + ) + assert "StartCalendarInterval" in out + assert "3" in out + assert "30" in out + assert "com.soup.local-rl" in out + + def test_write_scheduler_files(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + from soup_cli.utils.local_rl_scheduler import ( + LAUNCHD_PLIST_NAME, + SYSTEMD_SERVICE_NAME, + SYSTEMD_TIMER_NAME, + write_scheduler_files, + ) + + init_local_rl_db("db.sqlite") + written = write_scheduler_files( + "sched", + soup_python=sys.executable, + db_path="db.sqlite", + model="org/model", + train_method="dpo", + ) + for name in (SYSTEMD_SERVICE_NAME, SYSTEMD_TIMER_NAME, LAUNCHD_PLIST_NAME): + assert name in written + assert os.path.isfile(written[name]) + + def test_systemd_quote_escapes(self): + from soup_cli.utils.local_rl_scheduler import _systemd_quote + + assert _systemd_quote('a "b" c') == '"a \\"b\\" c"' + assert _systemd_quote("a\\b") == '"a\\\\b"' + + +class TestTrainCli: + def _import_app(self): + from soup_cli.commands.local_rl import app + + return app + + def test_once_runs_live(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _seed_db("db.sqlite", 5) + + import soup_cli.utils.local_rl as lr + + calls = [] + monkeypatch.setattr( + lr, "_default_train_fn", lambda **kw: calls.append(kw) + ) + res = runner.invoke( + self._import_app(), + [ + "train", + "--db", + "db.sqlite", + "--model", + "org/model", + "--train-method", + "dpo", + "--once", + "--min-pairs", + "1", + "--output", + "adapter", + ], + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert len(calls) == 1 + assert calls[0]["base_model"] == "org/model" + + def test_once_skip_insufficient(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + _seed_db("db.sqlite", 1) + import soup_cli.utils.local_rl as lr + + monkeypatch.setattr( + lr, "_default_train_fn", lambda **kw: pytest.fail("should not train") + ) + res = runner.invoke( + self._import_app(), + [ + "train", + "--db", + "db.sqlite", + "--model", + "org/model", + "--once", + "--min-pairs", + "10", + ], + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert "Skipped" in res.output + + def test_no_once_renders_scheduler(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + + init_local_rl_db("db.sqlite") + res = runner.invoke( + self._import_app(), + [ + "train", + "--db", + "db.sqlite", + "--model", + "org/model", + "--scheduler-dir", + "sched", + ], + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert os.path.isfile(str(tmp_path / "sched" / "soup-local-rl.timer")) + + def test_no_longer_deferred(self): + # The CLI must NOT surface the v0.68.1 deferred marker any more. + import soup_cli.utils.local_rl as lr + + assert "deferred" not in (lr.run_nightly_train.__doc__ or "").lower() + + +# =========================================================================== +# #226 — soup distill-prompt (live teacher/student dataset prep) +# =========================================================================== + + +def _write_traces(path: str, prompts): + with open(path, "w", encoding="utf-8") as fh: + for p in prompts: + fh.write(json.dumps({"prompt": p}) + "\n") + + +class TestExtractPrompt: + def test_prompt_field(self): + from soup_cli.utils.prompt_distill import extract_prompt + + assert extract_prompt({"prompt": "hi"}) == "hi" + + def test_input_instruction_question(self): + from soup_cli.utils.prompt_distill import extract_prompt + + assert extract_prompt({"input": "a"}) == "a" + assert extract_prompt({"instruction": "b"}) == "b" + assert extract_prompt({"question": "c"}) == "c" + + def test_messages_last_user(self): + from soup_cli.utils.prompt_distill import extract_prompt + + row = { + "messages": [ + {"role": "user", "content": "first"}, + {"role": "assistant", "content": "reply"}, + {"role": "user", "content": "second"}, + ] + } + assert extract_prompt(row) == "second" + + def test_no_prompt_returns_none(self): + from soup_cli.utils.prompt_distill import extract_prompt + + assert extract_prompt({"foo": "bar"}) is None + assert extract_prompt({"prompt": " "}) is None + assert extract_prompt("not a mapping") is None + + +class TestPrepareDistillDataset: + def _plan(self, tmp_path, strategy): + from soup_cli.utils.prompt_distill import build_distill_prompt_plan + + traces = str(tmp_path / "traces.jsonl") + _write_traces(traces, ["q1", "q2"]) + return build_distill_prompt_plan( + traces_path="traces.jsonl", + teacher="qwen2.5:0.5b", + student="smol", + strategy=strategy, + output_path="out.jsonl", + ) + + def test_sft_writes_messages(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "sft") + n = prepare_distill_dataset( + plan, teacher_fn=lambda p: {"text": f"T:{p}"} + ) + assert n == 2 + with open(tmp_path / "out.jsonl", encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + assert rows[0]["messages"][0] == {"role": "user", "content": "q1"} + assert rows[0]["messages"][1] == {"role": "assistant", "content": "T:q1"} + + def test_kl_same_shape_as_sft(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "kl") + n = prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": "T"}) + assert n == 2 + with open(tmp_path / "out.jsonl", encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + assert "messages" in rows[0] + + def test_preference_uses_student(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "preference") + n = prepare_distill_dataset( + plan, + teacher_fn=lambda p: {"text": "teach"}, + student_fn=lambda p: {"text": "stud"}, + ) + assert n == 2 + with open(tmp_path / "out.jsonl", encoding="utf-8") as fh: + rows = [json.loads(line) for line in fh if line.strip()] + assert rows[0] == {"prompt": "q1", "chosen": "teach", "rejected": "stud"} + + def test_empty_teacher_reply_skipped(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "sft") + n = prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": ""}) + assert n == 0 + + def test_max_rows_cap(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "sft") + n = prepare_distill_dataset( + plan, teacher_fn=lambda p: {"text": "T"}, max_rows=1 + ) + assert n == 1 + + def test_bad_max_rows(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "sft") + with pytest.raises(ValueError): + prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": "T"}, max_rows=0) + with pytest.raises(ValueError): + prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": "T"}, max_rows=True) + + def test_non_plan_rejected(self): + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + with pytest.raises(TypeError): + prepare_distill_dataset({"traces_path": "x"}) + + def test_teacher_exception_skipped(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import prepare_distill_dataset + + plan = self._plan(tmp_path, "sft") + + def boom(p): + raise RuntimeError("provider down") + + n = prepare_distill_dataset(plan, teacher_fn=boom) + assert n == 0 + + +class TestDistillPromptCli: + def test_cli_live(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.distill_prompt import distill_prompt_cmd + + _write_traces(str(tmp_path / "traces.jsonl"), ["a", "b"]) + import soup_cli.utils.prompt_distill as pd + + monkeypatch.setattr( + pd, + "_build_provider_fn", + lambda *a, **k: (lambda p: {"text": f"R:{p}"}), + ) + app = typer.Typer() + app.command()(distill_prompt_cmd) + res = runner.invoke( + app, + [ + "--traces", + "traces.jsonl", + "--teacher", + "qwen2.5:0.5b", + "--student", + "smol", + "--strategy", + "sft", + "--output", + "out.jsonl", + ], + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert os.path.isfile(str(tmp_path / "out.jsonl")) + + def test_cli_no_longer_deferred(self): + import soup_cli.utils.prompt_distill as pd + + assert "deferred" not in ( + pd.prepare_distill_dataset.__doc__ or "" + ).lower() + + +# =========================================================================== +# #225 — soup compile (DSPy / GEPA / TextGrad dispatcher) +# =========================================================================== + + +class TestLoadEvalExamples: + def test_json_list(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import load_eval_examples + + (tmp_path / "eval.json").write_text( + json.dumps([{"q": "1"}, {"q": "2"}, "skip-non-dict"]), + encoding="utf-8", + ) + ex = load_eval_examples("eval.json") + assert ex == [{"q": "1"}, {"q": "2"}] + + def test_jsonl(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import load_eval_examples + + (tmp_path / "eval.jsonl").write_text( + '{"q": "1"}\n\n{"q": "2"}\nbad-line\n', encoding="utf-8" + ) + ex = load_eval_examples("eval.jsonl") + assert ex == [{"q": "1"}, {"q": "2"}] + + def test_outside_cwd_rejected(self, tmp_path, monkeypatch): + sub = tmp_path / "work" + sub.mkdir() + monkeypatch.chdir(sub) + (tmp_path / "evil.json").write_text("[]", encoding="utf-8") + from soup_cli.utils.prompt_compile import load_eval_examples + + with pytest.raises(ValueError): + load_eval_examples(str(tmp_path / "evil.json")) + + +class TestRunCompileSeam: + def _plan(self, tmp_path): + from soup_cli.utils.prompt_compile import build_compile_plan + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + return build_compile_plan( + program_path="prog.py", + eval_suite_path="eval.jsonl", + optimizer="mipro", + max_iters=3, + output_path="out.py", + ) + + def test_override_seam(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + plan = self._plan(tmp_path) + monkeypatch.setattr( + pc, + "_OPTIMIZER_RUN_OVERRIDE", + lambda p: pc.CompileResult( + program_text="optimised!", score=0.9, iterations=3, converged=True + ), + ) + result = pc.run_compile(plan) + assert result.program_text == "optimised!" + assert result.score == 0.9 + + def test_override_must_return_result(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + plan = self._plan(tmp_path) + monkeypatch.setattr(pc, "_OPTIMIZER_RUN_OVERRIDE", lambda p: "nope") + with pytest.raises(TypeError): + pc.run_compile(plan) + + def test_non_plan_rejected(self): + from soup_cli.utils.prompt_compile import run_compile + + with pytest.raises(TypeError): + run_compile({"optimizer": "mipro"}) + + def test_missing_dspy_friendly_importerror(self, tmp_path, monkeypatch): + # dspy/textgrad/gepa are genuinely absent in this env -> the real + # branch raises a friendly ImportError naming the [compile] extra. + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import run_compile + + plan = self._plan(tmp_path) # optimizer=mipro -> dspy branch + with pytest.raises(ImportError) as exc: + run_compile(plan) + assert "soup-cli[compile]" in str(exc.value) + + def test_missing_textgrad_friendly(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import build_compile_plan, run_compile + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + plan = build_compile_plan( + program_path="prog.py", + eval_suite_path="eval.jsonl", + optimizer="textgrad", + max_iters=2, + output_path="out.py", + ) + with pytest.raises(ImportError) as exc: + run_compile(plan) + assert "TextGrad" in str(exc.value) + + def test_missing_gepa_friendly(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import build_compile_plan, run_compile + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + plan = build_compile_plan( + program_path="prog.py", + eval_suite_path="eval.jsonl", + optimizer="gepa", + max_iters=2, + output_path="out.py", + ) + with pytest.raises(ImportError) as exc: + run_compile(plan) + assert "GEPA" in str(exc.value) + + def test_resolve_program_via_get_program(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + (tmp_path / "prog2.py").write_text( + "def get_program():\n return 42\n", encoding="utf-8" + ) + mod = pc._load_program_module("prog2.py") + assert pc._resolve_program(mod) == 42 + + def test_resolve_program_missing(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + (tmp_path / "prog3.py").write_text("x = 1\n", encoding="utf-8") + mod = pc._load_program_module("prog3.py") + with pytest.raises(ValueError): + pc._resolve_program(mod) + + +class TestCompileCli: + def test_plan_only(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_cmd import compile_cmd + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + app = typer.Typer() + app.command()(compile_cmd) + res = runner.invoke( + app, + ["prog.py", "--eval", "eval.jsonl", "--optimizer", "mipro", "--plan-only"], + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + + def test_live_via_seam(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_cmd import compile_cmd + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + import soup_cli.utils.prompt_compile as pc + + monkeypatch.setattr( + pc, + "_OPTIMIZER_RUN_OVERRIDE", + lambda p: pc.CompileResult( + program_text="# compiled\n", score=0.5, iterations=3, converged=True + ), + ) + app = typer.Typer() + app.command()(compile_cmd) + res = runner.invoke( + app, ["prog.py", "--eval", "eval.jsonl", "--optimizer", "mipro"] + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert (tmp_path / "compiled_program.py").read_text( + encoding="utf-8" + ) == "# compiled\n" + + def test_live_missing_dep_exit2(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_cmd import compile_cmd + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + app = typer.Typer() + app.command()(compile_cmd) + res = runner.invoke( + app, ["prog.py", "--eval", "eval.jsonl", "--optimizer", "mipro"] + ) + assert res.exit_code == 2 + assert "compile" in res.output.lower() + + def test_unknown_optimizer_exit2(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_cmd import compile_cmd + + (tmp_path / "prog.py").write_text("program = 1\n", encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + app = typer.Typer() + app.command()(compile_cmd) + res = runner.invoke( + app, ["prog.py", "--eval", "eval.jsonl", "--optimizer", "nope"] + ) + assert res.exit_code == 2 + + +# =========================================================================== +# #227 — soup compile-tools (TextGrad / GEPA tool-schema optimiser) +# =========================================================================== + + +_OPENAPI_SPEC = { + "openapi": "3.0.0", + "info": {"title": "t", "version": "1"}, + "paths": { + "/widgets": { + "get": { + "operationId": "listWidgets", + "description": "List all the widgets", + }, + "post": { + "operationId": "createWidget", + "description": "Create a widget", + }, + } + }, +} + + +def _write_spec(tmp_path, name="spec.json"): + (tmp_path / name).write_text(json.dumps(_OPENAPI_SPEC), encoding="utf-8") + (tmp_path / "eval.jsonl").write_text('{"q":"1"}\n', encoding="utf-8") + + +class TestOptimiseDescription: + def test_override_returns_str(self, monkeypatch): + import soup_cli.utils.compile_tools as ct + + monkeypatch.setattr( + ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: d.upper() + ) + assert ct._optimise_description("hi", [], "textgrad") == "HI" + + def test_override_non_str_rejected(self, monkeypatch): + import soup_cli.utils.compile_tools as ct + + monkeypatch.setattr(ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: 5) + with pytest.raises(TypeError): + ct._optimise_description("hi", [], "textgrad") + + def test_missing_textgrad_friendly(self): + from soup_cli.utils.compile_tools import _optimise_description + + with pytest.raises(ImportError) as exc: + _optimise_description("hi", [], "textgrad") + assert "soup-cli[compile]" in str(exc.value) + + def test_missing_gepa_friendly(self): + from soup_cli.utils.compile_tools import _optimise_description + + with pytest.raises(ImportError) as exc: + _optimise_description("hi", [], "gepa") + assert "GEPA" in str(exc.value) + + +class TestRunToolCompile: + def _plan(self, tmp_path, output="out.json"): + from soup_cli.utils.compile_tools import build_tool_compile_plan + + _write_spec(tmp_path) + return build_tool_compile_plan( + spec_path="spec.json", + eval_suite_path="eval.jsonl", + optimizer="textgrad", + output_path=output, + ) + + def test_json_output(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.compile_tools as ct + + monkeypatch.setattr( + ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: f"OPT:{d}" + ) + plan = self._plan(tmp_path) + n = ct.run_tool_compile(plan) + assert n == 2 + out = json.loads((tmp_path / "out.json").read_text(encoding="utf-8")) + assert out["spec_kind"] == "openapi" + tools = {t["tool"]: t["description"] for t in out["tools"]} + assert tools["listWidgets"] == "OPT:List all the widgets" + + def test_yaml_output(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.compile_tools as ct + + monkeypatch.setattr(ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: "X") + plan = self._plan(tmp_path, output="out.yaml") + ct.run_tool_compile(plan) + import yaml + + out = yaml.safe_load((tmp_path / "out.yaml").read_text(encoding="utf-8")) + assert out["tools"][0]["description"] == "X" + + def test_non_plan_rejected(self): + from soup_cli.utils.compile_tools import run_tool_compile + + with pytest.raises(TypeError): + run_tool_compile({"spec_path": "x"}) + + def test_live_missing_dep(self, tmp_path, monkeypatch): + # Real branch (no override) -> textgrad absent -> friendly ImportError. + monkeypatch.chdir(tmp_path) + from soup_cli.utils.compile_tools import run_tool_compile + + plan = self._plan(tmp_path) + with pytest.raises(ImportError): + run_tool_compile(plan) + + +class TestCompileToolsCli: + def test_plan_only(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_tools import compile_tools_cmd + + _write_spec(tmp_path) + app = typer.Typer() + app.command()(compile_tools_cmd) + res = runner.invoke( + app, ["spec.json", "--eval", "eval.jsonl", "--plan-only"] + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + + def test_live_via_seam(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.compile_tools as ct + from soup_cli.commands.compile_tools import compile_tools_cmd + + _write_spec(tmp_path) + monkeypatch.setattr(ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: "OK") + app = typer.Typer() + app.command()(compile_tools_cmd) + res = runner.invoke( + app, ["spec.json", "--eval", "eval.jsonl", "--output", "out.json"] + ) + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert os.path.isfile(str(tmp_path / "out.json")) + + def test_live_missing_dep_exit2(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands.compile_tools import compile_tools_cmd + + _write_spec(tmp_path) + app = typer.Typer() + app.command()(compile_tools_cmd) + res = runner.invoke( + app, ["spec.json", "--eval", "eval.jsonl", "--output", "out.json"] + ) + assert res.exit_code == 2 + + +# =========================================================================== +# Review-fix coverage +# =========================================================================== + + +class TestReviewFixes: + def test_db_path_rejects_newline(self, tmp_path, monkeypatch): + # SEC HIGH: db_path flows into a systemd ExecStart -> reject newlines. + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import validate_db_path + + with pytest.raises(ValueError): + validate_db_path("a\nb.db") + with pytest.raises(ValueError): + validate_db_path("a\rb.db") + + def test_model_rejects_newline(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import LocalRLConfig, init_local_rl_db + + init_local_rl_db("db.sqlite") + with pytest.raises(ValueError): + LocalRLConfig( + backend="ollama", + model="org/model\nExecStartPre=/bin/sh -c evil", + db_path="db.sqlite", + train_method="dpo", + ) + + def test_scheduler_model_rejects_newline(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import init_local_rl_db + from soup_cli.utils.local_rl_scheduler import build_train_argv + + init_local_rl_db("db.sqlite") + with pytest.raises(ValueError): + build_train_argv( + soup_python=sys.executable, + db_path="db.sqlite", + model="m\nevil", + train_method="dpo", + ) + + def test_systemd_quote_rejects_newline(self): + from soup_cli.utils.local_rl_scheduler import _systemd_quote + + with pytest.raises(ValueError): + _systemd_quote("a\nb") + + def test_stamp_before_train_keeps_concurrent_thumbs( + self, tmp_path, monkeypatch + ): + # CORRECTNESS MEDIUM: thumbs recorded DURING the train window keep + # ts > last_train_at and are counted by the next run (not dropped). + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + LocalRLConfig, + count_new_thumbs_since, + get_state, + record_thumb, + run_nightly_train, + ) + + _seed_db("db.sqlite", 5) + cfg = LocalRLConfig( + backend="ollama", + model="org/model", + db_path="db.sqlite", + train_method="dpo", + ) + + def slow_train(*, base_model, pairs_path, output_dir, train_method): + # Simulate a thumb landing mid-train. + record_thumb( + db_path="db.sqlite", prompt="qZ", response="x", thumb="up" + ) + record_thumb( + db_path="db.sqlite", prompt="qZ", response="y", thumb="down" + ) + + run_nightly_train(cfg, min_pairs=1, train_fn=slow_train) + last = float(get_state("db.sqlite", "last_train_at")) + # The 2 thumbs recorded during the train survive the stamp. + assert count_new_thumbs_since("db.sqlite", last) == 2 + + def test_cli_once_subprocess_failure_exit_1(self, tmp_path, monkeypatch): + # CORRECTNESS MEDIUM: a failed soup-train subprocess -> friendly exit 1. + monkeypatch.chdir(tmp_path) + import subprocess + + import soup_cli.utils.local_rl as lr + from soup_cli.commands.local_rl import app + + _seed_db("db.sqlite", 5) + + def boom(**kw): + raise subprocess.CalledProcessError(1, ["soup", "train"]) + + monkeypatch.setattr(lr, "_default_train_fn", boom) + res = runner.invoke( + app, + [ + "train", + "--db", + "db.sqlite", + "--model", + "org/model", + "--once", + "--min-pairs", + "1", + ], + ) + assert res.exit_code == 1, (res.output, repr(res.exception)) + + def test_load_program_module_revalidates(self, tmp_path, monkeypatch): + # SEC LOW: _load_program_module re-validates the path before exec. + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + (tmp_path / "notpy.txt").write_text("program = 1\n", encoding="utf-8") + with pytest.raises(ValueError): + pc._load_program_module("notpy.txt") + + +# =========================================================================== +# TDD-review coverage gaps +# =========================================================================== + + +class TestCoverageGaps: + # --- #229 skip-ordering + harvest-dedup ------------------------------- + def _cfg(self, db, method="dpo"): + from soup_cli.utils.local_rl import LocalRLConfig + + return LocalRLConfig( + backend="ollama", model="org/model", db_path=db, train_method=method + ) + + def test_new_thumbs_but_insufficient_pairs(self, tmp_path, monkeypatch): + # last_train_at in the past + fresh thumbs that yield < min_pairs -> + # status must be insufficient_pairs (NOT no_new_thumbs). + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + init_local_rl_db, + record_thumb, + run_nightly_train, + set_state, + ) + + init_local_rl_db("db.sqlite") + set_state("db.sqlite", "last_train_at", "0.0") # far past + record_thumb(db_path="db.sqlite", prompt="q", response="g", thumb="up") + record_thumb(db_path="db.sqlite", prompt="q", response="b", thumb="down") + res = run_nightly_train( + self._cfg("db.sqlite"), + min_pairs=5, + train_fn=lambda **k: pytest.fail("must not train"), + ) + assert res.status == "skipped_insufficient_pairs" + + def test_harvest_dedup_one_pair_per_prompt(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + harvest_dpo_pairs, + init_local_rl_db, + record_thumb, + run_nightly_train, + ) + + init_local_rl_db("db.sqlite") + for _ in range(2): + record_thumb(db_path="db.sqlite", prompt="q", response="g", thumb="up") + record_thumb(db_path="db.sqlite", prompt="q", response="b", thumb="down") + assert len(harvest_dpo_pairs("db.sqlite")) == 1 + res = run_nightly_train( + self._cfg("db.sqlite"), + min_pairs=2, + train_fn=lambda **k: pytest.fail("dedup -> 1 pair < 2"), + ) + assert res.status == "skipped_insufficient_pairs" + + def test_count_new_thumbs_exact_ts_excluded(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.local_rl import ( + count_new_thumbs_since, + init_local_rl_db, + record_thumb, + ) + + init_local_rl_db("db.sqlite") + record_thumb(db_path="db.sqlite", prompt="q", response="g", thumb="up") + with sqlite3.connect(str(tmp_path / "db.sqlite")) as conn: + ts = conn.execute("SELECT MAX(ts) FROM thumbs").fetchone()[0] + assert count_new_thumbs_since("db.sqlite", ts) == 0 # strict > + + # --- #225 load_eval_examples + CompileResult ------------------------- + def test_eval_size_cap(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.prompt_compile as pc + + monkeypatch.setattr(pc, "_MAX_EVAL_BYTES", 8) + (tmp_path / "big.json").write_text("[" + "1," * 50 + "1]", encoding="utf-8") + with pytest.raises(ValueError, match="exceeds"): + pc.load_eval_examples("big.json") + + def test_eval_json_object_root(self, tmp_path, monkeypatch): + # A single top-level JSON object is read as one example (not rejected). + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import load_eval_examples + + (tmp_path / "obj.json").write_text('{"q": "1"}', encoding="utf-8") + assert load_eval_examples("obj.json") == [{"q": "1"}] + + def test_eval_pretty_json_array(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_compile import load_eval_examples + + (tmp_path / "arr.json").write_text( + "[\n {\"q\": \"1\"},\n {\"q\": \"2\"}\n]", encoding="utf-8" + ) + assert load_eval_examples("arr.json") == [{"q": "1"}, {"q": "2"}] + + def test_compile_result_validation(self): + from soup_cli.utils.prompt_compile import CompileResult + + with pytest.raises(ValueError): + CompileResult( + program_text="x", score=float("nan"), iterations=1, converged=True + ) + with pytest.raises(ValueError): + CompileResult( + program_text="x", score=0.0, iterations=-1, converged=True + ) + with pytest.raises(TypeError): + CompileResult( + program_text="x", score=True, iterations=1, converged=True + ) + + # --- #226 distill edge cases ----------------------------------------- + def test_extract_prompt_query_key(self): + from soup_cli.utils.prompt_distill import extract_prompt + + assert extract_prompt({"query": "z"}) == "z" + + def test_extract_prompt_truncation(self): + from soup_cli.utils.prompt_distill import _MAX_PROMPT_CHARS, extract_prompt + + out = extract_prompt({"prompt": "x" * (_MAX_PROMPT_CHARS + 50)}) + assert len(out) == _MAX_PROMPT_CHARS + + def test_preference_empty_student_skipped(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import ( + build_distill_prompt_plan, + prepare_distill_dataset, + ) + + _write_traces(str(tmp_path / "t.jsonl"), ["q"]) + plan = build_distill_prompt_plan( + traces_path="t.jsonl", + teacher="t", + student="s", + strategy="preference", + output_path="o.jsonl", + ) + n = prepare_distill_dataset( + plan, + teacher_fn=lambda p: {"text": "T"}, + student_fn=lambda p: {"text": ""}, + ) + assert n == 0 + + def test_preference_student_exception_skipped(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.prompt_distill import ( + build_distill_prompt_plan, + prepare_distill_dataset, + ) + + _write_traces(str(tmp_path / "t.jsonl"), ["q"]) + plan = build_distill_prompt_plan( + traces_path="t.jsonl", + teacher="t", + student="s", + strategy="preference", + output_path="o.jsonl", + ) + + def boom(p): + raise RuntimeError("down") + + n = prepare_distill_dataset( + plan, teacher_fn=lambda p: {"text": "T"}, student_fn=boom + ) + assert n == 0 + + # --- #227 compile-tools ---------------------------------------------- + def test_spec_path_extension_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.utils.compile_tools import validate_spec_path + + (tmp_path / "spec.txt").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="extension"): + validate_spec_path("spec.txt") + + def test_tool_full_shape(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + import soup_cli.utils.compile_tools as ct + + monkeypatch.setattr(ct, "_TOOL_OPTIMIZER_OVERRIDE", lambda d, e, o: "D") + _write_spec(tmp_path) + plan = ct.build_tool_compile_plan( + spec_path="spec.json", + eval_suite_path="eval.jsonl", + optimizer="textgrad", + output_path="out.json", + ) + ct.run_tool_compile(plan) + out = json.loads((tmp_path / "out.json").read_text(encoding="utf-8")) + tool = next(t for t in out["tools"] if t["tool"] == "listWidgets") + assert set(tool) == {"tool", "description", "method", "path", "parameters"} + assert tool["method"] == "get" + + +# =========================================================================== +# Patch invariants +# =========================================================================== + + +class TestPatchInvariants: + def test_version_bumped(self): + import soup_cli + + parts = tuple(int(x) for x in soup_cli.__version__.split(".")[:3]) + assert parts >= (0, 71, 13)