mirror of https://github.com/razor-ai/soup.git
feat(prompt-compile): live soup compile / distill-prompt / compile-tools / local-rl train (v0.71.13)
Lift the v0.68.0 deferred-stub family to live (closes #225, #226, #227, #229): - #229 local-rl train --once: harvest thumbs -> DPO/KTO/ORPO train via a soup train subprocess (argv list, no shell); state table tracks last_train_at (skip-on-no-new-thumbs + skip-on-insufficient-pairs); no --once renders a systemd/launchd nightly scheduler scaffold. New local_rl_scheduler.py. - #226 distill-prompt: call the teacher once per trace (Ollama/Anthropic/vLLM) and write a real dataset (sft/kl -> messages; preference -> chosen/rejected). - #225 compile / #227 compile-tools: live DSPy/GEPA/TextGrad dispatch behind the new [compile] extra with a friendly ImportError when absent; injectable seams. Security: reject \n/\r in the model id + shell-quote ExecStart args (systemd injection defence). Fix: render train output as a plain string (schema-valid), with a regression test against SoupConfig. Tests 13329 -> 13424. Smoked end-to-end: real DPO train on SmolLM2-135M (RTX 3050) + real Ollama teacher distillation.
This commit is contained in:
parent
18e642ccc7
commit
f528da5328
38
CHANGELOG.md
38
CHANGELOG.md
|
|
@ -12,6 +12,44 @@ reproducing 70+ versions of notes.
|
|||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.71.13] - 2026-06-04
|
||||
|
||||
### Added
|
||||
- **Prompt-compile family — live wiring** (closes #225, #226, #227, #229). Four
|
||||
`soup` commands that shipped as deferred-stub `NotImplementedError` in v0.68.0
|
||||
are now real, validated end-to-end (real DPO train on SmolLM2-135M + real
|
||||
Ollama teacher distillation on RTX 3050).
|
||||
- **`soup local-rl train` runs a real nightly DPO/KTO/ORPO train** (#229).
|
||||
`--once` harvests the latest thumbs-up/down DPO pairs from the local-RL SQLite
|
||||
and trains them via a `soup train` subprocess (argv list, no shell); a `state`
|
||||
table tracks `last_train_at` so a re-run with no new feedback skips, and a run
|
||||
with fewer than `--min-pairs` (default 10) skips. Without `--once` it renders a
|
||||
systemd `.service`/`.timer` + launchd `.plist` scheduler scaffold into
|
||||
`--scheduler-dir` for the user to install. New flags: `--once`, `--min-pairs`,
|
||||
`--output/-o`, `--scheduler-dir`, `--hour`, `--minute`.
|
||||
- **`soup distill-prompt` prepares a real distillation dataset** (#226). For
|
||||
each prompt in the traces JSONL the teacher is called once via the v0.20
|
||||
provider helpers (Ollama / Anthropic / vLLM); `sft`/`kl` emit
|
||||
`{messages:[user, assistant=teacher]}` and `preference` emits
|
||||
`{prompt, chosen=teacher, rejected=student}`. New flags: `--provider`,
|
||||
`--base-url`, `--temperature`, `--max-rows`.
|
||||
- **`soup compile` runs DSPy / GEPA / TextGrad prompt-program optimisation** (#225)
|
||||
and **`soup compile-tools` runs the TextGrad / GEPA tool-schema optimiser** (#227),
|
||||
both lazy-importing the optimiser libraries behind the new `[compile]` extra
|
||||
(`pip install 'soup-cli[compile]'`) with a friendly `ImportError` naming the
|
||||
extra when absent. `--plan-only` still renders the plan and exits 0.
|
||||
|
||||
### Security
|
||||
- **systemd / launchd injection defence** (#229). `local-rl` and the scheduler
|
||||
renderers reject `\n` / `\r` in the model id and shell-quote every `ExecStart`
|
||||
argument, so a crafted model id cannot inject extra unit directives.
|
||||
|
||||
### Fixed
|
||||
- **`local-rl` train config rendered `output` as a mapping** (#229). The nightly
|
||||
`soup train` YAML now emits `output: <dir>` (a plain string the schema accepts)
|
||||
instead of `output: {dir: <dir>}`; a regression test validates the rendered
|
||||
config against `SoupConfig`.
|
||||
|
||||
## [0.71.12] - 2026-06-04
|
||||
|
||||
### Added
|
||||
|
|
|
|||
|
|
@ -116,11 +116,11 @@ src/soup_cli/
|
|||
cans/ - Shareable .can artifact format + run/publish orchestrator (v0.26.0 + v0.33.0)
|
||||
data/traces/ - Trace-to-Preference harvester (v0.26.0)
|
||||
data/collators.py - CrossDocCollator for sample packing (v0.33.0)
|
||||
utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats, deploy_measure, advise, advise_history, adapter_diff, adapter_merge, blame, adapter_branch, unlearning, unlearning_eval, knowledge_edit, edit_governor, edit_diff, ra_dit, steering, citation_faithful, grace_codebook, ingest_sources, prune_prompt, active_sampler, ab_test, drift_alarm, tunability, terraform_plan, env_lock, hardware_fit, completions, license_advisor, behavior_battery, capability_suite, checklist_dsl, irt, sae_diff, sleeper_probe, interference, probe_pack, cmaes_merge, vector_bank, mole_routing, adapter_pr, soup_lock, adapter_bisect, prompt_compile, prompt_distill, compile_tools, apple_adapter, local_rl, build_dag, expectations, magpie, persona_hub, brain_rot, reward_hacking, uld, minillm, rl_checkpoint, iterative_dpo, echo_trap, mod
|
||||
utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats, deploy_measure, advise, advise_history, adapter_diff, adapter_merge, blame, adapter_branch, unlearning, unlearning_eval, knowledge_edit, edit_governor, edit_diff, ra_dit, steering, citation_faithful, grace_codebook, ingest_sources, prune_prompt, active_sampler, ab_test, drift_alarm, tunability, terraform_plan, env_lock, hardware_fit, completions, license_advisor, behavior_battery, capability_suite, checklist_dsl, irt, sae_diff, sleeper_probe, interference, probe_pack, cmaes_merge, vector_bank, mole_routing, adapter_pr, soup_lock, adapter_bisect, prompt_compile, prompt_distill, compile_tools, apple_adapter, local_rl, build_dag, expectations, magpie, persona_hub, brain_rot, reward_hacking, uld, minillm, rl_checkpoint, iterative_dpo, echo_trap, mod, local_rl_scheduler
|
||||
templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0)
|
||||
ui/ - Web UI (FastAPI + HTML/JS SPA)
|
||||
|
||||
tests/ - Test suite (283 files, 13329 tests)
|
||||
tests/ - Test suite (284 files, 13424 tests)
|
||||
examples/ - Real-world config examples and datasets
|
||||
```
|
||||
|
||||
|
|
|
|||
24
README.md
24
README.md
|
|
@ -49,20 +49,18 @@ infrastructure instead of improving models. Soup fixes that.
|
|||
|
||||
## What's New
|
||||
|
||||
**v0.71.12 — Architecture, distillation & adapter-training (live).** Seven schema-only surfaces
|
||||
from earlier releases are now real, validated end-to-end on tiny models:
|
||||
**v0.71.13 — Prompt-compile family (live).** Four commands that shipped as deferred stubs in
|
||||
v0.68.0 are now real, validated end-to-end on tiny models:
|
||||
|
||||
- **`soup serve --bank <bank.json>`** — multi-tenant VeRA / VB-LoRA serving: load N personas at
|
||||
KB-per-user (shared projection + per-user vectors) and pick the active one per request via the
|
||||
`X-User-Id` header. An unknown / absent id is a zero-delta no-op (no cross-request leak).
|
||||
- **`task: moe_lora_routing`** — MoLE per-token routing trains a small gating network over N frozen
|
||||
task LoRAs (`mole_task_adapters`, `mole_top_k`, `mole_temperature`); only the router trains.
|
||||
- **`task: distill` with `distill_mode: sequence`** — sequence-level KD trains the student on the
|
||||
teacher's generated continuations (cross-tokenizer-friendly), alongside the existing token logit-KL.
|
||||
- **`task: classifier` with a `lora` section** — train a frozen encoder + LoRA adapter classifier
|
||||
instead of the full model.
|
||||
- **`use_mod` (Mixture-of-Depths) · `expand_layers` (LLaMA Pro) · `use_longlora` (S² attention)** —
|
||||
the architecture knobs that were schema-only are now live for Llama / Qwen / Mistral (+ Phi for LongLoRA).
|
||||
- **`soup local-rl train --once`** — harvest your latest 👍/👎 feedback into DPO pairs and train a
|
||||
real DPO/KTO/ORPO round via `soup train` (argv list, no shell). A `state` table tracks the last
|
||||
train so a re-run with no new feedback skips. Without `--once` it renders a systemd/launchd nightly
|
||||
scheduler scaffold you can install.
|
||||
- **`soup distill-prompt`** — call a teacher model (Ollama / Anthropic / vLLM) once per trace and
|
||||
write a real distillation dataset: `sft`/`kl` → `{messages}`, `preference` → `{prompt, chosen, rejected}`.
|
||||
- **`soup compile`** — DSPy / GEPA / TextGrad prompt-program optimisation, and **`soup compile-tools`**
|
||||
— TextGrad / GEPA tool-schema optimisation, both behind the new `[compile]` extra
|
||||
(`pip install 'soup-cli[compile]'`).
|
||||
|
||||
Full history: [CHANGELOG.md](CHANGELOG.md) · [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases).
|
||||
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ in-scope issues:
|
|||
dataset, or artifact paths;
|
||||
- SSRF in the synthetic-data providers, inference server, or hub/endpoint
|
||||
validators;
|
||||
- command, Modelfile, or Jinja chat-template injection;
|
||||
- command, Modelfile, Jinja chat-template, or systemd/launchd unit injection;
|
||||
- secret leakage in logs, crash bundles, or generated artifacts;
|
||||
- sandbox escape in the RLVR code-execution reward path.
|
||||
|
||||
|
|
|
|||
|
|
@ -216,15 +216,16 @@ soup adapters bisect <ckpt>... --eval-command "..." Binary search over training
|
|||
soup lock write --base-sha <h> --dataset-sha <h> --env-hash <h> Write soup.lock (v0.67.0)
|
||||
soup lock write --base-sha <h> --dataset-sha <h> --env-lock soup-env.lock Auto-derive --env-hash from soup-env.lock (v0.71.1)
|
||||
soup lock show / soup lock check Show + drift-check (exit 3 on drift)
|
||||
soup compile <program.py> --eval <suite> [--optimizer mipro|gepa|textgrad|copro|bootstrap_fewshot] DSPy / GEPA prompt-program compiler (v0.68.0)
|
||||
soup distill-prompt --traces <jsonl> --teacher <m> --student <m> --strategy sft|preference|kl Distill prompt-heavy traces to small FT (v0.68.0)
|
||||
soup compile-tools <spec.json|yaml> --eval <jsonl> [--optimizer textgrad|gepa] TextGrad / GEPA tool-schema optimiser (v0.68.0)
|
||||
soup compile <program.py> --eval <suite> [--optimizer mipro|gepa|textgrad|copro|bootstrap_fewshot] [--plan-only] DSPy / GEPA / TextGrad prompt-program compiler — live (v0.71.13; pip install 'soup-cli[compile]')
|
||||
soup distill-prompt --traces <jsonl> --teacher <m> --student <m> --strategy sft|preference|kl [--provider ollama|anthropic|vllm] [--base-url <url>] [--temperature F] [--max-rows N] Distill prompt-heavy traces via a live teacher (v0.71.13)
|
||||
soup compile-tools <spec.json|yaml> --eval <jsonl> [--optimizer textgrad|gepa] [--plan-only] TextGrad / GEPA tool-schema optimiser — live (v0.71.13; pip install 'soup-cli[compile]')
|
||||
soup apple-adapter <source-dir> --direction hf-to-mlx|mlx-to-hf|hf-to-apple|mlx-to-apple --output <dir> [--sign] HF / MLX / Apple FoundationModels adapter conversion (v0.68.0)
|
||||
soup local-rl init --db <path> Create personal-LLM flywheel SQLite schema (v0.68.0)
|
||||
soup local-rl status --db <path> Print interactions / thumbs-up / thumbs-down counters
|
||||
soup local-rl record --db <path> --prompt <q> --response <r> --thumb up|down Append thumbs record
|
||||
soup local-rl harvest --db <path> -o <pairs.jsonl> Harvest DPO pairs from thumbs into JSONL
|
||||
soup local-rl train --db <path> --backend ollama|mlx --model <id> Nightly DPO/KTO/ORPO train (v0.68.1)
|
||||
soup local-rl train --db <path> --model <id> --once [--train-method dpo|kto|orpo] [--min-pairs N] [-o <dir>] Ad-hoc DPO/KTO/ORPO train from harvested thumbs — live (v0.71.13)
|
||||
soup local-rl train --db <path> --model <id> [--scheduler-dir <dir>] [--hour H] [--minute M] Render a systemd/launchd nightly-train scaffold (no --once) (v0.71.13)
|
||||
soup build <manifest.yaml> [--dry-run] [--output-dir <dir>] dbt-for-SFT DAG: validate + plan + live materialise (v0.69.0; live v0.71.6)
|
||||
soup expect <data.jsonl> <suite.yaml> Expectations suite: PII / token-length / refusal / judge (v0.69.0)
|
||||
soup data gen-magpie --base <m> --provider ollama|vllm --target N --output <jsonl> [--base-url <url>] [--quality-filter] Magpie synthetic generator — live (v0.69.0; live v0.71.6)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "soup-cli"
|
||||
version = "0.71.12"
|
||||
version = "0.71.13"
|
||||
description = "Fine-tune LLMs in one command. No SSH, no config hell."
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
|
|
@ -94,6 +94,10 @@ pdf = ["reportlab>=4.0.0"]
|
|||
# v0.71.3 #180 — codecarbon offline energy/CO2 measurement for
|
||||
# `soup train --track-energy` (offline; no IP-geolocation network call).
|
||||
carbon = ["codecarbon>=2.0.0"]
|
||||
# v0.71.13 #225/#227 — prompt-program / tool-schema compilers
|
||||
# (`soup compile` / `soup compile-tools`). Lazy-imported with a friendly
|
||||
# ImportError; not installed by default (heavy dependency trees).
|
||||
compile = ["dspy-ai>=2.5.0", "textgrad>=0.1.0", "gepa>=0.0.1"]
|
||||
|
||||
[project.scripts]
|
||||
soup = "soup_cli.cli:run"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
"""Soup CLI — Fine-tune LLMs in one command."""
|
||||
|
||||
__version__ = "0.71.12"
|
||||
__version__ = "0.71.13"
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
"""``soup compile`` — DSPy / GEPA prompt-program compiler CLI (v0.68.0 Part A).
|
||||
"""``soup compile`` — DSPy / GEPA / TextGrad prompt-program compiler CLI.
|
||||
|
||||
Renders a ``CompilePlan`` panel and, when ``--plan-only`` is omitted,
|
||||
invokes the deferred-live ``run_compile`` (raises NotImplementedError
|
||||
with a v0.68.1 marker). Mirrors v0.61.0 / v0.62.0 stub-then-live CLI policy.
|
||||
Renders a ``CompilePlan`` panel and, when ``--plan-only`` is omitted, runs
|
||||
the live optimizer (v0.71.13 #225). A missing optimizer library surfaces a
|
||||
friendly exit-2 advisory naming ``pip install 'soup-cli[compile]'``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -74,14 +74,14 @@ def compile_cmd(
|
|||
|
||||
try:
|
||||
result = run_compile(plan)
|
||||
except NotImplementedError as exc:
|
||||
except ImportError as exc:
|
||||
console.print(
|
||||
Panel(
|
||||
f"[yellow]{escape(str(exc))}[/]",
|
||||
title="Live compile deferred",
|
||||
title="compile — missing dependency",
|
||||
)
|
||||
)
|
||||
raise typer.Exit(3) from exc
|
||||
raise typer.Exit(2) from exc
|
||||
except (TypeError, ValueError) as exc:
|
||||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
|
|
|
|||
|
|
@ -62,12 +62,23 @@ def compile_tools_cmd(
|
|||
return
|
||||
|
||||
try:
|
||||
run_tool_compile(plan)
|
||||
except NotImplementedError as exc:
|
||||
n = run_tool_compile(plan)
|
||||
except ImportError as exc:
|
||||
console.print(
|
||||
Panel(
|
||||
f"[yellow]{escape(str(exc))}[/]",
|
||||
title="Live compile-tools deferred",
|
||||
title="compile-tools — missing dependency",
|
||||
)
|
||||
)
|
||||
raise typer.Exit(3) from exc
|
||||
raise typer.Exit(2) from exc
|
||||
except (TypeError, ValueError, FileNotFoundError) as exc:
|
||||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
|
||||
console.print(
|
||||
Panel(
|
||||
f"Tools: [bold]{n}[/]\n"
|
||||
f"Output: [bold]{escape(plan.output_path)}[/]",
|
||||
title="soup compile-tools — done",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -29,6 +29,20 @@ def distill_prompt_cmd(
|
|||
help="Distill strategy. Allowed: "
|
||||
+ ", ".join(sorted(SUPPORTED_DISTILL_STRATEGIES)),
|
||||
),
|
||||
provider: str = typer.Option(
|
||||
"ollama",
|
||||
"--provider",
|
||||
help="Teacher/student provider: ollama / anthropic / vllm.",
|
||||
),
|
||||
base_url: str = typer.Option(
|
||||
None, "--base-url", help="Provider base URL (ollama / vllm)."
|
||||
),
|
||||
temperature: float = typer.Option(
|
||||
0.0, "--temperature", help="Sampling temperature for teacher/student."
|
||||
),
|
||||
max_rows: int = typer.Option(
|
||||
None, "--max-rows", help="Cap the number of distilled rows."
|
||||
),
|
||||
output: str = typer.Option(
|
||||
"distilled.jsonl", "--output", "-o", help="Output JSONL path"
|
||||
),
|
||||
|
|
@ -55,6 +69,7 @@ def distill_prompt_cmd(
|
|||
f"Teacher: [bold]{escape(plan.teacher)}[/]\n"
|
||||
f"Student: [bold]{escape(plan.student)}[/]\n"
|
||||
f"Strategy: [bold]{escape(plan.strategy)}[/]\n"
|
||||
f"Provider: [bold]{escape(provider)}[/]\n"
|
||||
f"Output: [bold]{escape(plan.output_path)}[/]",
|
||||
title="soup distill-prompt — plan",
|
||||
)
|
||||
|
|
@ -64,12 +79,29 @@ def distill_prompt_cmd(
|
|||
return
|
||||
|
||||
try:
|
||||
prepare_distill_dataset(plan)
|
||||
except NotImplementedError as exc:
|
||||
n = prepare_distill_dataset(
|
||||
plan,
|
||||
provider=provider,
|
||||
base_url=base_url,
|
||||
temperature=temperature,
|
||||
max_rows=max_rows,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
except ImportError as exc:
|
||||
console.print(
|
||||
Panel(
|
||||
f"[yellow]{escape(str(exc))}[/]",
|
||||
title="Live distill-prompt deferred",
|
||||
title="distill-prompt — missing dependency",
|
||||
)
|
||||
)
|
||||
raise typer.Exit(3) from exc
|
||||
raise typer.Exit(2) from exc
|
||||
|
||||
console.print(
|
||||
Panel(
|
||||
f"Rows: [bold]{n}[/]\n"
|
||||
f"Output: [bold]{escape(plan.output_path)}[/]",
|
||||
title="soup distill-prompt — done",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ Subcommands:
|
|||
- ``status`` — print counters
|
||||
- ``record`` — append a thumbs-up/down record
|
||||
- ``harvest`` — emit DPO pairs as JSONL
|
||||
- ``train`` — nightly DPO/KTO/ORPO train (deferred to v0.68.1)
|
||||
- ``train`` — run (``--once``) or schedule the nightly DPO/KTO/ORPO train (v0.71.13 #229)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -53,7 +53,7 @@ def init_cmd(
|
|||
console.print(
|
||||
Panel(
|
||||
f"DB: [bold]{escape(db)}[/]\n"
|
||||
f"Tables: [bold]interactions[/], [bold]thumbs[/]",
|
||||
f"Tables: [bold]interactions[/], [bold]thumbs[/], [bold]state[/]",
|
||||
title="soup local-rl init",
|
||||
)
|
||||
)
|
||||
|
|
@ -166,14 +166,36 @@ def train_cmd(
|
|||
"--backend",
|
||||
help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_BACKENDS)),
|
||||
),
|
||||
model: str = typer.Option(..., "--model", help="Model id (Ollama tag or MLX path)"),
|
||||
model: str = typer.Option(
|
||||
...,
|
||||
"--model",
|
||||
help="Training base — HF repo id or local path (NOT the Ollama tag)",
|
||||
),
|
||||
train_method: str = typer.Option(
|
||||
"dpo",
|
||||
"--train-method",
|
||||
help="Allowed: " + ", ".join(sorted(SUPPORTED_LOCAL_RL_TRAIN_METHODS)),
|
||||
),
|
||||
once: bool = typer.Option(
|
||||
False,
|
||||
"--once",
|
||||
help="Run the train now (ad-hoc); skip scheduler install.",
|
||||
),
|
||||
min_pairs: int = typer.Option(
|
||||
10, "--min-pairs", help="Skip training when fewer pairs are harvested."
|
||||
),
|
||||
output: str = typer.Option(
|
||||
"local_rl_adapter", "--output", "-o", help="Adapter output directory"
|
||||
),
|
||||
scheduler_dir: str = typer.Option(
|
||||
"local-rl-scheduler",
|
||||
"--scheduler-dir",
|
||||
help="Directory to render the systemd / launchd scaffold into.",
|
||||
),
|
||||
hour: int = typer.Option(3, "--hour", help="Daily train hour (local time)."),
|
||||
minute: int = typer.Option(0, "--minute", help="Daily train minute."),
|
||||
) -> None:
|
||||
"""Trigger the nightly DPO/KTO/ORPO train. Deferred to v0.68.1."""
|
||||
"""Run (``--once``) or schedule the nightly DPO/KTO/ORPO train."""
|
||||
try:
|
||||
cfg = LocalRLConfig(
|
||||
backend=backend,
|
||||
|
|
@ -185,13 +207,73 @@ def train_cmd(
|
|||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
|
||||
try:
|
||||
run_nightly_train(cfg)
|
||||
except NotImplementedError as exc:
|
||||
if not once:
|
||||
# Render the scheduler scaffold; never run systemctl / launchctl.
|
||||
import sys
|
||||
|
||||
from soup_cli.utils.local_rl_scheduler import write_scheduler_files
|
||||
|
||||
try:
|
||||
written = write_scheduler_files(
|
||||
scheduler_dir,
|
||||
soup_python=sys.executable,
|
||||
db_path=db,
|
||||
model=model,
|
||||
train_method=train_method,
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
)
|
||||
except (TypeError, ValueError) as exc:
|
||||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
files = "\n".join(f" [bold]{escape(n)}[/]" for n in sorted(written))
|
||||
console.print(
|
||||
Panel(
|
||||
f"[yellow]{escape(str(exc))}[/]",
|
||||
title="Live local-rl train deferred",
|
||||
f"Rendered the nightly-train scaffold into "
|
||||
f"[bold]{escape(scheduler_dir)}[/]:\n{files}\n\n"
|
||||
"[dim]Linux:[/] cp soup-local-rl.{service,timer} ~/.config/systemd/user/ "
|
||||
"&& systemctl --user enable --now soup-local-rl.timer\n"
|
||||
"[dim]macOS:[/] cp com.soup.local-rl.plist ~/Library/LaunchAgents/ "
|
||||
"&& launchctl load ~/Library/LaunchAgents/com.soup.local-rl.plist\n"
|
||||
"[dim]Now:[/] re-run with [bold]--once[/] for an ad-hoc train.",
|
||||
title="soup local-rl train — scheduler scaffold",
|
||||
)
|
||||
)
|
||||
raise typer.Exit(3) from exc
|
||||
return
|
||||
|
||||
import subprocess
|
||||
|
||||
try:
|
||||
result = run_nightly_train(
|
||||
cfg, once=True, min_pairs=min_pairs, output_dir=output
|
||||
)
|
||||
except (TypeError, ValueError, FileNotFoundError) as exc:
|
||||
console.print(f"[red]{escape(str(exc))}[/]")
|
||||
raise typer.Exit(2) from exc
|
||||
except subprocess.SubprocessError as exc:
|
||||
# The underlying `soup train` subprocess failed (e.g. OOM / bad base).
|
||||
console.print(
|
||||
Panel(
|
||||
f"[red]Training subprocess failed:[/] {escape(str(exc))}",
|
||||
title="soup local-rl train — failed",
|
||||
)
|
||||
)
|
||||
raise typer.Exit(1) from exc
|
||||
|
||||
if result.status.startswith("skipped"):
|
||||
console.print(
|
||||
Panel(
|
||||
f"[yellow]Skipped:[/] {escape(result.reason)}",
|
||||
title="soup local-rl train — skipped",
|
||||
)
|
||||
)
|
||||
return
|
||||
|
||||
console.print(
|
||||
Panel(
|
||||
f"Trained: [bold]{result.num_pairs}[/] pairs via "
|
||||
f"[bold]{escape(cfg.train_method)}[/]\n"
|
||||
f"Output: [bold]{escape(result.output_dir or output)}[/]",
|
||||
title="soup local-rl train — done",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,7 +1,9 @@
|
|||
"""``soup compile-tools`` — TextGrad / GEPA tool-schema optimizer (v0.68.0 Part C).
|
||||
"""``soup compile-tools`` — TextGrad / GEPA tool-schema optimizer.
|
||||
|
||||
Generate tool schemas + descriptions optimized via textual gradients.
|
||||
Schema-only release: live optimizer pass lands in v0.68.1.
|
||||
Schema + validators from v0.68.0 Part C; the live optimizer pass lands in
|
||||
v0.71.13 (#227), lazy-importing TextGrad / GEPA with a friendly
|
||||
``ImportError`` (``pip install 'soup-cli[compile]'``).
|
||||
|
||||
Composes with v0.46 Agent Forge (OpenAPI / MCP / GraphQL parser) — Agent
|
||||
Forge produces the spec, ``compile-tools`` optimises the descriptions.
|
||||
|
|
@ -9,15 +11,29 @@ Forge produces the spec, ``compile-tools`` optimises the descriptions.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, List, Optional
|
||||
|
||||
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
|
||||
from soup_cli.utils.paths import (
|
||||
atomic_write_text,
|
||||
enforce_under_cwd_and_no_symlink,
|
||||
)
|
||||
|
||||
SUPPORTED_TOOL_OPTIMIZERS: frozenset = frozenset({"textgrad", "gepa"})
|
||||
_SUPPORTED_SPEC_EXTENSIONS: frozenset = frozenset({".json", ".yaml", ".yml"})
|
||||
SUPPORTED_TOOL_OPTIMIZERS: frozenset[str] = frozenset({"textgrad", "gepa"})
|
||||
_SUPPORTED_SPEC_EXTENSIONS: frozenset[str] = frozenset({".json", ".yaml", ".yml"})
|
||||
|
||||
_MAX_OPTIMIZER_NAME_LEN = 32
|
||||
_INSTALL_HINT = (
|
||||
"Run: pip install 'soup-cli[compile]' (installs textgrad / gepa)"
|
||||
)
|
||||
|
||||
# Injectable seam: tests set this to a ``(description, examples, optimizer)
|
||||
# -> str`` callable so the parse -> iterate -> write orchestration is
|
||||
# exercised without the TextGrad / GEPA libraries (mirrors v0.71.13 #225
|
||||
# ``prompt_compile._OPTIMIZER_RUN_OVERRIDE``).
|
||||
_TOOL_OPTIMIZER_OVERRIDE: "Optional[Callable[[str, list, str], str]]" = None
|
||||
|
||||
|
||||
def validate_tool_optimizer(name: object) -> str:
|
||||
|
|
@ -109,13 +125,110 @@ def build_tool_compile_plan(
|
|||
)
|
||||
|
||||
|
||||
def run_tool_compile(plan: ToolCompilePlan) -> None:
|
||||
"""Live optimiser pass. Deferred to v0.68.1."""
|
||||
# ---------------------------------------------------------------------------
|
||||
# Live runner (v0.71.13 #227) — TextGrad / GEPA tool-schema optimiser
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _optimise_description(
|
||||
description: str, examples: List[dict], optimizer: str
|
||||
) -> str:
|
||||
"""Optimise one tool description via the chosen textual-gradient method.
|
||||
|
||||
The ``_TOOL_OPTIMIZER_OVERRIDE`` seam short-circuits to a test fake.
|
||||
Otherwise the real branch lazy-imports the optimiser library and raises a
|
||||
friendly ``ImportError`` naming the ``[compile]`` extra when absent.
|
||||
"""
|
||||
if _TOOL_OPTIMIZER_OVERRIDE is not None:
|
||||
result = _TOOL_OPTIMIZER_OVERRIDE(description, examples, optimizer)
|
||||
if not isinstance(result, str):
|
||||
raise TypeError("tool optimizer override must return a str")
|
||||
return result
|
||||
|
||||
if optimizer == "textgrad":
|
||||
try:
|
||||
import textgrad as tg
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
f"TextGrad is required for the 'textgrad' optimizer. {_INSTALL_HINT}"
|
||||
) from exc
|
||||
variable = tg.Variable(
|
||||
description,
|
||||
requires_grad=True,
|
||||
role_description="tool description",
|
||||
)
|
||||
opt = tg.TGD(parameters=[variable])
|
||||
for _ in range(min(8, max(1, len(examples)))):
|
||||
opt.zero_grad()
|
||||
opt.step()
|
||||
return str(variable.value)
|
||||
|
||||
# gepa
|
||||
try:
|
||||
import gepa # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
f"GEPA is required for the 'gepa' optimizer. {_INSTALL_HINT}"
|
||||
) from exc
|
||||
optimised = gepa.optimize( # type: ignore[attr-defined]
|
||||
seed_candidate=description,
|
||||
trainset=examples,
|
||||
max_metric_calls=max(1, len(examples)),
|
||||
)
|
||||
return str(getattr(optimised, "best_candidate", optimised))
|
||||
|
||||
|
||||
def _serialise_tools(out: dict, output_path: str) -> None:
|
||||
"""Write the optimised tool catalog as JSON / YAML per the extension."""
|
||||
lower = output_path.lower()
|
||||
if lower.endswith((".yaml", ".yml")):
|
||||
import yaml
|
||||
|
||||
text = yaml.safe_dump(out, default_flow_style=False, sort_keys=False)
|
||||
else: # default JSON
|
||||
text = json.dumps(out, ensure_ascii=False, indent=2) + "\n"
|
||||
atomic_write_text(text, output_path, field="output_path")
|
||||
|
||||
|
||||
def run_tool_compile(plan: ToolCompilePlan) -> int:
|
||||
"""Optimise every tool description in the spec (v0.71.13 #227).
|
||||
|
||||
Reuses v0.46 ``agent_forge.parse_spec`` to lift OpenAPI / MCP / GraphQL
|
||||
into ``Endpoint`` objects, runs the textual-gradient optimiser on each
|
||||
tool's description (scored against the eval suite), and writes a flat
|
||||
optimised tool catalog (``{"tools": [...]}``) to ``plan.output_path`` as
|
||||
JSON / YAML matching the output extension. Returns the tool count.
|
||||
|
||||
Validates the plan type so a bare dict raises cleanly.
|
||||
"""
|
||||
if not isinstance(plan, ToolCompilePlan):
|
||||
raise TypeError("plan must be ToolCompilePlan")
|
||||
raise NotImplementedError(
|
||||
"compile-tools live optimisation is deferred to v0.68.1"
|
||||
)
|
||||
|
||||
from soup_cli.utils.agent_forge import load_spec_file, parse_spec
|
||||
from soup_cli.utils.prompt_compile import load_eval_examples
|
||||
|
||||
spec = load_spec_file(plan.spec_path)
|
||||
endpoints, report = parse_spec(spec)
|
||||
examples = load_eval_examples(plan.eval_suite_path)
|
||||
|
||||
tools: list[dict] = []
|
||||
for ep in endpoints:
|
||||
new_desc = _optimise_description(
|
||||
ep.description, examples, plan.optimizer
|
||||
)
|
||||
tools.append(
|
||||
{
|
||||
"tool": ep.tool,
|
||||
"description": new_desc,
|
||||
"method": ep.method,
|
||||
"path": ep.path,
|
||||
"parameters": list(ep.parameters),
|
||||
}
|
||||
)
|
||||
|
||||
out = {"spec_kind": report.spec_kind, "tools": tools}
|
||||
_serialise_tools(out, plan.output_path)
|
||||
return len(tools)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
|
|||
|
|
@ -5,8 +5,9 @@ and (in v0.68.1) DPO-train nightly via systemd / launchd. Smaller-scope
|
|||
cousin of v0.58 ``soup loop`` — runs locally on a single workstation,
|
||||
trains the user's personal model from their own feedback.
|
||||
|
||||
Schema + thumbs recording + DPO-pair harvester are LIVE in v0.68.0; the
|
||||
nightly train scheduler is the deferred stub.
|
||||
Schema + thumbs recording + DPO-pair harvester are LIVE from v0.68.0; the
|
||||
nightly DPO/KTO/ORPO train runner + systemd/launchd scheduler scaffold land
|
||||
in v0.71.13 (#229).
|
||||
|
||||
Public surface:
|
||||
|
||||
|
|
@ -14,10 +15,12 @@ Public surface:
|
|||
- ``SUPPORTED_LOCAL_RL_TRAIN_METHODS`` (``dpo``/``kto``/``orpo``)
|
||||
- ``validate_local_rl_backend`` / ``validate_local_rl_train_method``
|
||||
- ``LocalRLConfig`` frozen dataclass
|
||||
- ``init_local_rl_db(db_path)`` — atomic table creation
|
||||
- ``init_local_rl_db(db_path)`` — atomic table creation (interactions/thumbs/state)
|
||||
- ``record_thumb(...)`` — append a thumbs-up/down record
|
||||
- ``harvest_dpo_pairs(db_path)`` — pair up/down responses to same prompt
|
||||
- ``run_nightly_train(config)`` — NotImplementedError stub w/ v0.68.1 marker
|
||||
- ``get_state`` / ``set_state`` / ``count_new_thumbs_since`` — state-table I/O
|
||||
- ``pairs_to_rows(pairs, method)`` — DPO pairs -> dpo/orpo/kto JSONL rows
|
||||
- ``run_nightly_train(config)`` — live DPO/KTO/ORPO train runner (v0.71.13 #229)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -27,13 +30,13 @@ import sqlite3
|
|||
import stat
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Tuple
|
||||
from typing import Callable, Optional, Tuple
|
||||
|
||||
from soup_cli.utils.paths import is_under_cwd
|
||||
|
||||
SUPPORTED_LOCAL_RL_BACKENDS: frozenset = frozenset({"ollama", "mlx"})
|
||||
SUPPORTED_LOCAL_RL_TRAIN_METHODS: frozenset = frozenset({"dpo", "kto", "orpo"})
|
||||
_VALID_THUMBS: frozenset = frozenset({"up", "down"})
|
||||
SUPPORTED_LOCAL_RL_BACKENDS: frozenset[str] = frozenset({"ollama", "mlx"})
|
||||
SUPPORTED_LOCAL_RL_TRAIN_METHODS: frozenset[str] = frozenset({"dpo", "kto", "orpo"})
|
||||
_VALID_THUMBS: frozenset[str] = frozenset({"up", "down"})
|
||||
|
||||
MAX_PROMPT_LEN = 16_384
|
||||
MAX_RESPONSE_LEN = 16_384
|
||||
|
|
@ -41,6 +44,14 @@ _MAX_BACKEND_LEN = 32
|
|||
_MAX_TRAIN_METHOD_LEN = 32
|
||||
_MAX_MODEL_LEN = 512
|
||||
|
||||
# v0.71.13 #229 — nightly train scheduler defaults.
|
||||
MIN_PAIRS_DEFAULT = 10
|
||||
_MAX_MIN_PAIRS = 1_000_000
|
||||
_LAST_TRAIN_KEY = "last_train_at"
|
||||
# task -> data format the trainer consumes (dpo/orpo read chosen+rejected;
|
||||
# kto reads prompt+completion+label).
|
||||
_METHOD_FORMAT = {"dpo": "dpo", "orpo": "dpo", "kto": "kto"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Validators
|
||||
|
|
@ -100,6 +111,10 @@ def _validate_model(value: object) -> str:
|
|||
raise ValueError("model must be non-empty")
|
||||
if "\x00" in value:
|
||||
raise ValueError("model must not contain null bytes")
|
||||
# Reject newlines / CR (systemd unit-file injection defence — the model
|
||||
# flows into a scheduler ExecStart line via local_rl_scheduler).
|
||||
if "\n" in value or "\r" in value:
|
||||
raise ValueError("model must not contain newline / carriage return")
|
||||
if len(value) > _MAX_MODEL_LEN:
|
||||
raise ValueError(f"model length {len(value)} > {_MAX_MODEL_LEN}")
|
||||
return value
|
||||
|
|
@ -126,6 +141,10 @@ def validate_db_path(path: object) -> str:
|
|||
raise ValueError("db_path must be non-empty")
|
||||
if "\x00" in path:
|
||||
raise ValueError("db_path must not contain null bytes")
|
||||
# Newline / CR rejection: db_path flows into a scheduler ExecStart line
|
||||
# via local_rl_scheduler (systemd unit-file injection defence).
|
||||
if "\n" in path or "\r" in path:
|
||||
raise ValueError("db_path must not contain newline / carriage return")
|
||||
if not is_under_cwd(path):
|
||||
raise ValueError(
|
||||
f"db_path {os.path.basename(path)!r} must stay under cwd"
|
||||
|
|
@ -218,6 +237,14 @@ CREATE TABLE IF NOT EXISTS thumbs (
|
|||
)
|
||||
"""
|
||||
|
||||
# v0.71.13 #229 — key/value state table (tracks last_train_at across runs).
|
||||
_SCHEMA_STATE = """
|
||||
CREATE TABLE IF NOT EXISTS state (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def init_local_rl_db(db_path: str) -> None:
|
||||
"""Atomically create the local-RL SQLite schema. Idempotent."""
|
||||
|
|
@ -228,6 +255,7 @@ def init_local_rl_db(db_path: str) -> None:
|
|||
with sqlite3.connect(real) as conn:
|
||||
conn.execute(_SCHEMA_INTERACTIONS)
|
||||
conn.execute(_SCHEMA_THUMBS)
|
||||
conn.execute(_SCHEMA_STATE)
|
||||
conn.commit()
|
||||
# Best-effort 0o600 perms (matches v0.26.0 registry.db policy on POSIX).
|
||||
if os.name == "posix":
|
||||
|
|
@ -275,8 +303,8 @@ def harvest_dpo_pairs(db_path: str) -> Tuple[DpoPair, ...]:
|
|||
rows = conn.execute(
|
||||
"SELECT prompt, response, thumb, ts FROM thumbs ORDER BY ts ASC"
|
||||
).fetchall()
|
||||
last_up: dict = {}
|
||||
last_down: dict = {}
|
||||
last_up: dict[str, str] = {}
|
||||
last_down: dict[str, str] = {}
|
||||
for prompt, response, thumb, _ts in rows:
|
||||
if thumb == "up":
|
||||
last_up[prompt] = response
|
||||
|
|
@ -295,17 +323,278 @@ def harvest_dpo_pairs(db_path: str) -> Tuple[DpoPair, ...]:
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Live train stub (v0.68.1)
|
||||
# State table I/O (v0.71.13 #229)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_nightly_train(config: LocalRLConfig) -> None:
|
||||
"""Run the nightly DPO/KTO/ORPO train. Deferred to v0.68.1."""
|
||||
def _ensure_state_table(conn: sqlite3.Connection) -> None:
|
||||
"""Create the ``state`` table if a pre-v0.71.13 DB lacks it."""
|
||||
conn.execute(_SCHEMA_STATE)
|
||||
|
||||
|
||||
def get_state(db_path: str, key: str) -> "str | None":
|
||||
"""Return the stored value for ``key`` or ``None`` when absent."""
|
||||
validate_db_path(db_path)
|
||||
if not isinstance(key, str) or not key:
|
||||
raise ValueError("key must be a non-empty string")
|
||||
real = os.path.abspath(db_path)
|
||||
if not os.path.exists(real):
|
||||
raise FileNotFoundError(f"db_path not found: {db_path!r}")
|
||||
with sqlite3.connect(real) as conn:
|
||||
_ensure_state_table(conn)
|
||||
row = conn.execute(
|
||||
"SELECT value FROM state WHERE key = ?", (key,)
|
||||
).fetchone()
|
||||
return row[0] if row is not None else None
|
||||
|
||||
|
||||
def set_state(db_path: str, key: str, value: str) -> None:
|
||||
"""Upsert ``key`` -> ``value`` in the ``state`` table."""
|
||||
validate_db_path(db_path)
|
||||
if not isinstance(key, str) or not key:
|
||||
raise ValueError("key must be a non-empty string")
|
||||
if not isinstance(value, str):
|
||||
raise TypeError("value must be a string")
|
||||
real = os.path.abspath(db_path)
|
||||
with sqlite3.connect(real) as conn:
|
||||
_ensure_state_table(conn)
|
||||
conn.execute(
|
||||
"INSERT INTO state (key, value) VALUES (?, ?) "
|
||||
"ON CONFLICT(key) DO UPDATE SET value = excluded.value",
|
||||
(key, value),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def count_new_thumbs_since(db_path: str, since_ts: "float | None") -> int:
|
||||
"""Count thumbs with ``ts > since_ts`` (all thumbs when ``since_ts`` is None)."""
|
||||
validate_db_path(db_path)
|
||||
if since_ts is not None and (
|
||||
isinstance(since_ts, bool) or not isinstance(since_ts, (int, float))
|
||||
):
|
||||
raise TypeError("since_ts must be a number or None")
|
||||
real = os.path.abspath(db_path)
|
||||
if not os.path.exists(real):
|
||||
raise FileNotFoundError(f"db_path not found: {db_path!r}")
|
||||
with sqlite3.connect(real) as conn:
|
||||
if since_ts is None:
|
||||
row = conn.execute("SELECT COUNT(*) FROM thumbs").fetchone()
|
||||
else:
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM thumbs WHERE ts > ?", (float(since_ts),)
|
||||
).fetchone()
|
||||
return int(row[0])
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Pair -> training rows (v0.71.13 #229)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def pairs_to_rows(
|
||||
pairs: "Tuple[DpoPair, ...]", train_method: str
|
||||
) -> "list[dict]":
|
||||
"""Convert harvested DPO pairs to JSONL rows for ``train_method``.
|
||||
|
||||
- ``dpo`` / ``orpo`` -> one ``{prompt, chosen, rejected}`` row per pair.
|
||||
- ``kto`` -> two unpaired ``{prompt, completion, label}`` rows per pair
|
||||
(chosen -> label True, rejected -> label False).
|
||||
"""
|
||||
method = validate_local_rl_train_method(train_method)
|
||||
rows: "list[dict]" = []
|
||||
for pair in pairs:
|
||||
if not isinstance(pair, DpoPair):
|
||||
raise TypeError("pairs must contain DpoPair objects")
|
||||
if method == "kto":
|
||||
rows.append(
|
||||
{"prompt": pair.prompt, "completion": pair.chosen, "label": True}
|
||||
)
|
||||
rows.append(
|
||||
{"prompt": pair.prompt, "completion": pair.rejected, "label": False}
|
||||
)
|
||||
else: # dpo / orpo
|
||||
rows.append(
|
||||
{
|
||||
"prompt": pair.prompt,
|
||||
"chosen": pair.chosen,
|
||||
"rejected": pair.rejected,
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Nightly train runner (v0.71.13 #229) — lifts the v0.68.1 stub
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NightlyTrainResult:
|
||||
"""Outcome of a ``run_nightly_train`` invocation."""
|
||||
|
||||
status: str # "trained" / "skipped_no_new_thumbs" / "skipped_insufficient_pairs"
|
||||
num_pairs: int
|
||||
output_dir: "str | None"
|
||||
reason: str
|
||||
|
||||
|
||||
def _default_train_fn(
|
||||
*,
|
||||
base_model: str,
|
||||
pairs_path: str,
|
||||
output_dir: str,
|
||||
train_method: str,
|
||||
) -> None:
|
||||
"""Train one round via a ``soup train`` subprocess (no shell).
|
||||
|
||||
Mirrors v0.71.11 ``iterative_dpo._default_train_fn``: render a YAML via
|
||||
``yaml.safe_dump`` (no value can inject extra keys) and invoke
|
||||
``python -m soup_cli.cli train --config <tmp> --yes`` with an argv list.
|
||||
"""
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import yaml
|
||||
|
||||
data_format = _METHOD_FORMAT[train_method]
|
||||
yaml_text = yaml.safe_dump(
|
||||
{
|
||||
"base": base_model,
|
||||
"task": train_method,
|
||||
"data": {"train": pairs_path, "format": data_format, "max_length": 256},
|
||||
"training": {"epochs": 1, "batch_size": 1},
|
||||
"output": output_dir,
|
||||
},
|
||||
default_flow_style=False,
|
||||
sort_keys=False,
|
||||
)
|
||||
fd, tmp_yaml = tempfile.mkstemp(
|
||||
suffix=".yaml", prefix=".soup_localrl_", dir=os.getcwd()
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(yaml_text)
|
||||
subprocess.run( # noqa: S603 — argv list, no shell
|
||||
[
|
||||
sys.executable,
|
||||
"-m",
|
||||
"soup_cli.cli",
|
||||
"train",
|
||||
"--config",
|
||||
tmp_yaml,
|
||||
"--yes",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
os.remove(tmp_yaml)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def run_nightly_train(
|
||||
config: LocalRLConfig,
|
||||
*,
|
||||
once: bool = False,
|
||||
min_pairs: int = MIN_PAIRS_DEFAULT,
|
||||
output_dir: str = "local_rl_adapter",
|
||||
train_fn: Optional[Callable[..., None]] = None,
|
||||
) -> NightlyTrainResult:
|
||||
"""Harvest the latest DPO pairs and train them (v0.71.13 #229).
|
||||
|
||||
The flow: read ``last_train_at`` from the SQLite ``state`` table; if a
|
||||
prior train exists and no new thumbs landed since, skip. Otherwise
|
||||
harvest pairs and — when at least ``min_pairs`` are available — write a
|
||||
tmp JSONL in the right shape for ``config.train_method`` and invoke the
|
||||
DPO/KTO/ORPO trainer via ``soup train`` (or an injected ``train_fn`` for
|
||||
tests). ``last_train_at`` is stamped only after a real train.
|
||||
|
||||
``config.model`` is the **training base** here (an HF repo id or local
|
||||
path) — distinct from the Ollama tag used by the record/harvest loop.
|
||||
|
||||
The ``once`` flag is accepted for symmetry with the CLI (the runner is
|
||||
the same whether invoked ad-hoc or by the scheduler); it is recorded but
|
||||
does not change the harvest/skip logic.
|
||||
"""
|
||||
if not isinstance(config, LocalRLConfig):
|
||||
raise TypeError("config must be LocalRLConfig")
|
||||
raise NotImplementedError(
|
||||
"local-rl nightly train is deferred to v0.68.1 — "
|
||||
"harvest DPO pairs today, train them once the runner lands"
|
||||
if not isinstance(once, bool):
|
||||
raise TypeError("once must be bool")
|
||||
if isinstance(min_pairs, bool) or not isinstance(min_pairs, int):
|
||||
raise TypeError("min_pairs must be int")
|
||||
if min_pairs < 1 or min_pairs > _MAX_MIN_PAIRS:
|
||||
raise ValueError(f"min_pairs must be in [1, {_MAX_MIN_PAIRS}]")
|
||||
if not isinstance(output_dir, str) or not output_dir or "\x00" in output_dir:
|
||||
raise ValueError("output_dir must be a non-empty NUL-free string")
|
||||
|
||||
runner = train_fn if train_fn is not None else _default_train_fn
|
||||
if not callable(runner):
|
||||
raise TypeError("train_fn must be callable")
|
||||
|
||||
db = config.db_path
|
||||
last_raw = get_state(db, _LAST_TRAIN_KEY)
|
||||
last_ts: "float | None" = None
|
||||
if last_raw is not None:
|
||||
try:
|
||||
last_ts = float(last_raw)
|
||||
except (TypeError, ValueError):
|
||||
last_ts = None
|
||||
|
||||
if last_ts is not None:
|
||||
new = count_new_thumbs_since(db, last_ts)
|
||||
if new == 0:
|
||||
return NightlyTrainResult(
|
||||
status="skipped_no_new_thumbs",
|
||||
num_pairs=0,
|
||||
output_dir=None,
|
||||
reason=f"no new thumbs since last train at {last_ts}",
|
||||
)
|
||||
|
||||
# Stamp the run start (BEFORE harvest), not the post-train wall-clock, so
|
||||
# any thumbs recorded *during* the train window keep ``ts > last_train_at``
|
||||
# and are counted by the next run instead of being silently dropped
|
||||
# (code-review MEDIUM fix).
|
||||
run_started = time.time()
|
||||
pairs = harvest_dpo_pairs(db)
|
||||
if len(pairs) < min_pairs:
|
||||
return NightlyTrainResult(
|
||||
status="skipped_insufficient_pairs",
|
||||
num_pairs=len(pairs),
|
||||
output_dir=None,
|
||||
reason=f"{len(pairs)} pairs < min_pairs {min_pairs}",
|
||||
)
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
|
||||
rows = pairs_to_rows(pairs, config.train_method)
|
||||
fd, tmp_jsonl = tempfile.mkstemp(
|
||||
suffix=".jsonl", prefix=".soup_localrl_pairs_", dir=os.getcwd()
|
||||
)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
for row in rows:
|
||||
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
||||
runner(
|
||||
base_model=config.model,
|
||||
pairs_path=tmp_jsonl,
|
||||
output_dir=output_dir,
|
||||
train_method=config.train_method,
|
||||
)
|
||||
finally:
|
||||
try:
|
||||
os.remove(tmp_jsonl)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
set_state(db, _LAST_TRAIN_KEY, repr(run_started))
|
||||
return NightlyTrainResult(
|
||||
status="trained",
|
||||
num_pairs=len(pairs),
|
||||
output_dir=output_dir,
|
||||
reason=f"trained {len(pairs)} pairs via {config.train_method}",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -314,13 +603,19 @@ __all__ = [
|
|||
"SUPPORTED_LOCAL_RL_TRAIN_METHODS",
|
||||
"MAX_PROMPT_LEN",
|
||||
"MAX_RESPONSE_LEN",
|
||||
"MIN_PAIRS_DEFAULT",
|
||||
"validate_local_rl_backend",
|
||||
"validate_local_rl_train_method",
|
||||
"validate_db_path",
|
||||
"LocalRLConfig",
|
||||
"DpoPair",
|
||||
"NightlyTrainResult",
|
||||
"init_local_rl_db",
|
||||
"record_thumb",
|
||||
"harvest_dpo_pairs",
|
||||
"get_state",
|
||||
"set_state",
|
||||
"count_new_thumbs_since",
|
||||
"pairs_to_rows",
|
||||
"run_nightly_train",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,276 @@
|
|||
"""``soup local-rl train`` scheduler scaffold (v0.71.13 #229).
|
||||
|
||||
Render a systemd-user ``local-rl.service`` + ``.timer`` (Linux) or a
|
||||
``com.soup.local-rl.plist`` (macOS launchd) that invokes
|
||||
``soup local-rl train --once`` daily at a configurable local time.
|
||||
|
||||
This module only **renders** the unit files (and writes them to a chosen
|
||||
directory) — it never runs ``systemctl`` / ``launchctl`` itself, so it is
|
||||
fully testable on any OS. The CLI prints the manual install command. This
|
||||
mirrors v0.46 deploy-autopilot / v0.68 apple-adapter (render-not-execute).
|
||||
|
||||
Public surface:
|
||||
|
||||
- ``build_train_argv(...)`` — argv list the scheduler invokes
|
||||
- ``render_systemd_service(...)`` / ``render_systemd_timer(...)``
|
||||
- ``render_launchd_plist(...)``
|
||||
- ``write_scheduler_files(target_dir, ...)`` — atomic write of the units
|
||||
- ``SYSTEMD_SERVICE_NAME`` / ``SYSTEMD_TIMER_NAME`` / ``LAUNCHD_PLIST_NAME``
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Dict, List
|
||||
|
||||
from soup_cli.utils.local_rl import (
|
||||
validate_db_path,
|
||||
validate_local_rl_train_method,
|
||||
)
|
||||
from soup_cli.utils.paths import atomic_write_text
|
||||
|
||||
SYSTEMD_SERVICE_NAME = "soup-local-rl.service"
|
||||
SYSTEMD_TIMER_NAME = "soup-local-rl.timer"
|
||||
LAUNCHD_PLIST_NAME = "com.soup.local-rl.plist"
|
||||
|
||||
_MIN_HOUR = 0
|
||||
_MAX_HOUR = 23
|
||||
_MIN_MINUTE = 0
|
||||
_MAX_MINUTE = 59
|
||||
_MAX_MODEL_LEN = 512
|
||||
|
||||
|
||||
def _validate_model(value: object) -> str:
|
||||
if isinstance(value, bool):
|
||||
raise TypeError("model must not be bool")
|
||||
if not isinstance(value, str):
|
||||
raise TypeError("model must be str")
|
||||
if not value:
|
||||
raise ValueError("model must be non-empty")
|
||||
if "\x00" in value or "\n" in value or "\r" in value:
|
||||
raise ValueError(
|
||||
"model must not contain NUL / newline / carriage return"
|
||||
)
|
||||
if len(value) > _MAX_MODEL_LEN:
|
||||
raise ValueError(f"model length {len(value)} > {_MAX_MODEL_LEN}")
|
||||
return value
|
||||
|
||||
|
||||
def _validate_hm(hour: object, minute: object) -> "tuple[int, int]":
|
||||
for name, value, lo, hi in (
|
||||
("hour", hour, _MIN_HOUR, _MAX_HOUR),
|
||||
("minute", minute, _MIN_MINUTE, _MAX_MINUTE),
|
||||
):
|
||||
if isinstance(value, bool):
|
||||
raise TypeError(f"{name} must not be bool")
|
||||
if not isinstance(value, int):
|
||||
raise TypeError(f"{name} must be int")
|
||||
if value < lo or value > hi:
|
||||
raise ValueError(f"{name} must be in [{lo}, {hi}]")
|
||||
return int(hour), int(minute) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _validate_soup_python(value: object) -> str:
|
||||
"""The python executable that runs ``-m soup_cli.cli``."""
|
||||
if isinstance(value, bool):
|
||||
raise TypeError("soup_python must not be bool")
|
||||
if not isinstance(value, str):
|
||||
raise TypeError("soup_python must be str")
|
||||
if not value:
|
||||
raise ValueError("soup_python must be non-empty")
|
||||
if "\x00" in value or "\n" in value or "\r" in value:
|
||||
raise ValueError("soup_python must not contain NUL / newline")
|
||||
return value
|
||||
|
||||
|
||||
def build_train_argv(
|
||||
*,
|
||||
soup_python: str,
|
||||
db_path: str,
|
||||
model: str,
|
||||
train_method: str,
|
||||
) -> List[str]:
|
||||
"""Return the argv list the scheduler invokes (no shell)."""
|
||||
soup_python = _validate_soup_python(soup_python)
|
||||
validate_db_path(db_path)
|
||||
model = _validate_model(model)
|
||||
method = validate_local_rl_train_method(train_method)
|
||||
return [
|
||||
soup_python,
|
||||
"-m",
|
||||
"soup_cli.cli",
|
||||
"local-rl",
|
||||
"train",
|
||||
"--once",
|
||||
"--db",
|
||||
db_path,
|
||||
"--model",
|
||||
model,
|
||||
"--train-method",
|
||||
method,
|
||||
]
|
||||
|
||||
|
||||
def _systemd_quote(arg: str) -> str:
|
||||
"""Double-quote an argv element for a systemd ``ExecStart`` line.
|
||||
|
||||
systemd splits ``ExecStart`` on whitespace unless quoted; ``"`` and
|
||||
``\\`` inside a quoted token must be escaped. Defence-in-depth: refuse
|
||||
any newline / CR (a quoted token cannot span lines — this would inject a
|
||||
new directive). The argv builder already rejects these on db_path/model.
|
||||
"""
|
||||
if "\n" in arg or "\r" in arg:
|
||||
raise ValueError("systemd argument must not contain newline / CR")
|
||||
escaped = arg.replace("\\", "\\\\").replace('"', '\\"')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def render_systemd_service(
|
||||
*,
|
||||
soup_python: str,
|
||||
db_path: str,
|
||||
model: str,
|
||||
train_method: str,
|
||||
) -> str:
|
||||
"""Render the systemd-user ``.service`` unit."""
|
||||
argv = build_train_argv(
|
||||
soup_python=soup_python,
|
||||
db_path=db_path,
|
||||
model=model,
|
||||
train_method=train_method,
|
||||
)
|
||||
exec_start = " ".join(_systemd_quote(a) for a in argv)
|
||||
return (
|
||||
"[Unit]\n"
|
||||
"Description=Soup local-RL nightly DPO/KTO/ORPO train\n"
|
||||
"\n"
|
||||
"[Service]\n"
|
||||
"Type=oneshot\n"
|
||||
f"WorkingDirectory={_systemd_quote(os.getcwd())}\n"
|
||||
f"ExecStart={exec_start}\n"
|
||||
)
|
||||
|
||||
|
||||
def render_systemd_timer(*, hour: int = 3, minute: int = 0) -> str:
|
||||
"""Render the systemd-user ``.timer`` unit (daily at ``hour:minute``)."""
|
||||
hour, minute = _validate_hm(hour, minute)
|
||||
return (
|
||||
"[Unit]\n"
|
||||
"Description=Soup local-RL nightly train timer\n"
|
||||
"\n"
|
||||
"[Timer]\n"
|
||||
f"OnCalendar=*-*-* {hour:02d}:{minute:02d}:00\n"
|
||||
"Persistent=true\n"
|
||||
"\n"
|
||||
"[Install]\n"
|
||||
"WantedBy=timers.target\n"
|
||||
)
|
||||
|
||||
|
||||
def render_launchd_plist(
|
||||
*,
|
||||
soup_python: str,
|
||||
db_path: str,
|
||||
model: str,
|
||||
train_method: str,
|
||||
hour: int = 3,
|
||||
minute: int = 0,
|
||||
) -> str:
|
||||
"""Render a macOS launchd ``.plist`` (daily ``StartCalendarInterval``)."""
|
||||
from xml.sax.saxutils import escape as _xml_escape
|
||||
|
||||
hour, minute = _validate_hm(hour, minute)
|
||||
argv = build_train_argv(
|
||||
soup_python=soup_python,
|
||||
db_path=db_path,
|
||||
model=model,
|
||||
train_method=train_method,
|
||||
)
|
||||
args_xml = "\n".join(
|
||||
f" <string>{_xml_escape(a)}</string>" for a in argv
|
||||
)
|
||||
label = LAUNCHD_PLIST_NAME[: -len(".plist")]
|
||||
return (
|
||||
'<?xml version="1.0" encoding="UTF-8"?>\n'
|
||||
'<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" '
|
||||
'"http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n'
|
||||
'<plist version="1.0">\n'
|
||||
"<dict>\n"
|
||||
" <key>Label</key>\n"
|
||||
f" <string>{_xml_escape(label)}</string>\n"
|
||||
" <key>ProgramArguments</key>\n"
|
||||
" <array>\n"
|
||||
f"{args_xml}\n"
|
||||
" </array>\n"
|
||||
" <key>WorkingDirectory</key>\n"
|
||||
f" <string>{_xml_escape(os.getcwd())}</string>\n"
|
||||
" <key>StartCalendarInterval</key>\n"
|
||||
" <dict>\n"
|
||||
" <key>Hour</key>\n"
|
||||
f" <integer>{hour}</integer>\n"
|
||||
" <key>Minute</key>\n"
|
||||
f" <integer>{minute}</integer>\n"
|
||||
" </dict>\n"
|
||||
"</dict>\n"
|
||||
"</plist>\n"
|
||||
)
|
||||
|
||||
|
||||
def write_scheduler_files(
|
||||
target_dir: str,
|
||||
*,
|
||||
soup_python: str,
|
||||
db_path: str,
|
||||
model: str,
|
||||
train_method: str,
|
||||
hour: int = 3,
|
||||
minute: int = 0,
|
||||
) -> Dict[str, str]:
|
||||
"""Render both the systemd units and the launchd plist into ``target_dir``.
|
||||
|
||||
All three are written so an operator on either OS finds the right scaffold.
|
||||
Returns ``{filename: realpath}``. Writes are atomic + cwd-contained +
|
||||
symlink-rejected via ``atomic_write_text``.
|
||||
"""
|
||||
if not isinstance(target_dir, str) or not target_dir or "\x00" in target_dir:
|
||||
raise ValueError("target_dir must be a non-empty NUL-free string")
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
|
||||
service = render_systemd_service(
|
||||
soup_python=soup_python,
|
||||
db_path=db_path,
|
||||
model=model,
|
||||
train_method=train_method,
|
||||
)
|
||||
timer = render_systemd_timer(hour=hour, minute=minute)
|
||||
plist = render_launchd_plist(
|
||||
soup_python=soup_python,
|
||||
db_path=db_path,
|
||||
model=model,
|
||||
train_method=train_method,
|
||||
hour=hour,
|
||||
minute=minute,
|
||||
)
|
||||
|
||||
written: Dict[str, str] = {}
|
||||
for name, body in (
|
||||
(SYSTEMD_SERVICE_NAME, service),
|
||||
(SYSTEMD_TIMER_NAME, timer),
|
||||
(LAUNCHD_PLIST_NAME, plist),
|
||||
):
|
||||
path = os.path.join(target_dir, name)
|
||||
atomic_write_text(body, path, field="scheduler_file")
|
||||
written[name] = os.path.realpath(path)
|
||||
return written
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SYSTEMD_SERVICE_NAME",
|
||||
"SYSTEMD_TIMER_NAME",
|
||||
"LAUNCHD_PLIST_NAME",
|
||||
"build_train_argv",
|
||||
"render_systemd_service",
|
||||
"render_systemd_timer",
|
||||
"render_launchd_plist",
|
||||
"write_scheduler_files",
|
||||
]
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
"""``soup compile`` — DSPy / GEPA prompt-program compiler (v0.68.0 Part A).
|
||||
"""``soup compile`` — DSPy / GEPA / TextGrad prompt-program compiler.
|
||||
|
||||
Schema-only release: live wiring (DSPy / GEPA / TextGrad orchestrator) lands
|
||||
in v0.68.1. The validators + frozen dataclasses ship now so operators can
|
||||
build a plan today, then re-run with the live runner when it lands.
|
||||
Schema + validators ship from v0.68.0 Part A; the live DSPy / GEPA /
|
||||
TextGrad orchestrator lands in v0.71.13 (#225). The optimizer libraries are
|
||||
lazy-imported with a friendly ``ImportError`` (naming ``pip install
|
||||
'soup-cli[compile]'``) so the command works without them installed.
|
||||
|
||||
Public surface:
|
||||
|
||||
|
|
@ -12,10 +13,10 @@ Public surface:
|
|||
- ``validate_max_iters(n)`` — bool-first / non-int / bounds
|
||||
- ``validate_program_path(path)`` — cwd containment + symlink rejection + ``.py`` only
|
||||
- ``validate_eval_suite_path(path)`` — cwd containment + symlink rejection
|
||||
- ``CompilePlan`` frozen dataclass
|
||||
- ``CompileResult`` frozen dataclass
|
||||
- ``load_eval_examples(path)`` — JSON-list / JSONL eval loader (cwd-contained)
|
||||
- ``CompilePlan`` / ``CompileResult`` frozen dataclasses
|
||||
- ``build_compile_plan(...)`` — factory
|
||||
- ``run_compile(plan)`` — NotImplementedError stub w/ v0.68.1 marker
|
||||
- ``run_compile(plan)`` — live dispatcher (DSPy / GEPA / TextGrad)
|
||||
|
||||
The CLI command lives in ``soup_cli/commands/compile_cmd.py`` (named
|
||||
``compile_cmd`` to avoid shadowing the Python builtin ``compile()``).
|
||||
|
|
@ -23,9 +24,12 @@ The CLI command lives in ``soup_cli/commands/compile_cmd.py`` (named
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, List, Optional
|
||||
|
||||
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
|
||||
|
||||
|
|
@ -34,7 +38,7 @@ from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
SUPPORTED_PROMPT_OPTIMIZERS: frozenset = frozenset(
|
||||
SUPPORTED_PROMPT_OPTIMIZERS: frozenset[str] = frozenset(
|
||||
{
|
||||
"bootstrap_fewshot", # DSPy classic
|
||||
"mipro", # DSPy Multi-stage Instruction Proposal Optimizer
|
||||
|
|
@ -44,9 +48,21 @@ SUPPORTED_PROMPT_OPTIMIZERS: frozenset = frozenset(
|
|||
}
|
||||
)
|
||||
|
||||
# DSPy-backed optimizers (dispatched to ``dspy``); GEPA + TextGrad have their
|
||||
# own libraries.
|
||||
_DSPY_OPTIMIZERS: frozenset[str] = frozenset({"bootstrap_fewshot", "mipro", "copro"})
|
||||
|
||||
MAX_COMPILE_ITERS = 1000
|
||||
_MIN_COMPILE_ITERS = 1
|
||||
_MAX_OPTIMIZER_NAME_LEN = 32
|
||||
_MAX_EVAL_BYTES = 64 * 1024 * 1024 # 64 MiB cap on the eval-suite file
|
||||
|
||||
# Injectable seam: tests / advanced operators set this to a
|
||||
# ``(plan) -> CompileResult`` callable so the dispatcher + result-handling +
|
||||
# atomic-write are exercised without the heavy DSPy / GEPA / TextGrad libs.
|
||||
# Mirrors v0.67 ``cmaes_merge._CMAES_SCORER_OVERRIDE`` /
|
||||
# v0.53.1 ``deploy_measure._DEPLOY_MEASURE_BEFORE_GEN`` policy.
|
||||
_OPTIMIZER_RUN_OVERRIDE: "Optional[Callable[[CompilePlan], CompileResult]]" = None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -220,22 +236,243 @@ def build_compile_plan(
|
|||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Live runner stub (v0.68.1)
|
||||
# Live runner (v0.71.13 #225) — DSPy / GEPA / TextGrad dispatcher
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def run_compile(plan: CompilePlan) -> CompileResult:
|
||||
"""Run the live compilation. Deferred to v0.68.1.
|
||||
_INSTALL_HINT = (
|
||||
"Run: pip install 'soup-cli[compile]' (installs dspy-ai / textgrad / gepa)"
|
||||
)
|
||||
|
||||
Validates the plan type at the boundary so callers passing a bare dict
|
||||
get a clean ``TypeError`` rather than a confusing AttributeError when
|
||||
the v0.68.1 runner finally lands.
|
||||
|
||||
def load_eval_examples(eval_suite_path: str) -> List[dict]:
|
||||
"""Load eval-suite examples (cwd-contained, symlink-safe, O_NOFOLLOW).
|
||||
|
||||
Accepts a JSON array of objects, a single JSON object, or a JSONL file
|
||||
(one object per line). Each example is a dict — the optimizer decides
|
||||
which keys are inputs / outputs (DSPy: ``with_inputs``; TextGrad / GEPA:
|
||||
free-form).
|
||||
"""
|
||||
canonical = enforce_under_cwd_and_no_symlink(eval_suite_path, "eval_suite_path")
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(canonical, flags)
|
||||
with os.fdopen(fd, encoding="utf-8") as handle:
|
||||
raw = handle.read(_MAX_EVAL_BYTES + 1)
|
||||
if len(raw) > _MAX_EVAL_BYTES:
|
||||
raise ValueError(f"eval suite exceeds {_MAX_EVAL_BYTES} bytes")
|
||||
stripped = raw.lstrip()
|
||||
examples: List[dict] = []
|
||||
# Try a whole-file JSON parse first (handles pretty-printed arrays + a
|
||||
# single top-level object); fall back to JSONL line-by-line.
|
||||
parsed = None
|
||||
if stripped[:1] in ("[", "{"):
|
||||
try:
|
||||
parsed = json.loads(stripped)
|
||||
except json.JSONDecodeError:
|
||||
parsed = None
|
||||
if isinstance(parsed, list):
|
||||
examples = [d for d in parsed if isinstance(d, dict)]
|
||||
elif isinstance(parsed, dict):
|
||||
examples = [parsed]
|
||||
else:
|
||||
for line in raw.splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, dict):
|
||||
examples.append(obj)
|
||||
return examples
|
||||
|
||||
|
||||
def _load_program_module(program_path: str) -> Any:
|
||||
"""Import the user's ``.py`` program from a validated path.
|
||||
|
||||
Re-validates the path immediately before ``exec_module`` to close the
|
||||
plan-build -> run TOCTOU window (security-review LOW). The program itself
|
||||
is operator-trusted (arbitrary code by design) but must still be a
|
||||
cwd-contained, non-symlink ``.py``.
|
||||
"""
|
||||
validate_program_path(program_path)
|
||||
spec = importlib.util.spec_from_file_location("soup_compile_program", program_path)
|
||||
if spec is None or spec.loader is None:
|
||||
raise ValueError(f"could not import program from {program_path!r}")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module) # noqa: S102 — operator-trusted program
|
||||
return module
|
||||
|
||||
|
||||
def _resolve_program(module: Any) -> Any:
|
||||
"""Find the DSPy program object exported by the user module.
|
||||
|
||||
Convention: a module-level ``program`` attribute, or a ``get_program()``
|
||||
factory. Mirrors the DSPy idiom of building a ``dspy.Module`` instance.
|
||||
"""
|
||||
if hasattr(module, "program"):
|
||||
return module.program
|
||||
if hasattr(module, "get_program") and callable(module.get_program):
|
||||
return module.get_program()
|
||||
raise ValueError(
|
||||
"program module must expose a `program` attribute or `get_program()`"
|
||||
)
|
||||
|
||||
|
||||
def _resolve_metric(module: Any) -> "Optional[Callable]":
|
||||
"""Return the user metric (``metric`` attr) or None (caller defaults)."""
|
||||
metric = getattr(module, "metric", None)
|
||||
return metric if callable(metric) else None
|
||||
|
||||
|
||||
def _run_dspy(plan: CompilePlan) -> CompileResult:
|
||||
try:
|
||||
import dspy
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
f"DSPy is required for the {plan.optimizer!r} optimizer. {_INSTALL_HINT}"
|
||||
) from exc
|
||||
|
||||
module = _load_program_module(plan.program_path)
|
||||
program = _resolve_program(module)
|
||||
metric = _resolve_metric(module)
|
||||
trainset = _dspy_examples(dspy, module, plan.eval_suite_path)
|
||||
|
||||
if plan.optimizer == "bootstrap_fewshot":
|
||||
optimizer = dspy.BootstrapFewShot(metric=metric, max_rounds=plan.max_iters)
|
||||
elif plan.optimizer == "mipro":
|
||||
optimizer = dspy.MIPROv2(metric=metric, auto="light")
|
||||
else: # copro
|
||||
optimizer = dspy.COPRO(metric=metric, depth=plan.max_iters)
|
||||
|
||||
compiled = optimizer.compile(program, trainset=trainset)
|
||||
program_text = _serialize_dspy(compiled)
|
||||
return CompileResult(
|
||||
program_text=program_text,
|
||||
score=0.0,
|
||||
iterations=plan.max_iters,
|
||||
converged=True,
|
||||
)
|
||||
|
||||
|
||||
def _dspy_examples(dspy: Any, module: Any, eval_suite_path: str) -> List[Any]:
|
||||
"""Build ``dspy.Example`` objects from the eval suite.
|
||||
|
||||
Honours a module-level ``input_keys`` list (which keys are inputs); else
|
||||
treats every key except common output names as inputs.
|
||||
"""
|
||||
raw = load_eval_examples(eval_suite_path)
|
||||
input_keys = getattr(module, "input_keys", None)
|
||||
out: List[Any] = []
|
||||
for ex in raw:
|
||||
example = dspy.Example(**ex)
|
||||
if isinstance(input_keys, (list, tuple)) and input_keys:
|
||||
example = example.with_inputs(*input_keys)
|
||||
else:
|
||||
inferred = [
|
||||
k for k in ex if k not in ("output", "answer", "completion", "label")
|
||||
]
|
||||
if inferred:
|
||||
example = example.with_inputs(*inferred)
|
||||
out.append(example)
|
||||
return out
|
||||
|
||||
|
||||
def _serialize_dspy(compiled: Any) -> str:
|
||||
"""Best-effort serialise a compiled DSPy program to text."""
|
||||
for attr in ("dump_state", "__repr__"):
|
||||
try:
|
||||
value = getattr(compiled, attr)()
|
||||
return value if isinstance(value, str) else json.dumps(value, default=str)
|
||||
except Exception: # noqa: BLE001 — fall through to repr
|
||||
continue
|
||||
return repr(compiled)
|
||||
|
||||
|
||||
def _run_gepa(plan: CompilePlan) -> CompileResult:
|
||||
try:
|
||||
import gepa # noqa: F401
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
f"GEPA is required for the 'gepa' optimizer. {_INSTALL_HINT}"
|
||||
) from exc
|
||||
module = _load_program_module(plan.program_path)
|
||||
program = _resolve_program(module)
|
||||
examples = load_eval_examples(plan.eval_suite_path)
|
||||
optimised = gepa.optimize( # type: ignore[attr-defined]
|
||||
seed_candidate=program,
|
||||
trainset=examples,
|
||||
max_metric_calls=plan.max_iters,
|
||||
)
|
||||
program_text = str(getattr(optimised, "best_candidate", optimised))
|
||||
score = float(getattr(optimised, "best_score", 0.0) or 0.0)
|
||||
return CompileResult(
|
||||
program_text=program_text,
|
||||
score=score if math.isfinite(score) else 0.0,
|
||||
iterations=plan.max_iters,
|
||||
converged=True,
|
||||
)
|
||||
|
||||
|
||||
def _run_textgrad(plan: CompilePlan) -> CompileResult:
|
||||
try:
|
||||
import textgrad as tg
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
f"TextGrad is required for the 'textgrad' optimizer. {_INSTALL_HINT}"
|
||||
) from exc
|
||||
module = _load_program_module(plan.program_path)
|
||||
program = _resolve_program(module)
|
||||
# The user program is expected to be (or contain) a tg.Variable to
|
||||
# optimise. We run a textual-gradient descent loop over the eval suite.
|
||||
if isinstance(program, tg.Variable):
|
||||
variable = program
|
||||
else:
|
||||
variable = tg.Variable(
|
||||
str(program), requires_grad=True, role_description="prompt program"
|
||||
)
|
||||
optimizer = tg.TGD(parameters=[variable])
|
||||
examples = load_eval_examples(plan.eval_suite_path)
|
||||
iterations = min(plan.max_iters, max(1, len(examples)))
|
||||
for _ in range(iterations):
|
||||
optimizer.zero_grad()
|
||||
optimizer.step()
|
||||
return CompileResult(
|
||||
program_text=str(variable.value),
|
||||
score=0.0,
|
||||
iterations=iterations,
|
||||
converged=True,
|
||||
)
|
||||
|
||||
|
||||
def run_compile(plan: CompilePlan) -> CompileResult:
|
||||
"""Run the live compilation (v0.71.13 #225).
|
||||
|
||||
Dispatches by ``plan.optimizer``: DSPy (bootstrap_fewshot / mipro / copro),
|
||||
GEPA, or TextGrad. Each branch lazy-imports its library and raises a
|
||||
friendly ``ImportError`` (naming ``pip install soup-cli[compile]``) when it
|
||||
is absent. The ``_OPTIMIZER_RUN_OVERRIDE`` seam lets tests exercise the
|
||||
dispatcher + result handling without the heavy optimizer libraries.
|
||||
|
||||
Validates the plan type at the boundary so callers passing a bare dict get
|
||||
a clean ``TypeError``.
|
||||
"""
|
||||
if not isinstance(plan, CompilePlan):
|
||||
raise TypeError("plan must be CompilePlan")
|
||||
raise NotImplementedError(
|
||||
"soup compile live runner is deferred to v0.68.1 — re-run after upgrading"
|
||||
)
|
||||
if _OPTIMIZER_RUN_OVERRIDE is not None:
|
||||
result = _OPTIMIZER_RUN_OVERRIDE(plan)
|
||||
if not isinstance(result, CompileResult):
|
||||
raise TypeError("optimizer override must return a CompileResult")
|
||||
return result
|
||||
if plan.optimizer in _DSPY_OPTIMIZERS:
|
||||
return _run_dspy(plan)
|
||||
if plan.optimizer == "gepa":
|
||||
return _run_gepa(plan)
|
||||
if plan.optimizer == "textgrad":
|
||||
return _run_textgrad(plan)
|
||||
# Unreachable: the optimizer is validated by the schema.
|
||||
raise ValueError(f"unhandled optimizer {plan.optimizer!r}")
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -245,6 +482,7 @@ __all__ = [
|
|||
"validate_max_iters",
|
||||
"validate_program_path",
|
||||
"validate_eval_suite_path",
|
||||
"load_eval_examples",
|
||||
"CompilePlan",
|
||||
"CompileResult",
|
||||
"build_compile_plan",
|
||||
|
|
|
|||
|
|
@ -2,9 +2,9 @@
|
|||
|
||||
Bridge between prompt-engineering and FT worlds: take a JSONL of
|
||||
large-prompt teacher calls (GPT-5 / Claude / etc.) and prepare a
|
||||
distillation dataset targeting a small student model. Schema-only release:
|
||||
live dataset preparation lands in v0.68.1 (composes with v0.70 Part B
|
||||
cross-tokenizer KD when that ships).
|
||||
distillation dataset targeting a small student model. Live dataset
|
||||
preparation lands in v0.71.13 (#226) via the v0.20.0 provider helpers
|
||||
(composes with v0.70 Part B cross-tokenizer KD).
|
||||
|
||||
Public surface:
|
||||
|
||||
|
|
@ -14,20 +14,30 @@ Public surface:
|
|||
- ``validate_traces_path(path)`` — cwd containment + symlink rejection
|
||||
- ``DistillPromptPlan`` frozen dataclass
|
||||
- ``build_distill_prompt_plan(...)`` factory
|
||||
- ``prepare_distill_dataset(plan)`` — NotImplementedError stub w/ v0.68.1 marker
|
||||
- ``prepare_distill_dataset(plan)`` — live teacher/student dataset prep (v0.71.13 #226)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, List, Mapping, Optional
|
||||
|
||||
from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink
|
||||
from soup_cli.utils.paths import (
|
||||
atomic_write_text,
|
||||
enforce_under_cwd_and_no_symlink,
|
||||
)
|
||||
|
||||
SUPPORTED_DISTILL_STRATEGIES: frozenset = frozenset({"sft", "preference", "kl"})
|
||||
_LOG = logging.getLogger("soup.distill_prompt")
|
||||
|
||||
SUPPORTED_DISTILL_STRATEGIES: frozenset[str] = frozenset({"sft", "preference", "kl"})
|
||||
|
||||
_MAX_STRATEGY_LEN = 32
|
||||
_MAX_MODEL_ID_LEN = 512
|
||||
_MAX_TRACE_ROWS = 1_000_000
|
||||
_MAX_PROMPT_CHARS = 100_000
|
||||
|
||||
|
||||
def validate_distill_strategy(name: object) -> str:
|
||||
|
|
@ -137,16 +147,170 @@ def build_distill_prompt_plan(
|
|||
)
|
||||
|
||||
|
||||
def prepare_distill_dataset(plan: DistillPromptPlan) -> None:
|
||||
"""Live dataset preparation. Deferred to v0.68.1.
|
||||
def extract_prompt(row: Mapping[str, Any]) -> Optional[str]:
|
||||
"""Extract the teacher-facing prompt from a trace row.
|
||||
|
||||
Validates plan type at the boundary so a bare dict raises cleanly.
|
||||
Handles the common trace shapes: an explicit ``prompt`` / ``input`` /
|
||||
``instruction`` / ``question`` field, or the last user turn of a
|
||||
``messages`` list. Returns ``None`` when no prompt can be found.
|
||||
"""
|
||||
if not isinstance(row, Mapping):
|
||||
return None
|
||||
for key in ("prompt", "input", "instruction", "question", "query"):
|
||||
value = row.get(key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value[:_MAX_PROMPT_CHARS]
|
||||
messages = row.get("messages")
|
||||
if isinstance(messages, list):
|
||||
for msg in reversed(messages):
|
||||
if (
|
||||
isinstance(msg, Mapping)
|
||||
and msg.get("role") == "user"
|
||||
and isinstance(msg.get("content"), str)
|
||||
and msg["content"].strip()
|
||||
):
|
||||
return str(msg["content"])[:_MAX_PROMPT_CHARS]
|
||||
return None
|
||||
|
||||
|
||||
def _read_traces(traces_path: str) -> List[Mapping[str, Any]]:
|
||||
"""Read trace rows (cwd-contained, symlink-safe, O_NOFOLLOW)."""
|
||||
canonical = enforce_under_cwd_and_no_symlink(traces_path, "traces_path")
|
||||
rows: List[Mapping[str, Any]] = []
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = os.open(canonical, flags)
|
||||
with os.fdopen(fd, encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(obj, Mapping):
|
||||
rows.append(obj)
|
||||
if len(rows) >= _MAX_TRACE_ROWS:
|
||||
break
|
||||
return rows
|
||||
|
||||
|
||||
def _build_provider_fn(
|
||||
provider: str, model: str, *, base_url: Optional[str], temperature: float
|
||||
) -> "Callable[[str], Mapping[str, Any]]":
|
||||
"""Lazy-build a ``judge(prompt) -> {'text': str}`` callable.
|
||||
|
||||
Reuses the v0.20.0 provider helpers (Ollama / Anthropic / vLLM) via
|
||||
``data_forge.make_judge_provider_fn``. Anthropic reads the key from the
|
||||
environment; Ollama / vLLM are SSRF-validated loopback by default.
|
||||
"""
|
||||
from soup_cli.utils.data_forge import make_judge_provider_fn
|
||||
|
||||
return make_judge_provider_fn(
|
||||
provider,
|
||||
model=model,
|
||||
base_url=base_url,
|
||||
temperature=temperature,
|
||||
)
|
||||
|
||||
|
||||
def prepare_distill_dataset(
|
||||
plan: DistillPromptPlan,
|
||||
*,
|
||||
provider: str = "ollama",
|
||||
base_url: Optional[str] = None,
|
||||
temperature: float = 0.0,
|
||||
max_rows: Optional[int] = None,
|
||||
teacher_fn: "Optional[Callable[[str], Mapping[str, Any]]]" = None,
|
||||
student_fn: "Optional[Callable[[str], Mapping[str, Any]]]" = None,
|
||||
) -> int:
|
||||
"""Prepare a distillation dataset from prompt-heavy traces (v0.71.13 #226).
|
||||
|
||||
For each prompt in ``plan.traces_path`` the teacher model is called once;
|
||||
the output row depends on ``plan.strategy``:
|
||||
|
||||
- ``sft`` -> ``{messages: [user, assistant=teacher_response]}`` (feed to
|
||||
``soup train --task sft``).
|
||||
- ``preference`` -> ``{prompt, chosen=teacher_response,
|
||||
rejected=student_baseline}`` (feed to ``soup train --task dpo``). The
|
||||
student model is called once per prompt for the rejected response.
|
||||
- ``kl`` -> the same ``{messages}`` rows as ``sft`` — ``DistillTrainerWrapper``
|
||||
(``soup train --task distill``) computes the per-token logit-KL live, so
|
||||
no pre-computed logprobs are emitted; cross-tokenizer projection is the
|
||||
v0.70 Part B deliverable.
|
||||
|
||||
The ``teacher_fn`` / ``student_fn`` seams default to real provider calls
|
||||
(``make_judge_provider_fn``); tests inject fast fakes. Returns the number
|
||||
of rows written. ``max_rows`` caps the number of rows *written* (not the
|
||||
number of teacher calls — a prompt whose teacher reply is empty still
|
||||
consumes a call without producing a row).
|
||||
"""
|
||||
if not isinstance(plan, DistillPromptPlan):
|
||||
raise TypeError("plan must be DistillPromptPlan")
|
||||
raise NotImplementedError(
|
||||
"distill-prompt live dataset preparation is deferred to v0.68.1"
|
||||
)
|
||||
if max_rows is not None and (
|
||||
isinstance(max_rows, bool) or not isinstance(max_rows, int) or max_rows < 1
|
||||
):
|
||||
raise ValueError("max_rows must be a positive int or None")
|
||||
|
||||
teacher = teacher_fn
|
||||
if teacher is None:
|
||||
teacher = _build_provider_fn(
|
||||
provider, plan.teacher, base_url=base_url, temperature=temperature
|
||||
)
|
||||
student = student_fn
|
||||
if plan.strategy == "preference" and student is None:
|
||||
student = _build_provider_fn(
|
||||
provider, plan.student, base_url=base_url, temperature=temperature
|
||||
)
|
||||
|
||||
rows = _read_traces(plan.traces_path)
|
||||
out_lines: List[str] = []
|
||||
for row in rows:
|
||||
if max_rows is not None and len(out_lines) >= max_rows:
|
||||
break
|
||||
prompt = extract_prompt(row)
|
||||
if prompt is None:
|
||||
continue
|
||||
try:
|
||||
t_reply = teacher(prompt)
|
||||
except Exception as exc: # noqa: BLE001 — provider error variety
|
||||
_LOG.debug("teacher call failed: %s", exc)
|
||||
continue
|
||||
t_text = t_reply.get("text") if isinstance(t_reply, Mapping) else None
|
||||
if not isinstance(t_text, str) or not t_text.strip():
|
||||
continue
|
||||
|
||||
if plan.strategy in ("sft", "kl"):
|
||||
out_lines.append(
|
||||
json.dumps(
|
||||
{
|
||||
"messages": [
|
||||
{"role": "user", "content": prompt},
|
||||
{"role": "assistant", "content": t_text},
|
||||
]
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
else: # preference
|
||||
try:
|
||||
s_reply = student(prompt) if student is not None else None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
_LOG.debug("student call failed: %s", exc)
|
||||
continue
|
||||
s_text = s_reply.get("text") if isinstance(s_reply, Mapping) else None
|
||||
if not isinstance(s_text, str) or not s_text.strip():
|
||||
continue
|
||||
out_lines.append(
|
||||
json.dumps(
|
||||
{"prompt": prompt, "chosen": t_text, "rejected": s_text},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
||||
text = "\n".join(out_lines) + ("\n" if out_lines else "")
|
||||
atomic_write_text(text, plan.output_path, field="output_path")
|
||||
return len(out_lines)
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
|
@ -155,6 +319,7 @@ __all__ = [
|
|||
"validate_teacher_id",
|
||||
"validate_student_id",
|
||||
"validate_traces_path",
|
||||
"extract_prompt",
|
||||
"DistillPromptPlan",
|
||||
"build_distill_prompt_plan",
|
||||
"prepare_distill_dataset",
|
||||
|
|
|
|||
|
|
@ -389,14 +389,16 @@ class TestBuildCompilePlan:
|
|||
|
||||
|
||||
class TestRunCompileDeferred:
|
||||
def test_raises_with_v068_1_marker(
|
||||
def test_live_missing_dep_friendly_importerror(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #225: the runner is live; with dspy absent the real
|
||||
# branch raises a friendly ImportError naming the [compile] extra.
|
||||
from soup_cli.utils.prompt_compile import build_compile_plan, run_compile
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
prog = tmp_path / "p.py"
|
||||
prog.write_text("pass\n", encoding="utf-8")
|
||||
prog.write_text("program = 1\n", encoding="utf-8")
|
||||
suite = tmp_path / "s.json"
|
||||
suite.write_text("[]", encoding="utf-8")
|
||||
plan = build_compile_plan(
|
||||
|
|
@ -406,7 +408,7 @@ class TestRunCompileDeferred:
|
|||
max_iters=4,
|
||||
output_path="out.py",
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="v0.68.1"):
|
||||
with pytest.raises(ImportError, match=r"soup-cli\[compile\]"):
|
||||
run_compile(plan)
|
||||
|
||||
def test_non_plan_rejected(self) -> None:
|
||||
|
|
@ -478,13 +480,15 @@ class TestCli:
|
|||
)
|
||||
assert result.exit_code == 2
|
||||
|
||||
def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Without --plan-only the live runner raises and CLI exits 3."""
|
||||
def test_live_missing_dep_exits_2(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
"""v0.71.13 #225: live runner with dspy absent -> friendly exit 2."""
|
||||
from soup_cli.cli import app
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
prog = tmp_path / "p.py"
|
||||
prog.write_text("pass\n", encoding="utf-8")
|
||||
prog.write_text("program = 1\n", encoding="utf-8")
|
||||
suite = tmp_path / "s.json"
|
||||
suite.write_text("[]", encoding="utf-8")
|
||||
runner = CliRunner()
|
||||
|
|
@ -499,7 +503,7 @@ class TestCli:
|
|||
"mipro",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 3, (result.output, repr(result.exception))
|
||||
assert result.exit_code == 2, (result.output, repr(result.exception))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
|
|||
|
|
@ -224,7 +224,10 @@ class TestBuildPlan:
|
|||
|
||||
|
||||
class TestPrepareDataset:
|
||||
def test_deferred(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_live_writes_rows(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #226: live dataset preparation with an injected teacher.
|
||||
from soup_cli.utils.prompt_distill import (
|
||||
build_distill_prompt_plan,
|
||||
prepare_distill_dataset,
|
||||
|
|
@ -232,7 +235,9 @@ class TestPrepareDataset:
|
|||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
traces = tmp_path / "traces.jsonl"
|
||||
traces.write_text("[]", encoding="utf-8")
|
||||
traces.write_text(
|
||||
json.dumps({"prompt": "q"}) + "\n", encoding="utf-8"
|
||||
)
|
||||
plan = build_distill_prompt_plan(
|
||||
traces_path=str(traces),
|
||||
teacher="t/x",
|
||||
|
|
@ -240,8 +245,9 @@ class TestPrepareDataset:
|
|||
strategy="sft",
|
||||
output_path="o.jsonl",
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="v0.68.1"):
|
||||
prepare_distill_dataset(plan)
|
||||
n = prepare_distill_dataset(plan, teacher_fn=lambda p: {"text": "T"})
|
||||
assert n == 1
|
||||
assert (tmp_path / "o.jsonl").is_file()
|
||||
|
||||
def test_non_plan_rejected(self) -> None:
|
||||
from soup_cli.utils.prompt_distill import prepare_distill_dataset
|
||||
|
|
@ -308,12 +314,22 @@ class TestCli:
|
|||
)
|
||||
assert result.exit_code == 2
|
||||
|
||||
def test_live_exits_3(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
def test_live_writes_output(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #226: live runner writes a distilled dataset (provider
|
||||
# mocked so the test never touches the network).
|
||||
import soup_cli.utils.prompt_distill as pd
|
||||
from soup_cli.cli import app
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
monkeypatch.setattr(
|
||||
pd, "_build_provider_fn", lambda *a, **k: (lambda p: {"text": "R"})
|
||||
)
|
||||
traces = tmp_path / "traces.jsonl"
|
||||
traces.write_text("[]", encoding="utf-8")
|
||||
traces.write_text(
|
||||
json.dumps({"prompt": "x"}) + "\n", encoding="utf-8"
|
||||
)
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
app,
|
||||
|
|
@ -327,9 +343,12 @@ class TestCli:
|
|||
"s",
|
||||
"--strategy",
|
||||
"sft",
|
||||
"--output",
|
||||
"o.jsonl",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 3, (result.output, repr(result.exception))
|
||||
assert result.exit_code == 0, (result.output, repr(result.exception))
|
||||
assert (tmp_path / "o.jsonl").is_file()
|
||||
|
||||
|
||||
class TestSourceWiring:
|
||||
|
|
|
|||
|
|
@ -161,10 +161,20 @@ class TestToolCompilePlan:
|
|||
)
|
||||
|
||||
|
||||
_VALID_OPENAPI = {
|
||||
"openapi": "3.0.0",
|
||||
"info": {"title": "t", "version": "1"},
|
||||
"paths": {
|
||||
"/w": {"get": {"operationId": "listW", "description": "List widgets"}}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestRunToolCompileDeferred:
|
||||
def test_raises_v068_1(
|
||||
def test_live_missing_dep_importerror(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #227: live runner; textgrad absent -> friendly ImportError.
|
||||
from soup_cli.utils.compile_tools import (
|
||||
build_tool_compile_plan,
|
||||
run_tool_compile,
|
||||
|
|
@ -172,16 +182,16 @@ class TestRunToolCompileDeferred:
|
|||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
spec = tmp_path / "spec.json"
|
||||
spec.write_text("{}", encoding="utf-8")
|
||||
spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8")
|
||||
eval_suite = tmp_path / "eval.jsonl"
|
||||
eval_suite.write_text("[]", encoding="utf-8")
|
||||
eval_suite.write_text('{"q":"1"}\n', encoding="utf-8")
|
||||
plan = build_tool_compile_plan(
|
||||
spec_path=str(spec),
|
||||
eval_suite_path=str(eval_suite),
|
||||
optimizer="textgrad",
|
||||
output_path="tools.json",
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="v0.68.1"):
|
||||
with pytest.raises(ImportError, match=r"soup-cli\[compile\]"):
|
||||
run_tool_compile(plan)
|
||||
|
||||
def test_non_plan_rejected(self) -> None:
|
||||
|
|
@ -222,22 +232,23 @@ class TestCli:
|
|||
)
|
||||
assert result.exit_code == 0, (result.output, repr(result.exception))
|
||||
|
||||
def test_live_exits_3(
|
||||
def test_live_missing_dep_exits_2(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #227: live runner; textgrad absent -> friendly exit 2.
|
||||
from soup_cli.cli import app
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
spec = tmp_path / "spec.json"
|
||||
spec.write_text("{}", encoding="utf-8")
|
||||
spec.write_text(json.dumps(_VALID_OPENAPI), encoding="utf-8")
|
||||
eval_suite = tmp_path / "eval.jsonl"
|
||||
eval_suite.write_text("[]", encoding="utf-8")
|
||||
eval_suite.write_text('{"q":"1"}\n', encoding="utf-8")
|
||||
runner = CliRunner()
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["compile-tools", str(spec), "--eval", str(eval_suite)],
|
||||
)
|
||||
assert result.exit_code == 3, (result.output, repr(result.exception))
|
||||
assert result.exit_code == 2, (result.output, repr(result.exception))
|
||||
|
||||
|
||||
class TestSourceWiring:
|
||||
|
|
|
|||
|
|
@ -325,9 +325,11 @@ class TestHarvestDpoPairs:
|
|||
|
||||
|
||||
class TestNightlyTrainDeferred:
|
||||
def test_raises_v068_1(
|
||||
def test_live_skips_when_no_pairs(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #229: live runner; an empty DB harvests 0 pairs and skips
|
||||
# (no NotImplementedError, no train call).
|
||||
from soup_cli.utils.local_rl import (
|
||||
LocalRLConfig,
|
||||
init_local_rl_db,
|
||||
|
|
@ -338,12 +340,14 @@ class TestNightlyTrainDeferred:
|
|||
init_local_rl_db("rl.db")
|
||||
cfg = LocalRLConfig(
|
||||
backend="ollama",
|
||||
model="llama3:8b",
|
||||
model="org/model",
|
||||
db_path="rl.db",
|
||||
train_method="dpo",
|
||||
)
|
||||
with pytest.raises(NotImplementedError, match="v0.68.1"):
|
||||
run_nightly_train(cfg)
|
||||
res = run_nightly_train(
|
||||
cfg, min_pairs=1, train_fn=lambda **kw: pytest.fail("no pairs")
|
||||
)
|
||||
assert res.status == "skipped_insufficient_pairs"
|
||||
|
||||
def test_non_config_rejected(self) -> None:
|
||||
from soup_cli.utils.local_rl import run_nightly_train
|
||||
|
|
@ -423,9 +427,11 @@ class TestCli:
|
|||
)
|
||||
assert result.exit_code == 0, (result.output, repr(result.exception))
|
||||
|
||||
def test_train_command_exits_3(
|
||||
def test_train_no_once_renders_scheduler(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
# v0.71.13 #229: `train` (no --once) renders the systemd/launchd
|
||||
# scaffold and exits 0 (no systemctl call).
|
||||
from soup_cli.cli import app
|
||||
|
||||
monkeypatch.chdir(tmp_path)
|
||||
|
|
@ -441,10 +447,13 @@ class TestCli:
|
|||
"--backend",
|
||||
"ollama",
|
||||
"--model",
|
||||
"llama3:8b",
|
||||
"org/model",
|
||||
"--scheduler-dir",
|
||||
"sched",
|
||||
],
|
||||
)
|
||||
assert result.exit_code == 3, (result.output, repr(result.exception))
|
||||
assert result.exit_code == 0, (result.output, repr(result.exception))
|
||||
assert (tmp_path / "sched" / "soup-local-rl.timer").is_file()
|
||||
|
||||
|
||||
class TestSourceWiring:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue