From 5652215d4a60e914c847411a88662bfd9f2de987 Mon Sep 17 00:00:00 2001 From: Alpamys Date: Tue, 2 Jun 2026 12:25:01 +0500 Subject: [PATCH] =?UTF-8?q?feat(adapters,loop):=20v0.71.4=20=E2=80=94=20li?= =?UTF-8?q?ve=20canary=20verdict=20+=20cmaes=20merge=20+=20PR=20push=20+?= =?UTF-8?q?=20pre-wired=20loop=20+=20can=20lineage=20+=20branch=E2=86=94re?= =?UTF-8?q?gistry?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #172, #173, #176, #177, #220, #223. - #172 soup adapters merge --canary/--strict-verdict: live OK/MINOR/MAJOR verdict (was UNKNOWN stub) - #220 soup adapters merge --strategy cmaes: live merge→score→write-best loop (was plan-only) - #223 soup adapters pr --push owner/repo#N: post PR comment via gh api - #176 soup loop --pre-wired: real traces→DPO→eval-gate→canary stages - #177 soup loop --pack-cans / replay --extract: iterations as Soup Cans + Registry lineage DAG - #173 soup adapters branch --from-registry / --attach-to-registry Security: backdoor-scan + license gates now run for ALL merge strategies (incl cmaes); loop canary deploy restricted to loopback/RFC1918; gh child env from allowlist; canary read uses O_NOFOLLOW+fstat (TOCTOU); pack-entry failure rolls back registry entry. Tests: 12342 → 12474 (+130 in tests/test_v0714.py). --- CHANGELOG.md | 45 + CONTRIBUTING.md | 2 +- README.md | 26 +- docs/adapters-and-governance.md | 36 +- docs/commands.md | 13 +- pyproject.toml | 2 +- src/soup_cli/__init__.py | 2 +- src/soup_cli/commands/adapters.py | 275 +++- src/soup_cli/commands/history.py | 14 + src/soup_cli/commands/loop.py | 78 +- src/soup_cli/registry/store.py | 2 + src/soup_cli/utils/adapter_branch.py | 120 +- src/soup_cli/utils/adapter_merge.py | 152 ++- src/soup_cli/utils/adapter_pr.py | 152 ++- src/soup_cli/utils/cmaes_merge.py | 149 +++ src/soup_cli/utils/loop_daemon.py | 68 + src/soup_cli/utils/loop_iteration.py | 104 ++ src/soup_cli/utils/loop_stages.py | 418 ++++++ src/soup_cli/utils/loop_state.py | 7 + tests/test_v0714.py | 1774 ++++++++++++++++++++++++++ 20 files changed, 3354 insertions(+), 85 deletions(-) create mode 100644 src/soup_cli/utils/loop_stages.py create mode 100644 tests/test_v0714.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d5bfd96..68d6507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,51 @@ reproducing 70+ versions of notes. ## [Unreleased] +## [0.71.4] - 2026-06-02 + +### Added +- **Live canary verdict for `soup adapters merge`** — `--canary ` + scores the merged adapter against the first input and classifies + **OK / MINOR / MAJOR** using the Quant-Lobotomy taxonomy (drop <2% OK, <5% + MINOR, else MAJOR). `--strict-verdict` exits 2 on MAJOR. Pre-scored + `{"baseline_scores","candidate_scores"}` suites run with no model load; a + `{"tasks":[...]}` suite uses an injectable scorer. Replaces the v0.57 `UNKNOWN` + stub. +- **Live evolutionary merge** — `soup adapters merge --strategy cmaes --eval + --budget ` now runs the full CMA-ES loop: each candidate is merged, + materialised, scored against the eval suite, and the best-weighted merge is + written to `--output`. Replaces the v0.67 plan-only stub. +- **Publish an adapter PR to GitHub** — `soup adapters pr --base-sha + <hex> --adapter <path> --push owner/repo#N` posts the rendered PR Markdown as a + GitHub PR comment via `gh api` (argv-list, body over JSON stdin; no shell). + Token resolves from `GITHUB_TOKEN` / `GH_TOKEN`. +- **Pre-wired `soup loop` production stages** — `soup loop init --pre-wired` (or + `soup loop watch --pre-wired`) swaps the v0.58 no-op stage stubs for real + harvest (traces → preference pairs) → DPO train → eval-gate → canary-deploy + callables. `soup loop status` now shows the `pre_wired` flag. +- **Loop iterations as Soup Cans + Registry lineage** — `soup loop watch + --pack-cans` packs each successful iteration as a v0.26 Soup Can and appends a + Registry entry (tag `loop-iter`), chaining a real lineage DAG across + iterations visible through `soup history`. `soup loop replay <id> --extract + <dir>` unpacks a recorded iteration. +- **Branch pointers into the Registry** — `soup adapters branch <name> + --attach-to-registry <id>` links a branch snapshot to a Registry entry (shown + as a `branches` node in `soup history`); `soup adapters branch <name> + --from-registry <id>` derives a fresh snapshot's config + base from an entry. + +### Security +- The backdoor-scan gate (v0.71.2 #192) and license-conflict gate (v0.60 Part E) + now run for **all** merge strategies, including `--strategy cmaes` (previously + bypassed because cmaes returned before the gates). +- `soup loop` canary deploy restricts `SOUP_LOOP_SERVE_ENDPOINT` to loopback / + RFC1918-private hosts (a serve endpoint is the operator's own box/LAN), beyond + the general webhook SSRF policy which permits any HTTPS host. +- `soup adapters pr --push` builds the `gh` child environment from an allowlist + so unrelated secrets (`HF_TOKEN` / `OPENAI_API_KEY` / …) never reach the + subprocess. +- The canary-suite JSON read uses `O_NOFOLLOW` + `os.fstat` (size cap enforced on + the same fd) to close the symlink/size-cap TOCTOU window. + ## [0.71.3] - 2026-06-01 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1093676..15bf211 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,7 +120,7 @@ src/soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (273 files, 12342 tests) +tests/ - Test suite (274 files, 12474 tests) examples/ - Real-world config examples and datasets ``` diff --git a/README.md b/README.md index 2808aea..76d4bd9 100644 --- a/README.md +++ b/README.md @@ -49,19 +49,21 @@ infrastructure instead of improving models. Soup fixes that. ## What's New -**v0.71.3 — Compliance / annex / audit / energy.** The EU AI Act + carbon + audit surfaces go live: +**v0.71.4 — Adapter lifecycle + loop wiring.** The merge, PR, and continuous-loop surfaces go live: -- **Energy & CO2 measurement** — `soup train --track-energy` measures the training window with an - offline carbon tracker (no network) and reports kWh / CO2, feeding them into the Annex doc. - Install with `pip install soup-cli[carbon]`. -- **PDF compliance docs** — `soup train --annex-xi report.pdf` now renders a real PDF of the - EU AI Act Annex XI/XII (a `.md` path still gives markdown). The top crawled domains of your - training corpus are auto-listed. `pip install soup-cli[pdf]`. -- **Per-command audit log** — every `soup` command appends one HIPAA/SOC2 record to - `~/.soup/audit.jsonl` (secrets redacted). Opt out with `--no-audit-log` or `SOUP_NO_AUDIT_LOG=1`. -- **Signed Soup Cans + airgap receipts** — `soup can pack --attest statement.json` embeds in-toto - attestations into a can; `soup airgap-bundle --repro-receipt receipt.json` ships a reproducibility - receipt inside the offline bundle. +- **Canary verdict on merge** — `soup adapters merge … --canary suite.json` scores the merged + adapter and reports **OK / MINOR / MAJOR**; `--strict-verdict` exits non-zero on a MAJOR + regression. Works with no model load using a pre-scored canary suite. +- **Evolutionary merge for real** — `soup adapters merge --strategy cmaes --eval suite --budget 1h` + now runs the full CMA-ES search (merge → score → optimise) and writes the best blend, instead of + just printing a plan. +- **Publish an adapter PR** — `soup adapters pr <title> --base-sha <hex> --adapter <path> --push + owner/repo#42` posts the rendered PR straight to a GitHub PR comment. +- **Continuous fine-tuning loop, wired up** — `soup loop watch --pre-wired` runs the real + traces → DPO → eval-gate → canary pipeline; `--pack-cans` snapshots every iteration as a + shareable Soup Can with Registry lineage (`soup loop replay <id> --extract dir`). +- **Branches ↔ Registry** — `soup adapters branch <name> --attach-to-registry <id>` / + `--from-registry <id>` links training-env snapshots into the Registry lineage DAG. Full history: [CHANGELOG.md](CHANGELOG.md) · [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). diff --git a/docs/adapters-and-governance.md b/docs/adapters-and-governance.md index e0cad41..4c11ed0 100644 --- a/docs/adapters-and-governance.md +++ b/docs/adapters-and-governance.md @@ -34,6 +34,8 @@ v0.57 shipped `adapters diff / merge / blame / branch`. v0.67 finishes the lifec ```bash # 1. Evolutionary merge: search the simplex of merge weights via CMA-ES. +# v0.71.4 makes this a LIVE loop — each candidate is merged, scored against +# --eval, and the best blend is written to --output. soup adapters merge \ adapter-finance/ adapter-medical/ adapter-legal/ \ --strategy cmaes \ @@ -43,6 +45,12 @@ soup adapters merge \ --max-generations 20 \ --output merged/ +# 1b. One-shot merge with a live canary verdict (v0.71.4). Scores the merged +# adapter vs the first input → OK / MINOR / MAJOR; --strict-verdict exits 2 +# on a MAJOR regression. A pre-scored canary suite needs no model load. +soup adapters merge adapter-a/ adapter-b/ -o merged/ \ + --canary evals/canary.json --strict-verdict + # 2. Render the merge as a GitHub PR for review (eval deltas + sample diffs). soup adapters pr "merge: 3-domain blend" \ --base-sha $(git rev-parse HEAD) \ @@ -52,6 +60,12 @@ soup adapters pr "merge: 3-domain blend" \ --dataset-diff data/diff.txt \ --format markdown -o pr.md +# 2b. Post it straight to a GitHub PR comment (v0.71.4). Auth via GITHUB_TOKEN +# / GH_TOKEN; uses `gh api` under the hood (no secret leaks to the child). +soup adapters pr "merge: 3-domain blend" \ + --base-sha $(git rev-parse HEAD) --adapter merged/ \ + --eval evals/deltas.json --push your-org/your-repo#42 + # 3. Lock a reproducible run state. Closure = sha(base + dataset + env). soup env lock # v0.64 — capture env hash soup lock write \ @@ -93,6 +107,16 @@ soup loop status # Run the daemon (foreground) soup loop watch --poll-interval 300 +# Pre-wired production stages (v0.71.4): real traces → DPO → eval-gate → canary, +# instead of the v0.58 no-op stubs. Opt in once via `loop init --pre-wired` or +# per-run via --pre-wired. Pack each iteration as a shareable Soup Can with +# Registry lineage via --pack-cans. +soup loop init registry://abc12 --eval evals/lock.json --baseline registry://prod --pre-wired +soup loop watch --pre-wired --pack-cans --poll-interval 300 + +# Unpack a recorded iteration's Soup Can for what-if analysis (v0.71.4) +soup loop replay iter-20260515T120000-abcdef01 --extract ./iter-dump + # Background subprocess (writes PID, no shell) soup loop watch --detach @@ -240,6 +264,14 @@ soup adapters blame ./run-v18 --dataset train.jsonl --layer q_proj.7 \ soup adapters branch v18 --config soup.yaml --base meta-llama/Llama-3.1-8B \ --dataset train.jsonl +# Link a branch into the Registry lineage DAG (v0.71.4) — shows as a +# `branches` node under the entry in `soup history`. +soup adapters branch v18 --config soup.yaml --base meta-llama/Llama-3.1-8B \ + --attach-to-registry reg_20260601_abc123 + +# Or derive a fresh snapshot's config + base straight from a Registry entry (v0.71.4) +soup adapters branch v18-from-reg --from-registry reg_20260601_abc123 + # Restore the snapshot's config (refuses if source SHA drifted) soup adapters checkout v18 --output soup.yaml @@ -264,7 +296,9 @@ soup adapters branches - Branch pointers live under `~/.soup/branches/` (override via `SOUP_BRANCHES_DIR`, constrained to `$HOME` / `$CWD` / `$TMPDIR`). - `soup adapters checkout` SHA-checks the source config — refuses to restore when the source has drifted from the snapshot, so reproducibility never silently lies. -**v0.66.0:** `soup adapters blame` is now LIVE — the v0.57 `NotImplementedError` stub (#171) is lifted via a DataInf-style influence-function approximation. Pass `--top-k 50` to control the reported top-influencer count; pass a real `probe_fn` (Python API) to feed real gradients, or use the default deterministic synthetic probe for offline planning. `MergeReport.verdict` remains the `UNKNOWN` stub (live canary eval in v0.57.1). +**v0.66.0:** `soup adapters blame` is now LIVE — the v0.57 `NotImplementedError` stub (#171) is lifted via a DataInf-style influence-function approximation. Pass `--top-k 50` to control the reported top-influencer count; pass a real `probe_fn` (Python API) to feed real gradients, or use the default deterministic synthetic probe for offline planning. + +**v0.71.4:** the merge verdict is now LIVE — `soup adapters merge … --canary suite.json` lifts the `MergeReport.verdict` `UNKNOWN` stub. A pre-scored `{"baseline_scores","candidate_scores"}` suite classifies the blend OK / MINOR / MAJOR with no model load; a `{"tasks":[...]}` suite uses an injectable scorer. `--strict-verdict` exits 2 on MAJOR. The backdoor-scan and license-conflict gates now run for **every** strategy, including `cmaes`. `soup adapters branch <name> --from-registry <id>` / `--attach-to-registry <id>` link training-env snapshots into the Registry lineage DAG (shown as a `branches` node in `soup history`). ## Soup Cans (Shareable Recipes) diff --git a/docs/commands.md b/docs/commands.md index cd1e5a2..8d192b5 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -109,12 +109,12 @@ soup cost --config soup.yaml --gpu H100 Estimate training cost for specific soup adapters list ./output/ Scan for LoRA adapters soup adapters info ./output/checkpoint-500/ Show adapter metadata soup adapters compare adapter1/ adapter2/ Compare two adapters -soup loop init <model> --eval <s> --baseline <b> Create .soup/loop.yaml (data flywheel) -soup loop status Counters + status (traces / pairs / runs / shipped) -soup loop watch [--detach] [--max-iter N] Harvest → train → gate → deploy daemon +soup loop init <model> --eval <s> --baseline <b> [--pre-wired] Create .soup/loop.yaml (data flywheel; --pre-wired = real stages) +soup loop status Counters + status + pre_wired flag +soup loop watch [--detach] [--max-iter N] [--pre-wired] [--pack-cans] Harvest → train → gate → deploy daemon (pre-wired stages + Soup Can packing) soup loop pause / soup loop resume Atomic status flip soup loop canary <adapter> --traffic 5% Promote canary + auto-rollback on MAJOR -soup loop replay [<iter-id>] Replay a recorded iteration manifest +soup loop replay [<iter-id>] [--extract <dir>] Replay / unpack a recorded iteration manifest soup serve --model m --adapters chat=./c code=./d Multi-adapter serving soup migrate --from llamafactory config.yaml Import config from LLaMA-Factory soup migrate --from axolotl config.yml Import config from Axolotl @@ -188,8 +188,11 @@ soup probe interference <losses.json> [--output o.json] Pairwise N×N adapter i soup probe pack <base> [--output o.json] Per-base calibrated probe pack manifest (v0.66.0) soup probe pack --list List bundled probe-pack bases (v0.66.0) soup adapters blame ... --top-k 50 Live DataInf-style influence runner (v0.66.0, closes #171) -soup adapters merge ... --strategy cmaes --eval <s> --budget 1h CMA-ES evolutionary merge (v0.67.0) +soup adapters merge ... --strategy cmaes --eval <s> --budget 1h CMA-ES evolutionary merge — live loop (v0.67.0 schema / v0.71.4 live) +soup adapters merge ... --canary <suite.json> [--strict-verdict] Live OK/MINOR/MAJOR canary verdict, exit 2 on MAJOR (v0.71.4) soup adapters pr <title> --base-sha <hex> --adapter <path> GitHub-shaped adapter PR Markdown / JSON (v0.67.0) +soup adapters pr <title> ... --push owner/repo#N Post the PR as a GitHub comment via gh api (v0.71.4) +soup adapters branch <name> --from-registry <id> | --attach-to-registry <id> Branch ↔ Registry lineage (v0.71.4) soup adapters bisect <ckpt>... --eval-command "..." Binary search over training history (v0.67.0) soup lock write --base-sha <h> --dataset-sha <h> --env-hash <h> Write soup.lock (v0.67.0) soup lock write --base-sha <h> --dataset-sha <h> --env-lock soup-env.lock Auto-derive --env-hash from soup-env.lock (v0.71.1) diff --git a/pyproject.toml b/pyproject.toml index 54dae3d..cc22aa0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.71.3" +version = "0.71.4" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" diff --git a/src/soup_cli/__init__.py b/src/soup_cli/__init__.py index 2352afb..f05ab25 100644 --- a/src/soup_cli/__init__.py +++ b/src/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.71.3" +__version__ = "0.71.4" diff --git a/src/soup_cli/commands/adapters.py b/src/soup_cli/commands/adapters.py index ff19608..a283721 100644 --- a/src/soup_cli/commands/adapters.py +++ b/src/soup_cli/commands/adapters.py @@ -382,6 +382,18 @@ def merge( "that cannot be scanned)." ), ), + canary: Optional[str] = typer.Option( + None, "--canary", + help=( + "Canary-suite JSON to compute a live OK/MINOR/MAJOR verdict for " + "the merged adapter vs the first input (v0.71.4 #172). Shape: " + '{"baseline_scores": [...], "candidate_scores": [...]}.' + ), + ), + strict_verdict: bool = typer.Option( + False, "--strict-verdict", + help="Exit 2 when the canary verdict is MAJOR (CI gate).", + ), ): """Merge LoRA adapters via linear / ties / dare / svd (v0.57.0, v0.60.0 license gate).""" from soup_cli.utils.adapter_merge import SUPPORTED_STRATEGIES, merge_adapters @@ -403,11 +415,11 @@ def merge( console.print("[red]Need at least 2 adapter paths to merge[/]") raise typer.Exit(2) - # v0.67.0 Part A: cmaes evolutionary search requires an eval suite + - # budget. Render the planned strategy and emit a deferred-live advisory. - # Operator-supplied eval_fn lives in `soup_cli.utils.cmaes_merge.run_cmaes_merge`; - # the CLI surface here ships the plan + validation. Live `soup eval` - # auto-wiring against a registry run is deferred to v0.67.1. + # cmaes argument validation runs FIRST (before any file I/O / scanning): + # a user who forgot --eval should get that error immediately, not a scan + # error on their inputs. The cmaes *search* itself still runs after the + # security gates below (v0.71.4 review — preserves the "reject missing + # --eval before file-handling" contract while gating the merge). if strategy == "cmaes": if eval_suite is None: console.print( @@ -415,41 +427,18 @@ def merge( "(path to a YAML/JSONL eval).[/]" ) raise typer.Exit(2) - from soup_cli.utils.cmaes_merge import build_cmaes_plan - - try: - cmaes_plan = build_cmaes_plan( - adapters=adapters, - eval_suite=eval_suite, - budget_spec=budget, - population_size=population, - max_generations=max_generations, - seed=seed, + if canary is not None or strict_verdict: + console.print( + "[red]--canary / --strict-verdict are not supported with " + "--strategy cmaes; the --eval suite already drives the search.[/]" ) - except (FileNotFoundError, TypeError, ValueError) as exc: - console.print(f"[red]Invalid cmaes plan: {escape(str(exc))}[/]") - raise typer.Exit(2) from exc - panel = Panel( - f"Strategy: [bold]cmaes[/]\n" - f"Adapters: {len(cmaes_plan.adapters)}\n" - f"Eval suite: [bold]{escape(os.path.basename(cmaes_plan.eval_suite))}[/]\n" - f"Budget: [bold]{cmaes_plan.budget_seconds}s[/]\n" - f"Population: [bold]{cmaes_plan.population_size}[/]\n" - f"Max generations: [bold]{cmaes_plan.max_generations}[/]\n" - f"Live cmaes loop with auto-wired eval suite is deferred to " - f"[yellow]v0.67.1[/]. The plan is validated and ready to " - f"hand to an operator-supplied `eval_fn` via " - f"`run_cmaes_merge(plan, eval_fn=...)`.", - title="Adapter merge — cmaes (plan)", - ) - console.print(panel) - return + raise typer.Exit(2) # v0.71.2 #192 — backdoor-scan gate. Refuse to merge any input whose # spectral scan returns FAIL (or that cannot be scanned at all) unless the - # operator passes --allow-unscanned. WARN is advisory-only. Runs AFTER the - # cmaes plan-only branch (which doesn't merge weights) so it gates only the - # strategies that actually write a merged adapter. + # operator passes --allow-unscanned. WARN is advisory-only. Runs BEFORE the + # strategy dispatch (incl. cmaes, which v0.71.4 #220 made weight-writing) so + # EVERY strategy that produces a merged adapter is gated uniformly. if not allow_unscanned: from soup_cli.utils.adapter_scan import scan_adapter @@ -481,7 +470,8 @@ def merge( # per adapter (--license, repeatable) OR rely on v0.71.2 #187 # auto-extraction from adapter_config.json / config.json / model-card # frontmatter. On a conflict, --license-override <reason> proceeds and the - # decision is recorded to the audit log (#190). + # decision is recorded to the audit log (#190). Runs before the strategy + # dispatch so cmaes (#220, weight-writing) is gated like every other merge. gate_licenses: Optional[list[str]] = None if license_ids: if len(license_ids) != len(adapters): @@ -539,6 +529,81 @@ def merge( f"[dim]Reason: {escape(cleared)} (recorded to audit log)[/]" ) + # v0.67.0 Part A: cmaes evolutionary search requires an eval suite + + # budget. v0.71.4 #220 lifts this from plan-only to a live loop: the eval + # suite is auto-wired into an eval_fn that materialises + scores each + # candidate merge, and the best-weighted merge is written to --output. + # (Arg validation already happened above, before the security gates.) + if strategy == "cmaes": + assert eval_suite is not None # narrowed by the early arg-check + from soup_cli.utils.cmaes_merge import ( + build_cmaes_eval_fn, + build_cmaes_plan, + run_cmaes_merge, + ) + + try: + cmaes_plan = build_cmaes_plan( + adapters=adapters, + eval_suite=eval_suite, + budget_spec=budget, + population_size=population, + max_generations=max_generations, + seed=seed, + ) + except (FileNotFoundError, TypeError, ValueError) as exc: + console.print(f"[red]Invalid cmaes plan: {escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + console.print( + f"[cyan]Running CMA-ES merge[/] " + f"(pop={cmaes_plan.population_size}, " + f"max_gen={cmaes_plan.max_generations}, " + f"budget={cmaes_plan.budget_seconds}s)..." + ) + console.print( + "[dim]The default scorer reloads the base model per candidate " + "(pop x generations loads); pass a small --population for large " + "models, or wire a cached scorer.[/]" + ) + try: + eval_fn = build_cmaes_eval_fn(cmaes_plan) + result = run_cmaes_merge(cmaes_plan, eval_fn=eval_fn) + except (FileNotFoundError, TypeError, ValueError, RuntimeError, OSError) as exc: + console.print(f"[red]cmaes merge failed: {escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + # Write the best-weighted merge to --output (linear with the winning + # simplex weights). + try: + report = merge_adapters( + adapters, output, strategy="linear", + weights=list(result.best_weights), + ) + except (FileNotFoundError, TypeError, ValueError, RuntimeError, OSError) as exc: + console.print(f"[red]Could not write merged adapter: {escape(str(exc))}[/]") + raise typer.Exit(2) from exc + + conv_color = "green" if result.converged else "yellow" + best_w = ", ".join(f"{w:.3f}" for w in result.best_weights) + console.print( + Panel( + f"Strategy: [bold]cmaes[/]\n" + f"Adapters: {len(cmaes_plan.adapters)}\n" + f"Eval suite: [bold]{escape(os.path.basename(cmaes_plan.eval_suite))}[/]\n" + f"Generations: [bold]{result.generations_run}[/] / " + f"{cmaes_plan.max_generations}\n" + f"Evaluations: [bold]{result.evaluations}[/]\n" + f"Best score: [bold]{result.best_score:.4f}[/]\n" + f"Best weights: [dim]{escape(best_w)}[/]\n" + f"Converged: [{conv_color}]{result.converged}[/]\n" + f"Merged layers: [bold]{report.merged_layers}[/]\n" + f"Output: [bold]{escape(report.output_dir)}[/]", + title="Adapter merge — cmaes", + ) + ) + return + parsed_weights = None if weights: try: @@ -564,17 +629,40 @@ def merge( console.print(f"[red]{escape(str(exc))}[/]") raise typer.Exit(2) from exc + # v0.71.4 #172 — live canary verdict. With --canary, score the merged + # adapter vs the first input and classify OK/MINOR/MAJOR; without it the + # verdict stays UNKNOWN (advisory). + verdict = report.verdict + verdict_suffix = " (pass --canary <suite.json> to compute)" + if canary is not None: + from soup_cli.utils.adapter_merge import predict_merged_verdict + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + try: + enforce_under_cwd_and_no_symlink(canary, "canary") + verdict = predict_merged_verdict(report, canary) + except (TypeError, ValueError) as exc: + console.print(f"[red]Canary verdict failed: {escape(str(exc))}[/]") + raise typer.Exit(2) from exc + verdict_suffix = "" + + verdict_color = {"OK": "green", "MINOR": "yellow", "MAJOR": "red"}.get( + verdict, "yellow" + ) panel = Panel( f"Strategy: [bold]{escape(report.strategy)}[/]\n" f"Inputs: {len(report.adapters)}\n" f"Merged layers: [bold]{report.merged_layers}[/]\n" f"Skipped layers: {len(report.skipped_layers)}\n" - f"Verdict: [yellow]{report.verdict}[/] (live eval in v0.57.1)\n" + f"Verdict: [{verdict_color}]{verdict}[/]{verdict_suffix}\n" f"Output: [bold]{escape(report.output_dir)}[/]", title="Adapter merge", ) console.print(panel) + if strict_verdict and verdict == "MAJOR": + raise typer.Exit(2) + @app.command() def blame( @@ -672,16 +760,62 @@ def blame( @app.command() def branch( name: str = typer.Argument(..., help="Branch name (alphanumeric + ._-)"), - config: str = typer.Option(..., "--config", "-c", help="Path to soup.yaml"), - base: str = typer.Option(..., "--base", help="Base model id"), + config: Optional[str] = typer.Option( + None, "--config", "-c", + help="Path to soup.yaml (required unless --from-registry)", + ), + base: Optional[str] = typer.Option( + None, "--base", help="Base model id (required unless --from-registry)", + ), dataset: str = typer.Option(None, "--dataset", help="Training dataset path (optional)"), + attach_to_registry: Optional[str] = typer.Option( + None, "--attach-to-registry", + help=( + "Registry entry id/ref to attach this branch pointer to as a " + "branch_ref artifact + link via registry_entry_id (v0.71.4)." + ), + ), + from_registry: Optional[str] = typer.Option( + None, "--from-registry", + help=( + "Derive config + base_model + dataset hash from a Registry entry " + "instead of --config/--base (v0.71.4)." + ), + ), ): - """Snapshot a training environment as a comparable branch (v0.57.0).""" - from soup_cli.utils.adapter_branch import create_branch + """Snapshot a training environment as a comparable branch (v0.57.0). + + Pass ``--from-registry <id>`` to derive everything from a v0.26 Registry + entry, or ``--attach-to-registry <id>`` to link a fresh snapshot into the + lineage DAG (v0.71.4 #173). + """ + from soup_cli.utils.adapter_branch import ( + attach_branch_to_registry, + branch_from_registry, + create_branch, + load_branch, + ) try: - snap = create_branch(name, config_path=config, base_model=base, - dataset_path=dataset) + if from_registry is not None: + snap = branch_from_registry(name, from_registry) + else: + if not config: + console.print( + "[red]--config is required (or use --from-registry)[/]" + ) + raise typer.Exit(2) + if not base: + console.print( + "[red]--base is required (or use --from-registry)[/]" + ) + raise typer.Exit(2) + snap = create_branch( + name, config_path=config, base_model=base, dataset_path=dataset, + ) + if attach_to_registry is not None: + attach_branch_to_registry(name, attach_to_registry) + snap = load_branch(name) except FileNotFoundError as exc: console.print(f"[red]{escape(str(exc))}[/]") raise typer.Exit(1) from exc @@ -689,6 +823,11 @@ def branch( console.print(f"[red]{escape(str(exc))}[/]") raise typer.Exit(2) from exc + reg_line = ( + f"\nRegistry: [dim]{escape(snap.registry_entry_id)}[/]" + if snap.registry_entry_id + else "" + ) console.print( Panel( f"Name: [bold]{escape(snap.name)}[/]\n" @@ -697,7 +836,7 @@ def branch( f"SHA: [dim]{snap.config_sha256[:16]}...[/]\n" f"Dataset SHA: [dim]" f"{snap.dataset_sha256[:16] + '...' if snap.dataset_sha256 else '—'}[/]\n" - f"Version: {snap.soup_version}", + f"Version: {snap.soup_version}{reg_line}", title="Adapter branch", ) ) @@ -1020,15 +1159,25 @@ def adapter_pr( format_: str = typer.Option( "markdown", "--format", "-f", help="markdown | json", ), + push: Optional[str] = typer.Option( + None, "--push", + help=( + "Post the rendered Markdown as a comment on a GitHub PR: " + "owner/repo#N (e.g. MakazhanAlpamys/Soup#42). Auth via " + "GITHUB_TOKEN / GH_TOKEN (v0.71.4)." + ), + ), ): """Render a GitHub-style PR for an adapter (v0.67.0 Part D). The PR = ``{base SHA, dataset diff, adapter file, eval report}`` rendered as a Markdown document with eval-delta tables and sample - diffs, ready to drop into a GitHub PR description or comment. + diffs, ready to drop into a GitHub PR description or comment. Pass + ``--push owner/repo#N`` to post it directly as a PR comment (v0.71.4). """ from soup_cli.utils.adapter_pr import ( build_adapter_pr, + post_pr_comment, render_pr_json, render_pr_markdown, write_pr_markdown, @@ -1040,11 +1189,16 @@ def adapter_pr( raise typer.Exit(2) # Load deltas + samples + dataset_diff lazily; each accepts None. + pr_input_cap = 8 * 1024 * 1024 # 8 MiB per --eval/--samples/--dataset-diff + def _load_json_list(path: Optional[str], field: str) -> list: if path is None: return [] enforce_under_cwd_and_no_symlink(path, field=field) - with open(os.path.realpath(path), encoding="utf-8") as fh: + real = os.path.realpath(path) + if os.path.getsize(real) > pr_input_cap: + raise ValueError(f"{field} exceeds 8 MiB cap") + with open(real, encoding="utf-8") as fh: raw = json.load(fh) if not isinstance(raw, list): raise ValueError(f"{field} must contain a JSON list") @@ -1058,9 +1212,10 @@ def adapter_pr( enforce_under_cwd_and_no_symlink( dataset_diff_path, field="dataset_diff" ) - with open( - os.path.realpath(dataset_diff_path), encoding="utf-8" - ) as fh: + real_diff = os.path.realpath(dataset_diff_path) + if os.path.getsize(real_diff) > pr_input_cap: + raise ValueError("dataset_diff exceeds 8 MiB cap") + with open(real_diff, encoding="utf-8") as fh: dataset_diff = fh.read() pr = build_adapter_pr( title=title, @@ -1079,6 +1234,26 @@ def adapter_pr( else: rendered = render_pr_markdown(pr) + # v0.71.4 #223 — publish the rendered Markdown as a GitHub PR comment. + # Always posts Markdown (a JSON blob is not a useful PR comment), even + # when --format json was chosen for the local render/write. + if push is not None: + try: + url = post_pr_comment(push, render_pr_markdown(pr)) + except RuntimeError as exc: + # Missing token / gh failure — user-actionable, exit 1. + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(1) from exc + except (TypeError, ValueError) as exc: + console.print(f"[red]{escape(str(exc))}[/]") + raise typer.Exit(2) from exc + console.print( + f"[green]Posted PR comment to {escape(push)}[/]" + + (f" -> {escape(url)}" if url else "") + ) + if output is None: + return + if output is None: console.print(rendered) return diff --git a/src/soup_cli/commands/history.py b/src/soup_cli/commands/history.py index 77249b7..7fa78dd 100644 --- a/src/soup_cli/commands/history.py +++ b/src/soup_cli/commands/history.py @@ -2,6 +2,8 @@ from __future__ import annotations +import os + import typer from rich.console import Console from rich.markup import escape @@ -68,6 +70,18 @@ def history( f"[cyan]{escape(desc['id'])}[/] " f"[yellow]({escape(desc.get('relation', ''))})[/]" ) + # v0.71.4 #173 — surface attached branch pointers as a distinct + # lineage edge so `soup adapters branch --attach-to-registry` + # snapshots show up in the DAG view. + branch_refs = [ + art for art in store.get_artifacts(entry["id"]) + if art.get("kind") == "branch_ref" + ] + if branch_refs: + br_node = node.add("[dim]branches[/]") + for art in branch_refs: + label = os.path.basename(art.get("path", "")) or "branch" + br_node.add(f"[green]branch[/] [cyan]{escape(label)}[/]") console.print(tree) diff --git a/src/soup_cli/commands/loop.py b/src/soup_cli/commands/loop.py index 995d94f..ff605f7 100644 --- a/src/soup_cli/commands/loop.py +++ b/src/soup_cli/commands/loop.py @@ -74,6 +74,13 @@ def init_cmd( max_runs_per_day: Optional[int] = typer.Option( None, "--max-runs-per-day", help="Cap on iteration starts per UTC day." ), + pre_wired: bool = typer.Option( + False, "--pre-wired", + help=( + "Use the pre-wired harvest/train/gate/deploy stages " + "(utils/loop_stages.py) on watch instead of no-op stubs (v0.71.4)." + ), + ), force: bool = typer.Option(False, "--force", help="Overwrite existing loop.yaml."), ) -> None: """Create the .soup/loop.yaml control file (one-time setup).""" @@ -96,6 +103,7 @@ def init_cmd( baseline=baseline, monthly_budget_usd=budget_usd, max_runs_per_day=max_runs_per_day, + pre_wired=pre_wired, force=force, ) except (FileExistsError, FileNotFoundError, TypeError, ValueError) as exc: @@ -123,6 +131,7 @@ def status_cmd() -> None: table.add_row("served_model", escape(state.served_model)) table.add_row("eval_suite", escape(state.eval_suite)) table.add_row("baseline", escape(state.baseline)) + table.add_row("pre_wired", "yes" if state.pre_wired else "no") table.add_row("traces_collected", str(state.traces_collected)) table.add_row("pairs_distilled", str(state.pairs_distilled)) table.add_row("runs_gated", str(state.runs_gated)) @@ -191,12 +200,20 @@ def watch_cmd( poll_interval: float = typer.Option( 60.0, "--poll-interval", help="Seconds between iterations [1, 3600]." ), + pre_wired: bool = typer.Option( + False, "--pre-wired", + help="Force the pre-wired production stages even if loop.yaml didn't set it.", + ), + pack_cans: bool = typer.Option( + False, "--pack-cans", + help="Pack each iteration as a v0.26 Soup Can + Registry entry (v0.71.4).", + ), ) -> None: """Run the harvest → train → gate → deploy daemon.""" if detach and foreground: console.print("[red]--detach and --foreground are mutually exclusive[/]") raise typer.Exit(code=2) - _ = _safe_read() # ensure state exists before forking + state = _safe_read() # ensure state exists before forking if detach: argv = [ sys.executable, @@ -210,6 +227,10 @@ def watch_cmd( ] if max_iterations is not None: argv.extend(["--max-iterations", str(max_iterations)]) + if pre_wired: + argv.append("--pre-wired") + if pack_cans: + argv.append("--pack-cans") proc = subprocess.Popen( # noqa: S603 — argv is internal, no shell argv, stdout=subprocess.DEVNULL, @@ -218,11 +239,31 @@ def watch_cmd( ) console.print(f"[green]watch detached[/] pid={proc.pid}") return + use_prewired = pre_wired or state.pre_wired + base_model = ( + state.served_model + if state.served_model and not state.served_model.startswith("registry://") + else "unknown" + ) try: - cfg = WatchConfig( - poll_interval_sec=float(poll_interval), - max_iterations=max_iterations, - ) + if use_prewired: + from soup_cli.utils.loop_stages import build_prewired_watch_config + + cfg = build_prewired_watch_config( + poll_interval_sec=float(poll_interval), + max_iterations=max_iterations, + pack_iterations=pack_cans, + served_model=state.served_model, + base_model=base_model, + ) + else: + cfg = WatchConfig( + poll_interval_sec=float(poll_interval), + max_iterations=max_iterations, + pack_iterations=pack_cans, + served_model=state.served_model, + base_model=base_model, + ) except (TypeError, ValueError) as exc: console.print(f"[red]invalid watch config:[/] {escape(str(exc))}") raise typer.Exit(code=2) @@ -281,6 +322,10 @@ def replay_cmd( iteration_id: Optional[str] = typer.Argument( None, help="Iteration id (omit to list all)." ), + extract: Optional[str] = typer.Option( + None, "--extract", + help="Extract the iteration's .can to this directory for a what-if re-run (v0.71.4).", + ), ) -> None: """Replay a recorded loop iteration manifest.""" if iteration_id is None: @@ -295,6 +340,29 @@ def replay_cmd( except (FileNotFoundError, TypeError, ValueError) as exc: console.print(f"[red]replay failed:[/] {escape(str(exc))}") raise typer.Exit(code=2) + + if extract is not None: + from soup_cli.cans.unpack import extract_can + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + can_path = os.path.join(".soup-loops", iteration_id, "iteration.can") + if not os.path.isfile(can_path): + console.print( + f"[red]no .can for {escape(iteration_id)} " + "(was it run with `watch --pack-cans`?)[/]" + ) + raise typer.Exit(code=1) + try: + enforce_under_cwd_and_no_symlink(extract, "extract") + dest = extract_can(can_path, extract) + except (FileNotFoundError, ValueError, TypeError) as exc: + console.print(f"[red]extract failed:[/] {escape(str(exc))}") + raise typer.Exit(code=2) + console.print( + f"[green]extracted {escape(iteration_id)} -> {escape(str(dest))}[/]" + ) + return + table = Table(title=f"replay {escape(record.iteration_id)}", show_header=False) table.add_column("field", style="bold") table.add_column("value") diff --git a/src/soup_cli/registry/store.py b/src/soup_cli/registry/store.py index d279437..f328e8a 100644 --- a/src/soup_cli/registry/store.py +++ b/src/soup_cli/registry/store.py @@ -45,6 +45,8 @@ _VALID_KINDS = frozenset( "steering_vector", # v0.71.1 #214 — pairwise-judge calibration reports. "judge_calibration", + # v0.71.4 #173 — adapter branch-pointer JSON files. + "branch_ref", } ) _VALID_RELATIONS = frozenset( diff --git a/src/soup_cli/utils/adapter_branch.py b/src/soup_cli/utils/adapter_branch.py index a0648b2..714aff4 100644 --- a/src/soup_cli/utils/adapter_branch.py +++ b/src/soup_cli/utils/adapter_branch.py @@ -27,7 +27,7 @@ import re import stat import tempfile import time -from dataclasses import asdict, dataclass +from dataclasses import asdict, dataclass, replace from pathlib import Path from typing import Optional, Tuple @@ -48,6 +48,10 @@ class Branch: base_model: str created_at: float soup_version: str + # v0.71.4 #173 — optional link to a v0.26 Registry entry so a branch + # pointer participates in the lineage DAG. ``None`` preserves back-compat + # with v0.57.0 pointers that have no Registry edge. + registry_entry_id: Optional[str] = None def _validate_name(name: object) -> str: @@ -118,17 +122,30 @@ def create_branch( config_path: str, base_model: str, dataset_path: Optional[str] = None, + registry_entry_id: Optional[str] = None, + dataset_sha256: Optional[str] = None, ) -> Branch: """Snapshot a training environment as an immutable branch pointer. The config file is hashed (SHA-256) and the dataset file (if provided) is hashed too so callers can detect "same config, different data" drift. + + ``registry_entry_id`` (v0.71.4 #173) optionally links the pointer to a + v0.26 Registry entry. ``dataset_sha256`` lets a caller (e.g. + ``branch_from_registry``) record a known dataset hash directly instead + of re-hashing a file — mutually exclusive with ``dataset_path``. """ from soup_cli import __version__ name = _validate_name(name) enforce_under_cwd_and_no_symlink(config_path, "config_path") _validate_str_field(base_model, "base_model", max_len=512) + if registry_entry_id is not None: + registry_entry_id = _validate_str_field( + registry_entry_id, "registry_entry_id", max_len=128 + ) + if dataset_path is not None and dataset_sha256 is not None: + raise ValueError("pass dataset_path OR dataset_sha256, not both") config_full = Path(config_path) if not config_full.is_file(): @@ -147,6 +164,10 @@ def create_branch( if not ds_full.is_file(): raise FileNotFoundError(f"dataset not found: {ds_full.name}") dataset_sha = _hash_file(ds_full) + elif dataset_sha256 is not None: + dataset_sha = _validate_str_field( + dataset_sha256, "dataset_sha256", max_len=128 + ) branches_dir = _branches_dir() existing = sorted(p for p in branches_dir.glob("*.json")) @@ -161,6 +182,7 @@ def create_branch( base_model=base_model, created_at=time.time(), soup_version=__version__, + registry_entry_id=registry_entry_id, ) _atomic_write_branch(branches_dir, branch) return branch @@ -220,6 +242,14 @@ def load_branch(name: str) -> Branch: base_model=_validate_str_field(raw["base_model"], "base_model"), created_at=float(raw["created_at"]), soup_version=_validate_str_field(raw["soup_version"], "soup_version", max_len=64), + # ``.get`` so v0.57.0 pointers (no registry_entry_id key) load cleanly. + registry_entry_id=( + _validate_str_field( + raw["registry_entry_id"], "registry_entry_id", max_len=128 + ) + if raw.get("registry_entry_id") is not None + else None + ), ) @@ -238,6 +268,94 @@ def delete_branch(name: str) -> bool: return True +def branch_pointer_path(name: str) -> Path: + """Return the on-disk JSON pointer path for a branch (must exist).""" + name = _validate_name(name) + target = _branches_dir() / f"{name}.json" + if not target.is_file(): + raise FileNotFoundError(f"branch not found: {name}") + return target + + +def attach_branch_to_registry(name: str, registry_ref: str) -> int: + """Attach a branch pointer to a v0.26 Registry entry as ``branch_ref``. + + Resolves ``registry_ref`` (id / prefix / name:tag), attaches the branch + JSON file as a ``branch_ref`` artifact (``enforce_cwd=False`` since the + pointer lives in ``~/.soup/branches`` outside cwd — a library-produced + path), then re-writes the pointer with ``registry_entry_id`` set so the + edge is recorded on both sides. Returns the artifact rowid. + """ + from soup_cli.registry.store import RegistryStore + + branch = load_branch(name) + pointer = branch_pointer_path(name) + branches_dir = _branches_dir() + with RegistryStore() as store: + resolved = store.resolve(registry_ref) + if resolved is None: + raise ValueError(f"registry entry not found: {registry_ref}") + rowid = store.add_artifact( + entry_id=resolved, + kind="branch_ref", + path=str(pointer), + enforce_cwd=False, + ) + # Re-write the pointer with the resolved id so the edge is bidirectional. + _atomic_write_branch(branches_dir, replace(branch, registry_entry_id=resolved)) + return rowid + + +def branch_from_registry( + name: str, + registry_ref: str, + *, + config_out: Optional[str] = None, +) -> Branch: + """Create a branch by deriving config + base_model + dataset from a Registry entry. + + Materialises the entry's stored ``config_json`` to a cwd file (default + ``<name>.from-registry.yaml``) so the snapshot points at a real config, + derives ``base_model`` from the entry, and records the entry's + ``data_hash`` as the branch's ``dataset_sha256`` (the registry keeps the + file hash, not a path). The branch is linked back via ``registry_entry_id``. + """ + import yaml + + from soup_cli.registry.store import RegistryStore + + name = _validate_name(name) + with RegistryStore() as store: + resolved = store.resolve(registry_ref) + if resolved is None: + raise ValueError(f"registry entry not found: {registry_ref}") + entry = store.get(resolved) + if entry is None: + raise ValueError(f"registry entry not found: {registry_ref}") + base_model = entry.get("base_model") or "unknown" + data_hash = entry.get("data_hash") + try: + config = json.loads(entry.get("config_json") or "{}") + except (TypeError, ValueError): + config = {} + + out_name = config_out if config_out is not None else f"{name}.from-registry.yaml" + enforce_under_cwd_and_no_symlink(out_name, "config_out") + out_path = Path(out_name) + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text( + yaml.safe_dump(config, sort_keys=False), encoding="utf-8" + ) + + return create_branch( + name, + config_path=str(out_path), + base_model=base_model, + registry_entry_id=resolved, + dataset_sha256=data_hash if data_hash else None, + ) + + def write_checkout(branch: Branch, target_path: str) -> Path: """Copy the snapshotted config into cwd so the user can re-run.""" if not isinstance(branch, Branch): diff --git a/src/soup_cli/utils/adapter_merge.py b/src/soup_cli/utils/adapter_merge.py index 8903123..5860a12 100644 --- a/src/soup_cli/utils/adapter_merge.py +++ b/src/soup_cli/utils/adapter_merge.py @@ -21,11 +21,14 @@ import stat import tempfile from dataclasses import dataclass from pathlib import Path -from typing import Any, FrozenSet, Literal, Mapping, Sequence, Tuple +from typing import Any, Callable, FrozenSet, Literal, Mapping, Optional, Sequence, Tuple from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink MergeStrategy = Literal["linear", "ties", "dare", "svd", "cmaes"] +# A canary scorer: ``scorer(role, tasks) -> per-prompt scores`` where ``role`` +# is "baseline" / "candidate" and ``tasks`` is the parsed canary task list. +CanaryScorer = Callable[[str, Sequence[Mapping[str, Any]]], Sequence[float]] # frozenset for O(1) membership + immutability (mirrors v0.41.0 / v0.51.0 policy); # tuple alias preserved for caller code that iterates in canonical order. # v0.67.0 Part A: added "cmaes" — evolutionary search dispatched separately @@ -275,6 +278,15 @@ def merge_adapters( raise ValueError( f"strategy must be one of {SUPPORTED_STRATEGIES}, got {strategy!r}" ) + if strategy == "cmaes": + # cmaes is an evolutionary *search* over linear weights, not a + # one-shot tensor merge — routing it through this function would + # silently fall into the svd branch. Fail loudly with the real path. + raise ValueError( + "cmaes is an evolutionary strategy; use run_cmaes_merge / " + "`soup adapters merge --strategy cmaes --eval ...`, " + "not merge_adapters(strategy='cmaes')" + ) if not isinstance(adapter_paths, Sequence) or isinstance(adapter_paths, str): raise TypeError("adapter_paths must be a sequence of strings") if len(adapter_paths) < _MIN_ADAPTERS: @@ -300,7 +312,7 @@ def merge_adapters( else: # svd merged, skipped = merge_svd(weights_list, coeffs, rank=rank) - _write_merged_adapter(output_dir, adapter_paths[0], merged) + write_merged_adapter(output_dir, adapter_paths[0], merged) return MergeReport( strategy=strategy, @@ -341,7 +353,7 @@ def _atomic_write_bytes(target: Path, data: bytes) -> None: raise -def _write_merged_adapter( +def write_merged_adapter( output_dir: str, template_source: str, weights: Mapping[str, Any], @@ -402,17 +414,143 @@ def _write_merged_adapter( ) +# Back-compat alias: ``_write_merged_adapter`` was private through v0.71.4; +# promoted to the public ``write_merged_adapter`` so cmaes_merge imports a +# public name (review MEDIUM-3). Keep the old name for any external caller. +_write_merged_adapter = write_merged_adapter + + +_MAX_CANARY_BYTES = 16 * 1024 * 1024 # 16 MiB cap on the canary-suite JSON +_VERDICT_MINOR = 0.02 +_VERDICT_MAJOR = 0.05 + + +def _classify_drop(delta: float) -> str: + """OK / MINOR / MAJOR per the v0.26.0 Quant-Lobotomy taxonomy. + + ``delta`` is ``candidate_mean - baseline_mean``: positive (improvement) + or a small drop is OK, a 2-5 % drop is MINOR, a >5 % drop is MAJOR. + """ + if delta >= 0: + return "OK" + drop = -delta + if drop < _VERDICT_MINOR: + return "OK" + if drop < _VERDICT_MAJOR: + return "MINOR" + return "MAJOR" + + +def _require_score_list(value: Any, field: str) -> list[float]: + if not isinstance(value, list) or isinstance(value, (str, bytes)): + raise ValueError(f"{field} must be a JSON list of numbers") + out: list[float] = [] + for v in value: + if isinstance(v, bool) or not isinstance(v, (int, float)): + raise ValueError(f"{field} entries must be numbers") + vf = float(v) + if not math.isfinite(vf): + raise ValueError(f"{field} entries must be finite") + out.append(vf) + if not out: + raise ValueError(f"{field} must be non-empty") + return out + + +def _load_canary_scores( + canary_suite: str, + scorer: Optional[CanaryScorer], +) -> Tuple[list[float], list[float]]: + """Resolve baseline + candidate per-prompt scores from a canary suite. + + Two supported shapes: + + - ``{"baseline_scores": [...], "candidate_scores": [...]}`` — pre-scored + (no model load; the no-GPU workflow). Operators run ``soup eval custom`` + against the baseline and merged adapters, then assemble the two arrays. + - ``{"tasks": [{"prompt", "expected"}, ...]}`` — requires an injectable + ``scorer(role, tasks) -> list[float]`` (the live path). + """ + # Defence-in-depth cwd containment + symlink rejection at the read site so + # the public ``predict_merged_verdict`` entry point is safe even when a + # non-CLI caller skips the CLI-boundary check (mirrors every other read + # surface in this changeset). + enforce_under_cwd_and_no_symlink(canary_suite, "canary_suite") + path = Path(canary_suite) + if not path.is_file(): + raise ValueError(f"canary suite not found: {path.name}") + # Open ONCE with O_NOFOLLOW and enforce the size cap on the same fd + # (os.fstat), so the symlink-rejection + 16 MiB cap cannot be defeated by a + # local file swap between the lstat/stat and the read (TOCTOU) — matches the + # O_NOFOLLOW+fstat read pattern used elsewhere in the codebase. + try: + fd = os.open(str(path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + except OSError as exc: + raise ValueError(f"canary suite unreadable: {type(exc).__name__}") from exc + fh = None + try: + if os.fstat(fd).st_size > _MAX_CANARY_BYTES: + raise ValueError("canary suite exceeds 16 MiB cap") + fh = os.fdopen(fd, "r", encoding="utf-8") + raw = fh.read() + finally: + if fh is not None: + fh.close() + else: + os.close(fd) + try: + data = json.loads(raw) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"canary suite is not valid JSON: {exc}") from exc + if not isinstance(data, dict): + raise ValueError("canary suite root must be a JSON object") + + if "baseline_scores" in data and "candidate_scores" in data: + baseline = _require_score_list(data["baseline_scores"], "baseline_scores") + candidate = _require_score_list(data["candidate_scores"], "candidate_scores") + else: + tasks = data.get("tasks") + if not isinstance(tasks, list) or not tasks: + raise ValueError( + "canary suite must contain 'baseline_scores'+'candidate_scores' " + "or a non-empty 'tasks' list" + ) + if scorer is None or not callable(scorer): + raise ValueError( + "canary suite has 'tasks' but no scorer; supply pre-scored " + "'baseline_scores'/'candidate_scores' arrays or pass scorer=" + ) + baseline = _require_score_list(scorer("baseline", tasks), "baseline scorer output") + candidate = _require_score_list(scorer("candidate", tasks), "candidate scorer output") + + if len(baseline) != len(candidate): + raise ValueError( + f"baseline ({len(baseline)}) and candidate ({len(candidate)}) " + "score counts must match" + ) + return baseline, candidate + + def predict_merged_verdict( report: MergeReport, canary_suite: str | None = None, + *, + scorer: Optional[CanaryScorer] = None, ) -> str: - """Stub for v0.57.1: live canary-eval verdict via v0.55 gate. + """Live canary verdict for a merged adapter (v0.71.4 #172). - Today returns the existing report.verdict. The signature is forward-compatible - so v0.57.1 can lift this to a real OK/MINOR/MAJOR classification. + Returns the merged adapter's OK / MINOR / MAJOR verdict by comparing + per-prompt canary scores against the first input adapter (the "baseline" + by convention) using the v0.26.0 Quant-Lobotomy thresholds. When + ``canary_suite`` is ``None`` the existing ``report.verdict`` (``UNKNOWN`` + for a fresh merge) is returned unchanged — back-compat with v0.57.0. """ if not isinstance(report, MergeReport): raise TypeError("report must be MergeReport") if canary_suite is not None and not isinstance(canary_suite, str): raise TypeError("canary_suite must be str or None") - return report.verdict + if canary_suite is None: + return report.verdict + baseline, candidate = _load_canary_scores(canary_suite, scorer) + delta = (sum(candidate) / len(candidate)) - (sum(baseline) / len(baseline)) + return _classify_drop(delta) diff --git a/src/soup_cli/utils/adapter_pr.py b/src/soup_cli/utils/adapter_pr.py index 69ade86..0a79784 100644 --- a/src/soup_cli/utils/adapter_pr.py +++ b/src/soup_cli/utils/adapter_pr.py @@ -19,9 +19,10 @@ from __future__ import annotations import json import math +import os import re from dataclasses import asdict, dataclass -from typing import Any, Iterable, Mapping, Tuple +from typing import Any, Callable, Iterable, Mapping, Optional, Tuple from soup_cli.utils.paths import atomic_write_text @@ -343,3 +344,152 @@ def write_pr_markdown(pr: AdapterPR, path: str) -> str: raise TypeError("pr must be AdapterPR") text = render_pr_markdown(pr) return atomic_write_text(text, path, field="pr markdown path") + + +# --------------------------------------------------------------------------- +# GitHub PR publisher (v0.71.4 #223) +# --------------------------------------------------------------------------- + +# owner/repo#<number> — owner + repo are GitHub name-safe (alnum + ._-), +# number is a positive integer. +_PR_TARGET_RE = re.compile( + r"^([A-Za-z0-9][A-Za-z0-9._-]*)/([A-Za-z0-9][A-Za-z0-9._-]*)#([0-9]+)$" +) +_MAX_PR_BODY_BYTES = 60_000 # GitHub caps issue-comment bodies at 65_536 bytes + +# Env keys passed through to the `gh` child — everything else (HF_TOKEN / +# OPENAI_API_KEY / ANTHROPIC_API_KEY / ...) is filtered out so a publish call +# never leaks unrelated secrets to the subprocess (v0.71.4 review HIGH fix, +# mirrors v0.44.0 _LLAMA_ENV_ALLOWLIST). +_GH_ENV_ALLOWLIST = frozenset( + { + "PATH", "HOME", "USERPROFILE", "APPDATA", "LOCALAPPDATA", + "SYSTEMROOT", "SystemRoot", "TEMP", "TMP", "TMPDIR", + "GH_TOKEN", "GITHUB_TOKEN", "GH_ENTERPRISE_TOKEN", "GITHUB_API_URL", + "GH_HOST", "GH_CONFIG_DIR", + "XDG_CONFIG_HOME", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy", + } +) + + +def parse_pr_target(target: object) -> Tuple[str, str, int]: + """Parse ``"owner/repo#42"`` into ``(owner, repo, pr_number)``. + + Raises ``TypeError`` for non-strings and ``ValueError`` for any string + that does not match the ``owner/repo#<positive-int>`` shape. + """ + if isinstance(target, bool) or not isinstance(target, str): + raise TypeError("target must be str") + match = _PR_TARGET_RE.match(target.strip()) + if not match: + raise ValueError( + "target must be 'owner/repo#<number>' " + "(e.g. MakazhanAlpamys/Soup#42)" + ) + num = int(match.group(3)) + if num < 1: + raise ValueError("PR number must be >= 1") + return match.group(1), match.group(2), num + + +def resolve_github_token(env: Optional[Mapping[str, str]] = None) -> str: + """Resolve a GitHub token from ``GITHUB_TOKEN`` / ``GH_TOKEN`` env. + + Mirrors the v0.29.0 HF token-resolution policy: env only, first-match + wins, blank values treated as missing. Raises ``RuntimeError`` (a + user-actionable error the CLI renders) when neither is set. + """ + source = env if env is not None else os.environ + for key in ("GITHUB_TOKEN", "GH_TOKEN"): + val = source.get(key) + if val and val.strip(): + return val.strip() + raise RuntimeError( + "no GitHub token found; set GITHUB_TOKEN (or GH_TOKEN) " + "to publish a PR comment" + ) + + +def post_pr_comment( + target: str, + body: str, + *, + env: Optional[Mapping[str, str]] = None, + runner: Optional[Callable[..., Any]] = None, +) -> str: + """Post ``body`` as a comment on the GitHub PR named by ``target``. + + Uses ``gh api`` (no PyGithub dependency) with the body sent over + JSON stdin so multiline / markdown content is never shell-interpolated. + Auth resolves via ``GITHUB_TOKEN`` / ``GH_TOKEN`` (gh reads the same + vars). Returns the created comment's ``html_url`` (best-effort, may be + empty). ``runner`` is an injectable ``subprocess.run`` for testing. + """ + owner, repo, num = parse_pr_target(target) + if not isinstance(body, str): + raise TypeError("body must be str") + if not body.strip(): + raise ValueError("body must be non-empty") + if "\x00" in body: + raise ValueError("body must not contain null bytes") + if len(body.encode("utf-8")) > _MAX_PR_BODY_BYTES: + raise ValueError( + f"body exceeds {_MAX_PR_BODY_BYTES} byte GitHub comment cap" + ) + # Fail fast if no token before spawning the subprocess. + token = resolve_github_token(env) + + import subprocess # noqa: S404 — argv list mode, no shell + + argv = [ + "gh", + "api", + "--method", + "POST", + f"repos/{owner}/{repo}/issues/{num}/comments", + "--input", + "-", + ] + stdin = json.dumps({"body": body}) + # gh resolves GH_TOKEN / GITHUB_TOKEN from its own environment; thread + # the resolved token through. When ``env`` is supplied (tests / explicit) + # use it verbatim; otherwise build a MINIMAL env from an allowlist so we + # never leak HF_TOKEN / OPENAI_API_KEY / ANTHROPIC_API_KEY etc. into the + # gh child (mirrors the v0.44.0 _LLAMA_ENV_ALLOWLIST policy). + if env is not None: + base_env = dict(env) + else: + base_env = { + k: v for k, v in os.environ.items() if k in _GH_ENV_ALLOWLIST + } + base_env.setdefault("GH_TOKEN", token) + run = runner if runner is not None else subprocess.run + try: + result = run( + argv, + input=stdin, + capture_output=True, + text=True, + timeout=30, + check=False, + env=base_env, + ) + except FileNotFoundError as exc: + raise RuntimeError( + "`gh` CLI not found; install GitHub CLI or drop --push " + "and use --output to write the Markdown" + ) from exc + except subprocess.TimeoutExpired as exc: + raise RuntimeError("gh api timed out posting the PR comment") from exc + + if getattr(result, "returncode", 1) != 0: + stderr = (getattr(result, "stderr", "") or "").strip()[:512] + raise RuntimeError( + f"gh api failed (rc={result.returncode}): {stderr or 'no detail'}" + ) + try: + data = json.loads(getattr(result, "stdout", "") or "{}") + return str(data.get("html_url") or "") + except (ValueError, TypeError): + return "" diff --git a/src/soup_cli/utils/cmaes_merge.py b/src/soup_cli/utils/cmaes_merge.py index 170f6b9..dfe9b57 100644 --- a/src/soup_cli/utils/cmaes_merge.py +++ b/src/soup_cli/utils/cmaes_merge.py @@ -36,6 +36,8 @@ from __future__ import annotations import math import os +import shutil +import tempfile import time from dataclasses import dataclass from typing import Callable, Optional, Sequence, Tuple @@ -43,6 +45,12 @@ from typing import Callable, Optional, Sequence, Tuple from soup_cli.utils.blame import parse_budget from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink +# Test / smoke escape hatch (mirrors v0.53.1 #109 deploy_measure pattern): +# when set, build_cmaes_eval_fn uses this scorer instead of loading a model. +# Production callers leave it None and either pass scorer= explicitly or let +# the default lazy model-loading scorer run. +_CMAES_SCORER_OVERRIDE: Optional[Callable[[str, str], float]] = None + # --------------------------------------------------------------------------- # Bounds (closed, locked at module load) # --------------------------------------------------------------------------- @@ -56,6 +64,7 @@ _MIN_ADAPTERS = 2 _MAX_ADAPTERS = 16 # mirrors v0.57.0 adapter_merge cap _SIMPLEX_TOL = 1e-6 _FAILED_EVAL_SENTINEL = -1.0e9 # very negative so failed candidates never win +_MAX_ADAPTER_CONFIG_BYTES = 256 * 1024 # mirrors adapter_merge config-read cap # --------------------------------------------------------------------------- @@ -422,6 +431,146 @@ def run_cmaes_merge( ) +# --------------------------------------------------------------------------- +# Live eval-suite auto-wiring (v0.71.4 #220) +# --------------------------------------------------------------------------- + + +def _load_adapter_generator(merged_dir: str) -> Callable[[str], str]: + """Build a ``generate_fn(prompt) -> str`` from a merged LoRA adapter. + + Loads the base model named in the adapter's ``adapter_config.json`` and + applies the merged adapter via PEFT. Heavy imports stay inside the + function (project lazy-import policy + the cmaes_merge no-top-level-torch + grep guard). + """ + import json + import os + import stat + from pathlib import Path + + from peft import PeftModel + from transformers import AutoModelForCausalLM, AutoTokenizer + + # On the wired cmaes path ``merged_dir`` is always a Soup-written mkdtemp, + # but this is a public helper — guard the config read with a symlink + # rejection + size cap so a caller-supplied dir can't smuggle a symlink or + # a multi-GB JSON (defence-in-depth, parity with every other config read). + cfg_path = Path(merged_dir) / "adapter_config.json" + try: + cst = os.lstat(cfg_path) + except OSError as exc: + raise ValueError( + f"merged adapter_config.json unreadable: {type(exc).__name__}" + ) from exc + if stat.S_ISLNK(cst.st_mode): + raise ValueError("merged adapter_config.json must not be a symlink") + if cst.st_size > _MAX_ADAPTER_CONFIG_BYTES: + raise ValueError("merged adapter_config.json exceeds size cap") + cfg = json.loads(cfg_path.read_text(encoding="utf-8")) + base = cfg.get("base_model_name_or_path") + if not base: + raise ValueError( + "merged adapter_config.json has no base_model_name_or_path" + ) + tokenizer = AutoTokenizer.from_pretrained(base, trust_remote_code=False) + model = AutoModelForCausalLM.from_pretrained(base, trust_remote_code=False) + model = PeftModel.from_pretrained(model, merged_dir) + model.eval() + + def _gen(prompt: str) -> str: + if not prompt: + return "" + inputs = tokenizer(prompt, return_tensors="pt", truncation=True) + outputs = model.generate( + **inputs, + max_new_tokens=64, + do_sample=False, + pad_token_id=tokenizer.eos_token_id, + ) + new_tokens = outputs[0][inputs["input_ids"].shape[1]:] + return tokenizer.decode(new_tokens, skip_special_tokens=True) + + return _gen + + +def _default_cmaes_scorer(merged_dir: str, eval_suite: str) -> float: + """Score a merged adapter against an eval suite (mean score in [0, 1]). + + The live default — loads the base + merged adapter and runs the + v0.25.0 ``soup_cli.eval.custom`` scorers. Slow (one model load); use + the ``scorer=`` injection / ``_CMAES_SCORER_OVERRIDE`` escape hatch for + fast offline runs. + """ + from soup_cli.eval.custom import load_eval_tasks, score_task + + generate_fn = _load_adapter_generator(merged_dir) + tasks = load_eval_tasks(eval_suite) + if not tasks: + return 0.0 + total = 0.0 + for task in tasks: + output = generate_fn(task.prompt) + total += float(score_task(task, output).score) + return total / len(tasks) + + +def _resolve_cmaes_scorer( + scorer: Optional[Callable[[str, str], float]], +) -> Callable[[str, str], float]: + if scorer is not None: + if not callable(scorer): + raise TypeError("scorer must be callable") + return scorer + if _CMAES_SCORER_OVERRIDE is not None: + return _CMAES_SCORER_OVERRIDE + return _default_cmaes_scorer + + +def build_cmaes_eval_fn( + plan: CmaesPlan, + *, + scorer: Optional[Callable[[str, str], float]] = None, +) -> Callable[[Tuple[float, ...]], float]: + """Return an ``eval_fn(weights) -> float`` for :func:`run_cmaes_merge`. + + The closure linearly merges ``plan.adapters`` with the candidate weights, + materialises the merged LoRA to a temp dir, scores it via ``scorer`` + (default: live model-loading scorer), and cleans up. The *adapter* weights + are loaded from disk once up front so per-generation cost is merge + score. + + Perf note: the default :func:`_default_cmaes_scorer` reloads the *base + model* into a fresh PEFT wrapper on every candidate evaluation (it is + stateless by design). For ``population × max_generations`` candidates that + is many base-model loads — expensive for non-tiny models. Pass a custom + ``scorer=`` that caches the base model (or set ``_CMAES_SCORER_OVERRIDE``) + to amortise it; the injection also avoids any model load for tests / smokes. + + ``scorer(merged_dir, eval_suite) -> float`` returns a mean score in + ``[0, 1]``. + """ + if not isinstance(plan, CmaesPlan): + raise TypeError("plan must be CmaesPlan") + resolved = _resolve_cmaes_scorer(scorer) + + from soup_cli.utils.adapter_diff import load_adapter_weights + from soup_cli.utils.adapter_merge import merge_linear, write_merged_adapter + + weights_list = [load_adapter_weights(p) for p in plan.adapters] + template_source = plan.adapters[0] + + def _eval_fn(weights: Tuple[float, ...]) -> float: + merged, _skipped = merge_linear(weights_list, list(weights)) + tmp_dir = tempfile.mkdtemp(prefix=".soup_cmaes_") + try: + write_merged_adapter(tmp_dir, template_source, merged) + return float(resolved(tmp_dir, plan.eval_suite)) + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + return _eval_fn + + # --------------------------------------------------------------------------- # Deterministic RNG (no numpy dependency at module top) # --------------------------------------------------------------------------- diff --git a/src/soup_cli/utils/loop_daemon.py b/src/soup_cli/utils/loop_daemon.py index f139a74..f0dc2e4 100644 --- a/src/soup_cli/utils/loop_daemon.py +++ b/src/soup_cli/utils/loop_daemon.py @@ -105,6 +105,12 @@ class WatchConfig: deploy_fn: DeployFn = default_deploy cost_fn: CostFn = default_cost on_iteration: Optional[Callable[[IterationRecord], None]] = None + # v0.71.4 #177 — pack each successful iteration as a v0.26 Soup Can + + # append a Registry entry (default off so existing tests / stub watchers + # have no registry side effects). + pack_iterations: bool = False + served_model: Optional[str] = None + base_model: str = "unknown" def __post_init__(self) -> None: v = self.poll_interval_sec @@ -121,6 +127,21 @@ class WatchConfig: raise ValueError(f"{fname} must be callable") if self.on_iteration is not None and not callable(self.on_iteration): raise ValueError("on_iteration must be callable or None") + if not isinstance(self.pack_iterations, bool): + raise ValueError("pack_iterations must be bool") + # Validate the #177 registry-packing identity fields like every other + # field, so a NUL/oversize value fails at construction rather than deep + # inside store.push (review LOW). They flow into registry_name_from / + # store.push(base_model=...). + if self.served_model is not None: + sm = self.served_model + if not isinstance(sm, str) or "\x00" in sm or len(sm) > 512: + raise ValueError( + "served_model must be a NUL-free str <= 512 chars or None" + ) + bm = self.base_model + if not isinstance(bm, str) or not bm or "\x00" in bm or len(bm) > 512: + raise ValueError("base_model must be a non-empty NUL-free str <= 512 chars") def run_once( @@ -232,6 +253,9 @@ def watch(config: WatchConfig) -> "tuple[LoopState, int]": pass iterations = 0 + # v0.71.4 #177 — chain Registry entries across iterations so the loop + # forms a real lineage DAG (parent links to the prior iteration's entry). + prev_registry_id: Optional[str] = None state = read_state(config.state_path) # Only promote `stopped` → `running` automatically; `paused` must # survive a `soup loop watch` invocation so a SIGTERM + restart @@ -261,10 +285,16 @@ def watch(config: WatchConfig) -> "tuple[LoopState, int]": # iteration_count to match the manifest count (code-review # HIGH #3 fix). The state still records the skip in notes. if decision.proceed: + wrote = False try: write_iteration(record, base_dir=config.iteration_dir) + wrote = True except (OSError, ValueError) as exc: _LOG.warning("iteration write failed: %s", type(exc).__name__) + if wrote and config.pack_iterations: + prev_registry_id = _pack_iteration_safely( + record, config, prev_registry_id + ) if config.on_iteration is not None: try: config.on_iteration(record) @@ -293,6 +323,44 @@ def watch(config: WatchConfig) -> "tuple[LoopState, int]": return state, iterations +def _pack_iteration_safely( + record: IterationRecord, + config: WatchConfig, + prev_registry_id: Optional[str], +) -> Optional[str]: + """Pack one iteration as a Soup Can; never crash the daemon. + + Returns the new Registry entry id (to chain as the next iteration's + parent) on success, or the unchanged ``prev_registry_id`` on failure. + On failure the iteration manifest is re-written with a ``pack-failed:`` + note appended (best-effort) so the operator can see what happened + (#177 acceptance: swallow at WARNING + record in iteration notes). + """ + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + try: + _, entry_id = pack_iteration_as_can( + record.iteration_id, + base_dir=config.iteration_dir, + served_model=config.served_model, + base_model=config.base_model, + parent_registry_id=prev_registry_id, + ) + return entry_id + except Exception as exc: # noqa: BLE001 — instrumentation must not crash + _LOG.warning("iteration pack failed: %s", type(exc).__name__) + try: + prefix = record.notes + " | " if record.notes else "" + note = (prefix + f"pack-failed: {type(exc).__name__}")[:4096] + write_iteration( + replace(record, notes=note), + base_dir=config.iteration_dir, + ) + except (OSError, ValueError): + pass + return prev_registry_id + + def _state_with(state: LoopState, **kwargs: object) -> LoopState: """Return a copy with overrides applied (escape hatch around ``replace``). diff --git a/src/soup_cli/utils/loop_iteration.py b/src/soup_cli/utils/loop_iteration.py index 04418b1..1cab0c7 100644 --- a/src/soup_cli/utils/loop_iteration.py +++ b/src/soup_cli/utils/loop_iteration.py @@ -14,6 +14,7 @@ from __future__ import annotations import json import os +import re import stat import tempfile import uuid @@ -198,6 +199,109 @@ def read_iteration( raise ValueError(f"manifest contents invalid: {exc}") from exc +def registry_name_from(served_model: str) -> str: + """Sanitise a served-model id into a valid Registry entry name. + + ``served_model`` is typically ``registry://<id>`` or a model path with + ``/`` separators — neither is a valid Registry name (``^[A-Za-z0-9][...]``). + Strips the scheme, replaces every non-``[A-Za-z0-9_.-]`` char with ``-``, + drops leading non-alphanumerics, and caps at 128 chars. Falls back to + ``"loop"`` when nothing usable survives. + """ + raw = (served_model or "").replace("registry://", "") + cleaned = re.sub(r"[^A-Za-z0-9_.\-]", "-", raw) + cleaned = re.sub(r"^[^A-Za-z0-9]+", "", cleaned) + if not cleaned: + cleaned = "loop" + return cleaned[:128] + + +def pack_iteration_as_can( + iteration_id: str, + *, + base_dir: Optional[str] = None, + served_model: Optional[str] = None, + base_model: str = "unknown", + task: str = "dpo", + parent_registry_id: Optional[str] = None, +) -> Tuple[str, str]: + """Pack a loop iteration as a v0.26 Soup Can + append a Registry entry. + + Reads ``<base_dir>/<iteration_id>/iteration.json``, pushes a Registry + entry (name derived from ``served_model``, tag ``loop-iter``), links it + to ``parent_registry_id`` via a ``forked_from`` lineage edge so the loop + forms a real DAG visible through ``soup history``, and writes + ``<base_dir>/<iteration_id>/iteration.can``. Returns ``(can_path, + registry_entry_id)``. + """ + from soup_cli.cans.pack import pack_entry + from soup_cli.registry.store import RegistryStore + + record = read_iteration(iteration_id, base_dir=base_dir) + parent_dir = base_dir if base_dir is not None else _DEFAULT_DIR + iter_dir = os.path.join(parent_dir, iteration_id) + _check_dir(iter_dir) + if not os.path.isdir(iter_dir): + raise FileNotFoundError(f"iteration dir not found: {iteration_id}") + + name = registry_name_from(served_model or "loop") + config = { + "iteration_id": iteration_id, + "run_id": record.run_id, + "gate_verdict": record.gate_verdict, + "canary_verdict": record.canary_verdict, + "shipped": record.shipped, + } + with RegistryStore() as store: + entry_id = store.push( + name=name, + tag="loop-iter", + base_model=base_model or "unknown", + task=task or "dpo", + run_id=record.run_id, + config=config, + notes=f"loop iteration {iteration_id}", + ) + if parent_registry_id is not None: + # Best-effort lineage edge — a missing parent / cycle / FK error + # must not prevent the new entry from being created (daemon + # resilience). ``add_lineage`` raises ``ValueError`` for the + # documented cycle/self-ref/missing-parent cases and may surface a + # ``sqlite3.Error`` (e.g. FK violation on a since-deleted parent). + import sqlite3 + + try: + store.add_lineage( + child_id=entry_id, + parent_id=parent_registry_id, + relation="forked_from", + ) + except (ValueError, sqlite3.Error): + pass + + can_path = os.path.join(iter_dir, "iteration.can") + try: + pack_entry( + entry_id=entry_id, + out_path=can_path, + author="soup-loop", + description=f"loop iteration {iteration_id}", + ) + except Exception: + # All-or-nothing: a failed can write must not leave an orphaned + # Registry entry behind. Roll it back so the watch-loop lineage + # chain (which links the *next* iteration to the *prior successful* + # entry) stays consistent — a half-created iteration never enters + # the DAG. Best-effort rollback; the original error still propagates. + try: + with RegistryStore() as rollback_store: + rollback_store.delete(entry_id) + except Exception: # noqa: BLE001 — rollback is best-effort + pass + raise + return can_path, entry_id + + def list_iterations(base_dir: Optional[str] = None) -> Tuple[str, ...]: """Return iteration ids sorted by name (timestamp-prefixed).""" parent = base_dir if base_dir is not None else _DEFAULT_DIR diff --git a/src/soup_cli/utils/loop_stages.py b/src/soup_cli/utils/loop_stages.py new file mode 100644 index 0000000..47711dc --- /dev/null +++ b/src/soup_cli/utils/loop_stages.py @@ -0,0 +1,418 @@ +"""Pre-wired production stages for `soup loop watch` (v0.71.4 #176). + +v0.58.0 shipped the watch daemon with no-op default stage callbacks; this +module supplies real harvest / train / gate / deploy callables that compose +the existing v0.26.0 trace-to-preference, eval-gate, and v0.30.0 multi-adapter +deploy surfaces. Operators opt in via ``soup loop init --pre-wired`` (or +``soup loop watch --pre-wired``). + +Each callable matches the ``HarvestFn`` / ``TrainFn`` / ``GateFn`` / +``DeployFn`` protocols from ``loop_daemon``. Heavy deps (transformers / peft / +trl / httpx) are lazy-imported inside the callables so importing this module +stays cold-start cheap (project lazy-import policy). + +Module-level escape hatches (mirrors v0.53.1 #109 deploy_measure) let tests +and offline smokes inject behaviour without a GPU or a network: + +- ``_TRACE_DIR_RESOLVER(state) -> Optional[str]`` +- ``_TRAIN_RUNNER(argv, ...)`` (a ``subprocess.run`` stand-in) +- ``_GATE_GENERATE_FACTORY(adapter_dir) -> generate_fn`` +- ``_DEPLOY_POSTER(endpoint, name) -> bool`` +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import sys +import uuid +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Mapping, Optional + +from soup_cli.utils.loop_state import LoopState +from soup_cli.utils.paths import atomic_write_text + +if TYPE_CHECKING: + from soup_cli.utils.loop_daemon import WatchConfig + +_LOG = logging.getLogger(__name__) + +_PAIRS_DIR = ".soup-loops/pairs" +_ADAPTERS_DIR = ".soup-loops/adapters" +_DEFAULT_TRACE_DIR = ".soup-loops/traces" + +# Test / smoke escape hatches (default None → real behaviour). +_TRACE_DIR_RESOLVER: Optional[Callable[[LoopState], Optional[str]]] = None +_TRAIN_RUNNER: Optional[Callable[..., object]] = None +_GATE_GENERATE_FACTORY: Optional[Callable[[str], Callable[[str], str]]] = None +_DEPLOY_POSTER: Optional[Callable[[str, str], bool]] = None + + +# --------------------------------------------------------------------------- +# Harvest — production traces → preference pairs (v0.26.0 trace-to-pref) +# --------------------------------------------------------------------------- + + +def _resolve_trace_dir(state: LoopState) -> Optional[str]: + """Resolve the directory of ``*.jsonl`` serve logs to harvest from. + + Resolution order: ``SOUP_LOOP_TRACE_DIR`` env → ``served_model`` if it is + an existing local dir → ``.soup-loops/traces``. Returns ``None`` when no + usable directory exists. + + Note: this is the one read surface in this module that intentionally does + NOT enforce cwd-containment — operators legitimately keep serve logs + outside the project dir (e.g. ``/var/log/soup-serve``). The directory is + operator-supplied config (env / state), not untrusted input, and harvested + content is only used to build DPO pairs (never exec'd). + """ + if _TRACE_DIR_RESOLVER is not None: + return _TRACE_DIR_RESOLVER(state) + env = os.environ.get("SOUP_LOOP_TRACE_DIR") + candidates = [] + if env: + candidates.append(env) + if state.served_model and not state.served_model.startswith("registry://"): + candidates.append(state.served_model) + candidates.append(_DEFAULT_TRACE_DIR) + for cand in candidates: + if cand and os.path.isdir(cand): + return cand + return None + + +def harvest_from_traces(state: LoopState) -> Mapping[str, object]: + """Scan serve traces for thumbs/regeneration signals → preference pairs. + + Writes the harvested pairs to ``.soup-loops/pairs/<uuid>.jsonl`` (cwd- + contained) and returns ``{pairs_harvested, pairs_path, traces_collected}``. + Yields zero pairs (no path) when no trace directory is configured. + """ + from soup_cli.data.traces.pair_builder import build_pairs + from soup_cli.data.traces.parsers import parse_soup_serve + + trace_dir = _resolve_trace_dir(state) + if not trace_dir: + return {"pairs_harvested": 0, "pairs_path": None, "traces_collected": 0} + + traces = list(parse_soup_serve(trace_dir)) + pairs = list(build_pairs(traces, signal="thumbs_up")) + if not pairs: + return { + "pairs_harvested": 0, + "pairs_path": None, + "traces_collected": len(traces), + } + + pairs_dir = Path(_PAIRS_DIR) + pairs_dir.mkdir(parents=True, exist_ok=True) + pairs_path = str(pairs_dir / f"pairs-{uuid.uuid4().hex[:12]}.jsonl") + lines = "\n".join(json.dumps(p.to_jsonl_dict()) for p in pairs) + "\n" + # Atomic + cwd-contained + symlink-rejected (mirrors the rest of the + # changeset's write policy). + atomic_write_text(lines, pairs_path, field="pairs_path") + return { + "pairs_harvested": len(pairs), + "pairs_path": pairs_path, + "traces_collected": len(traces), + } + + +# --------------------------------------------------------------------------- +# Train — generated DPO YAML → `soup train` subprocess +# --------------------------------------------------------------------------- + + +def _render_dpo_yaml(state: LoopState, pairs_path: str, output_dir: str) -> str: + """Render a minimal DPO ``soup.yaml`` referencing the harvested pairs.""" + import yaml + + base = ( + state.served_model + if state.served_model and not state.served_model.startswith("registry://") + else "hf-internal-testing/tiny-random-gpt2" + ) + config = { + "base": base, + "task": "dpo", + "data": {"train": pairs_path, "format": "dpo"}, + "training": {"epochs": 1, "batch_size": 1, "lr": 5.0e-5}, + "output": output_dir, + } + return yaml.safe_dump(config, sort_keys=False) + + +def train_dpo_from_pairs( + state: LoopState, ctx: Mapping[str, object] +) -> Mapping[str, object]: + """Train a DPO adapter from harvested pairs via a ``soup train`` subprocess. + + Returns ``{run_id, skipped, adapter_path}``. Skips (no run) when the + harvest produced no pairs. The subprocess is argv-list (no shell); + ``_TRAIN_RUNNER`` injects a stand-in for tests. + """ + pairs_path = ctx.get("pairs_path") + pairs_n = int(ctx.get("pairs_harvested", 0) or 0) + if not pairs_path or pairs_n <= 0: + return {"run_id": None, "skipped": True, "adapter_path": None} + + run_id = f"loop-train-{uuid.uuid4().hex[:8]}" + output_dir = os.path.join(_ADAPTERS_DIR, run_id) + yaml_dir = Path(_ADAPTERS_DIR) + yaml_dir.mkdir(parents=True, exist_ok=True) + yaml_path = str(yaml_dir / f"{run_id}.yaml") + atomic_write_text( + _render_dpo_yaml(state, str(pairs_path), output_dir), + yaml_path, + field="train yaml", + ) + + argv = [ + sys.executable, + "-m", + "soup_cli.cli", + "train", + "--config", + yaml_path, + "--yes", + ] + runner = _TRAIN_RUNNER + if runner is None: + import subprocess # noqa: S404 — argv list mode, no shell + + runner = subprocess.run + try: + result = runner( # noqa: S603 — internal argv, no shell + argv, capture_output=True, timeout=24 * 3600, check=False + ) + except Exception as exc: # noqa: BLE001 — train failure must not crash loop + _LOG.warning("train subprocess failed: %s", type(exc).__name__) + return {"run_id": None, "skipped": True, "adapter_path": None} + + rc = getattr(result, "returncode", 1) + if rc != 0: + _LOG.warning("soup train exited rc=%s", rc) + return {"run_id": None, "skipped": True, "adapter_path": None} + return {"run_id": run_id, "skipped": False, "adapter_path": output_dir} + + +# --------------------------------------------------------------------------- +# Gate — eval-gate against the registered baseline (v0.26.0 Part B) +# --------------------------------------------------------------------------- + + +def _build_gate_generator(adapter_dir: str) -> Callable[[str], str]: + """Build a ``generate_fn`` for the trained adapter (live model load).""" + if _GATE_GENERATE_FACTORY is not None: + return _GATE_GENERATE_FACTORY(adapter_dir) + # Live default — load the merged/trained adapter as a full model dir. + from soup_cli.eval.quant_check import make_model_generator + + return make_model_generator(adapter_dir) + + +def gate_against_baseline( + state: LoopState, ctx: Mapping[str, object] +) -> Mapping[str, object]: + """Run the v0.26.0 eval gate for the trained adapter vs the baseline. + + Returns ``{gate_verdict}`` — ``SKIPPED`` when training was skipped or the + adapter dir is missing, otherwise ``OK`` / ``MAJOR`` from the gate result. + """ + if ctx.get("skipped"): + return {"gate_verdict": "SKIPPED"} + adapter_path = ctx.get("adapter_path") + if not adapter_path or not os.path.isdir(str(adapter_path)): + return {"gate_verdict": "SKIPPED"} + + from soup_cli.eval.gate import load_suite, resolve_baseline, run_gate + + try: + suite = load_suite(state.eval_suite) + except (FileNotFoundError, ValueError, TypeError, OSError) as exc: + _LOG.warning("gate suite load failed: %s", type(exc).__name__) + return {"gate_verdict": "SKIPPED"} + # An unresolvable baseline degrades to a threshold-only gate (no + # regression check) rather than failing the whole stage — the baseline + # may not exist yet on the first iteration. + try: + baseline = resolve_baseline(state.baseline) + except (FileNotFoundError, ValueError, TypeError, OSError): + baseline = {} + try: + generate_fn = _build_gate_generator(str(adapter_path)) + result = run_gate(suite, generate_fn=generate_fn, baseline=baseline) + except (FileNotFoundError, ValueError, TypeError, OSError) as exc: + _LOG.warning("gate run failed: %s", type(exc).__name__) + return {"gate_verdict": "SKIPPED"} + verdict = "OK" if result.passed else "MAJOR" + return {"gate_verdict": verdict} + + +# --------------------------------------------------------------------------- +# Deploy — promote to a canary via /v1/adapters/activate (v0.30.0) +# --------------------------------------------------------------------------- + + +# Adapter-name shape accepted by the v0.30.0 /v1/adapters/activate route. +_ADAPTER_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._\-]*$") + +_LOOPBACK_HOSTS = frozenset({"localhost", "127.0.0.1", "::1"}) + + +def _endpoint_is_local(endpoint: str) -> bool: + """True iff ``endpoint``'s host is loopback or a private/link-local IP. + + The deploy surface must only POST to the operator's own box / LAN, never + an arbitrary public host (the webhook validator permits any HTTPS host). + Loopback hostnames are accepted by name; any other host must parse as a + private / loopback / link-local IP literal. A non-IP public hostname is + rejected (we do not resolve DNS — project policy). + """ + import ipaddress + from urllib.parse import urlparse + + host = (urlparse(endpoint).hostname or "").strip("[]").lower() + if not host: + return False + if host in _LOOPBACK_HOSTS: + return True + try: + ip = ipaddress.ip_address(host) + except ValueError: + return False + return ip.is_loopback or ip.is_private or ip.is_link_local + + +def _post_activate(endpoint: str, name: str) -> bool: + """POST to ``<endpoint>/v1/adapters/activate/<name>``; True on 2xx. + + ``endpoint`` is already SSRF-validated by the caller; ``name`` is + re-validated here against the activate-route pattern before URL + interpolation (defence-in-depth against a crafted adapter dir name). + """ + if _DEPLOY_POSTER is not None: + return _DEPLOY_POSTER(endpoint, name) + if not _ADAPTER_NAME_RE.match(name): + return False + try: + import httpx + except ImportError: + return False + url = endpoint.rstrip("/") + f"/v1/adapters/activate/{name}" + try: + resp = httpx.post(url, timeout=5.0) + return 200 <= resp.status_code < 300 + except Exception: # noqa: BLE001 — deploy must never crash the loop + return False + + +def deploy_to_canary( + state: LoopState, ctx: Mapping[str, object] +) -> Mapping[str, object]: + """Promote the trained adapter as a canary when the gate verdict is OK. + + Returns ``{deployed, canary_verdict}``. No-op (not deployed) unless the + gate passed, an adapter is available, and a serve endpoint is configured + via ``SOUP_LOOP_SERVE_ENDPOINT``. The endpoint is SSRF-validated + (loopback-only HTTP, scheme allowlist, private-IP rejection) via the + shared ``validate_webhook_url`` helper before any POST. + """ + if ctx.get("gate_verdict") != "OK": + return {"deployed": False, "canary_verdict": None} + adapter_path = ctx.get("adapter_path") + if not adapter_path: + return {"deployed": False, "canary_verdict": None} + raw_endpoint = os.environ.get("SOUP_LOOP_SERVE_ENDPOINT") + if not raw_endpoint: + return { + "deployed": False, + "canary_verdict": None, + "notes": "no SOUP_LOOP_SERVE_ENDPOINT configured", + } + from soup_cli.utils.drift_alarm import validate_webhook_url + + try: + endpoint = validate_webhook_url(raw_endpoint) + except (TypeError, ValueError): + _LOG.warning("SOUP_LOOP_SERVE_ENDPOINT rejected by SSRF guard") + return { + "deployed": False, + "canary_verdict": None, + "notes": "SOUP_LOOP_SERVE_ENDPOINT rejected (SSRF guard)", + } + # A serve endpoint is the operator's own box / LAN, never an arbitrary + # remote host. `validate_webhook_url` (a *webhook* policy) permits any + # HTTPS host; tighten further for the deploy surface to loopback + RFC1918 + # private + link-local only, so a poisoned SOUP_LOOP_SERVE_ENDPOINT cannot + # POST adapter names to an attacker-controlled HTTPS server (v0.71.4 + # review MEDIUM-4). Non-IP hostnames can't be verified private without DNS + # (project policy: no DNS resolution), so the safe default rejects them. + if not _endpoint_is_local(endpoint): + _LOG.warning("SOUP_LOOP_SERVE_ENDPOINT must be loopback/LAN (SSRF guard)") + return { + "deployed": False, + "canary_verdict": None, + "notes": "SOUP_LOOP_SERVE_ENDPOINT must be loopback/LAN (SSRF guard)", + } + name = os.path.basename(str(adapter_path).rstrip("/\\")) or "canary" + ok = _post_activate(endpoint, name) + return { + "deployed": ok, + "canary_verdict": "OK" if ok else None, + "notes": "" if ok else "activate POST failed", + } + + +# --------------------------------------------------------------------------- +# Cost — placeholder estimate (v0.34.0 run_cost wiring is a follow-up) +# --------------------------------------------------------------------------- + + +def estimate_cost(state: LoopState) -> float: + """Per-iteration cost estimate (USD) — currently a hard ``0.0`` placeholder. + + KNOWN LIMITATION: returning ``0.0`` means the v0.58 budget gate + (``monthly_budget_usd`` / ``max_runs_per_day`` in ``run_once``) never trips + on *cost* for a pre-wired loop — only the daily run-count cap is effective. + Wiring a real estimate (v0.34.0 ``utils/run_cost.estimate_run_cost_usd``, + which needs the trained run's GPU + duration) is a tracked follow-up; until + then operators relying on a dollar budget should set ``max_runs_per_day``. + """ + return 0.0 + + +# --------------------------------------------------------------------------- +# Config builder +# --------------------------------------------------------------------------- + + +def build_prewired_watch_config( + *, + max_iterations: Optional[int] = None, + poll_interval_sec: float = 60.0, + state_path: Optional[str] = None, + iteration_dir: Optional[str] = None, + pack_iterations: bool = False, + served_model: Optional[str] = None, + base_model: str = "unknown", +) -> "WatchConfig": + """Return a ``WatchConfig`` wired with the pre-wired production stages.""" + from soup_cli.utils.loop_daemon import WatchConfig + + return WatchConfig( + poll_interval_sec=poll_interval_sec, + max_iterations=max_iterations, + state_path=state_path, + iteration_dir=iteration_dir, + harvest_fn=harvest_from_traces, + train_fn=train_dpo_from_pairs, + gate_fn=gate_against_baseline, + deploy_fn=deploy_to_canary, + cost_fn=estimate_cost, + pack_iterations=pack_iterations, + served_model=served_model, + base_model=base_model, + ) diff --git a/src/soup_cli/utils/loop_state.py b/src/soup_cli/utils/loop_state.py index ff5b7ed..7064024 100644 --- a/src/soup_cli/utils/loop_state.py +++ b/src/soup_cli/utils/loop_state.py @@ -51,6 +51,9 @@ class LoopState: eval_suite: str baseline: str status: str = "stopped" + # v0.71.4 #176 — use the pre-wired harvest/train/gate/deploy stages + # (utils/loop_stages.py) instead of the no-op default callbacks. + pre_wired: bool = False traces_collected: int = 0 pairs_distilled: int = 0 runs_gated: int = 0 @@ -110,6 +113,8 @@ class LoopState: raise ValueError("max_runs_per_day must be a positive int or None") if not isinstance(self.canary_autoroll_on_regress, bool): raise ValueError("canary_autoroll_on_regress must be bool") + if not isinstance(self.pre_wired, bool): + raise ValueError("pre_wired must be bool") if self.canary_active is not None: _require_str("canary_active", self.canary_active, allow_empty=False) if self.last_iteration_id is not None: @@ -273,6 +278,7 @@ def init_state( *, monthly_budget_usd: Optional[float] = None, max_runs_per_day: Optional[int] = None, + pre_wired: bool = False, path: Optional[str] = None, force: bool = False, ) -> Tuple[LoopState, str]: @@ -300,6 +306,7 @@ def init_state( eval_suite=eval_suite, baseline=baseline, status="stopped", + pre_wired=pre_wired, monthly_budget_usd=monthly_budget_usd, max_runs_per_day=max_runs_per_day, created_at=now, diff --git a/tests/test_v0714.py b/tests/test_v0714.py new file mode 100644 index 0000000..380a037 --- /dev/null +++ b/tests/test_v0714.py @@ -0,0 +1,1774 @@ +"""v0.71.4 — Adapter lifecycle + loop wiring. + +Closes #172, #173, #176, #177, #220, #223. + +- #223 soup adapters pr --push (GitHub PR publisher via `gh api`) +- #173 branch pointers into the v0.26 Registry lineage DAG +- #172 live canary verdict for soup adapters merge (replace UNKNOWN stub) +- #220 live eval-suite auto-wiring for soup adapters merge --strategy cmaes +- #176 pre-wired stage callbacks for soup loop watch +- #177 pack each soup loop iteration as a Soup Can + Registry lineage +""" + +from __future__ import annotations + +import json +import os +import re +import sys + +import pytest +from typer.testing import CliRunner + +runner = CliRunner() + +POSIX_ONLY = pytest.mark.skipif( + sys.platform == "win32", reason="symlink rejection is POSIX-only here" +) + + +@pytest.fixture() +def temp_registry(tmp_path, monkeypatch): + """Isolated registry + branches dir under cwd, returns a push helper.""" + monkeypatch.chdir(tmp_path) + db = tmp_path / "registry.db" + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(db)) + branches = tmp_path / "branches" + branches.mkdir() + monkeypatch.setenv("SOUP_BRANCHES_DIR", str(branches)) + + from soup_cli.registry.store import RegistryStore + + def _push(name="mymodel", tag="v1", base="meta-llama/x", task="dpo", + config=None, data_path=None): + with RegistryStore() as store: + return store.push( + name=name, tag=tag, base_model=base, task=task, + run_id="run-1", config=config or {"base": base, "task": task}, + data_path=data_path, + ) + + return _push + + +# =========================================================================== +# #173 — branch pointers into the v0.26 Registry lineage DAG +# =========================================================================== + + +def _write_cfg(tmp_path, name="soup.yaml", body="base: meta-llama/x\ntask: dpo\n"): + p = tmp_path / name + p.write_text(body, encoding="utf-8") + return str(p) + + +class TestBranchRefKind: + def test_branch_ref_in_valid_kinds(self): + from soup_cli.registry.store import _VALID_KINDS + + assert "branch_ref" in _VALID_KINDS + + def test_attach_artifact_branch_ref(self, temp_registry, tmp_path): + from soup_cli.registry.attach import attach_artifact + + entry_id = temp_registry() + ptr = tmp_path / "ptr.json" + ptr.write_text('{"k": "v"}', encoding="utf-8") + rowid = attach_artifact( + entry_id, path=str(ptr), kind="branch_ref", enforce_cwd=False + ) + assert isinstance(rowid, int) + + +class TestBranchSchema: + def test_registry_entry_id_field_default_none(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import create_branch, load_branch + + cfg = _write_cfg(tmp_path) + snap = create_branch("b1", config_path=cfg, base_model="meta-llama/x") + assert snap.registry_entry_id is None + reloaded = load_branch("b1") + assert reloaded.registry_entry_id is None + + def test_create_with_registry_entry_id_roundtrips(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import create_branch, load_branch + + cfg = _write_cfg(tmp_path) + create_branch( + "b2", config_path=cfg, base_model="meta-llama/x", + registry_entry_id="reg_20260601_abc123", + ) + assert load_branch("b2").registry_entry_id == "reg_20260601_abc123" + + def test_create_with_dataset_sha256_direct(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import create_branch, load_branch + + cfg = _write_cfg(tmp_path) + sha = "c" * 64 + create_branch( + "b3", config_path=cfg, base_model="meta-llama/x", dataset_sha256=sha, + ) + assert load_branch("b3").dataset_sha256 == sha + + def test_dataset_path_and_sha_mutually_exclusive(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import create_branch + + cfg = _write_cfg(tmp_path) + ds = tmp_path / "data.jsonl" + ds.write_text("{}\n", encoding="utf-8") + with pytest.raises(ValueError): + create_branch( + "b4", config_path=cfg, base_model="meta-llama/x", + dataset_path=str(ds), dataset_sha256="d" * 64, + ) + + def test_registry_entry_id_null_byte_rejected(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import create_branch + + cfg = _write_cfg(tmp_path) + with pytest.raises(ValueError): + create_branch( + "b5", config_path=cfg, base_model="meta-llama/x", + registry_entry_id="reg\x00bad", + ) + + def test_back_compat_v057_pointer(self, temp_registry, tmp_path): + """A v0.57.0 pointer with no registry_entry_id key loads cleanly.""" + from soup_cli.utils.adapter_branch import _branches_dir, load_branch + + legacy = { + "name": "legacy", + "config_path": str(tmp_path / "soup.yaml"), + "config_sha256": "a" * 64, + "dataset_sha256": None, + "base_model": "meta-llama/x", + "created_at": 1.0, + "soup_version": "0.57.0", + } + (_branches_dir() / "legacy.json").write_text( + json.dumps(legacy), encoding="utf-8" + ) + snap = load_branch("legacy") + assert snap.registry_entry_id is None + assert snap.base_model == "meta-llama/x" + + +class TestAttachBranchToRegistry: + def test_happy(self, temp_registry, tmp_path): + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.adapter_branch import ( + attach_branch_to_registry, + create_branch, + load_branch, + ) + + entry_id = temp_registry() + cfg = _write_cfg(tmp_path) + create_branch("attached", config_path=cfg, base_model="meta-llama/x") + rowid = attach_branch_to_registry("attached", entry_id) + assert isinstance(rowid, int) + assert load_branch("attached").registry_entry_id == entry_id + with RegistryStore() as store: + arts = store.get_artifacts(entry_id) + kinds = {a["kind"] for a in arts} + assert "branch_ref" in kinds + + def test_missing_entry_friendly(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import ( + attach_branch_to_registry, + create_branch, + ) + + cfg = _write_cfg(tmp_path) + create_branch("x", config_path=cfg, base_model="meta-llama/x") + with pytest.raises(ValueError, match="not found"): + attach_branch_to_registry("x", "no-such-entry") + + +class TestBranchFromRegistry: + def test_derives_all_fields(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import branch_from_registry + + entry_id = temp_registry(base="org/big-model") + snap = branch_from_registry("derived", entry_id) + assert snap.base_model == "org/big-model" + assert snap.registry_entry_id == entry_id + assert os.path.isfile(snap.config_path) + assert len(snap.config_sha256) == 64 + + def test_with_data_hash(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import branch_from_registry + + ds = tmp_path / "data.jsonl" + ds.write_text('{"x": 1}\n', encoding="utf-8") + entry_id = temp_registry(data_path=str(ds)) + snap = branch_from_registry("derived2", entry_id) + assert snap.dataset_sha256 is not None + assert len(snap.dataset_sha256) == 64 + + def test_missing_entry(self, temp_registry, tmp_path): + from soup_cli.utils.adapter_branch import branch_from_registry + + with pytest.raises(ValueError, match="not found"): + branch_from_registry("nope", "no-such-id") + + +class TestBranchCli: + def _app(self): + from soup_cli.commands.adapters import app + + return app + + def test_attach_to_registry_cli(self, temp_registry, tmp_path): + entry_id = temp_registry() + cfg = _write_cfg(tmp_path) + result = runner.invoke( + self._app(), + [ + "branch", "clibranch", + "-c", cfg, "--base", "meta-llama/x", + "--attach-to-registry", entry_id, + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + from soup_cli.utils.adapter_branch import load_branch + + assert load_branch("clibranch").registry_entry_id == entry_id + + def test_from_registry_cli(self, temp_registry, tmp_path): + entry_id = temp_registry(base="org/from-reg") + result = runner.invoke( + self._app(), + ["branch", "fromreg", "--from-registry", entry_id], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + from soup_cli.utils.adapter_branch import load_branch + + snap = load_branch("fromreg") + assert snap.base_model == "org/from-reg" + assert snap.registry_entry_id == entry_id + + def test_from_registry_missing_friendly(self, temp_registry, tmp_path): + result = runner.invoke( + self._app(), + ["branch", "fr2", "--from-registry", "nope"], + ) + assert result.exit_code == 2 + assert "not found" in result.output.lower() + + def test_requires_config_when_not_from_registry(self, temp_registry, tmp_path): + result = runner.invoke( + self._app(), + ["branch", "needsconfig", "--base", "meta-llama/x"], + ) + assert result.exit_code == 2 + assert "config" in result.output.lower() + + +class TestHistoryBranchEdges: + def test_history_renders_branch_edge(self, temp_registry, tmp_path): + import typer + + from soup_cli.commands.history import history as history_cmd + from soup_cli.utils.adapter_branch import ( + attach_branch_to_registry, + create_branch, + ) + + entry_id = temp_registry(name="histmodel") + cfg = _write_cfg(tmp_path) + create_branch("hb", config_path=cfg, base_model="meta-llama/x") + attach_branch_to_registry("hb", entry_id) + + app = typer.Typer() + app.command()(history_cmd) + result = runner.invoke(app, ["histmodel"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "branch" in result.output.lower() + + +# =========================================================================== +# #172 — live canary verdict for soup adapters merge (replace UNKNOWN stub) +# =========================================================================== + + +def _make_adapter(dir_path, *, scale=1.0): + """Write a minimal LoRA-shaped adapter (safetensors + config).""" + import numpy as np + from safetensors.numpy import save_file + + dir_path.mkdir(parents=True, exist_ok=True) + rng = np.random.default_rng(0) + tensors = { + "base_model.model.layers.0.self_attn.q_proj.lora_A.weight": + (rng.standard_normal((8, 16)) * scale).astype("float32"), + "base_model.model.layers.0.self_attn.q_proj.lora_B.weight": + (rng.standard_normal((16, 8)) * scale).astype("float32"), + } + save_file(tensors, str(dir_path / "adapter_model.safetensors")) + (dir_path / "adapter_config.json").write_text( + json.dumps({ + "peft_type": "LORA", + "r": 8, + "lora_alpha": 16, + "base_model_name_or_path": "hf-internal-testing/tiny-random-gpt2", + }), + encoding="utf-8", + ) + return str(dir_path) + + +def _make_report(): + from soup_cli.utils.adapter_merge import MergeReport + + return MergeReport( + strategy="linear", adapters=("a", "b"), weights=(0.5, 0.5), + merged_layers=1, skipped_layers=(), output_dir="out", verdict="UNKNOWN", + ) + + +class TestPredictMergedVerdict: + def test_none_returns_unknown(self): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + assert predict_merged_verdict(_make_report()) == "UNKNOWN" + + def test_no_drop_is_ok(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [0.9, 0.8, 0.85, 0.9], + "candidate_scores": [0.9, 0.81, 0.86, 0.9], + }), encoding="utf-8") + assert predict_merged_verdict(_make_report(), str(suite)) == "OK" + + def test_minor_drop(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + # ~3% drop on average → MINOR + suite.write_text(json.dumps({ + "baseline_scores": [1.0, 1.0, 1.0, 1.0], + "candidate_scores": [0.97, 0.97, 0.97, 0.97], + }), encoding="utf-8") + assert predict_merged_verdict(_make_report(), str(suite)) == "MINOR" + + def test_major_drop(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [1.0, 1.0, 1.0, 1.0], + "candidate_scores": [0.85, 0.85, 0.85, 0.85], + }), encoding="utf-8") + assert predict_merged_verdict(_make_report(), str(suite)) == "MAJOR" + + def test_length_mismatch(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [1.0, 1.0], + "candidate_scores": [1.0], + }), encoding="utf-8") + with pytest.raises(ValueError): + predict_merged_verdict(_make_report(), str(suite)) + + def test_empty_scores(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [], "candidate_scores": [], + }), encoding="utf-8") + with pytest.raises(ValueError): + predict_merged_verdict(_make_report(), str(suite)) + + def test_non_numeric_score_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text( + '{"baseline_scores": [1.0], "candidate_scores": ["x"]}', + encoding="utf-8", + ) + with pytest.raises(ValueError): + predict_merged_verdict(_make_report(), str(suite)) + + def test_infinity_literal_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + # `json.loads` parses the non-standard `Infinity` / `NaN` literals by + # default, so the finite-check in _require_score_list is load-bearing. + suite.write_text( + '{"baseline_scores": [1.0], "candidate_scores": [Infinity]}', + encoding="utf-8", + ) + with pytest.raises(ValueError, match="finite"): + predict_merged_verdict(_make_report(), str(suite)) + + def test_tasks_without_scorer(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "tasks": [{"prompt": "2+2", "expected": "4"}], + }), encoding="utf-8") + with pytest.raises(ValueError, match="scorer"): + predict_merged_verdict(_make_report(), str(suite)) + + def test_tasks_with_scorer(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "tasks": [{"prompt": "p", "expected": "e"}, {"prompt": "p2", "expected": "e2"}], + }), encoding="utf-8") + + def scorer(role, tasks): + assert role in ("baseline", "candidate") + # 3% drop → MINOR (5% is the MAJOR boundary). + return [1.0 for _ in tasks] if role == "baseline" else [0.97 for _ in tasks] + + assert predict_merged_verdict( + _make_report(), str(suite), scorer=scorer + ) == "MINOR" + + def test_canary_must_be_str(self): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + with pytest.raises(TypeError): + predict_merged_verdict(_make_report(), 123) # type: ignore[arg-type] + + +class TestMergeCanaryCli: + def _app(self): + from soup_cli.commands.adapters import app + + return app + + def test_merge_with_canary_renders_verdict(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [0.9, 0.9], "candidate_scores": [0.9, 0.9], + }), encoding="utf-8") + result = runner.invoke(self._app(), [ + "merge", a, b, "-o", str(tmp_path / "out"), + "--allow-unscanned", "--canary", str(suite), + ]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "OK" in result.output + + def test_strict_verdict_major_exits_2(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": [1.0, 1.0], "candidate_scores": [0.8, 0.8], + }), encoding="utf-8") + result = runner.invoke(self._app(), [ + "merge", a, b, "-o", str(tmp_path / "out"), + "--allow-unscanned", "--canary", str(suite), "--strict-verdict", + ]) + assert result.exit_code == 2, (result.output, repr(result.exception)) + assert "MAJOR" in result.output + + def test_missing_canary_unknown_advisory(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + result = runner.invoke(self._app(), [ + "merge", a, b, "-o", str(tmp_path / "out"), "--allow-unscanned", + ]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "UNKNOWN" in result.output + + def test_canary_outside_cwd_rejected(self, tmp_path, monkeypatch): + sub = tmp_path / "work" + sub.mkdir() + monkeypatch.chdir(sub) + a = _make_adapter(sub / "a") + b = _make_adapter(sub / "b") + outside = tmp_path / "canary.json" + outside.write_text("{}", encoding="utf-8") + result = runner.invoke(self._app(), [ + "merge", a, b, "-o", str(sub / "out"), + "--allow-unscanned", "--canary", str(outside), + ]) + assert result.exit_code == 2 + assert "cwd" in result.output.lower() + + +# =========================================================================== +# #220 — live eval-suite auto-wiring for soup adapters merge --strategy cmaes +# =========================================================================== + + +def _cmaes_plan(tmp_path, adapters, *, pop=4, gens=6): + from soup_cli.utils.cmaes_merge import build_cmaes_plan + + suite = tmp_path / "eval.jsonl" + suite.write_text('{"prompt": "p", "expected": "e"}\n', encoding="utf-8") + return build_cmaes_plan( + adapters=adapters, eval_suite=str(suite), budget_spec="60s", + population_size=pop, max_generations=gens, + ) + + +class TestBuildCmaesEvalFn: + def test_eval_fn_merges_and_scores(self, tmp_path, monkeypatch): + from soup_cli.utils.cmaes_merge import build_cmaes_eval_fn + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a", scale=1.0) + b = _make_adapter(tmp_path / "b", scale=2.0) + plan = _cmaes_plan(tmp_path, [a, b]) + + seen = {} + + def scorer(merged_dir, eval_suite): + seen["dir"] = merged_dir + seen["suite"] = eval_suite + # The merged dir must contain a materialised safetensors file. + assert os.path.isfile(os.path.join(merged_dir, "adapter_model.safetensors")) + return 0.42 + + fn = build_cmaes_eval_fn(plan, scorer=scorer) + score = fn((0.5, 0.5)) + assert score == 0.42 + assert seen["suite"] == plan.eval_suite + + def test_eval_fn_cleans_up_temp(self, tmp_path, monkeypatch): + from soup_cli.utils.cmaes_merge import build_cmaes_eval_fn + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + plan = _cmaes_plan(tmp_path, [a, b]) + captured = [] + + def scorer(merged_dir, eval_suite): + captured.append(merged_dir) + return 0.5 + + fn = build_cmaes_eval_fn(plan, scorer=scorer) + fn((0.5, 0.5)) + # The per-generation temp dir is removed after scoring. + assert not os.path.exists(captured[0]) + + def test_run_cmaes_with_eval_fn_converges(self, tmp_path, monkeypatch): + from soup_cli.utils.cmaes_merge import build_cmaes_eval_fn, run_cmaes_merge + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + plan = _cmaes_plan(tmp_path, [a, b], pop=4, gens=20) + + # Constant scorer → plateau → converged after 3 flat generations. + fn = build_cmaes_eval_fn(plan, scorer=lambda d, s: 0.7) + result = run_cmaes_merge(plan, eval_fn=fn) + assert result.converged is True + assert result.generations_run < plan.max_generations + assert abs(sum(result.best_weights) - 1.0) < 1e-6 + + def test_eval_fn_isolates_scorer_failure(self, tmp_path, monkeypatch): + from soup_cli.utils.cmaes_merge import ( + _FAILED_EVAL_SENTINEL, + _eval_safely, + build_cmaes_eval_fn, + ) + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + plan = _cmaes_plan(tmp_path, [a, b]) + + def boom(merged_dir, eval_suite): + raise RuntimeError("scorer crashed") + + fn = build_cmaes_eval_fn(plan, scorer=boom) + # The eval_fn itself surfaces the error; run_cmaes_merge wraps in + # _eval_safely → sentinel. Confirm the isolation path. + assert _eval_safely(fn, (0.5, 0.5)) == _FAILED_EVAL_SENTINEL + + +class TestCmaesCliLive: + def _app(self): + from soup_cli.commands.adapters import app + + return app + + def test_cmaes_runs_loop_and_writes_output(self, tmp_path, monkeypatch): + import soup_cli.utils.cmaes_merge as cm + + monkeypatch.chdir(tmp_path) + # Inject a synthetic scorer so the CLI never loads a model. + monkeypatch.setattr(cm, "_CMAES_SCORER_OVERRIDE", lambda d, s: 0.6) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + suite = tmp_path / "eval.jsonl" + suite.write_text('{"prompt": "p", "expected": "e"}\n', encoding="utf-8") + out = tmp_path / "merged" + result = runner.invoke(self._app(), [ + "merge", a, b, "--strategy", "cmaes", + "--eval", str(suite), "--budget", "60s", + "--population", "4", "--max-generations", "6", + "-o", str(out), + ]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + # The best merge was materialised to --output. + assert (out / "adapter_model.safetensors").is_file() + # No more plan-only "deferred to v0.67.1" advisory. + assert "deferred to" not in result.output.lower() + + def test_cmaes_still_requires_eval(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + # Real adapters + --allow-unscanned so the input gates pass and we + # reach the cmaes arg-check (the scan gate now runs before dispatch). + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + result = runner.invoke(self._app(), [ + "merge", a, b, "--strategy", "cmaes", + "-o", str(tmp_path / "out"), "--allow-unscanned", + ]) + assert result.exit_code == 2 + assert "eval" in result.output.lower() + + +# =========================================================================== +# #176 — pre-wired stage callbacks for soup loop watch +# =========================================================================== + + +@pytest.fixture() +def loop_env(tmp_path, monkeypatch): + """Isolated cwd + registry DB for loop tests; returns a state factory.""" + monkeypatch.chdir(tmp_path) + monkeypatch.setenv("SOUP_REGISTRY_DB_PATH", str(tmp_path / "registry.db")) + monkeypatch.delenv("SOUP_LOOP_TRACE_DIR", raising=False) + monkeypatch.delenv("SOUP_LOOP_SERVE_ENDPOINT", raising=False) + + from soup_cli.utils.loop_state import LoopState + + def _state(served="mymodel", eval_suite="evals/gate.yaml", baseline="base-ref", + pre_wired=False): + return LoopState( + served_model=served, eval_suite=eval_suite, baseline=baseline, + status="running", pre_wired=pre_wired, + ) + + return _state + + +def _write_trace_dir(tmp_path): + trace_dir = tmp_path / "traces" + trace_dir.mkdir() + rows = [ + {"id": "1", "prompt": "Q", "response": "good", "feedback": {"rating": "up"}}, + {"id": "2", "prompt": "Q", "response": "bad", "feedback": {"rating": "down"}}, + ] + (trace_dir / "serve.jsonl").write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + return str(trace_dir) + + +class TestLoopStatePreWired: + def test_default_false(self, loop_env): + assert loop_env().pre_wired is False + + def test_non_bool_rejected(self): + from soup_cli.utils.loop_state import LoopState + + with pytest.raises(ValueError): + LoopState( + served_model="m", eval_suite="e", baseline="b", + pre_wired="yes", # type: ignore[arg-type] + ) + + def test_init_state_persists(self, loop_env, tmp_path): + from soup_cli.utils.loop_state import init_state, read_state + + state, _path = init_state("mymodel", "evals/gate.yaml", "base-ref", pre_wired=True) + assert state.pre_wired is True + assert read_state().pre_wired is True + + def test_to_dict_includes_pre_wired(self, loop_env): + from soup_cli.utils.loop_state import LoopState + + keys = set(loop_env().to_dict().keys()) + assert keys == set(LoopState.__dataclass_fields__.keys()) + assert "pre_wired" in keys + + +class TestHarvestStage: + def test_produces_pairs(self, loop_env, tmp_path, monkeypatch): + from soup_cli.utils.loop_stages import harvest_from_traces + + td = _write_trace_dir(tmp_path) + monkeypatch.setenv("SOUP_LOOP_TRACE_DIR", td) + out = harvest_from_traces(loop_env()) + assert out["pairs_harvested"] == 1 + assert out["pairs_path"] is not None + assert os.path.isfile(out["pairs_path"]) + + def test_no_trace_dir_zero(self, loop_env): + from soup_cli.utils.loop_stages import harvest_from_traces + + out = harvest_from_traces(loop_env()) + assert out["pairs_harvested"] == 0 + assert out["pairs_path"] is None + + def test_traces_present_but_no_pairs(self, loop_env, tmp_path, monkeypatch): + """The common steady state: traces exist but no down-vote → no pairs. + + Distinct from the no-dir case: traces_collected > 0 but + pairs_harvested == 0 and pairs_path is None (no file written).""" + from soup_cli.utils.loop_stages import harvest_from_traces + + trace_dir = tmp_path / "traces" + trace_dir.mkdir() + # All thumbs-up, no down → build_pairs(signal="thumbs_up") yields none. + rows = [ + {"id": "1", "prompt": "Q", "response": "a", + "feedback": {"rating": "up"}}, + {"id": "2", "prompt": "Q2", "response": "b", + "feedback": {"rating": "up"}}, + ] + (trace_dir / "s.jsonl").write_text( + "\n".join(json.dumps(r) for r in rows) + "\n", encoding="utf-8" + ) + monkeypatch.setenv("SOUP_LOOP_TRACE_DIR", str(trace_dir)) + out = harvest_from_traces(loop_env()) + assert out["pairs_harvested"] == 0 + assert out["pairs_path"] is None + assert out["traces_collected"] >= 1 + + +class TestTrainStage: + def test_skips_without_pairs(self, loop_env): + from soup_cli.utils.loop_stages import train_dpo_from_pairs + + out = train_dpo_from_pairs( + loop_env(), {"pairs_harvested": 0, "pairs_path": None} + ) + assert out["skipped"] is True + assert out["run_id"] is None + + def test_invokes_runner(self, loop_env, tmp_path, monkeypatch): + import soup_cli.utils.loop_stages as ls + + captured = {} + + class _R: + returncode = 0 + + def fake_runner(argv, **kwargs): + captured["argv"] = argv + return _R() + + monkeypatch.setattr(ls, "_TRAIN_RUNNER", fake_runner) + pairs = tmp_path / "p.jsonl" + pairs.write_text('{"prompt":"q","chosen":"a","rejected":"b"}\n', encoding="utf-8") + out = ls.train_dpo_from_pairs( + loop_env(), {"pairs_harvested": 1, "pairs_path": str(pairs)} + ) + assert out["skipped"] is False + assert out["run_id"].startswith("loop-train-") + argv = captured["argv"] + assert "train" in argv and "--yes" in argv + + def test_runner_failure_skips(self, loop_env, tmp_path, monkeypatch): + import soup_cli.utils.loop_stages as ls + + class _R: + returncode = 1 + + monkeypatch.setattr(ls, "_TRAIN_RUNNER", lambda *a, **k: _R()) + pairs = tmp_path / "p.jsonl" + pairs.write_text('{"prompt":"q","chosen":"a","rejected":"b"}\n', encoding="utf-8") + out = ls.train_dpo_from_pairs( + loop_env(), {"pairs_harvested": 1, "pairs_path": str(pairs)} + ) + assert out["skipped"] is True + + +class TestGateStage: + def test_skipped_when_train_skipped(self, loop_env): + from soup_cli.utils.loop_stages import gate_against_baseline + + out = gate_against_baseline(loop_env(), {"skipped": True}) + assert out["gate_verdict"] == "SKIPPED" + + def test_skipped_when_adapter_missing(self, loop_env): + from soup_cli.utils.loop_stages import gate_against_baseline + + out = gate_against_baseline( + loop_env(), {"skipped": False, "adapter_path": "nope/dir"} + ) + assert out["gate_verdict"] == "SKIPPED" + + def test_ok_verdict(self, loop_env, tmp_path, monkeypatch): + import soup_cli.utils.loop_stages as ls + + # Real eval suite + tasks; injected generate factory returns expected. + evals = tmp_path / "evals" + evals.mkdir() + tasks = evals / "tasks.jsonl" + tasks.write_text( + '{"prompt": "2+2", "expected": "4", "scoring": "exact"}\n', + encoding="utf-8", + ) + suite = evals / "gate.yaml" + suite.write_text( + "suite: loop\n" + "tasks:\n" + " - type: custom\n" + " name: math\n" + " threshold: 0.5\n" + f" tasks: {tasks.as_posix()}\n" + " scorer: exact\n", + encoding="utf-8", + ) + adapter = tmp_path / "adapter" + adapter.mkdir() + monkeypatch.setattr( + ls, "_GATE_GENERATE_FACTORY", lambda d: (lambda prompt: "4") + ) + state = loop_env(eval_suite=str(suite)) + out = ls.gate_against_baseline( + state, {"skipped": False, "adapter_path": str(adapter)} + ) + assert out["gate_verdict"] == "OK" + + +class TestDeployStage: + def test_noop_when_not_ok(self, loop_env): + from soup_cli.utils.loop_stages import deploy_to_canary + + out = deploy_to_canary(loop_env(), {"gate_verdict": "MAJOR"}) + assert out["deployed"] is False + assert out["canary_verdict"] is None + + def test_posts_when_ok(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + seen = {} + + def poster(endpoint, name): + seen["endpoint"] = endpoint + seen["name"] = name + return True + + monkeypatch.setattr(ls, "_DEPLOY_POSTER", poster) + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "http://localhost:8000") + out = ls.deploy_to_canary( + loop_env(), {"gate_verdict": "OK", "adapter_path": "adapters/run-1"} + ) + assert out["deployed"] is True + assert out["canary_verdict"] == "OK" + assert seen["name"] == "run-1" + + +class TestPrewiredConfig: + def test_build_returns_wired_config(self): + from soup_cli.utils.loop_stages import ( + build_prewired_watch_config, + deploy_to_canary, + gate_against_baseline, + harvest_from_traces, + train_dpo_from_pairs, + ) + + cfg = build_prewired_watch_config(max_iterations=1) + assert cfg.harvest_fn is harvest_from_traces + assert cfg.train_fn is train_dpo_from_pairs + assert cfg.gate_fn is gate_against_baseline + assert cfg.deploy_fn is deploy_to_canary + + +class TestPrewiredE2E: + def test_watch_harvests_and_trains(self, loop_env, tmp_path, monkeypatch): + import soup_cli.utils.loop_stages as ls + from soup_cli.utils.loop_iteration import list_iterations, read_iteration + from soup_cli.utils.loop_state import write_state + + td = _write_trace_dir(tmp_path) + monkeypatch.setenv("SOUP_LOOP_TRACE_DIR", td) + + class _R: + returncode = 0 + + train_calls = [] + monkeypatch.setattr( + ls, "_TRAIN_RUNNER", lambda argv, **k: train_calls.append(argv) or _R() + ) + write_state(loop_env(pre_wired=True)) + from soup_cli.utils.loop_stages import build_prewired_watch_config + + cfg = build_prewired_watch_config( + max_iterations=1, poll_interval_sec=1.0, + ) + from soup_cli.utils.loop_daemon import watch + + _final, ran = watch(cfg) + assert ran == 1 + # The DPO subprocess was invoked. + assert len(train_calls) == 1 + # The iteration recorded the harvested pairs. + ids = list_iterations() + assert len(ids) == 1 + rec = read_iteration(ids[0]) + assert rec.pairs_harvested == 1 + + +class TestLoopStagesSourceWiring: + def test_no_top_level_heavy_imports(self): + from pathlib import Path + + root = Path(__file__).resolve().parent.parent + src = ( + root / "src" / "soup_cli" / "utils" / "loop_stages.py" + ).read_text(encoding="utf-8") + head = "\n".join( + line for line in src.splitlines()[:60] + if line.strip() and not line.strip().startswith("#") + ) + for forbidden in ( + "import torch", "import transformers", "import peft", "import trl", + ): + assert forbidden not in head, f"top-level {forbidden!r} in loop_stages" + + +class TestLoopCliPrewired: + def _app(self): + from soup_cli.commands.loop import app + + return app + + def test_init_pre_wired_flag(self, loop_env, tmp_path): + result = runner.invoke(self._app(), [ + "init", "mymodel", "--eval", "evals/gate.yaml", + "--baseline", "base-ref", "--pre-wired", + ]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + from soup_cli.utils.loop_state import read_state + + assert read_state().pre_wired is True + + def test_watch_help_lists_pack_cans(self): + result = runner.invoke(self._app(), ["watch", "--help"]) + assert result.exit_code == 0 + out = result.stdout.replace("\n", " ") + assert "--pack-cans" in out + assert "--pre-wired" in out + + def test_status_shows_pre_wired(self, loop_env): + runner.invoke(self._app(), [ + "init", "mymodel", "--eval", "evals/gate.yaml", + "--baseline", "base-ref", "--pre-wired", + ]) + result = runner.invoke(self._app(), ["status"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + out = result.stdout.replace("\n", " ") + assert "pre_wired" in out + assert "yes" in out + + +# =========================================================================== +# #177 — pack each loop iteration as a v0.26 Soup Can + Registry lineage +# =========================================================================== + + +def _write_iteration(iteration_id, *, served="mymodel"): + from soup_cli.utils.loop_iteration import IterationRecord, write_iteration + + rec = IterationRecord( + iteration_id=iteration_id, + started_at="2026-06-01T00:00:00+00:00", + finished_at="2026-06-01T00:01:00+00:00", + pairs_harvested=3, + run_id="run-x", + gate_verdict="OK", + canary_verdict=None, + shipped=True, + rolled_back=False, + estimated_cost_usd=0.0, + notes="", + ) + write_iteration(rec) + return rec + + +class TestRegistryNameFrom: + @pytest.mark.parametrize("raw,expected_start", [ + ("mymodel", "mymodel"), + ("registry://abc12", "abc12"), + ("org/big-model", "org-big-model"), + ("//weird", "weird"), + ]) + def test_sanitises(self, raw, expected_start): + from soup_cli.utils.loop_iteration import registry_name_from + + out = registry_name_from(raw) + assert out[0].isalnum() + assert out.startswith(expected_start) or out == "loop" + + def test_empty_falls_back(self): + from soup_cli.utils.loop_iteration import registry_name_from + + assert registry_name_from("") == "loop" + assert registry_name_from("///") == "loop" + + +class TestPackIterationAsCan: + def test_writes_can_and_entry(self, loop_env, tmp_path): + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-aaa") + can_path, entry_id = pack_iteration_as_can( + "iter-aaa", served_model="mymodel" + ) + assert os.path.isfile(can_path) + assert can_path.endswith("iteration.can") + with RegistryStore() as store: + entry = store.get(entry_id) + assert entry is not None + assert entry["name"] == "mymodel" + + def test_can_verifies(self, loop_env, tmp_path): + from soup_cli.cans.unpack import inspect_can + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-bbb") + can_path, _ = pack_iteration_as_can("iter-bbb", served_model="mymodel") + manifest = inspect_can(can_path) + assert manifest.name == "mymodel" + + def test_parent_lineage(self, loop_env, tmp_path): + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-p1") + _write_iteration("iter-p2") + _can1, parent_id = pack_iteration_as_can("iter-p1", served_model="mymodel") + _can2, child_id = pack_iteration_as_can( + "iter-p2", served_model="mymodel", parent_registry_id=parent_id + ) + with RegistryStore() as store: + ancestors = store.get_ancestors(child_id) + assert any(a["id"] == parent_id for a in ancestors) + + def test_missing_parent_still_creates(self, loop_env, tmp_path): + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-mp") + _can, entry_id = pack_iteration_as_can( + "iter-mp", served_model="mymodel", parent_registry_id="no-such-parent" + ) + with RegistryStore() as store: + assert store.get(entry_id) is not None + + +class TestWatchPackCans: + def test_three_iterations_chain(self, loop_env, tmp_path, monkeypatch): + import soup_cli.utils.loop_stages as ls + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_daemon import watch + from soup_cli.utils.loop_state import write_state + + td = _write_trace_dir(tmp_path) + monkeypatch.setenv("SOUP_LOOP_TRACE_DIR", td) + + class _R: + returncode = 0 + + monkeypatch.setattr(ls, "_TRAIN_RUNNER", lambda *a, **k: _R()) + write_state(loop_env(pre_wired=True)) + from soup_cli.utils.loop_stages import build_prewired_watch_config + + cfg = build_prewired_watch_config( + max_iterations=3, poll_interval_sec=1.0, + pack_iterations=True, served_model="mymodel", base_model="mymodel", + ) + _final, ran = watch(cfg) + assert ran == 3 + # 3 registry entries named mymodel, each with a loop-iter tag. + with RegistryStore() as store: + entries = store.list_by_name("mymodel") + assert len(entries) == 3 + # Cans exist + verify. + import glob + + from soup_cli.cans.unpack import inspect_can + cans = glob.glob(os.path.join(".soup-loops", "*", "iteration.can")) + assert len(cans) == 3 + for c in cans: + inspect_can(c) # raises if invalid + + def test_replay_extract(self, loop_env, tmp_path): + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-ext") + pack_iteration_as_can("iter-ext", served_model="mymodel") + + from soup_cli.commands.loop import app + + result = runner.invoke( + app, ["replay", "iter-ext", "--extract", "extracted"] + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert os.path.isfile(os.path.join("extracted", "manifest.yaml")) + + def test_replay_extract_missing_can(self, loop_env, tmp_path): + _write_iteration("iter-nocan") + from soup_cli.commands.loop import app + + result = runner.invoke( + app, ["replay", "iter-nocan", "--extract", "extracted"] + ) + assert result.exit_code == 1 + assert "can" in result.output.lower() + + +# =========================================================================== +# #223 — soup adapters pr --push (GitHub PR publisher) +# =========================================================================== + + +class TestParsePrTarget: + def test_happy(self): + from soup_cli.utils.adapter_pr import parse_pr_target + + assert parse_pr_target("MakazhanAlpamys/Soup#42") == ( + "MakazhanAlpamys", + "Soup", + 42, + ) + + def test_strips_whitespace(self): + from soup_cli.utils.adapter_pr import parse_pr_target + + assert parse_pr_target(" owner/repo#7 ") == ("owner", "repo", 7) + + @pytest.mark.parametrize( + "bad", + [ + "owner/repo", # no #N + "owner#42", # no repo + "owner/repo#0", # PR 0 + "owner/repo#-1", + "owner/repo#abc", + "/repo#1", + "owner/#1", + "owner repo#1", + ], + ) + def test_rejects_bad(self, bad): + from soup_cli.utils.adapter_pr import parse_pr_target + + with pytest.raises(ValueError): + parse_pr_target(bad) + + def test_non_string(self): + from soup_cli.utils.adapter_pr import parse_pr_target + + with pytest.raises(TypeError): + parse_pr_target(42) # type: ignore[arg-type] + + def test_bool_rejected(self): + from soup_cli.utils.adapter_pr import parse_pr_target + + with pytest.raises(TypeError): + parse_pr_target(True) # type: ignore[arg-type] + + +class TestResolveGithubToken: + def test_github_token(self): + from soup_cli.utils.adapter_pr import resolve_github_token + + assert resolve_github_token({"GITHUB_TOKEN": "ghp_abc"}) == "ghp_abc" + + def test_gh_token_fallback(self): + from soup_cli.utils.adapter_pr import resolve_github_token + + assert resolve_github_token({"GH_TOKEN": "ghp_xyz"}) == "ghp_xyz" + + def test_strips_whitespace(self): + from soup_cli.utils.adapter_pr import resolve_github_token + + assert resolve_github_token({"GITHUB_TOKEN": " tok "}) == "tok" + + def test_missing_fails_fast(self): + from soup_cli.utils.adapter_pr import resolve_github_token + + with pytest.raises(RuntimeError, match="token"): + resolve_github_token({}) + + def test_blank_treated_as_missing(self): + from soup_cli.utils.adapter_pr import resolve_github_token + + with pytest.raises(RuntimeError): + resolve_github_token({"GITHUB_TOKEN": " "}) + + +class TestPostPrComment: + def test_happy_path_invokes_gh(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + captured = {} + + class _Result: + returncode = 0 + stdout = '{"html_url": "https://github.com/o/r/pull/1#issuecomment-9"}' + stderr = "" + + def fake_run(argv, **kwargs): + captured["argv"] = argv + captured["input"] = kwargs.get("input") + return _Result() + + url = post_pr_comment( + "o/r#1", "## Hello\nbody", env={"GITHUB_TOKEN": "tok"}, runner=fake_run + ) + assert url == "https://github.com/o/r/pull/1#issuecomment-9" + argv = captured["argv"] + assert argv[0] == "gh" + assert "api" in argv + assert "repos/o/r/issues/1/comments" in argv + assert "--method" in argv and "POST" in argv + # body posted via JSON stdin (handles multiline safely) + assert json.loads(captured["input"])["body"] == "## Hello\nbody" + + def test_missing_token_fails_before_subprocess(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + def fake_run(argv, **kwargs): # pragma: no cover — must not be reached + raise AssertionError("subprocess should not run without a token") + + with pytest.raises(RuntimeError, match="token"): + post_pr_comment("o/r#1", "body", env={}, runner=fake_run) + + def test_nonzero_exit_raises(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + class _Result: + returncode = 1 + stdout = "" + stderr = "HTTP 404: Not Found" + + with pytest.raises(RuntimeError, match="404"): + post_pr_comment( + "o/r#1", "body", env={"GH_TOKEN": "t"}, runner=lambda *a, **k: _Result() + ) + + def test_empty_body_rejected(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + with pytest.raises(ValueError): + post_pr_comment( + "o/r#1", " ", env={"GITHUB_TOKEN": "t"}, runner=lambda *a, **k: None + ) + + def test_null_byte_body_rejected(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + with pytest.raises(ValueError): + post_pr_comment( + "o/r#1", + "a\x00b", + env={"GITHUB_TOKEN": "t"}, + runner=lambda *a, **k: None, + ) + + def test_bad_target_rejected(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + with pytest.raises(ValueError): + post_pr_comment( + "not-a-target", "body", env={"GITHUB_TOKEN": "t"}, + runner=lambda *a, **k: None, + ) + + +class TestPrCliPush: + def _adapters_app(self): + from soup_cli.commands.adapters import app + + return app + + def test_push_flag_in_help(self): + result = runner.invoke(self._adapters_app(), ["pr", "--help"]) + assert result.exit_code == 0 + out = result.stdout.replace("\n", " ") + assert "--push" in out + + def test_push_missing_token(self, monkeypatch): + monkeypatch.delenv("GITHUB_TOKEN", raising=False) + monkeypatch.delenv("GH_TOKEN", raising=False) + sha = "a" * 64 + result = runner.invoke( + self._adapters_app(), + [ + "pr", + "my-title", + "--base-sha", + sha, + "--adapter", + "adapters/x", + "--push", + "owner/repo#42", + ], + ) + assert result.exit_code == 1, (result.output, repr(result.exception)) + assert "token" in result.output.lower() + + def test_push_happy(self, monkeypatch): + # Mock the post so we never touch the network. + import soup_cli.utils.adapter_pr as ap + + monkeypatch.setattr( + ap, "post_pr_comment", + lambda target, body, **kw: "https://github.com/o/r/pull/42#c-1", + ) + sha = "b" * 64 + result = runner.invoke( + self._adapters_app(), + [ + "pr", + "my-title", + "--base-sha", + sha, + "--adapter", + "adapters/x", + "--push", + "owner/repo#42", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "github.com/o/r/pull/42" in result.output + + +# =========================================================================== +# Review follow-ups (Step 5 — fix every finding CRITICAL→LOW) +# =========================================================================== + + +class TestVerdictBoundaries: + """Strict `<` boundaries: drop==0.02 → MINOR, drop==0.05 → MAJOR.""" + + def _verdict(self, tmp_path, monkeypatch, baseline, candidate): + from soup_cli.utils.adapter_merge import predict_merged_verdict + + monkeypatch.chdir(tmp_path) + suite = tmp_path / "canary.json" + suite.write_text(json.dumps({ + "baseline_scores": baseline, "candidate_scores": candidate, + }), encoding="utf-8") + return predict_merged_verdict(_make_report(), str(suite)) + + def test_drop_just_under_two_pct_is_ok(self, tmp_path, monkeypatch): + # 1.9% drop → still OK (boundary is strict `< 0.02`). + assert self._verdict( + tmp_path, monkeypatch, [1.0, 1.0], [0.981, 0.981] + ) == "OK" + + def test_drop_exactly_two_pct_is_minor(self, tmp_path, monkeypatch): + # Exactly 2.0% drop → MINOR (OK requires drop strictly < 0.02). + assert self._verdict( + tmp_path, monkeypatch, [1.0, 1.0], [0.98, 0.98] + ) == "MINOR" + + def test_drop_just_under_five_pct_is_minor(self, tmp_path, monkeypatch): + # 4.9% drop → still MINOR. + assert self._verdict( + tmp_path, monkeypatch, [1.0, 1.0], [0.951, 0.951] + ) == "MINOR" + + def test_drop_exactly_five_pct_is_major(self, tmp_path, monkeypatch): + # Exactly 5.0% drop → MAJOR (MINOR requires drop strictly < 0.05). + assert self._verdict( + tmp_path, monkeypatch, [1.0, 1.0], [0.95, 0.95] + ) == "MAJOR" + + def test_improvement_is_ok(self, tmp_path, monkeypatch): + # Candidate better than baseline → negative drop → OK. + assert self._verdict( + tmp_path, monkeypatch, [0.8, 0.8], [0.9, 0.9] + ) == "OK" + + +class TestPackIterationSafely: + def test_pack_failure_never_crashes_watch(self, loop_env, tmp_path, monkeypatch): + """A pack failure mid-watch must not kill the daemon; the iteration + manifest gets a ``pack-failed:`` note and the loop survives.""" + import soup_cli.utils.loop_iteration as li + from soup_cli.utils.loop_daemon import WatchConfig, watch + from soup_cli.utils.loop_state import init_state + + init_state("mymodel", "evals/gate.yaml", "base-ref") + + def boom(*a, **k): + raise OSError("disk full") + + monkeypatch.setattr(li, "pack_iteration_as_can", boom) + + cfg = WatchConfig( + poll_interval_sec=1.0, max_iterations=1, pack_iterations=True, + served_model="mymodel", + ) + final_state, n = watch(cfg) + assert n == 1 # the daemon completed its single iteration + # The manifest was re-written with the pack-failed note. + from soup_cli.utils.loop_iteration import list_iterations, read_iteration + ids = list_iterations() + assert len(ids) == 1 + rec = read_iteration(ids[0]) + assert "pack-failed" in rec.notes + + def test_pack_iterations_false_skips_packing(self, loop_env, tmp_path): + import glob + + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_daemon import WatchConfig, watch + from soup_cli.utils.loop_state import init_state + + init_state("mymodel", "evals/gate.yaml", "base-ref") + cfg = WatchConfig( + poll_interval_sec=1.0, max_iterations=1, pack_iterations=False, + ) + watch(cfg) + # No cans written, no registry rows created. + assert glob.glob(os.path.join(".soup-loops", "*", "iteration.can")) == [] + with RegistryStore() as store: + assert store.list_by_name("mymodel") == [] + + +class TestGhArgvAdjacency: + def test_method_and_input_flags_adjacent(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + captured = {} + + class _R: + returncode = 0 + stdout = '{"html_url": "u"}' + stderr = "" + + def fake_run(argv, **kw): + captured["argv"] = argv + return _R() + + post_pr_comment( + "o/r#1", "body", env={"GITHUB_TOKEN": "t"}, runner=fake_run + ) + argv = captured["argv"] + # `--method POST` and `--input -` must be value-adjacent. + assert argv[argv.index("--method") + 1] == "POST" + assert argv[argv.index("--input") + 1] == "-" + # No `shell=True` could ever apply — argv[0] is the bare binary. + assert argv[0] == "gh" + + +class TestPostPrCommentEnvAllowlist: + def test_secrets_not_leaked_to_child(self, monkeypatch): + """When ``env`` is None, the gh child env is built from an allowlist + so HF_TOKEN / OPENAI_API_KEY never reach the subprocess.""" + from soup_cli.utils.adapter_pr import post_pr_comment + + monkeypatch.setenv("GITHUB_TOKEN", "ghp_real") + monkeypatch.setenv("HF_TOKEN", "hf_secret") + monkeypatch.setenv("OPENAI_API_KEY", "sk-secret") + monkeypatch.setenv("PATH", "/usr/bin") + + captured = {} + + class _R: + returncode = 0 + stdout = "{}" + stderr = "" + + def fake_run(argv, **kw): + captured["env"] = kw.get("env") + return _R() + + post_pr_comment("o/r#1", "body", runner=fake_run) # env defaults to None + child_env = captured["env"] + assert child_env is not None + assert "HF_TOKEN" not in child_env + assert "OPENAI_API_KEY" not in child_env + assert child_env.get("GH_TOKEN") == "ghp_real" + assert "PATH" in child_env + + def test_body_over_cap_rejected(self): + from soup_cli.utils.adapter_pr import post_pr_comment + + big = "x" * 60_001 + with pytest.raises(ValueError, match="cap"): + post_pr_comment( + "o/r#1", big, env={"GITHUB_TOKEN": "t"}, + runner=lambda *a, **k: None, + ) + + +class TestCmaesTempCleanup: + def test_temp_removed_even_when_scorer_raises(self, tmp_path, monkeypatch): + from soup_cli.utils.cmaes_merge import build_cmaes_eval_fn + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + plan = _cmaes_plan(tmp_path, [a, b]) + seen = {} + + def scorer(merged_dir, eval_suite): + seen["dir"] = merged_dir + assert os.path.isdir(merged_dir) + raise RuntimeError("scorer blew up") + + fn = build_cmaes_eval_fn(plan, scorer=scorer) + with pytest.raises(RuntimeError): + fn((0.5, 0.5)) + # `finally: shutil.rmtree(..., ignore_errors=True)` removed the temp. + assert not os.path.exists(seen["dir"]) + + +class TestRegistryNameFromCap: + def test_caps_at_128_chars(self): + from soup_cli.utils.loop_iteration import registry_name_from + + out = registry_name_from("a" * 500) + assert len(out) == 128 + assert out[0].isalnum() + + +class TestDeploySsrfGuard: + def test_public_http_endpoint_rejected(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + # Plain HTTP to a public IP is rejected by validate_webhook_url. + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "http://8.8.8.8:8000") + out = ls.deploy_to_canary( + loop_env(), + {"gate_verdict": "OK", "adapter_path": ".soup-loops/adapters/x"}, + ) + assert out["deployed"] is False + assert "SSRF" in str(out.get("notes", "")) + + def test_public_https_endpoint_rejected(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + # HTTPS to a PUBLIC host passes the webhook validator but the deploy + # surface tightens to loopback/LAN only (review MEDIUM-4). + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "https://8.8.8.8:8000") + out = ls.deploy_to_canary( + loop_env(), + {"gate_verdict": "OK", "adapter_path": ".soup-loops/adapters/x"}, + ) + assert out["deployed"] is False + assert "loopback/LAN" in str(out.get("notes", "")) + + def test_public_hostname_rejected(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + # A non-IP hostname can't be verified private without DNS → rejected. + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "https://evil.example.com") + out = ls.deploy_to_canary( + loop_env(), + {"gate_verdict": "OK", "adapter_path": ".soup-loops/adapters/x"}, + ) + assert out["deployed"] is False + assert "loopback/LAN" in str(out.get("notes", "")) + + def test_loopback_endpoint_allowed_to_post(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "http://127.0.0.1:8000") + # Inject the poster so no real network call happens. + monkeypatch.setattr(ls, "_DEPLOY_POSTER", lambda ep, name: True) + out = ls.deploy_to_canary( + loop_env(), + {"gate_verdict": "OK", "adapter_path": ".soup-loops/adapters/win"}, + ) + assert out["deployed"] is True + assert out["canary_verdict"] == "OK" + + def test_private_lan_endpoint_allowed(self, loop_env, monkeypatch): + import soup_cli.utils.loop_stages as ls + + # RFC1918 over HTTPS is a legitimate LAN serve endpoint. + monkeypatch.setenv("SOUP_LOOP_SERVE_ENDPOINT", "https://192.168.1.5:8000") + monkeypatch.setattr(ls, "_DEPLOY_POSTER", lambda ep, name: True) + out = ls.deploy_to_canary( + loop_env(), + {"gate_verdict": "OK", "adapter_path": ".soup-loops/adapters/win"}, + ) + assert out["deployed"] is True + + +class TestNoHeavyTopLevelImports: + """Heavy deps must be lazy-imported inside functions (cold-start policy).""" + + @pytest.mark.parametrize("module", [ + "soup_cli.utils.cmaes_merge", + "soup_cli.utils.loop_stages", + "soup_cli.utils.loop_iteration", + "soup_cli.utils.adapter_pr", + ]) + def test_no_top_level_heavy_imports(self, module): + import importlib + + path = importlib.import_module(module).__file__ + with open(path, encoding="utf-8") as fh: + src = fh.read() + # Strip everything after the first `def`/`class` so we only inspect + # the module-top import block. + head = re.split(r"\n(?:def |class )", src, maxsplit=1)[0] + for heavy in ("torch", "transformers", "peft", "trl", "numpy", "httpx"): + assert f"\nimport {heavy}" not in head, (module, heavy) + assert f"\nfrom {heavy}" not in head, (module, heavy) + + +class TestLoopStagesAtomicWrites: + def test_uses_atomic_write_text(self): + import soup_cli.utils.loop_stages as ls + + path = ls.__file__ + with open(path, encoding="utf-8") as fh: + src = fh.read() + # Pairs + train-yaml writes go through the shared atomic helper. + assert "atomic_write_text(" in src + # No bare open(..., "w") for the artifact writes. + assert 'open(pairs_path, "w")' not in src + + +class TestMergeAdaptersCmaesGuard: + def test_cmaes_strategy_rejected_with_path_hint(self, tmp_path, monkeypatch): + from soup_cli.utils.adapter_merge import merge_adapters + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + with pytest.raises(ValueError, match="run_cmaes_merge"): + merge_adapters([a, b], str(tmp_path / "out"), strategy="cmaes") + + +class TestWriteMergedAdapterPublic: + def test_public_symbol_and_alias(self): + from soup_cli.utils import adapter_merge + + assert hasattr(adapter_merge, "write_merged_adapter") + # back-compat alias preserved for any external caller. + assert ( + adapter_merge._write_merged_adapter + is adapter_merge.write_merged_adapter + ) + + def test_cmaes_imports_public_name(self): + import soup_cli.utils.cmaes_merge as cm + + with open(cm.__file__, encoding="utf-8") as fh: + src = fh.read() + assert "write_merged_adapter" in src + assert "_write_merged_adapter" not in src + + +class TestPackEntryRollback: + def test_pack_failure_rolls_back_registry_entry( + self, loop_env, tmp_path, monkeypatch + ): + """If pack_entry fails after the Registry push, the entry is rolled + back so the watch-loop lineage chain never points at an orphan.""" + import soup_cli.cans.pack as pack_mod + from soup_cli.registry.store import RegistryStore + from soup_cli.utils.loop_iteration import pack_iteration_as_can + + _write_iteration("iter-rb") + + def boom(**kw): + raise OSError("disk full") + + monkeypatch.setattr(pack_mod, "pack_entry", boom) + with pytest.raises(OSError): + pack_iteration_as_can("iter-rb", served_model="mymodel") + # No orphaned entry left behind. + with RegistryStore() as store: + assert store.list_by_name("mymodel") == [] + + +class TestWatchConfigValidation: + def test_pack_iterations_must_be_bool(self): + from soup_cli.utils.loop_daemon import WatchConfig + + with pytest.raises(ValueError): + WatchConfig(pack_iterations="yes") # type: ignore[arg-type] + + def test_base_model_null_byte_rejected(self): + from soup_cli.utils.loop_daemon import WatchConfig + + with pytest.raises(ValueError): + WatchConfig(base_model="a\x00b") + + def test_base_model_empty_rejected(self): + from soup_cli.utils.loop_daemon import WatchConfig + + with pytest.raises(ValueError): + WatchConfig(base_model="") + + def test_served_model_oversize_rejected(self): + from soup_cli.utils.loop_daemon import WatchConfig + + with pytest.raises(ValueError): + WatchConfig(served_model="a" * 513) + + def test_served_model_none_ok(self): + from soup_cli.utils.loop_daemon import WatchConfig + + cfg = WatchConfig(served_model=None) + assert cfg.served_model is None + + +class TestCmaesScanGate: + def _app(self): + from soup_cli.commands.adapters import app + + return app + + def test_cmaes_respects_backdoor_scan_gate(self, tmp_path, monkeypatch): + """A FAIL scan must block --strategy cmaes too (review MEDIUM-5).""" + import soup_cli.utils.adapter_scan as scan_mod + + monkeypatch.chdir(tmp_path) + a = _make_adapter(tmp_path / "a") + b = _make_adapter(tmp_path / "b") + suite = tmp_path / "eval.jsonl" + suite.write_text('{"prompt": "p", "expected": "e"}\n', encoding="utf-8") + + class _Rep: + overall = "FAIL" + summary = "rank-1 dominance" + + monkeypatch.setattr(scan_mod, "scan_adapter", lambda p: _Rep()) + result = runner.invoke(self._app(), [ + "merge", a, b, "--strategy", "cmaes", + "--eval", str(suite), "-o", str(tmp_path / "out"), + ]) + assert result.exit_code == 3, (result.output, repr(result.exception)) + assert "FAIL" in result.output