From 21a2bf8e8c7fe7a92d802f9225406f1baa877731 Mon Sep 17 00:00:00 2001 From: Alpamys Date: Mon, 1 Jun 2026 19:12:24 +0500 Subject: [PATCH] =?UTF-8?q?feat(governance,energy):=20v0.71.3=20=E2=80=94?= =?UTF-8?q?=20annex=20PDF,=20audit=20auto-log,=20energy=20hook,=20can=20v3?= =?UTF-8?q?,=20airgap=20receipt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #180 #181 #182 #183 #184 #188. - #180 EnergyTracker (codecarbon offline) + `soup train --track-energy`; [carbon] extra - #181 PDF Annex XI/XII (reportlab) + paths.atomic_write_bytes; [pdf] extra - #182 Soup Can manifest v3 + attestations field + `can pack --attest` - #183 per-command audit-log auto-instrumentation (--no-audit-log / SOUP_NO_AUDIT_LOG) - #184 auto-populate Annex top_domains from the training JSONL - #188 embed repro-receipt into `soup airgap-bundle` New [pdf]+[carbon] extras (reportlab also in [dev]). +83 tests (12259 -> 12342), 79.10% coverage. Reviewed (security/code/python/tdd): security M1 raw-size gate on --attest before parse, python H1/H2 type hints, +22 negative tests. --- CHANGELOG.md | 35 + CONTRIBUTING.md | 4 +- README.md | 24 +- docs/adapters-and-governance.md | 44 +- docs/commands.md | 8 +- pyproject.toml | 9 +- src/soup_cli/__init__.py | 2 +- src/soup_cli/cans/pack.py | 4 + src/soup_cli/cans/schema.py | 51 +- src/soup_cli/cli.py | 113 ++- src/soup_cli/commands/airgap.py | 60 ++ src/soup_cli/commands/can.py | 41 +- src/soup_cli/commands/train.py | 85 ++- src/soup_cli/utils/airgap_bundle.py | 48 +- src/soup_cli/utils/annex_xi.py | 254 ++++++- src/soup_cli/utils/energy.py | 127 +++- src/soup_cli/utils/paths.py | 32 + tests/test_cans.py | 4 +- tests/test_part_a_wave2.py | 19 +- tests/test_v0713.py | 1036 +++++++++++++++++++++++++++ 20 files changed, 1942 insertions(+), 58 deletions(-) create mode 100644 tests/test_v0713.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cae135..d5bfd96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,41 @@ reproducing 70+ versions of notes. ## [Unreleased] +## [0.71.3] - 2026-06-01 + +### Added +- **Energy & CO2 measurement for training** — `soup train --track-energy` wraps + the training window in a codecarbon **offline** tracker (no IP-geolocation + network call) and reports kWh / CO2 / grid intensity, feeding those numbers + into `--annex-xi`. New `EnergyTracker` context manager; graceful no-op when + codecarbon is absent (`pip install soup-cli[carbon]`). `--energy-country` + picks the ISO-3166 alpha-3 grid for the CO2 estimate (default `USA`). +- **PDF Annex XI/XII documents** — `soup train --annex-xi report.pdf` now renders + a reportlab PDF (a `.md` path still renders markdown). `pip install + soup-cli[pdf]`. +- **Auto-populated training-corpus domains in Annex XI/XII** — the top crawled + domains (with shares) are now extracted from the training JSONL and listed in + the EU AI Act docs, replacing the previous empty placeholder. +- **Soup Can manifest v3 with embedded attestations** — `soup can pack --attest + ` (repeatable) embeds in-toto Statements into a v3 can + manifest; v1/v2 cans still load. Each statement is shape- and size-validated. +- **Local audit log auto-instrumentation** — every `soup` command now appends one + HIPAA/SOC2-shaped record to `~/.soup/audit.jsonl` (secrets redacted, args + capped). Opt out per-invocation with `--no-audit-log` or globally with + `SOUP_NO_AUDIT_LOG=1`. Tail/rotate with `soup audit-log`. +- **Reproducibility receipt in airgap bundles** — `soup airgap-bundle + --repro-receipt ` embeds an SR 11-7 receipt as + `repro-receipt.json`; auto-detected from `/repro-receipt.json` when not + supplied. + +### Security +- `soup can pack --attest` now rejects oversize attestation files by their raw + size *before* parsing them into memory (defence against memory-exhaustion). +- The new file-loading paths (attestation JSON, airgap receipt, training-corpus + scan, PDF write) are all cwd-contained + TOCTOU symlink-rejected and + size-capped; the audit auto-log redacts `hf_`/`sk-`/`Bearer` tokens and never + crashes the CLI on a broken log. + ## [0.71.2] - 2026-06-01 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d2d3c7..1093676 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -33,6 +33,8 @@ This installs: pulled in too (since v0.71.0 these are an opt-in extra, not core deps) - `cryptography` — `[dev]` also pulls it in (it's the `[sign]` extra) so the ed25519 signing tests (`soup adapters sign` / `soup attest`) run in CI +- `reportlab` — `[dev]` also pulls it in (it's the `[pdf]` extra) so the + Annex XI/XII PDF tests (`soup train --annex-xi *.pdf`) run in CI ### 3. Verify Setup @@ -118,7 +120,7 @@ src/soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (272 files, 12259 tests) +tests/ - Test suite (273 files, 12342 tests) examples/ - Real-world config examples and datasets ``` diff --git a/README.md b/README.md index 1ea9f8d..2808aea 100644 --- a/README.md +++ b/README.md @@ -49,19 +49,19 @@ infrastructure instead of improving models. Soup fixes that. ## What's New -**v0.71.2 — Governance & supply-chain live.** Real signatures and supply-chain gates: +**v0.71.3 — Compliance / annex / audit / energy.** The EU AI Act + carbon + audit surfaces go live: -- **ed25519 signing** — `soup adapters sign --backend ed25519 --key priv.pem` (or `--generate-key`) - produces a real detached signature over the adapter's Merkle root; `soup adapters verify - --public-key trusted.pem` does cryptographic authentication. Same for `soup attest emit - --sign ed25519` + the new `soup attest verify`. Install with `pip install soup-cli[sign]`. -- **Anti-AI-Jacking namespace pin** — Hub model downloads now refuse a repo whose author silently - changed (or whose creation date jumped backward), the classic namespace-re-creation attack. -- **License-conflict gate at merge** — `soup adapters merge` auto-detects each adapter's license - (from `adapter_config.json` / model card) and refuses incompatible combinations unless you pass - `--license-override ` (logged to the audit trail). -- **Backdoor-scan gate at merge** — refuses to merge any adapter that `soup adapters scan` flags - FAIL unless you pass `--allow-unscanned`. +- **Energy & CO2 measurement** — `soup train --track-energy` measures the training window with an + offline carbon tracker (no network) and reports kWh / CO2, feeding them into the Annex doc. + Install with `pip install soup-cli[carbon]`. +- **PDF compliance docs** — `soup train --annex-xi report.pdf` now renders a real PDF of the + EU AI Act Annex XI/XII (a `.md` path still gives markdown). The top crawled domains of your + training corpus are auto-listed. `pip install soup-cli[pdf]`. +- **Per-command audit log** — every `soup` command appends one HIPAA/SOC2 record to + `~/.soup/audit.jsonl` (secrets redacted). Opt out with `--no-audit-log` or `SOUP_NO_AUDIT_LOG=1`. +- **Signed Soup Cans + airgap receipts** — `soup can pack --attest statement.json` embeds in-toto + attestations into a can; `soup airgap-bundle --repro-receipt receipt.json` ships a reproducibility + receipt inside the offline bundle. Full history: [CHANGELOG.md](CHANGELOG.md) · [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). diff --git a/docs/adapters-and-governance.md b/docs/adapters-and-governance.md index c6a54aa..e0cad41 100644 --- a/docs/adapters-and-governance.md +++ b/docs/adapters-and-governance.md @@ -275,6 +275,9 @@ Share a reproducible recipe as a single `.can` file — a tarball of the manifes # Pack a registry entry into a .can soup can pack --entry-id llama31-chat-v1 --out ./llama31-chat.can +# Embed in-toto attestations into the manifest (v3 cans; repeatable) +soup can pack --entry-id llama31-chat-v1 --out ./signed.can --attest att.json + # Preview the manifest without extracting soup can inspect ./llama31-chat.can @@ -358,29 +361,45 @@ Render an EU AI Act Annex XI (technical documentation, Sections 1+2) or Annex XI (Article 53(1)(d) public training summary) directly from a training run: ```bash -soup train --config soup.yaml --annex-xi annex.md +soup train --config soup.yaml --annex-xi annex.md # markdown +soup train --config soup.yaml --annex-xi annex.pdf # PDF (pip install soup-cli[pdf]) +soup train --config soup.yaml --track-energy --annex-xi annex.md # + measured kWh/CO2 ``` Top-10 domains by share, modality breakdown, training compute / kWh / CO₂, model -description, base model, run id. Markdown body now; PDF in v0.59.1. Operator-controlled -fields are escape-neutralised (`|[](){}!<>` + newline / CR / tab) so a malicious model -name can't inject a forged heading into downstream PDF/HTML renderers. +description, base model, run id. A `.pdf` output path renders a reportlab PDF (a `.md` +path renders markdown). The **top crawled domains** are auto-extracted from the training +JSONL (`cfg.data.train`). With `--track-energy`, the measured energy is recorded in the +doc. Operator-controlled fields are escape-neutralised (`|[](){}!<>` + newline / CR / tab) +so a malicious model name can't inject a forged heading into downstream renderers. + +### Energy & CO₂ measurement (`--track-energy`) + +`soup train --track-energy` wraps the training window in a codecarbon **offline** tracker +(no IP-geolocation network call). It reports kWh / CO₂ / grid intensity and feeds them into +`--annex-xi`. `--energy-country ` picks the grid for the CO₂ estimate (default `USA`; +the kWh figure itself is country-independent). Requires `pip install soup-cli[carbon]`; +without it, `--track-energy` is a graceful no-op. ## Audit Log (`soup audit-log`) -Tail or rotate the HIPAA/SOC2-shaped JSONL audit log at `~/.soup/audit.jsonl` (override -via `SOUP_AUDIT_LOG_PATH`, containment-checked to `$HOME / $CWD / $TMPDIR`): +**Every `soup` command** now appends one HIPAA/SOC2-shaped record to the JSONL audit log at +`~/.soup/audit.jsonl` (override via `SOUP_AUDIT_LOG_PATH`, containment-checked to +`$HOME / $CWD / $TMPDIR`). Opt out per-invocation with `soup --no-audit-log ` or +globally with `SOUP_NO_AUDIT_LOG=1`. Tail or rotate: ```bash soup audit-log tail --limit 50 # Rich table view soup audit-log tail --json # raw JSONL for SIEM ingestion soup audit-log rotate --cap-mb 100 # force a rotation pass +soup --no-audit-log version # skip the audit line for this command ``` PII redaction across **every** string field (`hf_*` / `sk-*` / `Bearer …` → ``) via the v0.40.3 `_SECRET_RE` policy. POSIX `O_NOFOLLOW` + `0o600` perms, atomic-append, -rotation at 100 MiB with symlink rejection at the backup path. +rotation at 100 MiB with symlink rejection at the backup path. The auto-instrumentation is +best-effort — a broken audit log never crashes the CLI. ## Reproducibility Receipt (`soup train --repro-receipt`) @@ -490,7 +509,16 @@ merge with exit 3 unless you pass `--allow-unscanned`. Single signed tarball with model + datasets + wheels + CUDA kernels + embedded `manifest.json` listing SHA-256 per file. Sized for one-way -physical-media transfer through a data diode. Default 100 GiB cap; +physical-media transfer through a data diode. A reproducibility receipt is +embedded as a top-level `repro-receipt.json` when you pass +`--repro-receipt ` (or auto-detected from +`/repro-receipt.json`): + +```bash +soup airgap-bundle --model ./out --output bundle.tar --repro-receipt repro.json +``` + +Default 100 GiB cap; refuses oversize. Deterministic dataset labeling by sorted basename (NOT argv order) so the same inputs in different argv order produce identical manifests. TOCTOU defence: `os.lstat + S_ISLNK` re-check on diff --git a/docs/commands.md b/docs/commands.md index af5a25e..cd1e5a2 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -157,7 +157,13 @@ soup adapters check-safetensors [--strict] Refuse pickle / PyTorch-cl soup adapters merge ... [--license ] [--license-override ] [--allow-unscanned] License + backdoor-scan gates (auto-detect license; scan FAIL refused) soup attest emit ... [--sign ed25519 --key ] [-o att.json] in-toto/SLSA-3 attestation (+ .sig sidecar) soup attest verify --signature [--public-key ] Verify ed25519 attestation signature -soup airgap-bundle --model --output Signed tarball for data-diode transfer +soup airgap-bundle --model --output [--repro-receipt ] Signed tarball for data-diode transfer (embeds repro-receipt) +soup train --config soup.yaml --annex-xi EU AI Act Annex XI/XII doc (markdown or PDF; top_domains auto-filled) +soup train --config soup.yaml --track-energy [--energy-country USA] codecarbon offline kWh/CO2 → annex-xi (pip install soup-cli[carbon]) +soup train --config soup.yaml --repro-receipt SR 11-7 reproducibility receipt +soup can pack --entry-id --out r.can --attest Embed in-toto Statements into a v3 can manifest +soup audit-log tail / rotate Tail / rotate the per-command HIPAA/SOC2 audit log (~/.soup/audit.jsonl) +soup --no-audit-log / SOUP_NO_AUDIT_LOG=1 Opt out of the per-command audit line soup eval unlearning --benchmark tofu|muse|wmdp Forget Quality + Model Utility + PrivLeak verdict soup edit set --base --method rome|memit|alphaedit --subject "..." --target "..." Surgical knowledge edit (--plan-only available) soup edit diff --probes p.jsonl Knowledge-injection diff visualizer diff --git a/pyproject.toml b/pyproject.toml index aaf73c2..54dae3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.71.2" +version = "0.71.3" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" @@ -54,7 +54,7 @@ wandb = ["wandb>=0.15.0,<0.18.0"] # Self-references `[train]` so CI / contributors get the full training stack # (CI runs `pip install -e ".[dev]"`; without this every test would fail at # `import torch`). -dev = ["soup-cli[train]", "cryptography>=41.0.0", "pytest>=7.0", "ruff>=0.1.0", "pytest-cov>=4.0", "httpx>=0.24.0", "mypy>=1.8.0", "pre-commit>=3.5.0"] +dev = ["soup-cli[train]", "cryptography>=41.0.0", "reportlab>=4.0.0", "pytest>=7.0", "ruff>=0.1.0", "pytest-cov>=4.0", "httpx>=0.24.0", "mypy>=1.8.0", "pre-commit>=3.5.0"] ui = ["fastapi>=0.104.0", "uvicorn>=0.24.0"] serve = ["fastapi>=0.104.0", "uvicorn>=0.24.0"] serve-fast = ["vllm>=0.4.0", "fastapi>=0.104.0", "uvicorn>=0.24.0"] @@ -89,6 +89,11 @@ data-pro = ["langdetect>=1.0.9", "presidio-analyzer>=2.2.0"] # adapters sign`. Pure-offline; Sigstore keyless (OIDC + Fulcio/Rekor network) # stays infra-blocked and is NOT bundled here. sign = ["cryptography>=41.0.0"] +# v0.71.3 #181 — reportlab PDF rendering for `soup train --annex-xi *.pdf`. +pdf = ["reportlab>=4.0.0"] +# v0.71.3 #180 — codecarbon offline energy/CO2 measurement for +# `soup train --track-energy` (offline; no IP-geolocation network call). +carbon = ["codecarbon>=2.0.0"] [project.scripts] soup = "soup_cli.cli:run" diff --git a/src/soup_cli/__init__.py b/src/soup_cli/__init__.py index 4b64995..2352afb 100644 --- a/src/soup_cli/__init__.py +++ b/src/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.71.2" +__version__ = "0.71.3" diff --git a/src/soup_cli/cans/pack.py b/src/soup_cli/cans/pack.py index a10cdc1..f7b1485 100644 --- a/src/soup_cli/cans/pack.py +++ b/src/soup_cli/cans/pack.py @@ -29,6 +29,7 @@ def _add_text(tar: tarfile.TarFile, name: str, content: str) -> None: def pack_entry( *, entry_id: str, out_path: str, author: str = "unknown", description: Optional[str] = None, + attestations: Optional[list[dict]] = None, ) -> Path: """Pack a registry entry as a ``.can`` tarball. @@ -38,6 +39,8 @@ def pack_entry( at call time. author: author handle baked into the manifest. description: optional free-form description. + attestations: optional list of embedded in-toto Statements (v0.71.3 + #182). Each is shape-validated by the manifest schema. Returns: The absolute path to the written file. @@ -66,6 +69,7 @@ def pack_entry( base_hash=entry.get("entry_hash", ""), description=description or entry.get("notes"), tags=list(entry.get("tags", [])), + attestations=list(attestations or []), ) out.parent.mkdir(parents=True, exist_ok=True) diff --git a/src/soup_cli/cans/schema.py b/src/soup_cli/cans/schema.py index 9d988e0..eacd96f 100644 --- a/src/soup_cli/cans/schema.py +++ b/src/soup_cli/cans/schema.py @@ -2,18 +2,48 @@ from __future__ import annotations +import json import re from typing import Literal, Optional from pydantic import BaseModel, Field, field_validator -CAN_FORMAT_VERSION = 2 # v0.33.0: deploy_targets + env capture (additive over v1) -SUPPORTED_CAN_FORMAT_VERSIONS = (1, 2) +CAN_FORMAT_VERSION = 3 # v0.71.3 #182: attestations field (additive over v2) +SUPPORTED_CAN_FORMAT_VERSIONS = (1, 2, 3) _NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-.]{0,127}$") _HF_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-./]{0,127}$") _HF_REPO_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_\-.]{0,95}/[A-Za-z0-9][A-Za-z0-9_\-.]{0,95}$") +# v0.71.3 #182 — caps on embedded in-toto attestations. +_MAX_ATTESTATIONS = 64 +_MAX_ATTESTATION_BYTES = 1024 * 1024 # 1 MiB per statement + + +def validate_attestation_statement(stmt: object) -> dict: + """Validate one embedded in-toto Statement dict (v0.71.3 #182). + + Shape-only: must be a dict carrying ``_type`` (str) and ``predicateType`` + (str), and serialise to <= 1 MiB. Returns the statement unchanged. + """ + if not isinstance(stmt, dict): + raise ValueError("attestation must be a dict (in-toto Statement)") + type_field = stmt.get("_type") + predicate_type = stmt.get("predicateType") + if not isinstance(type_field, str) or not type_field: + raise ValueError("attestation missing a non-empty '_type' string") + if not isinstance(predicate_type, str) or not predicate_type: + raise ValueError("attestation missing a non-empty 'predicateType' string") + try: + size = len(json.dumps(stmt).encode("utf-8")) + except (TypeError, ValueError) as exc: + raise ValueError(f"attestation is not JSON-serialisable: {exc}") from exc + if size > _MAX_ATTESTATION_BYTES: + raise ValueError( + f"attestation too large ({size} > {_MAX_ATTESTATION_BYTES} bytes)" + ) + return stmt + class DeployTarget(BaseModel): """One declarative deploy target embedded in a can manifest (v2+). @@ -116,6 +146,23 @@ class Manifest(BaseModel): default_factory=list, description="Optional declarative deploy targets (v2+)", ) + attestations: list[dict] = Field( + default_factory=list, + description="Optional embedded in-toto Statements (v3+)", + ) + + @field_validator("attestations", mode="before") + @classmethod + def _valid_attestations(cls, value: object) -> list[dict]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("attestations must be a list of in-toto Statements") + if len(value) > _MAX_ATTESTATIONS: + raise ValueError( + f"too many attestations ({len(value)} > {_MAX_ATTESTATIONS})" + ) + return [validate_attestation_statement(s) for s in value] @field_validator("can_format_version") @classmethod diff --git a/src/soup_cli/cli.py b/src/soup_cli/cli.py index 9e54b76..36d4e3e 100644 --- a/src/soup_cli/cli.py +++ b/src/soup_cli/cli.py @@ -1,5 +1,6 @@ """Main CLI entry point — all commands registered here.""" +import os import sys # UTF-8 stdio bootstrap (v0.40.1 Part A) — must run before any Rich console @@ -71,6 +72,12 @@ console = Console() _verbose = False # Global log level (resolved string), set by main() callback _log_level = "normal" +# v0.71.3 #183 — audit-log opt-out, set by the --no-audit-log callback flag. +_audit_disabled = False + +# Global options that consume a following value (so the audit command-splitter +# does not mistake the value for the subcommand name). +_GLOBAL_VALUE_OPTS = frozenset({"--log-level"}) app = typer.Typer( name="soup", @@ -598,10 +605,20 @@ def main( "--log-level", help="Logging tier: quiet | normal | verbose | debug", ), + no_audit_log: bool = typer.Option( + False, + "--no-audit-log", + help=( + "Disable the local HIPAA/SOC2 audit log for this invocation " + "(also via SOUP_NO_AUDIT_LOG=1). Default: a one-line record per " + "command under ~/.soup/audit.jsonl. v0.71.3." + ), + ), ): """Soup — fine-tune LLMs in one command.""" - global _verbose, _log_level + global _verbose, _log_level, _audit_disabled _verbose = verbose + _audit_disabled = no_audit_log from soup_cli.utils.log_level import ( apply_logging_level, parse_log_level, @@ -621,24 +638,114 @@ def main( apply_logging_level(tier) +def _split_command_args(argv: list[str]) -> tuple[str, list[str]]: + """Split ``argv`` into (command, remaining_args) for the audit record. + + Skips the program name (argv[0]) and global options. Global options that + take a value (``--log-level``) consume their following token so the value + is not mistaken for the subcommand. Returns ``("(root)", [...])`` when no + subcommand is present. + """ + tokens = list(argv[1:]) + i = 0 + while i < len(tokens): + tok = tokens[i] + if tok in _GLOBAL_VALUE_OPTS: + i += 2 # skip the option AND its value (even if the value is "-"-like) + continue + if isinstance(tok, str) and tok.startswith("-"): + i += 1 + continue + return tok, tokens[i + 1:] + return "(root)", tokens + + +def _audit_env_opt_out() -> bool: + val = (os.environ.get("SOUP_NO_AUDIT_LOG") or "").strip().lower() + return val in {"1", "true", "yes", "on"} + + +def _emit_audit_event(argv: list[str], exit_code: int) -> None: + """Append one HIPAA/SOC2 audit record for this command. Best-effort. + + v0.71.3 #183 — auto-instrumentation. Disabled by ``--no-audit-log`` / + ``SOUP_NO_AUDIT_LOG``. Never raises: a broken audit log must never crash + the CLI (mirrors v0.59.0 audit-log fail-soft policy). + """ + if _audit_disabled or _audit_env_opt_out(): + return + try: + import getpass + import platform + from datetime import datetime, timezone + + from soup_cli.utils.audit_log import AuditEvent, append_audit_event + + command, args = _split_command_args(argv) + command = (command or "(root)")[:64] or "(root)" + try: + operator = getpass.getuser() or "unknown" + except (OSError, KeyError, ImportError): + # getpass.getuser() can raise when no pwd / env user is resolvable. + operator = "unknown" + host = (platform.node() or "unknown")[:128] or "unknown" + operator = (operator or "unknown")[:128] or "unknown" + # Defensive re-cap, mirroring audit_log._MAX_ARGS (256) / + # _MAX_ARG_LEN (1024) so AuditEvent.__post_init__ never rejects. + capped_args = tuple(str(a)[:1024] for a in args[:256]) + code = exit_code if isinstance(exit_code, int) and not isinstance( + exit_code, bool + ) else 1 + ev = AuditEvent( + timestamp=datetime.now(tz=timezone.utc).isoformat(), + command=command, + args=capped_args, + exit_code=code, + host_id=host, + operator_id=operator, + ) + append_audit_event(ev) + except Exception: # noqa: BLE001 — audit must never crash the CLI + pass + + def run(): """Entry point with friendly error handling.""" # v0.54.0 — rewrite `soup advise ` → `soup advise run `. sys.argv = _rewrite_advise_argv(sys.argv) + argv_snapshot = list(sys.argv) try: app() - except SystemExit: + except SystemExit as exc: + code = exc.code + if code is None: + resolved = 0 + elif isinstance(code, int) and not isinstance(code, bool): + resolved = code + else: + resolved = 1 + _emit_audit_event(argv_snapshot, resolved) raise - except typer.Exit: + except typer.Exit as exc: + # Defensive/unreachable under click standalone mode (typer.Exit is + # converted to SystemExit inside app()); kept so a future click + # behaviour change still audits exactly once. + _emit_audit_event(argv_snapshot, getattr(exc, "exit_code", 1) or 0) raise except KeyboardInterrupt: + _emit_audit_event(argv_snapshot, 130) console.print("\n[yellow]Interrupted.[/]") sys.exit(130) except Exception as exc: + _emit_audit_event(argv_snapshot, 1) from soup_cli.utils.errors import format_friendly_error format_friendly_error(exc, verbose=_verbose) sys.exit(1) + else: + # Defensive/unreachable: app() raises SystemExit(0) on success under + # click standalone mode, so this normal-return path rarely fires. + _emit_audit_event(argv_snapshot, 0) # When invoked via `soup` entry point, use run() for error handling. diff --git a/src/soup_cli/commands/airgap.py b/src/soup_cli/commands/airgap.py index 2ff02a1..1a256f3 100644 --- a/src/soup_cli/commands/airgap.py +++ b/src/soup_cli/commands/airgap.py @@ -2,6 +2,8 @@ from __future__ import annotations +import json +import os from typing import List, Optional import typer @@ -11,6 +13,51 @@ from rich.panel import Panel console = Console() +_MAX_RECEIPT_BYTES = 16 * 1024 * 1024 # 16 MiB + + +def _load_receipt_json(path: str) -> dict: + """Load + validate a receipt JSON (cwd-contained, symlink-rejected, capped).""" + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + real = enforce_under_cwd_and_no_symlink(path, "repro-receipt") + if os.path.getsize(real) > _MAX_RECEIPT_BYTES: + raise ValueError( + f"repro-receipt too large (> {_MAX_RECEIPT_BYTES} bytes)" + ) + with open(real, encoding="utf-8") as fh: + data = json.load(fh) + if not isinstance(data, dict): + raise ValueError("repro-receipt must be a JSON object") + return data + + +def _resolve_repro_receipt(explicit: Optional[str], model_dir: str) -> Optional[dict]: + """Resolve the receipt dict: explicit flag, else auto-detect in model dir. + + An explicit ``--repro-receipt`` load failure is a hard error (exit 2). The + auto-detected ``/repro-receipt.json`` path is best-effort: any + failure silently yields ``None`` (the bundle is still built without it). + """ + if explicit: + try: + return _load_receipt_json(explicit) + except (OSError, ValueError, json.JSONDecodeError) as exc: + console.print(f"[red]--repro-receipt {escape(explicit)}: {escape(str(exc))}[/]") + raise typer.Exit(2) from exc + # Auto-detect — best-effort. Note: when the receipt lives inside the model + # dir it is embedded twice (top-level ``repro-receipt.json`` AND + # ``model/repro-receipt.json`` via the model-dir walk). Both are faithful + # byte-copies with distinct arcnames; the duplication is a few KB and + # acceptable. + candidate = os.path.join(model_dir, "repro-receipt.json") + if os.path.isfile(candidate): + try: + return _load_receipt_json(candidate) + except (OSError, ValueError, json.JSONDecodeError): + return None + return None + def airgap_bundle( output: str = typer.Option(..., "--output", "-o", @@ -32,12 +79,22 @@ def airgap_bundle( 100.0, "--bundle-size-cap", help="Cap in GiB (default 100). Build aborts when exceeded.", ), + repro_receipt: Optional[str] = typer.Option( + None, "--repro-receipt", + help=( + "Embed a reproducibility receipt JSON (from `soup train " + "--repro-receipt`) into the bundle. Auto-detected from " + "/repro-receipt.json when not given. v0.71.3 #188." + ), + ), ): """Build a signed tarball with model + datasets + wheels + kernels (v0.60.0). Designed for one-way physical-media transfer through a data diode. Refuses to write a bundle larger than ``--bundle-size-cap`` GiB. Manifest is embedded inside as ``manifest.json`` with SHA-256 per file. + When a reproducibility receipt is supplied (or auto-detected in the model + dir) it is embedded as ``repro-receipt.json`` (v0.71.3 #188). """ from soup_cli.utils.airgap_bundle import ( AirgapBundlePlan, @@ -53,6 +110,8 @@ def airgap_bundle( if cap_bytes < 1: cap_bytes = 1 + receipt = _resolve_repro_receipt(repro_receipt, model) + try: plan = AirgapBundlePlan( output=output, @@ -61,6 +120,7 @@ def airgap_bundle( wheel_dirs=tuple(wheel or ()), kernel_dirs=tuple(kernel or ()), bundle_size_cap_bytes=cap_bytes, + repro_receipt=receipt, ) except (TypeError, ValueError) as exc: console.print(f"[red]Invalid plan: {escape(str(exc))}[/]") diff --git a/src/soup_cli/commands/can.py b/src/soup_cli/commands/can.py index 7aeaa2b..8c1fbfb 100644 --- a/src/soup_cli/commands/can.py +++ b/src/soup_cli/commands/can.py @@ -2,7 +2,9 @@ from __future__ import annotations -from typing import Optional +import json +import os +from typing import List, Optional import typer from rich.console import Console @@ -19,6 +21,34 @@ def _fail(message: str) -> None: raise typer.Exit(1) +def _load_attestation_files(paths: Optional[List[str]]) -> list[dict]: + """Load + shape-validate in-toto Statement JSON files (cwd-contained).""" + from soup_cli.cans.schema import ( + _MAX_ATTESTATION_BYTES, + validate_attestation_statement, + ) + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + statements: list[dict] = [] + for raw in paths or []: + try: + real = enforce_under_cwd_and_no_symlink(raw, "attest") + # Security: gate the raw file size BEFORE json.load so a multi-GB + # --attest file is rejected without being parsed into memory + # (the 1 MiB cap in validate_attestation_statement only fires + # AFTER the parse). + if os.path.getsize(real) > _MAX_ATTESTATION_BYTES: + raise ValueError( + f"attestation file too large (> {_MAX_ATTESTATION_BYTES} bytes)" + ) + with open(real, encoding="utf-8") as fh: + stmt = json.load(fh) + statements.append(validate_attestation_statement(stmt)) + except (OSError, ValueError, json.JSONDecodeError) as exc: + _fail(f"--attest {raw}: {exc}") + return statements + + @app.command(name="pack") def pack_cmd( entry_id: str = typer.Option( @@ -33,14 +63,23 @@ def pack_cmd( description: str = typer.Option( "", "--description", help="Free-form description", ), + attest: Optional[List[str]] = typer.Option( + None, "--attest", + help=( + "Embed an in-toto Statement JSON into the can manifest (v3, " + "repeatable). Each file is shape-validated. v0.71.3." + ), + ), ) -> None: """Pack a registry entry into a shareable .can file.""" from soup_cli.cans.pack import pack_entry + attestations = _load_attestation_files(attest) try: path = pack_entry( entry_id=entry_id, out_path=out, author=author, description=description or None, + attestations=attestations, ) except (ValueError, FileNotFoundError) as exc: _fail(str(exc)) diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.py index b70d34c..f5e5711 100644 --- a/src/soup_cli/commands/train.py +++ b/src/soup_cli/commands/train.py @@ -196,6 +196,23 @@ def train( "(seeds + kernel versions + GPU + OS) to the given path. v0.59.0." ), ), + track_energy: bool = typer.Option( + False, + "--track-energy", + help=( + "Measure the training window's energy + CO2 via codecarbon " + "(offline; requires `pip install soup-cli[carbon]`). Feeds the " + "kWh / CO2 into --annex-xi. v0.71.3." + ), + ), + energy_country: str = typer.Option( + "USA", + "--energy-country", + help=( + "ISO 3166-1 alpha-3 country code for the CO2 grid-intensity " + "estimate used by --track-energy (default USA)." + ), + ), ): """Start training from a soup.yaml config.""" config_path = Path(config) @@ -914,8 +931,23 @@ def train( f"{cfg.output}/profiles/{run_id}.trace.json (early-steps window)" ) + # v0.71.3 #180 — optional codecarbon energy/CO2 measurement around the + # training window. Lazy-built; a graceful no-op when codecarbon is absent. + energy_ctx = contextlib.nullcontext() + energy_tracker = None + if track_energy: + try: + from soup_cli.utils.energy import EnergyTracker + + energy_tracker = EnergyTracker(country_iso_code=energy_country) + energy_ctx = energy_tracker + except ValueError as exc: + console.print(f"[yellow]--track-energy disabled:[/] {exc}") + energy_tracker = None + energy_ctx = contextlib.nullcontext() + try: - with profiler_ctx: + with profiler_ctx, energy_ctx: result = trainer_wrapper.train( display=display, tracker=tracker, run_id=run_id, resume_from_checkpoint=resume_from, @@ -987,10 +1019,28 @@ def train( ) raise typer.Exit(1) from exc + # --- v0.71.3 #180 --track-energy: print the measured energy/CO2 ------- + energy_measurement = ( + energy_tracker.measurement if energy_tracker is not None else None + ) + if track_energy: + if energy_measurement is not None: + console.print( + f"[cyan]--track-energy:[/] {energy_measurement.energy_kwh:.4f} kWh" + f" / {energy_measurement.co2_kg:.4f} kg CO2" + f" (grid {energy_measurement.grid_intensity_g_per_kwh:.0f} g/kWh," + f" PUE {energy_measurement.pue})" + ) + else: + console.print( + "[yellow]--track-energy:[/] no reading " + "(install `pip install soup-cli[carbon]`)" + ) + # --- v0.59.0 --annex-xi: Annex XI/XII auto-doc ----------------------- if annex_xi and _should_run_diagnose_gate_on_rank(): try: - _write_annex_xi(annex_xi, run_id, cfg) + _write_annex_xi(annex_xi, run_id, cfg, energy=energy_measurement) except typer.Exit: raise except (OSError, ValueError) as exc: @@ -1010,29 +1060,44 @@ def train( ) -def _write_annex_xi(out_path: str, run_id: str, cfg) -> None: - """Render an Annex XI markdown using values from the resolved soup.yaml.""" +def _write_annex_xi(out_path: str, run_id: str, cfg, *, energy=None) -> None: + """Render an Annex XI doc using values from the resolved soup.yaml. + + v0.71.3 #180: the optional ``energy`` measurement populates the kWh / CO2 + fields. v0.71.3 #184: the top crawled domains are auto-extracted from the + training JSONL. v0.71.3 #181: a ``.pdf`` output path renders a PDF. + """ from datetime import datetime, timezone from soup_cli import __version__ - from soup_cli.utils.annex_xi import AnnexXIData, write_annex_doc + from soup_cli.utils.annex_xi import ( + AnnexXIData, + load_top_domains_from_jsonl, + write_annex_doc, + ) modality = getattr(cfg, "modality", "text") or "text" + energy_kwh = float(getattr(energy, "energy_kwh", 0.0)) if energy is not None else 0.0 + co2_kg = float(getattr(energy, "co2_kg", 0.0)) if energy is not None else 0.0 + train_path = str(getattr(cfg.data, "train", "") or "") + # #184 — best-effort extract the top crawled domains from the training data. + top_domains = load_top_domains_from_jsonl(train_path) + fmt = "pdf" if out_path.lower().endswith(".pdf") else "markdown" data = AnnexXIData( model_name=str(getattr(cfg, "output", run_id) or run_id), base_model=str(cfg.base), task=str(cfg.task), - dataset_summary=str(getattr(cfg.data, "train", "")), + dataset_summary=train_path, modalities=(modality,), train_compute_flops=0.0, - train_energy_kwh=0.0, - train_co2_kg=0.0, - top_domains=(), + train_energy_kwh=energy_kwh, + train_co2_kg=co2_kg, + top_domains=top_domains, soup_version=__version__, run_id=run_id, created_at=datetime.now(tz=timezone.utc).isoformat(), ) - written = write_annex_doc(data, "xi", out_path) + written = write_annex_doc(data, "xi", out_path, fmt=fmt) console.print(f"[green]--annex-xi[/] -> {written}") diff --git a/src/soup_cli/utils/airgap_bundle.py b/src/soup_cli/utils/airgap_bundle.py index fecce76..094e893 100644 --- a/src/soup_cli/utils/airgap_bundle.py +++ b/src/soup_cli/utils/airgap_bundle.py @@ -25,9 +25,10 @@ import os import stat import tarfile import tempfile +from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime, timezone -from typing import Tuple +from typing import Any, Optional, Tuple import soup_cli from soup_cli.utils.paths import ( @@ -37,6 +38,7 @@ from soup_cli.utils.paths import ( _DEFAULT_BUNDLE_CAP_BYTES = 100 * 1024 * 1024 * 1024 # 100 GiB _MANIFEST_FILENAME = "manifest.json" +_REPRO_RECEIPT_FILENAME = "repro-receipt.json" # v0.71.3 #188 _MAX_FILES = 100_000 _HASH_CHUNK = 1024 * 1024 _MAX_MANIFEST_BYTES = 64 * 1024 * 1024 # 64 MiB cap on inspect-side manifest read @@ -52,6 +54,8 @@ class AirgapBundlePlan: wheel_dirs: Tuple[str, ...] kernel_dirs: Tuple[str, ...] bundle_size_cap_bytes: int + # v0.71.3 #188 — optional reproducibility receipt to embed. + repro_receipt: Optional[Mapping[str, Any]] = None def __post_init__(self) -> None: for value, name in ( @@ -80,6 +84,10 @@ class AirgapBundlePlan: raise ValueError("bundle_size_cap_bytes must be int") if self.bundle_size_cap_bytes <= 0: raise ValueError("bundle_size_cap_bytes must be > 0") + if self.repro_receipt is not None and not isinstance( + self.repro_receipt, Mapping + ): + raise ValueError("repro_receipt must be a mapping or None") @dataclass(frozen=True) @@ -103,6 +111,8 @@ class BundleManifest: kernels: Tuple[str, ...] files: Tuple[BundleFileEntry, ...] total_bytes: int + # v0.71.3 #188 — embedded reproducibility receipt (None when absent). + repro_receipt: Optional[dict] = None def _hash_file(path: str) -> tuple[int, str]: @@ -198,9 +208,33 @@ def build_airgap_bundle(plan: AirgapBundlePlan) -> BundleManifest: if len(members) > _MAX_FILES: raise ValueError(f"bundle has > {_MAX_FILES} files") + # v0.71.3 #188 — serialise the optional reproducibility receipt up front so + # it counts toward the size cap and lands as a top-level manifest member. + receipt_dict: Optional[dict] = None + receipt_bytes: Optional[bytes] = None + if plan.repro_receipt is not None: + receipt_dict = dict(plan.repro_receipt) + receipt_bytes = json.dumps( + receipt_dict, indent=2, sort_keys=True + ).encode("utf-8") + # Hash + pre-size check (refuse early if cap exceeded). entries: list[BundleFileEntry] = [] total = 0 + if receipt_bytes is not None: + total += len(receipt_bytes) + if total > plan.bundle_size_cap_bytes: + raise ValueError( + f"bundle aggregate size {total} bytes exceeds cap " + f"{plan.bundle_size_cap_bytes} bytes" + ) + entries.append( + BundleFileEntry( + name=_REPRO_RECEIPT_FILENAME, + size=len(receipt_bytes), + sha256=hashlib.sha256(receipt_bytes).hexdigest(), + ) + ) for full, rel_tar in members: size, digest = _hash_file(full) total += size @@ -220,6 +254,7 @@ def build_airgap_bundle(plan: AirgapBundlePlan) -> BundleManifest: kernels=tuple(os.path.basename(os.path.normpath(k)) for k in plan.kernel_dirs), files=tuple(entries), total_bytes=total, + repro_receipt=receipt_dict, ) # Write tar via a temp file in the same dir, then atomic-rename. @@ -247,6 +282,12 @@ def build_airgap_bundle(plan: AirgapBundlePlan) -> BundleManifest: manifest_info.mtime = 0 import io as _io tar.addfile(manifest_info, _io.BytesIO(manifest_bytes)) + # v0.71.3 #188 — write the repro receipt as a top-level member. + if receipt_bytes is not None: + receipt_info = tarfile.TarInfo(name=_REPRO_RECEIPT_FILENAME) + receipt_info.size = len(receipt_bytes) + receipt_info.mtime = 0 + tar.addfile(receipt_info, _io.BytesIO(receipt_bytes)) for full, rel_tar in members: info = tar.gettarinfo(name=full, arcname=rel_tar) # Force regular-file type; suppress owner / group / mtime @@ -281,6 +322,7 @@ def _manifest_to_bytes(manifest: BundleManifest) -> bytes: "wheels": list(manifest.wheels), "kernels": list(manifest.kernels), "total_bytes": manifest.total_bytes, + "repro_receipt": manifest.repro_receipt, "files": [ {"name": e.name, "size": e.size, "sha256": e.sha256} for e in manifest.files @@ -301,6 +343,9 @@ def _manifest_from_payload(payload: dict) -> BundleManifest: ) for entry in files_raw ) + receipt = payload.get("repro_receipt") + if receipt is not None and not isinstance(receipt, dict): + receipt = None return BundleManifest( soup_version=str(payload.get("soup_version", "")), created_at=str(payload.get("created_at", "")), @@ -310,6 +355,7 @@ def _manifest_from_payload(payload: dict) -> BundleManifest: kernels=tuple(payload.get("kernels", [])), files=files, total_bytes=int(payload.get("total_bytes", 0)), + repro_receipt=receipt, ) diff --git a/src/soup_cli/utils/annex_xi.py b/src/soup_cli/utils/annex_xi.py index 2b076b6..06994a5 100644 --- a/src/soup_cli/utils/annex_xi.py +++ b/src/soup_cli/utils/annex_xi.py @@ -12,17 +12,34 @@ modality breakdown, training compute. from __future__ import annotations +import json +import logging import math +import os import re +from collections import Counter +from collections.abc import Iterable, Mapping from dataclasses import dataclass from typing import Tuple +from urllib.parse import urlsplit -from soup_cli.utils.paths import atomic_write_text +from soup_cli.utils.paths import atomic_write_text, enforce_under_cwd_and_no_symlink + +_LOG = logging.getLogger(__name__) _MAX_NAME = 256 _MAX_TEXT = 16384 _VALID_SECTIONS = ("xi", "xii") +# #184 — top-domain extraction caps (DoS defence). +_MAX_DOMAIN_ROWS = 200_000 +_MAX_ROW_CHARS = 1_000_000 +_MAX_URLS_PER_ROW = 5_000 +_MAX_JSONL_BYTES = 1024 * 1024 * 1024 # 1 GiB +# Linear-time, bounded URL matcher (ReDoS-safe — no nested quantifiers). +_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]{1,2048}") +_TEXT_FIELDS = ("text", "content", "output", "response", "prompt", "instruction") + def _validate_text(value: str, field_name: str, *, max_len: int = _MAX_NAME) -> str: if not isinstance(value, str): @@ -202,17 +219,240 @@ share of the data sources. """ -def write_annex_doc(data: AnnexXIData, section: str, output_path: str) -> str: - """Atomic write of an Annex XI or XII markdown to ``output_path``.""" +_VALID_FORMATS = ("markdown", "md", "pdf") + + +def _render_markdown(data: AnnexXIData, section_lc: str) -> str: + return ( + render_annex_xi_markdown(data) if section_lc == "xi" + else render_annex_xii_markdown(data) + ) + + +def render_annex_pdf(data: AnnexXIData, section: str) -> bytes: + """Render an Annex XI or XII document as PDF bytes. + + Lazy-imports ``reportlab`` (install via ``pip install soup-cli[pdf]``). + The PDF body is a block-level projection of the markdown rendering: ``#`` / + ``##`` / ``###`` become heading styles, ``- `` becomes a bullet, blank + lines become spacers (inline emphasis is not interpreted). Operator- + controlled strings stay XML-escaped so a crafted ``model_name`` cannot + inject reportlab markup. + """ + if not isinstance(data, AnnexXIData): + raise TypeError(f"data must be AnnexXIData, got {type(data).__name__}") if not isinstance(section, str) or section.lower() not in _VALID_SECTIONS: raise ValueError( f"section must be one of {_VALID_SECTIONS}, got {section!r}" ) + try: + import io # noqa: PLC0415 + from xml.sax.saxutils import escape as _xml_escape # noqa: PLC0415 + + from reportlab.lib.pagesizes import A4 # noqa: PLC0415 + from reportlab.lib.styles import getSampleStyleSheet # noqa: PLC0415 + from reportlab.platypus import ( # noqa: PLC0415 + ListFlowable, + ListItem, + Paragraph, + SimpleDocTemplate, + Spacer, + ) + except ImportError as exc: # pragma: no cover - exercised only without reportlab + raise RuntimeError( + "PDF rendering requires reportlab. Install with " + "`pip install soup-cli[pdf]` (or `pip install reportlab`)." + ) from exc + + markdown = _render_markdown(data, section.lower()) + styles = getSampleStyleSheet() + flowables: list[object] = [] + bullets: list[object] = [] + + def _flush_bullets() -> None: + if bullets: + flowables.append(ListFlowable(list(bullets), bulletType="bullet")) + bullets.clear() + + def _clean(raw: str) -> str: + # Undo the markdown backslash-escapes from _md_escape, then XML-escape + # for reportlab's mini-markup parser. + return _xml_escape(re.sub(r"\\([|\[\]()!<>])", r"\1", raw)) + + for line in markdown.splitlines(): + stripped = line.strip() + if not stripped: + _flush_bullets() + flowables.append(Spacer(1, 6)) + continue + if stripped.startswith("### "): + _flush_bullets() + flowables.append(Paragraph(_clean(stripped[4:]), styles["Heading3"])) + elif stripped.startswith("## "): + _flush_bullets() + flowables.append(Paragraph(_clean(stripped[3:]), styles["Heading2"])) + elif stripped.startswith("# "): + _flush_bullets() + flowables.append(Paragraph(_clean(stripped[2:]), styles["Title"])) + elif stripped.startswith("- "): + bullets.append( + ListItem(Paragraph(_clean(stripped[2:]), styles["BodyText"])) + ) + else: + _flush_bullets() + flowables.append(Paragraph(_clean(stripped), styles["BodyText"])) + _flush_bullets() + + buffer = io.BytesIO() + doc = SimpleDocTemplate(buffer, pagesize=A4, title=f"Annex {section.upper()}") + doc.build(flowables) + return buffer.getvalue() + + +def write_annex_doc( + data: AnnexXIData, + section: str, + output_path: str, + *, + fmt: str = "markdown", +) -> str: + """Atomic write of an Annex XI or XII doc to ``output_path``. + + ``fmt`` is ``markdown`` (default) / ``md`` for the markdown body, or + ``pdf`` (v0.71.3 #181) for a reportlab-rendered PDF. + """ + if not isinstance(section, str) or section.lower() not in _VALID_SECTIONS: + raise ValueError( + f"section must be one of {_VALID_SECTIONS}, got {section!r}" + ) + if not isinstance(fmt, str) or fmt.lower() not in _VALID_FORMATS: + raise ValueError(f"fmt must be one of {_VALID_FORMATS}, got {fmt!r}") section_lc = section.lower() - text = ( - render_annex_xi_markdown(data) if section_lc == "xi" - else render_annex_xii_markdown(data) - ) + if fmt.lower() == "pdf": + from soup_cli.utils.paths import atomic_write_bytes + + pdf_bytes = render_annex_pdf(data, section) + return atomic_write_bytes( + pdf_bytes, output_path, prefix=".annex.", suffix=".pdf.tmp", + ) + text = _render_markdown(data, section_lc) return atomic_write_text( text, output_path, prefix=".annex.", suffix=".md.tmp", ) + + +# --------------------------------------------------------------------------- +# #184 — auto-populate top_domains from the training corpus +# --------------------------------------------------------------------------- +def _row_text(row: object) -> str: + """Best-effort concatenation of a row's text-bearing fields.""" + if isinstance(row, str): + return row + if not isinstance(row, Mapping): + return "" + parts: list[str] = [] + for field in _TEXT_FIELDS: + val = row.get(field) + if isinstance(val, str): + parts.append(val) + messages = row.get("messages") + if isinstance(messages, (list, tuple)): + for msg in messages: + if isinstance(msg, Mapping): + content = msg.get("content") + if isinstance(content, str): + parts.append(content) + return " ".join(parts) + + +def _domain_of(url: str) -> str: + """Extract a bare lowercased hostname from a URL (port stripped).""" + try: + host = (urlsplit(url).hostname or "").lower() + except ValueError: + return "" + return host + + +def extract_top_domains( + rows: Iterable[object], *, top_n: int = 10 +) -> Tuple[Tuple[str, float], ...]: + """Count URL domains across ``rows`` and return the top-N by share. + + Returns a tuple of ``(domain, share)`` where ``share`` is the domain's + fraction of all extracted URLs. Deterministic: ties broken by domain + ascending. DoS-capped (``_MAX_DOMAIN_ROWS`` rows × ``_MAX_ROW_CHARS`` + chars × ``_MAX_URLS_PER_ROW`` URLs/row). + """ + if isinstance(top_n, bool) or not isinstance(top_n, int): + raise ValueError("top_n must be an int") + if top_n < 1: + raise ValueError("top_n must be >= 1") + if isinstance(rows, (str, bytes)) or not isinstance(rows, Iterable): + raise TypeError("rows must be an iterable of mappings/strings") + counter: Counter[str] = Counter() + total = 0 + for i, row in enumerate(rows): + if i >= _MAX_DOMAIN_ROWS: + break + text = _row_text(row) + if not text: + continue + if len(text) > _MAX_ROW_CHARS: + text = text[:_MAX_ROW_CHARS] + for j, match in enumerate(_URL_RE.finditer(text)): + if j >= _MAX_URLS_PER_ROW: + break + domain = _domain_of(match.group(0)) + if domain: + counter[domain] += 1 + total += 1 + if total == 0: + return () + ranked = sorted(counter.items(), key=lambda kv: (-kv[1], kv[0]))[:top_n] + return tuple((domain, count / total) for domain, count in ranked) + + +def load_top_domains_from_jsonl( + path: object, *, top_n: int = 10 +) -> Tuple[Tuple[str, float], ...]: + """Best-effort domain extraction from a cwd-contained JSONL file. + + Returns ``()`` on any failure (missing file, outside cwd, symlink, + unreadable, oversize) — an Annex doc must never fail because the corpus + is unavailable. cwd-contained + symlink-rejected via the shared helper. + """ + if not isinstance(path, str) or not path: + return () + try: + realpath = enforce_under_cwd_and_no_symlink(path, "data.train") + except (ValueError, OSError) as exc: + _LOG.debug("load_top_domains_from_jsonl: rejected %r: %s", path, exc) + return () + try: + if not os.path.isfile(realpath): + return () + if os.path.getsize(realpath) > _MAX_JSONL_BYTES: + return () + except OSError: + return () + + def _row_iter(): + try: + with open(realpath, encoding="utf-8") as fh: + for line in fh: + raw = line.strip() + if not raw: + continue + try: + yield json.loads(raw) + except (ValueError, TypeError): + continue + except OSError as exc: + _LOG.debug("load_top_domains_from_jsonl: read failed: %s", exc) + + try: + return extract_top_domains(_row_iter(), top_n=top_n) + except (ValueError, TypeError) as exc: + _LOG.debug("load_top_domains_from_jsonl: extract failed: %s", exc) + return () diff --git a/src/soup_cli/utils/energy.py b/src/soup_cli/utils/energy.py index aae4d03..9e59a66 100644 --- a/src/soup_cli/utils/energy.py +++ b/src/soup_cli/utils/energy.py @@ -13,16 +13,22 @@ char / oversize rejection. from __future__ import annotations import ipaddress +import logging import math import re from dataclasses import dataclass +from types import TracebackType from typing import Optional from urllib.parse import urlsplit +_LOG = logging.getLogger(__name__) + _MAX_ENDPOINT_LEN = 2048 _CTRL_RE = re.compile(r"[\x00-\x1f\x7f]") _LOOPBACK = frozenset({"localhost", "127.0.0.1", "::1"}) _SCHEMES = frozenset({"http", "https"}) +_COUNTRY_RE = re.compile(r"^[A-Za-z]{3}$") +_DEFAULT_COUNTRY = "USA" @dataclass(frozen=True) @@ -164,7 +170,122 @@ def measure_run_energy( # builder) can decide whether to omit the energy properties. return None - # Live wiring deferred to v0.59.1; return None until the codecarbon - # `EmissionsTracker` hook lands. The schema + endpoint validator + PUE - # math are live; the actual measurement is the v0.59.1 deliverable. + # ``measure_run_energy`` is the duration-only fallback: with only a wall- + # clock number it cannot read instantaneous power draw, so it returns None + # even when codecarbon IS installed. Use ``EnergyTracker`` (v0.71.3 #180) + # for a real start()/stop() measurement around the training window. return None + + +class EnergyTracker: + """Context manager that measures a training window's energy + CO2. + + Wraps codecarbon's ``OfflineEmissionsTracker`` (offline = no network / + no IP-geolocation call, so the privacy guarantee is preserved). Lazy- + imports codecarbon; when it is absent the tracker is a graceful no-op and + ``measurement`` stays ``None``. + + The energy reading (kWh) is country-independent — it is measured power × + time. Only the CO2 conversion uses the chosen country's grid intensity + (``country_iso_code``, default ``"USA"``). Both energy and CO2 are scaled + by ``pue`` (Power Usage Effectiveness) so data-centre overhead is counted. + + Usage:: + + with EnergyTracker(pue=1.1) as tracker: + trainer.train() + m = tracker.measurement # EnergyMeasurement or None + """ + + def __init__( + self, + *, + pue: float = 1.1, + grid_intensity_g_per_kwh: float = 400.0, + country_iso_code: str = _DEFAULT_COUNTRY, + ) -> None: + # Validate PUE up front (reuses the shared bounds checker). + adjust_for_pue(1.0, pue) + if isinstance(grid_intensity_g_per_kwh, bool) or not isinstance( + grid_intensity_g_per_kwh, (int, float) + ): + raise ValueError("grid_intensity_g_per_kwh must be numeric") + if ( + not math.isfinite(float(grid_intensity_g_per_kwh)) + or grid_intensity_g_per_kwh < 0 + ): + raise ValueError("grid_intensity_g_per_kwh must be finite and >= 0") + if not isinstance(country_iso_code, str) or not _COUNTRY_RE.match( + country_iso_code + ): + raise ValueError( + "country_iso_code must be a 3-letter ISO 3166-1 alpha-3 code " + f"(got {country_iso_code!r})" + ) + self._pue = float(pue) + self._grid_default = float(grid_intensity_g_per_kwh) + self._country = country_iso_code.upper() + self._tracker = None + self._measurement: Optional[EnergyMeasurement] = None + + @property + def measurement(self) -> Optional[EnergyMeasurement]: + return self._measurement + + def __enter__(self) -> "EnergyTracker": + try: + from codecarbon import OfflineEmissionsTracker # noqa: PLC0415 + + self._tracker = OfflineEmissionsTracker( + country_iso_code=self._country, + save_to_file=False, + log_level="error", + ) + self._tracker.start() + except Exception as exc: # noqa: BLE001 — never crash training + _LOG.debug("EnergyTracker: codecarbon unavailable/failed: %s", exc) + self._tracker = None + return self + + def __exit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + tb: TracebackType | None, + ) -> bool: + tracker = self._tracker + self._tracker = None + if tracker is None: + return False + try: + emissions = tracker.stop() + data = getattr(tracker, "final_emissions_data", None) + energy_raw = float(getattr(data, "energy_consumed", 0.0) or 0.0) + co2_raw = float( + emissions + if emissions is not None + else getattr(data, "emissions", 0.0) or 0.0 + ) + if not math.isfinite(energy_raw) or energy_raw < 0: + energy_raw = 0.0 + if not math.isfinite(co2_raw) or co2_raw < 0: + co2_raw = 0.0 + grid = ( + (co2_raw / energy_raw * 1000.0) + if energy_raw > 0 + else self._grid_default + ) + if not math.isfinite(grid) or grid < 0: + grid = self._grid_default + self._measurement = EnergyMeasurement( + energy_kwh=adjust_for_pue(energy_raw, self._pue), + co2_kg=co2_raw * self._pue, + pue=self._pue, + grid_intensity_g_per_kwh=grid, + source="codecarbon-offline", + ) + except Exception as exc: # noqa: BLE001 — never crash training + _LOG.debug("EnergyTracker: stop()/measurement failed: %s", exc) + self._measurement = None + # Always return False so a body exception propagates unmasked. + return False diff --git a/src/soup_cli/utils/paths.py b/src/soup_cli/utils/paths.py index 9e28711..da339bd 100644 --- a/src/soup_cli/utils/paths.py +++ b/src/soup_cli/utils/paths.py @@ -106,3 +106,35 @@ def atomic_write_text( except OSError: pass return os.path.realpath(output_path) + + +def atomic_write_bytes( + data: bytes, + output_path: str, + *, + prefix: str = ".soup.", + suffix: str = ".tmp", + field: str = "output", +) -> str: + """Atomically write ``data`` (bytes) to ``output_path`` under cwd containment. + + Binary sibling of :func:`atomic_write_text` (v0.71.3 #181 — used by the + Annex XI/XII PDF renderer). Same TOCTOU-safe pipeline. + """ + if not isinstance(data, (bytes, bytearray)): + raise TypeError("data must be bytes") + enforce_under_cwd_and_no_symlink(output_path, field) + parent = os.path.dirname(os.path.abspath(output_path)) or "." + os.makedirs(parent, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(prefix=prefix, suffix=suffix, dir=parent) + try: + with os.fdopen(fd, "wb") as fh: + fh.write(bytes(data)) + os.replace(tmp_path, output_path) + finally: + if os.path.exists(tmp_path): + try: + os.unlink(tmp_path) + except OSError: + pass + return os.path.realpath(output_path) diff --git a/tests/test_cans.py b/tests/test_cans.py index f8dce0a..de74731 100644 --- a/tests/test_cans.py +++ b/tests/test_cans.py @@ -108,8 +108,8 @@ class TestPackUnpack: manifest = inspect_can(str(out)) assert manifest.name == "recipe" - # v0.33.0 bumped CAN_FORMAT_VERSION 1 -> 2; v1 still loads on inspect. - assert manifest.can_format_version in (1, 2) + # v0.33.0 bumped 1 -> 2; v0.71.3 bumped 2 -> 3. Older still load. + assert manifest.can_format_version in (1, 2, 3) def test_verify_valid_can(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) diff --git a/tests/test_part_a_wave2.py b/tests/test_part_a_wave2.py index a18893b..eed8d25 100644 --- a/tests/test_part_a_wave2.py +++ b/tests/test_part_a_wave2.py @@ -97,20 +97,31 @@ class TestCanFormatVersionBump: ) assert m.can_format_version == 2 - def test_v3_rejected(self): + def test_v3_loads(self): + # v0.71.3 #182 bumped CAN_FORMAT_VERSION 2 -> 3 (attestations field). + from soup_cli.cans.schema import Manifest + + m = Manifest( + can_format_version=3, + name="x", author="a", created_at="2026-01-01", + base_hash="0" * 64, + ) + assert m.can_format_version == 3 + + def test_v4_rejected(self): from soup_cli.cans.schema import Manifest with pytest.raises(Exception, match="unknown can_format_version"): Manifest( - can_format_version=3, + can_format_version=4, name="x", author="a", created_at="2026-01-01", base_hash="0" * 64, ) - def test_pack_writes_v2(self, tmp_path, monkeypatch): + def test_pack_writes_v3(self, tmp_path, monkeypatch): from soup_cli.cans.schema import CAN_FORMAT_VERSION - assert CAN_FORMAT_VERSION == 2 + assert CAN_FORMAT_VERSION == 3 class TestDeployTargetValidation: diff --git a/tests/test_v0713.py b/tests/test_v0713.py new file mode 100644 index 0000000..230ba85 --- /dev/null +++ b/tests/test_v0713.py @@ -0,0 +1,1036 @@ +"""Tests for v0.71.3 — Compliance / annex / audit / energy. + +Closes: +- #180 Live CodeCarbon hook (EnergyTracker context manager) + `soup train --track-energy` +- #181 PDF rendering for Annex XI/XII (`write_annex_doc(..., fmt="pdf")`) +- #182 Soup Can manifest v3 `attestations` field + embed in-toto Statement +- #183 Audit-log auto-instrumentation via the top-level CLI entry point +- #184 Auto-populate AnnexXIData.top_domains from the data.train manifest +- #188 Auto-attach the repro-receipt into `soup airgap-bundle` + +Pure-Python / config / sqlite — no GPU required for these tests. The live +codecarbon energy reading + reportlab PDF body are exercised in the Step-6 +local smoke (codecarbon + reportlab installed); here we test the graceful / +schema / wiring surface. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest +import typer +from typer.testing import CliRunner + +runner = CliRunner() + + +# --------------------------------------------------------------------------- +# #180 — Live CodeCarbon hook: EnergyTracker context manager +# --------------------------------------------------------------------------- +class TestEnergyTracker: + def test_imports(self): + from soup_cli.utils.energy import EnergyMeasurement, EnergyTracker # noqa: F401 + + def test_tracker_graceful_when_codecarbon_missing(self, monkeypatch): + """When codecarbon is absent the tracker yields, never crashes, and + leaves measurement=None.""" + import builtins + + real_import = builtins.__import__ + + def _no_codecarbon(name, *args, **kwargs): + if name == "codecarbon" or name.startswith("codecarbon."): + raise ImportError("no codecarbon") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_codecarbon) + from soup_cli.utils.energy import EnergyTracker + + with EnergyTracker(pue=1.1) as tracker: + pass # no work + assert tracker.measurement is None + + def test_tracker_exit_swallows_exceptions(self, monkeypatch): + """A failure inside the codecarbon stop() must not crash __exit__ and + must NOT mask an exception raised in the body.""" + import builtins + + real_import = builtins.__import__ + + def _no_codecarbon(name, *args, **kwargs): + if name == "codecarbon" or name.startswith("codecarbon."): + raise ImportError("no codecarbon") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _no_codecarbon) + from soup_cli.utils.energy import EnergyTracker + + with pytest.raises(RuntimeError, match="boom"): + with EnergyTracker() as tracker: + raise RuntimeError("boom") + assert tracker.measurement is None + + def test_tracker_rejects_bad_pue(self): + from soup_cli.utils.energy import EnergyTracker + + with pytest.raises(ValueError): + EnergyTracker(pue=0.5) + + def test_tracker_rejects_bad_country(self): + from soup_cli.utils.energy import EnergyTracker + + with pytest.raises(ValueError): + EnergyTracker(country_iso_code="USAA") # not 3-letter + with pytest.raises(ValueError): + EnergyTracker(country_iso_code="12") + + def test_tracker_builds_measurement_from_fake_codecarbon(self, monkeypatch): + """Inject a fake codecarbon module so we exercise the real start/stop + lifecycle math without installing codecarbon.""" + import types + + fake = types.ModuleType("codecarbon") + + class _Data: + energy_consumed = 0.5 # kWh + emissions = 0.2 # kg + + class _FakeTracker: + def __init__(self, **kwargs): + self.final_emissions_data = _Data() + self._started = False + + def start(self): + self._started = True + + def stop(self): + return 0.2 # kg CO2 + + fake.OfflineEmissionsTracker = _FakeTracker + monkeypatch.setitem(sys.modules, "codecarbon", fake) + from soup_cli.utils.energy import EnergyTracker + + with EnergyTracker(pue=1.2, country_iso_code="USA") as tracker: + pass + m = tracker.measurement + assert m is not None + # energy is PUE-scaled: 0.5 * 1.2 + assert abs(m.energy_kwh - 0.6) < 1e-9 + # co2 PUE-scaled: 0.2 * 1.2 + assert abs(m.co2_kg - 0.24) < 1e-9 + # grid intensity computed from raw: 0.2 / 0.5 * 1000 = 400 g/kWh + assert abs(m.grid_intensity_g_per_kwh - 400.0) < 1e-6 + assert m.pue == 1.2 + assert m.source == "codecarbon-offline" + + def test_tracker_zero_energy_uses_default_grid(self, monkeypatch): + import types + + fake = types.ModuleType("codecarbon") + + class _Data: + energy_consumed = 0.0 + emissions = 0.0 + + class _FakeTracker: + def __init__(self, **kwargs): + self.final_emissions_data = _Data() + + def start(self): + pass + + def stop(self): + return 0.0 + + fake.OfflineEmissionsTracker = _FakeTracker + monkeypatch.setitem(sys.modules, "codecarbon", fake) + from soup_cli.utils.energy import EnergyTracker + + with EnergyTracker(grid_intensity_g_per_kwh=350.0) as tracker: + pass + m = tracker.measurement + assert m is not None + assert m.energy_kwh == 0.0 + assert m.grid_intensity_g_per_kwh == 350.0 + + def test_train_track_energy_flag_in_help(self): + from soup_cli.cli import app + + result = runner.invoke(app, ["train", "--help"]) + assert result.exit_code == 0 + out = "".join(ch for ch in result.output if ord(ch) < 128) + assert "track-energy" in out.replace("─", "") + + +# --------------------------------------------------------------------------- +# #184 — Auto-populate AnnexXIData.top_domains from the training corpus +# --------------------------------------------------------------------------- +class TestTopDomains: + def test_imports(self): + from soup_cli.utils.annex_xi import ( # noqa: F401 + extract_top_domains, + load_top_domains_from_jsonl, + ) + + def test_extract_top_domains_counts_and_shares(self): + from soup_cli.utils.annex_xi import extract_top_domains + + rows = [ + {"text": "see https://example.com/a and https://example.com/b"}, + {"text": "also http://other.org/x"}, + {"content": "visit https://example.com/c"}, + ] + out = extract_top_domains(rows) + # example.com appears 3x, other.org 1x → 4 urls total + assert out[0] == ("example.com", 0.75) + assert out[1] == ("other.org", 0.25) + + def test_extract_top_domains_strips_port_lowercases(self): + from soup_cli.utils.annex_xi import extract_top_domains + + rows = [{"text": "https://Example.COM:8443/x https://example.com/y"}] + out = extract_top_domains(rows) + assert out == (("example.com", 1.0),) + + def test_extract_top_domains_messages_field(self): + from soup_cli.utils.annex_xi import extract_top_domains + + rows = [ + {"messages": [{"role": "user", "content": "https://hf.co/datasets/x"}]}, + ] + out = extract_top_domains(rows) + assert out == (("hf.co", 1.0),) + + def test_extract_top_domains_empty_returns_empty(self): + from soup_cli.utils.annex_xi import extract_top_domains + + assert extract_top_domains([]) == () + assert extract_top_domains([{"text": "no urls here"}]) == () + + def test_extract_top_domains_caps_at_top_n(self): + from soup_cli.utils.annex_xi import extract_top_domains + + rows = [{"text": " ".join(f"https://d{i}.com/x" for i in range(20))}] + out = extract_top_domains(rows, top_n=5) + assert len(out) == 5 + + def test_extract_top_domains_non_iterable_rejected(self): + from soup_cli.utils.annex_xi import extract_top_domains + + with pytest.raises(TypeError): + extract_top_domains(42) + + def test_extract_top_domains_bad_top_n(self): + from soup_cli.utils.annex_xi import extract_top_domains + + with pytest.raises(ValueError): + extract_top_domains([], top_n=0) + with pytest.raises(ValueError): + extract_top_domains([], top_n=True) + + def test_load_from_jsonl(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import load_top_domains_from_jsonl + + monkeypatch.chdir(tmp_path) + p = tmp_path / "train.jsonl" + p.write_text( + '{"text": "https://example.com/a"}\n' + '{"text": "https://example.com/b https://other.org/c"}\n', + encoding="utf-8", + ) + out = load_top_domains_from_jsonl("train.jsonl") + assert out[0][0] == "example.com" + + def test_load_from_jsonl_missing_returns_empty(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import load_top_domains_from_jsonl + + monkeypatch.chdir(tmp_path) + assert load_top_domains_from_jsonl("nope.jsonl") == () + + def test_load_from_jsonl_outside_cwd_returns_empty(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import load_top_domains_from_jsonl + + monkeypatch.chdir(tmp_path) + # an absolute path outside cwd — best-effort returns () + outside = tmp_path.parent / "x.jsonl" + outside.write_text('{"text": "https://example.com/a"}\n', encoding="utf-8") + assert load_top_domains_from_jsonl(str(outside)) == () + + def test_load_from_jsonl_non_string_returns_empty(self): + from soup_cli.utils.annex_xi import load_top_domains_from_jsonl + + assert load_top_domains_from_jsonl(None) == () + assert load_top_domains_from_jsonl("") == () + + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlink semantics") + def test_load_from_jsonl_symlink_returns_empty(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import load_top_domains_from_jsonl + + monkeypatch.chdir(tmp_path) + real = tmp_path / "real.jsonl" + real.write_text('{"text": "https://example.com/a"}\n', encoding="utf-8") + link = tmp_path / "link.jsonl" + os.symlink(real, link) + assert load_top_domains_from_jsonl("link.jsonl") == () + + +# --------------------------------------------------------------------------- +# #181 — PDF rendering for Annex XI/XII +# --------------------------------------------------------------------------- +def _sample_annex_data(): + from soup_cli.utils.annex_xi import AnnexXIData + + return AnnexXIData( + model_name="my-model", + base_model="meta-llama/Llama-3.2-1B", + task="sft", + dataset_summary="data/train.jsonl", + modalities=("text",), + train_compute_flops=1.5e18, + train_energy_kwh=0.5, + train_co2_kg=0.2, + top_domains=(("example.com", 0.6), ("other.org", 0.4)), + soup_version="0.71.3", + run_id="run-1", + created_at="2026-06-01T00:00:00+00:00", + ) + + +class TestAnnexPdf: + def test_atomic_write_bytes_imports(self): + from soup_cli.utils.paths import atomic_write_bytes # noqa: F401 + + def test_atomic_write_bytes_roundtrip(self, tmp_path, monkeypatch): + from soup_cli.utils.paths import atomic_write_bytes + + monkeypatch.chdir(tmp_path) + out = atomic_write_bytes(b"%PDF-1.4\nhello", "x.pdf") + assert Path(out).read_bytes() == b"%PDF-1.4\nhello" + + def test_atomic_write_bytes_rejects_outside_cwd(self, tmp_path, monkeypatch): + from soup_cli.utils.paths import atomic_write_bytes + + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + atomic_write_bytes(b"x", str(tmp_path.parent / "y.pdf")) + + def test_render_annex_pdf_starts_with_pdf_magic(self): + from soup_cli.utils.annex_xi import render_annex_pdf + + data = _sample_annex_data() + pdf = render_annex_pdf(data, "xi") + assert isinstance(pdf, bytes) + assert pdf.startswith(b"%PDF") + + def test_render_annex_pdf_xii(self): + from soup_cli.utils.annex_xi import render_annex_pdf + + pdf = render_annex_pdf(_sample_annex_data(), "xii") + assert pdf.startswith(b"%PDF") + + def test_render_annex_pdf_rejects_non_data(self): + from soup_cli.utils.annex_xi import render_annex_pdf + + with pytest.raises(TypeError): + render_annex_pdf({"not": "data"}, "xi") + + def test_render_annex_pdf_rejects_bad_section(self): + from soup_cli.utils.annex_xi import render_annex_pdf + + with pytest.raises(ValueError): + render_annex_pdf(_sample_annex_data(), "zz") + + def test_write_annex_doc_markdown_default(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import write_annex_doc + + monkeypatch.chdir(tmp_path) + out = write_annex_doc(_sample_annex_data(), "xi", "annex.md") + body = Path(out).read_text(encoding="utf-8") + assert "Annex XI" in body + + def test_write_annex_doc_pdf_format(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import write_annex_doc + + monkeypatch.chdir(tmp_path) + out = write_annex_doc(_sample_annex_data(), "xi", "annex.pdf", fmt="pdf") + assert Path(out).read_bytes().startswith(b"%PDF") + + def test_write_annex_doc_rejects_bad_fmt(self, tmp_path, monkeypatch): + from soup_cli.utils.annex_xi import write_annex_doc + + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + write_annex_doc(_sample_annex_data(), "xi", "annex.x", fmt="docx") + + +# --------------------------------------------------------------------------- +# #182 — Soup Can manifest v3 `attestations` field +# --------------------------------------------------------------------------- +def _sample_statement(): + return { + "_type": "https://in-toto.io/Statement/v1", + "subject": [{"name": "model", "digest": {"sha256": "a" * 64}}], + "predicateType": "https://slsa.dev/provenance/v1", + "predicate": {"buildDefinition": {}, "runDetails": {}}, + } + + +class TestCanManifestV3: + def test_version_bumped_to_3(self): + from soup_cli.cans.schema import ( + CAN_FORMAT_VERSION, + SUPPORTED_CAN_FORMAT_VERSIONS, + ) + + assert CAN_FORMAT_VERSION == 3 + assert SUPPORTED_CAN_FORMAT_VERSIONS == (1, 2, 3) + + def test_v1_and_v2_still_load(self): + from soup_cli.cans.schema import Manifest + + for v in (1, 2, 3): + m = Manifest( + can_format_version=v, name="x", author="a", + created_at="2026-01-01", base_hash="h", + ) + assert m.can_format_version == v + + def test_attestations_default_empty(self): + from soup_cli.cans.schema import Manifest + + m = Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + ) + assert m.attestations == [] + + def test_attestations_accepts_valid_statement(self): + from soup_cli.cans.schema import Manifest + + m = Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=[_sample_statement()], + ) + assert len(m.attestations) == 1 + assert m.attestations[0]["_type"].endswith("Statement/v1") + + def test_attestations_none_coerced_to_empty(self): + from soup_cli.cans.schema import Manifest + + m = Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=None, + ) + assert m.attestations == [] + + def test_attestations_rejects_non_dict_entry(self): + from pydantic import ValidationError + + from soup_cli.cans.schema import Manifest + + with pytest.raises(ValidationError): + Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=["not-a-dict"], + ) + + def test_attestations_rejects_missing_type(self): + from pydantic import ValidationError + + from soup_cli.cans.schema import Manifest + + with pytest.raises(ValidationError): + Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=[{"predicateType": "x"}], + ) + + def test_attestations_too_many_rejected(self): + from pydantic import ValidationError + + from soup_cli.cans.schema import Manifest + + with pytest.raises(ValidationError): + Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=[_sample_statement() for _ in range(200)], + ) + + def test_validate_attestation_statement_helper(self): + from soup_cli.cans.schema import validate_attestation_statement + + out = validate_attestation_statement(_sample_statement()) + assert out["_type"] + with pytest.raises(ValueError): + validate_attestation_statement({"_type": "x"}) # missing predicateType + with pytest.raises(ValueError): + validate_attestation_statement("nope") + + def test_pack_entry_embeds_attestations(self, tmp_path, monkeypatch): + """pack_entry threads attestations into the written manifest.yaml.""" + import tarfile + + import yaml + + from soup_cli.cans import pack as pack_mod + + monkeypatch.chdir(tmp_path) + + class _FakeStore: + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def resolve(self, eid): + return "eid" + + def get(self, eid): + return { + "name": "my-recipe", "base_model": "b", "task": "sft", + "entry_hash": "h", "config_json": "{}", "notes": "n", + "tags": [], + } + + monkeypatch.setattr(pack_mod, "RegistryStore", lambda: _FakeStore()) + out = pack_mod.pack_entry( + entry_id="eid", out_path="x.can", author="a", + attestations=[_sample_statement()], + ) + with tarfile.open(out, "r:gz") as tar: + manifest = yaml.safe_load( + tar.extractfile("manifest.yaml").read().decode("utf-8") + ) + assert manifest["can_format_version"] == 3 + assert len(manifest["attestations"]) == 1 + + def test_can_pack_attest_flag_in_help(self): + from soup_cli.cli import app + + result = runner.invoke(app, ["can", "pack", "--help"]) + assert result.exit_code == 0 + out = "".join(ch for ch in result.output if ord(ch) < 128) + assert "attest" in out + + +# --------------------------------------------------------------------------- +# #183 — Audit-log auto-instrumentation via the top-level CLI entry point +# --------------------------------------------------------------------------- +class TestAuditInstrumentation: + def test_imports(self): + from soup_cli.cli import _emit_audit_event, _split_command_args # noqa: F401 + + def test_split_command_args_basic(self): + from soup_cli.cli import _split_command_args + + cmd, args = _split_command_args(["soup", "train", "--config", "x"]) + assert cmd == "train" + assert args == ["--config", "x"] + + def test_split_command_args_skips_log_level_value(self): + from soup_cli.cli import _split_command_args + + cmd, args = _split_command_args( + ["soup", "--log-level", "debug", "train", "--yes"] + ) + assert cmd == "train" + assert args == ["--yes"] + + def test_split_command_args_root_only(self): + from soup_cli.cli import _split_command_args + + cmd, args = _split_command_args(["soup"]) + assert cmd == "(root)" + assert args == [] + + def test_split_command_args_help_flag(self): + from soup_cli.cli import _split_command_args + + cmd, args = _split_command_args(["soup", "--help"]) + assert cmd == "(root)" + assert args == ["--help"] + + def test_emit_audit_event_writes_line(self, tmp_path, monkeypatch): + import soup_cli.cli as cli_mod + + log = tmp_path / "audit.jsonl" + monkeypatch.setenv("SOUP_AUDIT_LOG_PATH", str(log)) + monkeypatch.delenv("SOUP_NO_AUDIT_LOG", raising=False) + monkeypatch.setattr(cli_mod, "_audit_disabled", False) + cli_mod._emit_audit_event(["soup", "version"], 0) + rows = [json.loads(x) for x in log.read_text().splitlines() if x.strip()] + assert len(rows) == 1 + assert rows[0]["command"] == "version" + assert rows[0]["exit_code"] == 0 + + def test_emit_audit_event_env_opt_out(self, tmp_path, monkeypatch): + import soup_cli.cli as cli_mod + + log = tmp_path / "audit.jsonl" + monkeypatch.setenv("SOUP_AUDIT_LOG_PATH", str(log)) + monkeypatch.setenv("SOUP_NO_AUDIT_LOG", "1") + monkeypatch.setattr(cli_mod, "_audit_disabled", False) + cli_mod._emit_audit_event(["soup", "version"], 0) + assert not log.exists() + + def test_emit_audit_event_flag_opt_out(self, tmp_path, monkeypatch): + import soup_cli.cli as cli_mod + + log = tmp_path / "audit.jsonl" + monkeypatch.setenv("SOUP_AUDIT_LOG_PATH", str(log)) + monkeypatch.delenv("SOUP_NO_AUDIT_LOG", raising=False) + monkeypatch.setattr(cli_mod, "_audit_disabled", True) + cli_mod._emit_audit_event(["soup", "version"], 0) + assert not log.exists() + + def test_emit_audit_event_never_raises(self, monkeypatch): + import soup_cli.cli as cli_mod + + # Point at an impossible path via a broken default — must swallow. + monkeypatch.setenv("SOUP_AUDIT_LOG_PATH", "\x00bad") + monkeypatch.delenv("SOUP_NO_AUDIT_LOG", raising=False) + monkeypatch.setattr(cli_mod, "_audit_disabled", False) + # Should not raise even with a garbage path. + cli_mod._emit_audit_event(["soup", "x"], 1) + + def test_no_audit_log_flag_in_help(self): + from soup_cli.cli import app + + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + out = "".join(ch for ch in result.output if ord(ch) < 128) + assert "no-audit-log" in out.replace("─", "") + + def test_subprocess_emits_audit_line(self, tmp_path): + """End-to-end: a real `soup` invocation via run() writes an audit line.""" + log = tmp_path / "audit.jsonl" + env = dict(os.environ) + env["SOUP_AUDIT_LOG_PATH"] = str(log) + env.pop("SOUP_NO_AUDIT_LOG", None) + env["PYTHONUTF8"] = "1" + r = subprocess.run( + [sys.executable, "-m", "soup_cli", "version"], + env=env, capture_output=True, text=True, timeout=120, + ) + assert r.returncode == 0, (r.stdout, r.stderr) + assert log.exists(), (r.stdout, r.stderr) + rows = [json.loads(x) for x in log.read_text().splitlines() if x.strip()] + assert any(row["command"] == "version" for row in rows) + + def test_subprocess_no_audit_log_env_suppresses(self, tmp_path): + log = tmp_path / "audit.jsonl" + env = dict(os.environ) + env["SOUP_AUDIT_LOG_PATH"] = str(log) + env["SOUP_NO_AUDIT_LOG"] = "1" + env["PYTHONUTF8"] = "1" + r = subprocess.run( + [sys.executable, "-m", "soup_cli", "version"], + env=env, capture_output=True, text=True, timeout=120, + ) + assert r.returncode == 0, (r.stdout, r.stderr) + assert not log.exists() + + +# --------------------------------------------------------------------------- +# #188 — Auto-attach the repro-receipt into `soup airgap-bundle` +# --------------------------------------------------------------------------- +def _make_model_dir(tmp_path, *, with_receipt=False): + model = tmp_path / "model" + model.mkdir() + (model / "config.json").write_text('{"x": 1}', encoding="utf-8") + if with_receipt: + (model / "repro-receipt.json").write_text( + json.dumps({"run_id": "r1", "seeds": {"torch": 42}}), + encoding="utf-8", + ) + return model + + +class TestAirgapReproReceipt: + def test_plan_accepts_repro_receipt_field(self, tmp_path, monkeypatch): + from soup_cli.utils.airgap_bundle import AirgapBundlePlan + + monkeypatch.chdir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=1024 * 1024, + repro_receipt={"run_id": "r1"}, + ) + assert plan.repro_receipt == {"run_id": "r1"} + + def test_plan_repro_receipt_defaults_none(self, tmp_path, monkeypatch): + from soup_cli.utils.airgap_bundle import AirgapBundlePlan + + monkeypatch.chdir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=1024 * 1024, + ) + assert plan.repro_receipt is None + + def test_plan_repro_receipt_must_be_mapping(self, tmp_path, monkeypatch): + from soup_cli.utils.airgap_bundle import AirgapBundlePlan + + monkeypatch.chdir(tmp_path) + with pytest.raises(ValueError): + AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=1024 * 1024, + repro_receipt="not-a-dict", + ) + + def test_build_embeds_receipt_member(self, tmp_path, monkeypatch): + import tarfile + + from soup_cli.utils.airgap_bundle import ( + AirgapBundlePlan, + build_airgap_bundle, + inspect_airgap_bundle, + ) + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=10 * 1024 * 1024, + repro_receipt={"run_id": "r1", "seeds": {"torch": 42}}, + ) + manifest = build_airgap_bundle(plan) + # The receipt is a manifest member. + assert any(f.name == "repro-receipt.json" for f in manifest.files) + # Tar actually contains the file. + with tarfile.open("b.tar", "r") as tar: + names = tar.getnames() + assert "repro-receipt.json" in names + # inspect round-trips the receipt back. + read = inspect_airgap_bundle("b.tar") + assert read.repro_receipt == {"run_id": "r1", "seeds": {"torch": 42}} + + def test_build_without_receipt_has_none(self, tmp_path, monkeypatch): + from soup_cli.utils.airgap_bundle import ( + AirgapBundlePlan, + build_airgap_bundle, + inspect_airgap_bundle, + ) + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=10 * 1024 * 1024, + ) + build_airgap_bundle(plan) + read = inspect_airgap_bundle("b.tar") + assert read.repro_receipt is None + assert not any(f.name == "repro-receipt.json" for f in read.files) + + def test_cli_repro_receipt_flag_in_help(self): + from soup_cli.cli import app + + result = runner.invoke(app, ["airgap-bundle", "--help"]) + assert result.exit_code == 0 + out = "".join(ch for ch in result.output if ord(ch) < 128) + assert "repro-receipt" in out.replace("─", "") + + def test_cli_explicit_repro_receipt(self, tmp_path, monkeypatch): + from soup_cli.cli import app + from soup_cli.utils.airgap_bundle import inspect_airgap_bundle + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + receipt = tmp_path / "receipt.json" + receipt.write_text(json.dumps({"run_id": "explicit"}), encoding="utf-8") + result = runner.invoke( + app, + ["airgap-bundle", "-o", "b.tar", "--model", "model", + "--repro-receipt", "receipt.json"], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + read = inspect_airgap_bundle("b.tar") + assert read.repro_receipt == {"run_id": "explicit"} + + def test_cli_auto_detects_receipt_in_model_dir(self, tmp_path, monkeypatch): + from soup_cli.cli import app + from soup_cli.utils.airgap_bundle import inspect_airgap_bundle + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path, with_receipt=True) + result = runner.invoke( + app, ["airgap-bundle", "-o", "b.tar", "--model", "model"] + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + read = inspect_airgap_bundle("b.tar") + assert read.repro_receipt == {"run_id": "r1", "seeds": {"torch": 42}} + + +# --------------------------------------------------------------------------- +# Review follow-ups (security M1 + python H1/H2 + tdd missing tests) +# --------------------------------------------------------------------------- +class TestReviewFollowups: + # --- #180 EnergyTracker boundary (tdd HIGH 1 / LOW 16) --- + @pytest.mark.parametrize("bad", [-5.0, True, float("nan"), float("inf")]) + def test_energy_tracker_rejects_bad_grid(self, bad): + from soup_cli.utils.energy import EnergyTracker + + with pytest.raises(ValueError): + EnergyTracker(grid_intensity_g_per_kwh=bad) + + def test_energy_tracker_normalises_country_to_upper(self, monkeypatch): + import types + + captured = {} + fake = types.ModuleType("codecarbon") + + class _Data: + energy_consumed = 0.1 + emissions = 0.05 + + class _FakeTracker: + def __init__(self, **kwargs): + captured.update(kwargs) + self.final_emissions_data = _Data() + + def start(self): + pass + + def stop(self): + return 0.05 + + fake.OfflineEmissionsTracker = _FakeTracker + monkeypatch.setitem(sys.modules, "codecarbon", fake) + from soup_cli.utils.energy import EnergyTracker + + with EnergyTracker(country_iso_code="usa"): + pass + assert captured.get("country_iso_code") == "USA" + + # --- #184 extract_top_domains (tdd HIGH 2/3, LOW 18) --- + def test_extract_top_domains_skips_non_mapping_rows(self): + from soup_cli.utils.annex_xi import extract_top_domains + + out = extract_top_domains([42, None, "https://x.com/a", {"text": "no"}]) + assert out == (("x.com", 1.0),) + + def test_extract_top_domains_tie_break_domain_ascending(self): + from soup_cli.utils.annex_xi import extract_top_domains + + out = extract_top_domains([{"text": "https://z.com/1 https://a.com/2"}]) + # equal counts → ascending domain + assert out == (("a.com", 0.5), ("z.com", 0.5)) + + def test_extract_top_domains_urls_per_row_cap(self, monkeypatch): + import soup_cli.utils.annex_xi as ax + + monkeypatch.setattr(ax, "_MAX_URLS_PER_ROW", 2) + text = " ".join(f"https://d{i}.com/x" for i in range(10)) + out = ax.extract_top_domains([{"text": text}]) + # Only the first 2 URLs counted. + assert sum(round(s * 2) for _, s in out) == 2 + + # --- #181 atomic_write_bytes (tdd HIGH 4/5) --- + def test_atomic_write_bytes_rejects_non_bytes(self, tmp_path, monkeypatch): + from soup_cli.utils.paths import atomic_write_bytes + + monkeypatch.chdir(tmp_path) + with pytest.raises(TypeError): + atomic_write_bytes("not-bytes", "x.pdf") + + @pytest.mark.skipif(sys.platform == "win32", reason="POSIX symlink semantics") + def test_atomic_write_bytes_rejects_symlink_target(self, tmp_path, monkeypatch): + from soup_cli.utils.paths import atomic_write_bytes + + monkeypatch.chdir(tmp_path) + (tmp_path / "real").write_bytes(b"x") + os.symlink(tmp_path / "real", tmp_path / "link.pdf") + with pytest.raises(ValueError): + atomic_write_bytes(b"y", "link.pdf") + + def test_render_annex_pdf_escapes_operator_markup(self): + from soup_cli.utils.annex_xi import AnnexXIData, render_annex_pdf + + data = AnnexXIData( + model_name="evil & co", + base_model="b", task="sft", dataset_summary="d", + modalities=("text",), train_compute_flops=0.0, + train_energy_kwh=0.0, train_co2_kg=0.0, top_domains=(), + soup_version="0.71.3", run_id="r", + created_at="2026-06-01T00:00:00+00:00", + ) + # Must not raise — reportlab paragraph parser would choke on raw . + assert render_annex_pdf(data, "xi").startswith(b"%PDF") + + # --- #182 attestation caps (security M1 / tdd HIGH 6/7) --- + def test_validate_attestation_oversize_rejected(self): + from soup_cli.cans.schema import validate_attestation_statement + + big = {**_sample_statement(), "pad": "a" * (1024 * 1024 + 10)} + with pytest.raises(ValueError, match="too large"): + validate_attestation_statement(big) + + def test_validate_attestation_non_serialisable_rejected(self): + from soup_cli.cans.schema import validate_attestation_statement + + bad = {**_sample_statement(), "bad": {object()}} + with pytest.raises(ValueError): + validate_attestation_statement(bad) + + def test_manifest_rejects_oversize_attestation(self): + from pydantic import ValidationError + + from soup_cli.cans.schema import Manifest + + big = {**_sample_statement(), "pad": "a" * (1024 * 1024 + 10)} + with pytest.raises(ValidationError): + Manifest( + can_format_version=3, name="x", author="a", + created_at="2026-01-01", base_hash="h", + attestations=[big], + ) + + def test_load_attestation_file_oversize_rejected(self, tmp_path, monkeypatch): + from soup_cli.commands.can import _load_attestation_files + + monkeypatch.chdir(tmp_path) + big = tmp_path / "big.json" + big.write_text( + json.dumps({**_sample_statement(), "pad": "a" * (1024 * 1024 + 50)}), + encoding="utf-8", + ) + with pytest.raises(typer.Exit): + _load_attestation_files(["big.json"]) + + # --- #183 audit splitter + exit codes (tdd MED 8/9/10/11) --- + def test_split_command_args_value_opt_followed_by_flag(self): + from soup_cli.cli import _split_command_args + + cmd, args = _split_command_args( + ["soup", "--log-level", "--yes", "train", "--config", "x"] + ) + # --yes is consumed as the --log-level value; train is the command. + assert cmd == "train" + assert args == ["--config", "x"] + + def test_split_command_args_trailing_value_opt(self): + from soup_cli.cli import _split_command_args + + # Trailing value-opt: i += 2 overshoots, loop exits cleanly (no + # IndexError) and the leftover token is returned as args. + cmd, args = _split_command_args(["soup", "--log-level"]) + assert cmd == "(root)" + assert args == ["--log-level"] + + def test_emit_audit_event_nonzero_exit_code(self, tmp_path, monkeypatch): + import soup_cli.cli as cli_mod + + log = tmp_path / "audit.jsonl" + monkeypatch.setenv("SOUP_AUDIT_LOG_PATH", str(log)) + monkeypatch.delenv("SOUP_NO_AUDIT_LOG", raising=False) + monkeypatch.setattr(cli_mod, "_audit_disabled", False) + cli_mod._emit_audit_event(["soup", "train", "--config", "x"], 3) + rows = [json.loads(x) for x in log.read_text().splitlines() if x.strip()] + assert rows[0]["command"] == "train" + assert rows[0]["exit_code"] == 3 + assert "--config" in rows[0]["args"] + + def test_subprocess_no_audit_log_flag_suppresses(self, tmp_path): + """The --no-audit-log FLAG (not env) flows through the real run().""" + log = tmp_path / "audit.jsonl" + env = dict(os.environ) + env["SOUP_AUDIT_LOG_PATH"] = str(log) + env.pop("SOUP_NO_AUDIT_LOG", None) + env["PYTHONUTF8"] = "1" + r = subprocess.run( + [sys.executable, "-m", "soup_cli", "--no-audit-log", "version"], + env=env, capture_output=True, text=True, timeout=120, + ) + assert r.returncode == 0, (r.stdout, r.stderr) + assert not log.exists() + + # --- #188 airgap repro-receipt edges (tdd MED 12/13/14/15, LOW 17) --- + def test_build_receipt_over_size_cap_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.airgap_bundle import ( + AirgapBundlePlan, + build_airgap_bundle, + ) + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=10, + repro_receipt={"x": "a" * 1000}, + ) + with pytest.raises(ValueError, match="exceeds cap"): + build_airgap_bundle(plan) + + def test_plan_accepts_mappingproxy_receipt(self, tmp_path, monkeypatch): + from types import MappingProxyType + + from soup_cli.utils.airgap_bundle import ( + AirgapBundlePlan, + build_airgap_bundle, + inspect_airgap_bundle, + ) + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + plan = AirgapBundlePlan( + output="b.tar", model_dir="model", dataset_dirs=(), + wheel_dirs=(), kernel_dirs=(), + bundle_size_cap_bytes=10 * 1024 * 1024, + repro_receipt=MappingProxyType({"run_id": "proxy"}), + ) + build_airgap_bundle(plan) + assert inspect_airgap_bundle("b.tar").repro_receipt == {"run_id": "proxy"} + + def test_manifest_from_payload_coerces_bad_receipt_to_none(self): + from soup_cli.utils.airgap_bundle import _manifest_from_payload + + m = _manifest_from_payload({ + "soup_version": "0.71.3", "created_at": "x", "model_dir": "model", + "datasets": [], "wheels": [], "kernels": [], "files": [], + "total_bytes": 0, "repro_receipt": ["not", "a", "dict"], + }) + assert m.repro_receipt is None + + def test_cli_auto_detect_malformed_receipt_is_none(self, tmp_path, monkeypatch): + from soup_cli.cli import app + from soup_cli.utils.airgap_bundle import inspect_airgap_bundle + + monkeypatch.chdir(tmp_path) + model = _make_model_dir(tmp_path) + (model / "repro-receipt.json").write_text("not json{", encoding="utf-8") + result = runner.invoke( + app, ["airgap-bundle", "-o", "b.tar", "--model", "model"] + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert inspect_airgap_bundle("b.tar").repro_receipt is None + + def test_cli_explicit_missing_receipt_exits_2(self, tmp_path, monkeypatch): + from soup_cli.cli import app + + monkeypatch.chdir(tmp_path) + _make_model_dir(tmp_path) + result = runner.invoke( + app, + ["airgap-bundle", "-o", "b.tar", "--model", "model", + "--repro-receipt", "nope.json"], + ) + assert result.exit_code == 2, (result.output, repr(result.exception))