diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ea0537..d707c77 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,11 +107,11 @@ soup_cli/ cans/ - Shareable .can artifact format + run/publish orchestrator (v0.26.0 + v0.33.0) data/traces/ - Trace-to-Preference harvester (v0.26.0) data/collators.py - CrossDocCollator for sample packing (v0.33.0) - utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats + utils/ - GPU, errors, MoE, GaLore, QAT, Unsloth, vLLM, SGLang, Liger, FlashAttn, FSDP, Ring Attention, long-context, quality, curriculum, freeze, dataset-registry, mlx, peft_builder, paths, topology, launcher, mii, pipeline, cut_ce, fp8, gradient_ckpt, kernel_picker, cross_doc_attn, activation_offload, hf, spec_pairing, structured_output, metrics, tracing, auto_quant, lr_finder, grad_accum, mixed_precision, warmup, spike_recovery, convergence, v028_features, multipack_sampler, multipack, neat_packing, jinja_analyzer, quant_menu, relora, peft_patches, peft_wiring, dpo_variants, optimizer_zoo, lr_groups, loftq_init, block_expansion, tts, classifier, distill, bitnet, ebft_gdpo, moe_quant, reasoning_effort, gguf_quant, kv_cache, advanced_precision, save_formats, deploy_measure templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (180 files, 7610 tests) +tests/ - Test suite (184 files, 7722 tests) examples/ - Real-world config examples and datasets ``` @@ -263,6 +263,10 @@ pytest tests/ --cov=soup_cli --cov-report=html | test_v0500_part_d.py | v0.50.0 Part D — 7 stability/efficiency knobs (`ref_model_ema_alpha` / `replay_buffer_size` / `async_grpo_prefetch` / `tis_threshold` / `mask_truncated_completions` / `defer_rerolling` / `skip_zero_advantage` / `off_policy_mask_threshold`); explicit bool-rejection field_validator across all numeric fields (tdd-guide HIGH fix); `mask_truncated_completions` requires `tis_threshold` cross-validator; SoupConfig task-gate naming every offending field; `grpo_fp16` task-gate (code-review HIGH fix) (v0.50.0 Part D) | | test_v0500_part_e.py | v0.50.0 Part E — `task='prm'` (Process Reward Model) + `vision_grpo` flag; `validate_prm_compat` (data.format / modality / mlx gates); `validate_vision_grpo_compat` (task ∈ {grpo, ppo} / modality='vision' / non-mlx); `build_prm_trainer` deferred stub; SoupConfig integration with all rejection paths exercised (v0.50.0 Part E) | | test_v0520.py | v0.52.0 Modality II — TTS / classifier / distill / BitNet / EBFT-GDPO / MoE quant / reasoning_effort: TTS family allowlist + per-family emotion allowlists (Orpheus + Oute) + validate_tts_compat; classifier / reranker / cross_encoder tasks + num_labels (with field_validator bool guard) + label_names dedup + classifier-only field gates; distill divergence (kl alias canonicalised, Literal excludes alias) + teacher_model + distill_temperature bounds; BitNet 1.58 quant + bitnet/tq1_0 export-format stubs + Falcon-E recipe + is_bitnet_model org-prefix detect; EBFT (structured/strided) + GDPO (standard/length_normalized/margin) variant allowlists + task gates; MoE expert quant (nf4/int8_rowwise) + train_router_only requiring moe_lora=true; reasoning_effort + train_on_eot with SFT-family task gate; 6 new recipes (5 TTS + Falcon-E BitNet); review-fix coverage (num_labels bool guard, Oute emotion allowlist, lazy-import in classifier validator, task gates, oversize / NaN / Inf matrices). Test count: 272 (v0.52.0) | +| test_v0531_82.py | v0.53.1 #82 autopilot pre-quantized detection: `detect_prequantized_format` + `decide_quantization(prequantized=...)` + `detect_prequantized_format_from_path` with cwd-containment + config.json symlink rejection + name/config aliases + word-boundary regex (v0.53.1) | +| test_v0531_142.py | v0.53.1 #142 merge_4bit + export_torchao live wiring: BNB-4bit single-stage merge + TorchAO PTQ with per-scheme kwarg allowlist + CLI `--save-format` + `--quant-config` + `load_quant_config` (yaml.safe_load + 256 KB cap + extension allowlist) + path TOCTOU (v0.53.1) | +| test_v0531_139.py | v0.53.1 #139 export_advanced_gguf live: 3-stage llama.cpp pipeline (convert → imatrix → quantize) + UD-prefix strip + subprocess argv shape + `_prepare_calibration_text` JSONL alias fallback + null-byte strip + 50 MB cap + POSIX O_NOFOLLOW + `_safe_stderr` Rich escape (v0.53.1) | +| test_v0531_109.py | v0.53.1 #109 deploy autopilot --measure: `compute_cache_key` + `sha_of_file` + `measure_candidate` OK/MINOR/MAJOR bands + `pick_best` soft-fallback (max-by-delta) + cache round-trip with symlink rejection on load AND save + CLI integration + `_MAX_CANDIDATES=32` cap + `render_measure_table` markup escape regression (v0.53.1) | | test_v0530.py | v0.53.0 Quant Menu II — UD GGUFs + KV cache + NVFP4 + LF parity + save formats: Parts A+B GGUF (UD ladder 14 entries + IQ 12 + Apple/ARM 10 frozensets + non-overlap invariant + `validate_*` case-insensitive + rejection matrix + `is_advanced_gguf_format` union + `_LOWER_INDEX` O(1) lookup + MappingProxyType immutability + `validate_calibration_data_path` shape rejection + 4096-boundary + `export_advanced_gguf` v0.53.1 deferred stub); Part C KV cache (`KV_CACHE_TYPES` frozenset + `validate_kv_cache_type` case + bool/null/oversize/non-string rejection + `requires_hopper` delegates to spec + `get_kv_cache_spec` frozen + schema fp8-on-mlx rejected with specific message + q8_0-on-mlx allowed); Part D advanced precision (`fp8_attention` requires `quantization_aware='fp8'` BEFORE mlx-gate ordering + bool guards on every string param + schema rejects-without-fp8-qat; `nvfp4` mlx + vision rejection + bool guards; `unsloth_bnb_4bit` backend='unsloth' + quantization='4bit' rejection matrix; `apply_*` deferred); Part E LF parity (`bnb_4bit_use_double_quant` rejects none/8bit/gptq parametrize; `llm_int8` rejects default-none + 4bit; `quantize_ref_model` happy on dpo/grpo/kto + rejects sft/pretrain; `quantize_reward_model` happy on ppo/reward_model + rejects dpo; explicit `TypeError("v0.53.0 flag must be bool")` from `_validate_v053_bool_fields`; explicit-null surfaces as `valid boolean` ValidationError); Part F save formats (`MERGE_SAVE_FORMATS` lowercase normalisation + rejection matrix; `TORCHAO_PTQ_SCHEMES` CASE-SENSITIVE — `int4weightonly` rejected; `validate_quant_config_path` 4096-boundary; `MergeSaveSpec` + `TorchAOPTQSpec` frozen + MappingProxyType immutability; `merge_4bit` + `export_torchao` deferred); Cross-cutting (full 5-field YAML round-trip + cardinality invariant + tautological-assert replaced with allowlist + idempotent re-validate + `get_gguf_spec` unknown raises + bool guards on backend/modality/quantization across every Part D validator). Test count: 154 (v0.53.0) | | test_v0510.py | v0.51.0 Model Catalog Expansion + Alternative Model Hubs: Part E hubs.py (`SUPPORTED_HUBS` + `validate_hub_name` + `validate_hub_endpoint` SSRF parity / CRLF rejection / IPv6 mapped private rejected / IPv6 loopback ok / control chars; `resolve_endpoint` env-var override; `default_endpoint` + `endpoint_env_var` + `required_hub_package` + `is_hf` with bool guards; MappingProxyType immutability); TrainingConfig `hub` field (default + Literal accept + None reject + case-insensitive normalisation + YAML round-trip) + SoupConfig `_validate_hub_supported` (mlx + non-hf rejected; mlx + hf accepted; modelers + transformers accepted); Part D MULTIPACK_ARCHITECTURES extension (20 new arches parametrize + legacy preserved + exact count=38 + frozenset immutability); Parts A/B/C 26 new recipes (parametrize over every name × {get_recipe / RecipeMeta / SoupConfig load / yaml.safe_load / model id no null/whitespace/empty parts / max_length bounds / GRPO required fields}); baichuan-sft uses `hub: modelscope`; total recipe count >= 105 (v0.51.0) | diff --git a/README.md b/README.md index cc2860a..dcbb286 100644 --- a/README.md +++ b/README.md @@ -43,15 +43,14 @@ soup train Latest highlights only. Full history: [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases). -**v0.53.0 — Quant Menu II (UD GGUFs + KV cache + NVFP4 + LF parity + save formats)**: Unsloth Dynamic 2.0 GGUF ladder (UD-Q8_K_XL … UD-IQ1_M), IQ + Apple/ARM GGUF flavours, `kv_cache_type` (q8_0 / bf16 / f16 / fp8), FP8 attention, NVFP4 (Blackwell), explicit `unsloth_bnb_4bit`, BNB double-quant, ref/reward model quantization, merge-4bit save, TorchAO PTQ export schema — schema-only release; live llama.cpp imatrix + serve / merge / export wiring lands in v0.53.1. +**v0.53.1 — Quant Menu II + Export pipeline live**: Six v0.53.0 deferred stubs lifted — autopilot pre-quantized detection, single-stage BNB-4bit merge, TorchAO PTQ export, Unsloth Dynamic 2.0 GGUF ladder via llama.cpp `imatrix`, and `soup deploy autopilot --measure` Quant-Lobotomy scorecard. -- **Unsloth Dynamic 2.0 GGUF ladder.** 14-entry closed allowlist (UD-Q{8..2}_K_XL + UD-IQ{4_XS, 3_M, 3_XXS, 2_M, 2_XS, 2_XXS, 1_M, 1_S}) with `validate_ud_gguf_format` case-insensitive canonical normalisation. `--calibration-data ` flag shape-validates now; cwd-containment + TOCTOU symlink rejection land at CLI dispatch in v0.53.1. -- **IQ + Apple/ARM GGUF.** 12-entry IQ family (IQ1/2/3/4 — including IQ4_NL non-linear) + 10-entry Apple/ARM-friendly set (Q4_0_4_4 / Q4_NL / Q5_K_M / etc.) wrapped in `MappingProxyType` metadata. -- **KV cache types.** New `training.kv_cache_type: q8_0 | bf16 | f16 | fp8`. FP8 is Hopper-only — cross-validator rejects `fp8` on the MLX backend; the SM-capability check fires at serve construction. -- **FP8 attention + NVFP4 + native `unsloth_bnb_4bit`.** Three new bool flags. `fp8_attention=true` requires `quantization_aware='fp8'` and a non-MLX backend; `nvfp4=true` is gated to CUDA + text modality (Blackwell SM ≥ 12 check is runtime-only); `unsloth_bnb_4bit=true` requires `backend='unsloth'` + `quantization='4bit'`. -- **LF / Axolotl parity.** `bnb_4bit_use_double_quant` (requires `quantization='4bit'`), `llm_int8` (asserts `quantization='8bit'` — distinct from v0.41.0 `load_in_8bit` aliasing), `quantize_ref_model` (extends v0.40.5 Quant Menu to the ref model on DPO/IPO/SimPO/ORPO/BCO/KTO/GRPO/PPO/preference), `quantize_reward_model` (PPO + reward_model tasks). -- **Advanced save formats.** `soup merge --save-format 4bit | 4bit_forced` (single BNB-4bit merged checkpoint without dequant/merge/requant cycle) and `soup export --format torchao --quant-config ` (closed allowlist Int4WeightOnly / Int8DynActInt4 / Float8DynActFloat8 / NVFP4) — schema lands now; live writers land in v0.53.1. -- **+157 net new tests** (7453 → 7610) across 154 tests in `test_v0530.py`. Five review agents (python / code / security / tdd / verification) ran in parallel; every CRITICAL / HIGH / MEDIUM / LOW finding fixed or documented: O(1) `_LOWER_INDEX` for GGUF lookup, ref-task set extended with GRPO + KTO, `_validate_v053_bool_fields` no longer silently coerces `None`, `requires_hopper` reads from spec metadata, `fp8_attention` validator order swapped so the `quantization_aware='fp8'` error fires first, `validate_calibration_data_path` / `validate_quant_config_path` docstrings name the exact controls v0.53.1 CLI dispatch must add. +- **Autopilot detects pre-quantized bases.** `TheBloke/Llama-2-7B-Chat-GPTQ` now recommends `gptq` instead of stacking BNB-4bit on top. Name-regex + `config.json` `quantization_config.quant_method` probe with cwd-containment + symlink rejection on the on-disk path. BNB aliases (`bitsandbytes_4bit` / `nf4` / `bnb_8bit`) canonicalise to `4bit` / `8bit`. +- **`soup merge --save-format 4bit | 4bit_forced`.** Single BNB-4bit-quantized merged checkpoint without the dequant→merge→requant cycle. `4bit_forced` quantizes every Linear (including `lm_head`). Output path is cwd-contained + symlink-rejected at CLI dispatch. +- **`soup export --format torchao --quant-config `.** Live `torchao.quantize_` + `save_pretrained`. Closed per-scheme kwarg allowlist (Int4WeightOnly accepts `{group_size, inner_k_tiles}`, NVFP4 accepts nothing extra) defeats kwarg-injection through the YAML. +- **`soup export --format gguf-ud --gguf-flavour `.** Three-stage pipeline: HF → f16 GGUF → optional importance-matrix (UD ladder + low-bit IQ) → quantize. All subprocess calls use argv-list form + 30-min timeout. Calibration JSONL is sanitised (null-byte stripped, newlines collapsed, 8 KB per-line + 50 MB total cap). POSIX `O_NOFOLLOW` defeats the TOCTOU race between the dispatch-time symlink check and the actual open. +- **`soup deploy autopilot --measure --tasks `.** Loops every candidate quant through the v0.26.0 `eval/quant_check` scorer, renders OK/MINOR/MAJOR table, picks the best-by-delta candidate. Results cached at `~/.soup/deploy_autopilot_cache.json` (atomic write, 0o600 perms on POSIX, symlink-rejected on both load and save). Cache key is SHA-256 of `(base, profile, eval-tasks)`. +- **+112 net new tests** (7610 → 7722) across `test_v0531_82.py`, `test_v0531_109.py`, `test_v0531_139.py`, `test_v0531_142.py`. Four review agents (python / code / security / tdd) ran; every CRITICAL / HIGH / MEDIUM / LOW finding fixed: per-scheme TorchAO kwarg allowlist (rejects dunders + unknown keys), corrected BNB 4-bit skip-modules kwarg name, shared `enforce_under_cwd_and_no_symlink` in `utils/paths.py` (single source of truth), `pick_best` switches from `max(after)` to `max(delta)` matching the v0.33.0 #54 design intent, `_run_convert_to_f16` verifies the convert script stays inside `llama_cpp_dir` via realpath + commonpath, `_safe_stderr` Rich-escapes subprocess stderr before exception propagation. ## Why Soup? @@ -3641,6 +3640,40 @@ Cross-validator ordering picks the most actionable error: `quantization_aware='f `soup export --format torchao --quant-config ` is the planned PTQ export surface for `torchao.quantize_` + `save_pretrained`. Four schemes are allowlisted: `Int4WeightOnly`, `Int8DynActInt4`, `Float8DynActFloat8`, `NVFP4`. CASE-SENSITIVE — these are PyTorch class names and `torchao.quantize_` looks them up by exact name. Diverges from `--save-format` (lowercase-normalised) on purpose; documented at both validators. +## Quant Menu II + Export Pipeline (v0.53.1) + +v0.53.1 lifts the v0.53.0 schema-only stubs to live wiring: + +```bash +# Single-stage BNB-4bit merged checkpoint (no dequant/merge/requant) +soup merge -a ./adapter -o ./merged_4bit --save-format 4bit + +# TorchAO PTQ export — closed per-scheme kwarg allowlist +cat > q.yaml <` to redirect (constrained to home / cwd / tempdir). The recommended candidate uses soft-fallback: first `OK` by insertion order, else the candidate with the smallest delta (least drop relative to its own baseline). + ## Changelog See [GitHub Releases](https://github.com/MakazhanAlpamys/Soup/releases) for version history. diff --git a/SECURITY.md b/SECURITY.md index b86bc49..e7b42bb 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -9,14 +9,14 @@ We provide security updates for the following versions: - **Versions older than 3 minor versions:** No support Example: -- v0.53.0 -- Full support (latest) +- v0.53.1 -- Full support (latest) +- v0.53.0 -- Full support - v0.52.0 -- Full support - v0.51.0 -- Full support - v0.50.0 -- Full support -- v0.49.0 -- Full support +- v0.49.0 -- Bug-fix support only - v0.48.0 -- Bug-fix support only -- v0.47.0-v0.47.x -- Bug-fix support only -- v0.46.x and below -- No support +- v0.47.x and below -- No support ## Reporting a Vulnerability @@ -147,6 +147,7 @@ No known critical vulnerabilities in current releases. - **v0.32.0 — Training Stability & Auto-Tuning**: `--find-lr-output` containment via shared `utils/paths.is_under_cwd` (prevents writes outside cwd); `save_lr_finder_report` rejects NaN / Infinity floats in `lrs` / `losses` and serialises with `allow_nan=False` (keeps the report parser-safe); `compute_lr_schedule` rejects non-positive `start_lr`, inverted ranges, and `num_steps` outside `[2, 10_000]`; `pick_mixed_precision` rejects empty / null-byte / >200-char model names and resolves multi-version quirks (`qwen2.5` vs `qwen2`, `phi-3.5` vs `phi-3`) by longest-substring-first iteration so an added family can never accidentally make a more-specific entry dead code; `compute_warmup_steps` clamps to `[10, 1000]` with a `ratio==0.0` short-circuit matching HF Trainer's "no warmup" convention; `SpikeRecoveryStrategy` is `@dataclass(frozen=True)` (post-construction mutation cannot bypass validation), `max_attempts ∈ [1, 10]`, `lr_decay ∈ (0, 1)`, `min_lr > 0`; cross-validator `_validate_spike_recovery_requires_watchdog` rejects `loss_spike_recovery=true, loss_watchdog=false` at config-load (fails fast instead of never triggering); `convergence_window ∈ [5, 10_000]`, `convergence_rel_tol ∈ (0, 1]`, `recommend_action` reuses `detect_plateau` so plateau heuristic stays single-source-of-truth; `GradAccumMonitor.recommend()` caps doubled `accum` at `MAX_ACCUM=1024` so a runaway advisory loop cannot blow up DataLoader prefetch; `generate_config` validates BOTH the YAML output path AND the embedded `decisions["output"]` field via `is_under_cwd` (closes the gap where a crafted `decisions["output"]="../../etc"` would have silently propagated into the rendered YAML) - **v0.34.0 — Observability & Dev UX**: `.crash` bundle generator (`utils/crash.py`) recursively redacts `hf_*` / `sk-*` / `Bearer …` token-shaped strings in any captured `config` and metric tail before serialisation, so a `.crash` file shared on a public GitHub issue cannot leak credentials; `output_dir` is reduced to `os.path.basename` so `$HOME` doesn't leak; `write_crash_bundle` uses `os.path.realpath + commonpath` for cwd containment (Windows-safe; raises `ValueError` not `PermissionError` so callers cannot silently swallow with `except OSError`); filename appends `secrets.token_hex(4)` so two crashes in the same UTC second don't collide; bundle truncated to `MAX_BUNDLE_BYTES=1_000_000`. `train.py` crash-write surfaces failures to the user (no silent missing-bundle). `profiling.py` `resolve_trace_path` rejects empty / `.` / `..` / `/` / `\\` / null-byte `run_id` (closes the `output_dir/profiles/../trace.json` escape) and uses `os.path.realpath + is_under_cwd`; profiles dir is created only on successful torch import (no stale empty dirs on torch-less CI). `tracker.get_run` LIKE-prefix match escapes `%` / `_` / `\\` and uses `ESCAPE '\\'` so a crafted `run_id` cannot widen the match (mirrors v0.26.0 registry policy). Lazy schema migration (`_ensure_schema`) tolerates the "duplicate column" race when two CLI processes start simultaneously on a fresh DB (fork-based multi-GPU training, TUI auto-refresh). `runs.py show/replay/clean` switched user `run_id` rendering to `markup_escape` and switched `clean` containment from broken `Path.resolve() + relative_to()` to project-standard `os.path.realpath + is_under_cwd`. `tui_app.py` lazy-imports `ExperimentTracker` and `markup_escape`s every DB-sourced string before passing into Textual widgets so a crafted base_model / experiment_name cannot inject `[bold red]…[/]` markup. `run_cost.estimate_run_cost_usd` rejects `bool` in `num_gpus` (bool is a subclass of int — same defence as v0.30.0 `Candidate.__post_init__`); duration clamped to `[0, 1 year]`; unknown GPU returns `None` so callers render `—` instead of fabricating `$0.00`. `log_level.parse_log_level` rejects non-string + null-byte input. - **v0.33.0 — Live Wire**: RLVR `code_exec_reward` adds OS-level isolation (Linux best-effort `os.unshare(CLONE_NEWUSER|CLONE_NEWNET|CLONE_NEWPID)`, macOS `sandbox-exec` with default-deny `MACOS_SANDBOX_PROFILE` narrowed to a 3-name `mach-lookup` allowlist to prevent DNS / NSURLSession bypass of `(deny network*)`); `prune_checkpoints` switches to TOCTOU-safe `os.lstat + S_ISLNK` + `shutil.rmtree(onerror=_abort_on_symlink)` so a symlink encountered mid-walk aborts rather than escapes; `run_gate` wraps each task scorer in a typed `try/except` so backend failures produce `score=None, error=str(exc)` (never silent `score=1.0`); `_parse_judge_url` removes the bare `http://` catch-all (defence-in-depth after the Pydantic GateTask validator); `soup can run` requires `--yes` or explicit consent callback and raises `ValueError` (not `PermissionError`, which is an `OSError` subclass that broad `except` blocks would swallow); GGUF `rglob` result for ollama deploy is `realpath+commonpath` checked against extract_dir (prevents symlink escape from a crafted can); `DeployTarget.path` validator normalises mixed `\\`/`/` separators before splitting (closes a Windows `..` bypass); `CAN_FORMAT_VERSION` 1→2 (additive — v1 still loads); `soup can publish` validates `repo_id` via `utils/hf.validate_repo_id`, resolves token via `resolve_token`, sanitises commit messages (first-line, 200-char cap), uses HTTPS-only HfApi; `_write_spike_recovery_hint` adds `is_under_cwd` containment check on `args.output_dir` from raw HF `TrainingArguments`; `lookup_entry_by_output_dir` emits `ResourceWarning` when 1000-row scan limit is hit (no silent miss); `CrossDocCollator` no longer mutates input feature dicts (HF Dataset rows are cached and reused — mutation broke subsequent batches); `Candidate` rejects `bool` in `score`/`latency_ms` (was sneaking past `int` isinstance check); `evaluate_candidate` latency mean now divides by *completed* prompts (excludes crashed) so a broken candidate isn't artificially fast; `auto_quant.run_auto_quant_picker` soft-falls-back to highest-scored candidate when no candidate clears `min_score` (server still binds); `build_logits_processors` returns `[]` when neither `outlines` nor `lm-format-enforcer` is installed (server degrades to free-form rather than 500); MII server uses loopback-only CORS, max_tokens cap [1, 16384], stream rejection, generic 500 with no stack-trace leak; `os.execvp` auto-reexec uses list args (no shell), all forwarded flags pre-validated; `cleanup_extract_dir` uses `os.path.commonpath` (Windows-safe) instead of `startswith`; `_run_subprocess` catches `TimeoutExpired` and returns rc=124 (coreutils convention) instead of an unhandled traceback; new `eval_results` and `tensorrt` artifact kinds in `RegistryStore._VALID_KINDS` +- **v0.53.1 — Quant Menu II + Export pipeline live**: lifts six v0.53.0 deferred stubs to live wiring while keeping the project's hardening invariants. New shared helper `soup_cli/utils/paths.enforce_under_cwd_and_no_symlink` consolidates the v0.33.0 #22 TOCTOU pattern (cwd containment via `os.path.realpath + os.path.commonpath` + `os.lstat + S_ISLNK` rejection) — used by `commands/merge.py`, `commands/export.py`, `utils/save_formats.py`, and `utils/gguf_quant.py` so the same boundary check fires at every CLI dispatch point. `merge_4bit` and `export_torchao` (`utils/save_formats.py`): cwd containment + symlink rejection on `merged_dir` / `model_dir` / `output_dir`; `load_quant_config` enforces `yaml.safe_load` only + 256 KB cap + extension allowlist (`.yaml`/`.yml`); **per-scheme closed kwarg allowlist** rejects dunder keys + unknown params before the splat into `torchao.Config(**kwargs)` (security-review HIGH fix — `Int4WeightOnly` accepts `{group_size, inner_k_tiles}`, `NVFP4` accepts nothing extra). Corrected BNB-4bit skip-modules kwarg name from `llm_int8_skip_modules` to `bnb_4bit_skip_modules`. `export_advanced_gguf` (`utils/gguf_quant.py`): all three subprocess invocations (`convert_hf_to_gguf.py`, `llama-imatrix`, `llama-quantize`) use argv-list form with no shell, 30-min timeout, `sys.executable` for the convert script; `_run_convert_to_f16` realpath-verifies that `convert_hf_to_gguf.py` stays inside the `llama_cpp_dir` after resolution (security-review HIGH M5 fix — defends against a symlinked script escape). `_prepare_calibration_text` strips null bytes, collapses newlines to spaces, caps per-line at 8 KB + total at 50 MB (security-review M1), uses POSIX `O_NOFOLLOW` to refuse symlinks at the kernel level (security-review M3 — closes the TOCTOU window between the dispatch-time check and the actual `open()`); requires ≥ 1 usable row before invoking imatrix. `_safe_stderr` Rich-markup-escapes subprocess stderr before embedding in `RuntimeError` (security-review L4) so a crafted llama.cpp error cannot inject `[red]...[/]` into the operator-facing panel. UD-prefix stripped from flavour arg before passing to llama-quantize (`UD-Q4_K_XL` → `Q4_K_XL`). Calibration data path containment + symlink rejection fires at CLI dispatch in `commands/export.py::_export_gguf_advanced`. `detect_prequantized_format_from_path` (`autopilot/decisions.py`): cwd containment + `os.lstat + S_ISLNK` on `/config.json` (security-review HIGH H2 — out-of-cwd model paths silently return `None` to preserve soft-probe semantics so HF Hub repo IDs aren't rejected); null-byte rejection on `model_dir`. `commands/merge.py`: early `is_under_cwd(output)` check at CLI boundary (security-review M4) — consistent with the v0.20.0 / v0.40.2 containment-at-the-boundary policy. `deploy_measure.py`: cache file written atomically via `tempfile.mkstemp` + `os.replace` with `os.lstat + S_ISLNK` rejection on BOTH `load_cache` and `save_cache` (security-review M2 — was missing on the load side); env override `SOUP_DEPLOY_AUTOPILOT_CACHE` rejects null bytes + control chars before any path resolution and confines the override to home / cwd / tempdir; cache file gets best-effort 0o600 perms on POSIX (matches v0.26.0 registry.db policy); 1 MB cache-file cap. `_DEPLOY_MEASURE_BEFORE_GEN` / `_AFTER_FACTORY` module-level callables are documented as a non-public escape hatch (deferred until v0.46.1 live model-loader). Test surface: 4 new test files (`test_v0531_82.py` / `test_v0531_109.py` / `test_v0531_139.py` / `test_v0531_142.py`) carrying 112 new tests covering happy paths + failure modes + every security guard (POSIX symlink rejection, per-scheme kwarg allowlist, TOCTOU defences, `_MAX_CANDIDATES` cap, MINOR-verdict band, mxfp4 word boundary, BNB-alias detection, render-table markup escape). Known limitations: (1) `_DEPLOY_MEASURE_BEFORE_GEN` / `_AFTER_FACTORY` are a stop-gap until v0.46.1 ships first-party transformers / vLLM generator factories. (2) `#70` GGUF and `#72` AWQ/GPTQ manual QA smokes remain pending — require CUDA + llama.cpp build; recipes scripted in `tests/qa/v053_qa.md`. (3) BNB-4bit merge + TorchAO PTQ live happy-path is mock-covered only — CPU-only CI cannot execute the real BNB / torchao kernels. (4) `_prepare_calibration_text` accepts JSONL with `text` / `prompt` / `content` aliases + raw text fallback; other formats (parquet / markdown) are out of scope. (5) Cache key truncates `base_sha` to 16 hex chars at the call site (collision probability ≈ 1-in-2³² across ~4 billion entries). (6) Pre-quantized detection is heuristic — name regex + local `config.json` probe; HF Hub repo IDs without local download fall back to name-only matching. (7) `enforce_under_cwd_and_no_symlink` checks only the leaf path; deeper traversal relies on the per-file leaf check at each site. (v0.53.1) - **v0.53.0 — Quant Menu II (UD GGUFs + KV cache + NVFP4 + LF parity + save formats)**: 6 schema-only Parts; live wiring deferred to v0.53.1. Every new validator follows the project's established hardening policy: closed allowlists (`UD_GGUF_FORMATS`, `IQ_GGUF_FORMATS`, `APPLE_ARM_GGUF_FORMATS`, `KV_CACHE_TYPES`, `MERGE_SAVE_FORMATS`, `TORCHAO_PTQ_SCHEMES`) as `frozenset` so registries cannot be mutated; `_GGUF_METADATA` / `_KV_CACHE_METADATA` / `_MERGE_METADATA` / `_TORCHAO_METADATA` wrapped in `MappingProxyType`; `_LOWER_INDEX` for GGUF lookup is also `MappingProxyType`-wrapped (replaces O(N) walk with O(1) lookup — code-review MEDIUM fix). All string validators reject non-string / bool / empty / null-byte / oversize with case-insensitive normalisation (matches v0.41.0 `validate_optimizer_name` / v0.51.0 `validate_hub_name` policy); `validate_torchao_scheme` is INTENTIONALLY case-sensitive (PyTorch class names — `torchao.quantize_` looks them up by exact name) with the asymmetry documented at both validators (security-review LOW fix). `validate_calibration_data_path` + `validate_quant_config_path` are shape-only at this release; their docstrings name the exact controls a v0.53.1 CLI dispatch contributor MUST add (`os.path.realpath` + `os.path.commonpath` cwd containment, `os.lstat` + `stat.S_ISLNK` symlink rejection before `open()`, existence check, `yaml.safe_load`-only for quant configs) — closes the security-review MEDIUM "documentation gap at trust boundary" finding. SoupConfig cross-validators: `_validate_fp8_attention_compat` (requires `quantization_aware='fp8'` BEFORE the MLX gate so the more actionable error fires first — code-review MEDIUM fix); `_validate_nvfp4_compat` (non-MLX + `modality='text'`; Blackwell SM ≥ 12.0 runtime check fires at trainer construction); `_validate_unsloth_bnb_4bit_compat` (requires `backend='unsloth'` + `quantization='4bit'`); `_validate_bnb_4bit_double_quant` (requires `quantization='4bit'` — rejects `none`/`8bit`/Quant-Menu); `_validate_llm_int8_alias` (asserts `quantization='8bit'`, deliberately disjoint from v0.41.0 `load_in_8bit` aliasing); `_validate_quantize_ref_reward` (extended ref-task allowlist `{dpo, ipo, simpo, orpo, bco, kto, preference, grpo, ppo}` per code-review HIGH fix — first-cut omitted grpo + kto + ppo which all have reference policies); `_validate_kv_cache_type_supported` (only `fp8` gated to non-MLX in v0.53.0; q8_0/bf16/f16 pass-through documented at validator site so v0.53.1 contributor sees the gate immediately). `requires_hopper` reads from `_KV_CACHE_METADATA` spec — single source of truth so adding a Hopper-only type means flipping the spec field only (code-review MEDIUM fix). All 7 new bool fields share `_validate_v053_bool_fields` `field_validator(mode='before')` that rejects bool-as-int with explicit `TypeError("v0.53.0 flag must be bool")` and passes `None` through to Pydantic's `default=False` rather than silently coercing it (python-review MEDIUM fix — `fp8_attention: null` in YAML now surfaces as a "valid boolean" ValidationError instead of masquerading as `False`). Known limitations: (1) Every live wiring is deferred to v0.53.1 — `export_advanced_gguf`, `apply_kv_cache_type`, `apply_fp8_attention`, `apply_nvfp4`, `merge_4bit`, `export_torchao` all raise `NotImplementedError` with explicit `v0.53.1` markers. (2) `validate_calibration_data_path` + `validate_quant_config_path` are shape-only this release; CLI dispatch in v0.53.1 MUST add cwd-containment + TOCTOU symlink rejection. (3) `kv_cache_type` MLX permissive policy: only `fp8` is rejected, the other three pass-through; v0.53.1 may narrow further. (4) Hopper SM-capability check is runtime-only — schema accepts `kv_cache_type='fp8'` + `fp8_attention=true` without GPU probe. (5) NVFP4 + Blackwell (SM ≥ 12.0) check is runtime-only. (6) `bnb_4bit_use_double_quant` only gated against `quantization`, not against `quantization_aware` — the latter combination is already rejected by v0.28.0 Quant-Menu + QAT cross-validator. (7) `llm_int8` is an assertion not an aliaser — diverges from v0.41.0 `load_in_8bit` design on purpose. (v0.53.0) - **v0.52.0 — Modality II (TTS + Distillation + BitNet + EBFT-GDPO + MoE quant + reasoning_effort)**: 7 schema-only Parts; live trainer / loss / export wiring deferred to v0.52.1. Every new validator follows the project's established hardening policy: closed allowlist (`SUPPORTED_TTS_FAMILIES`, `CLASSIFIER_TASKS`, `DIVERGENCES`, `BITNET_QUANT_FORMATS`, `BITNET_EXPORT_FORMATS`, `EBFT_VARIANTS`, `GDPO_VARIANTS`, `MOE_EXPERT_QUANT_FORMATS`, `REASONING_EFFORT_LEVELS`, per-family `_FAMILY_EMOTIONS`) wrapped in `frozenset` / `MappingProxyType` so registries cannot be mutated at runtime; `validate_*` helpers reject non-string / bool / empty / null-byte / oversize / unknown inputs with case-insensitive normalisation (matches v0.41.0 `validate_optimizer_name` / v0.50.0 `grpo_variant` / v0.51.0 `hub` policy); float validators (`validate_distill_temperature`, `validate_ebft_temperature`) gate on `math.isfinite` to reject NaN AND `±inf` (matches v0.32.0 `save_lr_finder_report` policy). `field_validator(mode="before")` on `num_labels` (security-review HIGH fix) rejects `bool` before Pydantic's `ge=1` coercion silently treats `True` as `1`. Field validator on `reasoning_effort` routes through the shared `validate_reasoning_effort` helper so the schema and runtime validator agree on what's accepted (security-review MEDIUM fix). SoupConfig cross-validators: `_validate_tts_compat` (requires `task='tts'` + `modality='audio_out'` + non-MLX backend; per-family emotion allowlist via `_FAMILY_EMOTIONS`), `_validate_classifier_compat` (with lazy-import early-return — code-review HIGH fix — so SFT hot path doesn't pay import cost; requires `num_labels` on classifier tasks; rejects classifier-only fields outside the task family with named offenders), `_validate_distill_compat` (requires `teacher_model` when `task='distill'`; rejects distill-only fields outside the task), `_validate_bitnet_compat` (gates to non-MLX + text-modality + task ∈ {sft, pretrain, dpo}), `_validate_ebft_compat` + `_validate_gdpo_compat` (task-family gates), `_validate_moe_expert_quant_compat` (requires `moe_lora=true` to prevent silent no-op), `_validate_reasoning_effort_task_gate` (code-review HIGH fix — rejects `reasoning_effort` + `train_on_eot` outside the SFT-family task set with named offenders; mirrors v0.50.0 GRPO stability task-gate policy). Public `DIVERGENCES` frozenset is derived from `_DIVERGENCE_ALIASES` so adding a new alias updates both the accepted-input set and the error message in lockstep (review fix LOW). `validate_bitnet_export` enforces a closed-allowlist canonical form for `soup export --format `, both of which are CLI-registered with a yellow advisory panel + `Exit(0)` stub (no artifact written until v0.52.1 — the format flag is accepted so existing scripts pinned to v0.52.0 will not break). 6 new YAML recipes appended (5 TTS + Falcon-E BitNet) — every entry is exercised by `tests/test_v0520.py` for `load_config_from_string` round-trip + `_no_null_or_whitespace` model-id check (mirrors v0.51.0 review-fix LOW). Known limitations: (1) Every live trainer / loss / export path is deferred to v0.52.1 — `build_tts_trainer`, `build_classifier_trainer`, `build_distill_trainer`, `build_bitnet_trainer`, `export_bitnet_gguf`, `apply_ebft_loss`, `apply_gdpo_loss`, `apply_moe_expert_quant` all raise `NotImplementedError` with explicit `v0.52.1` markers; schema accepts every new task / quant / variant + the CLI stub for `soup export --format bitnet/tq1_0` prints a deferred-advisory panel and exits 0. (2) `modality='audio_out'` accepted on non-TTS tasks — design choice this release so future audio-output tasks (ASR / V2A) can reuse it; today's runtime trainer dispatch must check `task == 'tts'` to avoid silent routing into the deferred TTS path. (3) Oute emotion allowlist is a tight 6-entry subset (neutral / happy / sad / angry / calm / excited); operators wanting custom emotions will need a v0.52.1 patch to extend `OUTE_EMOTIONS`. (4) `is_bitnet_model` is best-effort heuristic over name prefixes (`bitnet`, `falcon-e`, `1bitllm`, `onebit`); a BitNet checkpoint published under an org without any of those prefixes returns False. This is detection, not gating — the trainer wrapper (v0.52.1) loads the model regardless of the heuristic. (5) `quantization='bitnet_1.58'` gated to task ∈ {sft, pretrain, dpo} — extending to GRPO / PPO / RewardModel requires upstream onebitllms RL kernels not yet shipped. (v0.52.0) - **v0.51.0 — Model Catalog Expansion + Alternative Model Hubs**: 5 release Parts. New `soup_cli/utils/hubs.py` ships closed allowlist `SUPPORTED_HUBS = frozenset({hf, modelscope, modelers})` + three `MappingProxyType`-wrapped registries (`_HUB_DEFAULT_ENDPOINTS` / `_HUB_ENDPOINT_ENV` / `_HUB_PACKAGE`) so the registry cannot be mutated at runtime (matches v0.36.0 `_REGISTRY` policy). `validate_hub_name` rejects non-string / bool / empty / null-byte / >32-char / unknown with case-insensitive normalisation (matches v0.41.0 `validate_optimizer_name` policy). `validate_hub_endpoint` is the SSRF kernel — full parity with v0.29.0 `utils/hf.resolve_endpoint`: scheme allowlist (`http`/`https` only), null-byte rejection, **control-character / CRLF rejection** added in v0.51.0 as a defence-in-depth review fix (defends against URL-as-HTTP-header injection if the URL ever flows into a raw HTTP client), `0.0.0.0` explicitly rejected, plain HTTP only for loopback `{localhost, 127.0.0.1, ::1}`, RFC1918 / link-local / cloud-metadata IPs (169.254.x) rejected via `ipaddress.ip_address` for plain HTTP. `resolve_endpoint(hub, *, env=None)` looks up the per-hub env var (`HF_ENDPOINT` / `MODELSCOPE_ENDPOINT` / `MODELERS_ENDPOINT`) and runs the override through `validate_hub_endpoint`; default endpoints are baked-in HTTPS URLs. `is_hf` rejects `bool` explicitly (review fix HIGH — bool is a subclass of int and would have silently fallen through `hub.lower() == "hf"` → `False`, which happens to be correct by accident but violates the contract; matches v0.30.0 `Candidate` / v0.34.0 `estimate_run_cost_usd` policy). `TrainingConfig.hub: Literal["hf","modelscope","modelers"]` field gets a `field_validator(mode="before")` `_normalize_hub` that delegates to `validate_hub_name` so `hub: HF` in YAML normalises to `"hf"` (review fix HIGH — first-cut had Pydantic Literal exact-match while `validate_hub_name` was case-insensitive, breaking the v0.41.0 `validate_optimizer_name` / v0.50.0 `grpo_variant` / `rollout_backend` policy of agreement between schema and shared validator). SoupConfig `_validate_hub_supported` cross-validator rejects `hub != 'hf'` on `backend == 'mlx'` with a distinct error message (review fix HIGH — `mlx-lm` only downloads from HF Hub; without this gate a `backend: mlx` + `hub: modelscope` config would silently pass schema load and fail at runtime with a confusing `mlx-lm` error). 26 new YAML recipes appended to `soup_cli/recipes/catalog.py` — every entry is exercised by `tests/test_v0510.py` via `load_config_from_string` round-trip + `yaml.safe_load` (no Python tags / no template injection / no credential leak in the YAML strings) + a `_no_null_or_whitespace` model-id check that rejects empty path components (review fix LOW — first-cut allowed `"/name"` leading-slash IDs to pass). Two non-`B` `size` strings (`"image"` / `"ocr"` / `"moe"` / `"medium"`) were normalised to `"N/A"` (review fix MEDIUM — `search_recipes(size=…)` would silently miss those entries, and the autopilot VRAM estimator could not parse them). Known limitations: (1) Live downloader / uploader / push integration deferred to v0.51.1 — `TrainingConfig.hub` schema lock-in ships now (Literal accept + MLX cross-validator + case-normalisation), but `soup data download --hub modelscope` and `soup push --hub modelers` still route through the existing HF Hub code path; the actual `modelscope-sdk` / `openmind-hub` adapters are the v0.51.1 deliverable. Same stub-then-live pattern as v0.27.0 MII / v0.37.0 multipack / v0.50.0 GRPO Plus. (2) Speculative / aspirational `base` model IDs in some Part A/C recipes — the catalog ships entries for `openai/gpt-oss-{20,120}b`, `THUDM/glm-5`, `Qwen/Qwen-Image`, `deepseek-ai/DeepSeek-OCR`, `PaddlePaddle/PaddleOCR-VL`, `google/embeddinggemma-300m` so users have ready-made recipes the moment those repos go live (matches the plan's "match Unsloth's day-zero coverage" directive). Recipes for not-yet-published repos will surface a clear HF Hub 404 when the user runs `soup train --recipe `. (3) DNS-resolved private hostnames not blocked — `validate_hub_endpoint` only rejects literal RFC1918 / link-local IP addresses; a hostname like `corp-proxy.internal` that DNS-resolves to a private IP is accepted at validation time (mirrors the v0.29.0 `HF_ENDPOINT` policy — DNS resolution is intentionally not performed in this local-tool threat model). (v0.51.0) diff --git a/pyproject.toml b/pyproject.toml index 599be67..6a78ed4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.53.0" +version = "0.53.1" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "Apache-2.0" diff --git a/soup_cli/__init__.py b/soup_cli/__init__.py index c6eaebe..762e583 100644 --- a/soup_cli/__init__.py +++ b/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.53.0" +__version__ = "0.53.1" diff --git a/soup_cli/autopilot/__init__.py b/soup_cli/autopilot/__init__.py index c30057c..da34747 100644 --- a/soup_cli/autopilot/__init__.py +++ b/soup_cli/autopilot/__init__.py @@ -17,6 +17,8 @@ from soup_cli.autopilot.decisions import ( decide_performance_flags, decide_quantization, decide_task, + detect_prequantized_format, + detect_prequantized_format_from_path, parse_gpu_budget, ) from soup_cli.autopilot.generate_config import build_soup_config, write_yaml @@ -37,6 +39,8 @@ __all__ = [ "decide_performance_flags", "decide_quantization", "decide_task", + "detect_prequantized_format", + "detect_prequantized_format_from_path", "parse_gpu_budget", "write_yaml", ] diff --git a/soup_cli/autopilot/decisions.py b/soup_cli/autopilot/decisions.py index fe8816a..676d6b3 100644 --- a/soup_cli/autopilot/decisions.py +++ b/soup_cli/autopilot/decisions.py @@ -2,8 +2,10 @@ from __future__ import annotations +import json +import os import re -from typing import Any, Literal +from typing import Any, Literal, Mapping, Optional GOAL_TO_TASK: dict[str, str] = { "chat": "sft", @@ -33,8 +35,211 @@ def decide_task(goal: str, dataset_profile: Any = None) -> str: return GOAL_TO_TASK[goal] -def decide_quantization(model_params_b: float, vram_gb: float) -> str: - """Pick a quantization tier from model size vs available VRAM.""" +# --- v0.53.1 #82 — pre-quantized base detection ----------------------------- + +# Quant Menu formats Autopilot can recommend back. Mirrors v0.38.0 + v0.40.5 +# canonical strings; ``hqq:Nbit`` is the only parameterised entry. +_VALID_PREQUANT_FORMATS: frozenset[str] = frozenset({ + "gptq", "awq", "aqlm", "eetq", "fp8", "mxfp4", +}) +# HQQ uses ``hqq:bit`` where N ∈ {1, 2, 3, 4, 8}. +_HQQ_VALID_BITS: frozenset[int] = frozenset({1, 2, 3, 4, 8}) + +# Word-boundary matchers — substring won't match ``agptqa``. +_PREQUANT_NAME_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + (re.compile(r"(?:^|[^a-z0-9])gptq(?:[^a-z0-9]|$)", re.IGNORECASE), "gptq"), + (re.compile(r"(?:^|[^a-z0-9])awq(?:[^a-z0-9]|$)", re.IGNORECASE), "awq"), + (re.compile(r"(?:^|[^a-z0-9])aqlm(?:[^a-z0-9]|$)", re.IGNORECASE), "aqlm"), + (re.compile(r"(?:^|[^a-z0-9])eetq(?:[^a-z0-9]|$)", re.IGNORECASE), "eetq"), + # FP8 is matched separately so we can distinguish from generic "8" + (re.compile(r"(?:^|[^a-z0-9])fp8(?:[^a-z0-9]|$)", re.IGNORECASE), "fp8"), + (re.compile(r"(?:^|[^a-z0-9])mxfp4(?:[^a-z0-9]|$)", re.IGNORECASE), "mxfp4"), +) + +_HQQ_NAME_RE = re.compile( + r"(?:^|[^a-z0-9])hqq(?:[-_]?(?P[1-9])bit)?(?:[^a-z0-9]|$)", + re.IGNORECASE, +) + +_MAX_BASE_NAME_LEN = 512 + + +def _check_base_name(name: object) -> str: + if isinstance(name, bool): + raise TypeError(f"base name must not be bool, got {name!r}") + if not isinstance(name, str): + raise TypeError(f"base name must be str, got {type(name).__name__}") + if not name: + raise ValueError("base name must be non-empty") + if "\x00" in name: + raise ValueError("base name must not contain null bytes") + if len(name) > _MAX_BASE_NAME_LEN: + raise ValueError(f"base name too long (max {_MAX_BASE_NAME_LEN} chars)") + return name + + +def _detect_from_name(name: str) -> Optional[str]: + """Return Quant Menu format string if the base name matches a known prefix.""" + hqq_match = _HQQ_NAME_RE.search(name) + if hqq_match: + bits = hqq_match.group("bits") + if bits is None: + return "hqq:4bit" + bits_int = int(bits) + if bits_int in _HQQ_VALID_BITS: + return f"hqq:{bits_int}bit" + return "hqq:4bit" # fall back to safe default + + for pattern, fmt in _PREQUANT_NAME_PATTERNS: + if pattern.search(name): + return fmt + return None + + +def _detect_from_config(hf_config: Any) -> Optional[str]: + """Probe an HF-style ``config.json`` dict for a ``quantization_config`` block.""" + if not isinstance(hf_config, Mapping): + return None + qc = hf_config.get("quantization_config") + if not isinstance(qc, Mapping): + return None + method_raw = qc.get("quant_method") + if not isinstance(method_raw, str): + return None + method = method_raw.lower() + if method == "hqq": + bits = qc.get("bits") + if isinstance(bits, bool) or not isinstance(bits, int): + return "hqq:4bit" + if bits in _HQQ_VALID_BITS: + return f"hqq:{bits}bit" + return "hqq:4bit" + if method in _VALID_PREQUANT_FORMATS: + return method + # Map common aliases + if method in {"bitsandbytes_4bit", "bnb_4bit", "nf4"}: + return "4bit" + if method in {"bitsandbytes_8bit", "bnb_8bit"}: + return "8bit" + return None + + +def detect_prequantized_format( + name: object, hf_config: Any = None, +) -> Optional[str]: + """Detect a pre-quantized base model's quant format. + + Returns a canonical Quant Menu format string (``gptq`` / ``awq`` / + ``hqq:4bit`` / ``aqlm`` / ``eetq`` / ``fp8`` / ``mxfp4`` / ``4bit`` / + ``8bit``) or ``None`` if no hint can be derived. + + Search order: + 1. ``hf_config`` ``quantization_config.quant_method`` (authoritative) + 2. Base-name regex with word-boundary anchoring (heuristic) + """ + validated = _check_base_name(name) + config_hit = _detect_from_config(hf_config) + if config_hit is not None: + return config_hit + return _detect_from_name(validated) + + +def detect_prequantized_format_from_path(model_dir: object) -> Optional[str]: + """Read ``/config.json`` and delegate to :func:`detect_prequantized_format`. + + Returns ``None`` on missing directory, missing file, or malformed JSON + (matches v0.36.0 ``model_requires_trust_remote_code`` probe semantics). + """ + if isinstance(model_dir, bool): + raise TypeError("model_dir must not be bool") + if not isinstance(model_dir, str): + raise TypeError( + f"model_dir must be str, got {type(model_dir).__name__}" + ) + if not model_dir: + return None + if "\x00" in model_dir: + return None + # Soft-probe semantics: callers can pass an HF repo id OR a local path. + # Only attempt the on-disk probe when ``model_dir`` resolves to a + # directory under cwd (containment defence). Out-of-cwd local paths + # silently fall through to name-only detection. + import stat as _stat + + from soup_cli.utils.paths import is_under_cwd + + if not is_under_cwd(model_dir): + return None + config_path = os.path.join(model_dir, "config.json") + if not os.path.isfile(config_path): + return None + # Reject symlinks at the config target (TOCTOU defence; mirrors + # v0.33.0 #22 / v0.43.0 / v0.46.0 / v0.47.0 policy). + try: + st = os.lstat(config_path) + except OSError: + return None + if _stat.S_ISLNK(st.st_mode): + return None + try: + with open(config_path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError, UnicodeDecodeError): + return None + return detect_prequantized_format( + os.path.basename(os.path.normpath(model_dir)) or "model", + data, + ) + + +def _validate_prequantized(value: object) -> str: + if isinstance(value, bool): + raise TypeError(f"prequantized must not be bool, got {value!r}") + if not isinstance(value, str): + raise TypeError( + f"prequantized must be str, got {type(value).__name__}" + ) + if not value: + raise ValueError("prequantized must be non-empty") + if "\x00" in value: + raise ValueError("prequantized must not contain null bytes") + canonical = value.lower() + if canonical.startswith("hqq:"): + # Validate hqq:Nbit shape + tail = canonical.split(":", 1)[1] + if not tail.endswith("bit"): + raise ValueError( + f"prequantized {value!r} invalid HQQ shape; expected hqq:Nbit" + ) + bits_str = tail[: -len("bit")] + if not bits_str.isdigit() or int(bits_str) not in _HQQ_VALID_BITS: + raise ValueError( + f"prequantized {value!r} HQQ bits invalid; " + f"expected one of {sorted(_HQQ_VALID_BITS)}" + ) + return canonical + if canonical in _VALID_PREQUANT_FORMATS or canonical in {"4bit", "8bit"}: + return canonical + raise ValueError( + f"prequantized {value!r} not a recognised Quant Menu format. " + f"Supported: {sorted(_VALID_PREQUANT_FORMATS)} | 4bit | 8bit | hqq:Nbit" + ) + + +def decide_quantization( + model_params_b: float, + vram_gb: float, + prequantized: Optional[str] = None, +) -> str: + """Pick a quantization tier from model size vs available VRAM. + + v0.53.1 #82 — when ``prequantized`` is set (e.g. ``"gptq"`` / ``"awq"`` / + ``"hqq:4bit"``), it takes precedence over the VRAM-based heuristic so we + don't silently stack a fresh BNB 4-bit on top of an already-quantized base. + """ + if prequantized is not None: + return _validate_prequantized(prequantized) + # Rough: 1 param byte each in 8bit, 0.5 in 4bit, 2 in fp16 model_gb_fp16 = model_params_b * 2.0 if vram_gb >= 2.5 * model_gb_fp16: @@ -50,15 +255,16 @@ def decide_quantization(model_params_b: float, vram_gb: float) -> str: ) -def decide_peft(data_size: int, model_size_b: float, vram_gb: float) -> dict: +def decide_peft( + data_size: int, model_size_b: float, vram_gb: float, +) -> dict[str, Any]: """Pick a LoRA rank and settings based on dataset + model + VRAM.""" if data_size < 1000: rank = 8 elif data_size < 10_000: rank = 16 - elif data_size < 100_000: - rank = 32 else: + # rank capped at 32 for all datasets >= 10k samples rank = 32 alpha = rank * 2 use_dora = data_size > 100_000 and vram_gb >= 2.0 * model_size_b diff --git a/soup_cli/autopilot/generate_config.py b/soup_cli/autopilot/generate_config.py index e12b4b4..531a0ff 100644 --- a/soup_cli/autopilot/generate_config.py +++ b/soup_cli/autopilot/generate_config.py @@ -20,6 +20,8 @@ from soup_cli.autopilot.decisions import ( decide_performance_flags, decide_quantization, decide_task, + detect_prequantized_format, + detect_prequantized_format_from_path, ) from soup_cli.config.schema import ( DataConfig, @@ -43,8 +45,15 @@ def build_soup_config( target_vram = vram_gb if vram_gb is not None else max(hardware_profile.vram_gb, 8.0) task = decide_task(goal, dataset_profile) + # v0.53.1 #82 — if the base is already quantized, route through the matching + # Quant Menu format instead of stacking a fresh BNB-4bit on top. + prequantized = detect_prequantized_format_from_path(model) + if prequantized is None: + prequantized = detect_prequantized_format(model) quantization = decide_quantization( - model_params_b=model_profile.params_b, vram_gb=target_vram + model_params_b=model_profile.params_b, + vram_gb=target_vram, + prequantized=prequantized, ) peft = decide_peft( data_size=dataset_profile.samples, diff --git a/soup_cli/commands/deploy.py b/soup_cli/commands/deploy.py index f7621d2..1b9dda1 100644 --- a/soup_cli/commands/deploy.py +++ b/soup_cli/commands/deploy.py @@ -2,10 +2,14 @@ from __future__ import annotations +import os from pathlib import Path -from typing import List, Optional +from typing import TYPE_CHECKING, Callable, List, Optional import typer + +if TYPE_CHECKING: + from soup_cli.utils.deploy_autopilot import DeployProfile from rich.console import Console from rich.panel import Panel from rich.table import Table @@ -594,11 +598,34 @@ def autopilot( "-l", help="List all known deploy profiles.", ), + measure: bool = typer.Option( + False, + "--measure", + help=( + "Run Quant-Lobotomy measurement on candidate quants. " + "Requires --tasks. Results cached at " + "~/.soup/deploy_autopilot_cache.json (v0.53.1 #109)." + ), + ), + tasks_file: Optional[str] = typer.Option( + None, + "--tasks", + help="JSONL eval tasks for --measure (one prompt + expected per line).", + ), + measure_candidates: Optional[str] = typer.Option( + None, + "--measure-candidates", + help=( + "Comma-separated quant candidates to measure (default: profile's " + "primary quant). Example: 4bit,gptq,awq" + ), + ), ): """Pick PEFT + quant + spec-decoding combo for a hardware target. Writes a ready-to-train ``soup.yaml`` recipe and a planned deploy - shell script. Live Quant-Lobotomy measurement deferred to v0.46.1. + shell script. Pass ``--measure --tasks `` to also run the + Quant-Lobotomy measurement loop across candidate quants (v0.53.1 #109). """ from soup_cli.utils.deploy_autopilot import ( get_profile, @@ -669,10 +696,20 @@ def autopilot( ) if profile.notes: console.print(f"[dim]Notes: {_escape(profile.notes)}[/]") - console.print( - "[yellow]Note:[/] Live Quant-Lobotomy auto-measure deferred to v0.46.1; " - "this release writes the canonical combo + recipe." - ) + + # v0.53.1 #109 — optional live measurement + if measure: + if not tasks_file: + console.print( + "[red]--measure requires --tasks .[/]" + ) + raise typer.Exit(2) + _run_deploy_autopilot_measure( + profile=profile, + base=base, + tasks_file=tasks_file, + measure_candidates=measure_candidates, + ) def _auto_detect_template() -> Optional[str]: @@ -697,3 +734,99 @@ def _auto_detect_template() -> Optional[str]: except (yaml.YAMLError, OSError, KeyError, ImportError): return None return None + + +# --- v0.53.1 #109 — deploy autopilot live measurement ----------------------- + + +def _run_deploy_autopilot_measure( + *, + profile: "DeployProfile", + base: str, + tasks_file: str, + measure_candidates: Optional[str], +) -> None: + """Lazy-import + invoke the measurement helper from utils.deploy_measure.""" + import hashlib + + from rich.markup import escape + + from soup_cli.utils.deploy_measure import ( + pick_best, + render_measure_table, + run_measure, + ) + from soup_cli.utils.paths import is_under_cwd + + if not is_under_cwd(tasks_file): + console.print( + f"[red]--tasks {escape(tasks_file)!r} must stay under cwd[/]" + ) + raise typer.Exit(2) + if not Path(tasks_file).is_file(): + console.print(f"[red]Tasks file not found: {escape(tasks_file)}[/]") + raise typer.Exit(2) + + # Determine candidate list + if measure_candidates: + candidates = [ + c.strip() for c in measure_candidates.split(",") if c.strip() + ] + if not candidates: + console.print("[red]--measure-candidates parsed to empty list.[/]") + raise typer.Exit(2) + else: + # Default: just the profile's primary quant + candidates = [profile.quant] + + # Build a base sha — local path → realpath; HF repo → name + base_sha_seed = base if not os.path.isdir(base) else os.path.realpath(base) + base_sha = hashlib.sha256(base_sha_seed.encode("utf-8")).hexdigest()[:16] + + console.print( + f"[dim]Measuring {len(candidates)} candidate(s) " + f"against {Path(tasks_file).name}...[/]" + ) + + def _placeholder_before(prompt: str) -> str: + # The full v0.46.1 live measurement plumbs in real + # transformers / vllm generators. v0.53.1 ships the orchestrator + # surface; callers / smoke runs can monkeypatch this in. + return "" + + def _placeholder_after_factory(candidate: str) -> Callable[[str], str]: + def _gen(prompt: str) -> str: + return "" + return _gen + + # Pull injected generators if the caller registered them via env (escape + # hatch for tests + advanced operator workflows) + from soup_cli.utils import deploy_measure as _dm + before_gen = getattr(_dm, "_DEPLOY_MEASURE_BEFORE_GEN", None) or _placeholder_before + after_factory = ( + getattr(_dm, "_DEPLOY_MEASURE_AFTER_FACTORY", None) + or _placeholder_after_factory + ) + + try: + results, cache_hit = run_measure( + profile_name=profile.name, + base_sha=base_sha, + candidates=candidates, + tasks_file=tasks_file, + before_gen=before_gen, + after_gen_factory=after_factory, + ) + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]Measure failed:[/] {escape(str(exc))}") + raise typer.Exit(1) from exc + + console.print(render_measure_table(results)) + if cache_hit: + console.print("[dim](cache hit — re-run with --no-cache to refresh)[/]") + best = pick_best(results) + if best is not None: + console.print( + f"[bold green]Recommended:[/] {escape(best.candidate)} " + f"(verdict={best.verdict}, delta={best.delta:+.3f})" + ) diff --git a/soup_cli/commands/export.py b/soup_cli/commands/export.py index f311f28..71450af 100644 --- a/soup_cli/commands/export.py +++ b/soup_cli/commands/export.py @@ -18,6 +18,11 @@ SUPPORTED_FORMATS = ( # v0.52.0 Part D — BitNet 1.58-bit + TQ1_0 GGUF. # Schema-only stubs in v0.52.0; live conversion lands in v0.52.1. "bitnet", "tq1_0", + # v0.53.1 #142 — TorchAO PTQ live wiring (Int4WeightOnly / Int8DynActInt4 / + # Float8DynActFloat8 / NVFP4). Requires --quant-config . + "torchao", + # v0.53.1 #139 — UD/IQ/Apple-ARM GGUFs via llama.cpp imatrix. + "gguf-ud", ) GGUF_QUANT_TYPES = ("q4_0", "q4_k_m", "q5_k_m", "q8_0", "f16", "f32") LLAMA_CPP_DIR_NAME = "llama.cpp" @@ -110,8 +115,24 @@ def export( "Default deny (v0.36.0). Only enable if you trust the source." ), ), + quant_config: Optional[str] = typer.Option( + None, + "--quant-config", + help=( + "Path to YAML for torchao PTQ export (v0.53.1 #142). " + "Required when --format=torchao." + ), + ), + gguf_flavour: Optional[str] = typer.Option( + None, + "--gguf-flavour", + help=( + "Advanced GGUF format flag — UD-Q*_K_XL / IQ*_M / Q4_0_4_4 / etc. " + "Required when --format=gguf-ud (v0.53.1 #139)." + ), + ), ): - """Export a model to GGUF, ONNX, TensorRT-LLM, AWQ, or GPTQ format.""" + """Export a model to GGUF, ONNX, TensorRT-LLM, AWQ, GPTQ, or TorchAO format.""" model_path = Path(model) # --- Validate --- @@ -152,6 +173,26 @@ def export( ) return + # --- TorchAO PTQ export path (v0.53.1 #142) --- + if fmt == "torchao": + _export_torchao_cli( + model_path, output, quant_config, trust_remote_code, + ) + return + + # --- Advanced GGUF export path (UD / IQ / Apple-ARM, v0.53.1 #139) --- + if fmt == "gguf-ud": + _export_gguf_advanced( + model_path=model_path, + output=output, + base=base, + gguf_flavour=gguf_flavour, + calibration_data=calibration_data, + llama_cpp_path=llama_cpp_path, + trust_remote_code=trust_remote_code, + ) + return + # --- BitNet 1.58-bit / TQ1_0 GGUF — schema-only stubs (v0.52.0) --- # Live conversion via onebitllms + llama.cpp TQ1_0 lands in v0.52.1. if fmt in ("bitnet", "tq1_0"): @@ -1099,3 +1140,199 @@ def _maybe_attach_export( console.print( f"[green]Attached export to registry entry '{entry_id}' as {kind}.[/]" ) + + +# --- v0.53.1 #142 — TorchAO PTQ export CLI dispatch ------------------------- + + +def _export_torchao_cli( + model_path: Path, + output: Optional[str], + quant_config: Optional[str], + trust_remote_code: bool, +) -> None: + """Dispatch ``soup export --format torchao``. + + Per v0.53.0 ``validate_quant_config_path`` docstring contract: + enforce cwd containment + ``os.lstat + S_ISLNK`` rejection at CLI + dispatch time, not in the schema validator. + """ + if quant_config is None: + console.print( + "[red]--format torchao requires --quant-config [/]\n" + "Example: [bold]soup export --format torchao " + "--quant-config q.yaml --model ./merged[/]" + ) + raise typer.Exit(2) + + from soup_cli.utils.save_formats import ( + export_torchao, + load_quant_config, + validate_torchao_scheme, + ) + + try: + cfg_data = load_quant_config(quant_config) + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(2) + + scheme_raw = cfg_data.get("scheme") + if not isinstance(scheme_raw, str): + console.print( + "[red]quant_config must declare a top-level 'scheme: ' field.[/]" + ) + raise typer.Exit(2) + try: + scheme = validate_torchao_scheme(scheme_raw) + except (TypeError, ValueError) as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(2) + + if output is None: + output_path = model_path.parent / f"{model_path.name}.torchao.{scheme}" + else: + output_path = Path(output) + + console.print(Panel( + f"Model: [bold]{model_path}[/]\n" + f"Scheme: [bold]{scheme}[/]\n" + f"Output: [bold]{output_path}[/]", + title="TorchAO PTQ Export", + )) + + try: + export_torchao( + model_dir=str(model_path), + output_dir=str(output_path), + scheme=scheme, + quant_config_data={k: v for k, v in cfg_data.items() if k != "scheme"}, + trust_remote_code=trust_remote_code, + ) + except (ImportError, RuntimeError) as exc: + console.print(f"[red]TorchAO export failed: {exc}[/]") + console.print( + "Try: [bold]pip install torchao[/] " + "(NVFP4 requires torchao>=0.5)" + ) + raise typer.Exit(1) + except (TypeError, ValueError, FileNotFoundError) as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(2) + + console.print(Panel( + f"Output: [bold]{output_path}[/]\n" + f"Scheme: [bold]{scheme}[/]", + title="[bold green]TorchAO Export Complete[/]", + )) + + +# --- v0.53.1 #139 — Advanced GGUF export CLI dispatch ----------------------- + + +def _export_gguf_advanced( + *, + model_path: Path, + output: Optional[str], + base: Optional[str], + gguf_flavour: Optional[str], + calibration_data: Optional[str], + llama_cpp_path: Optional[str], + trust_remote_code: bool, +) -> None: + """Dispatch ``soup export --format gguf-ud --gguf-flavour <...>``. + + Routes through llama.cpp's ``imatrix`` + ``quantize`` binaries. Supports + UD-Q*_K_XL ladder, IQ*_M family, Apple/ARM Q4_0_4_4 / Q4_NL etc. + """ + if gguf_flavour is None: + console.print( + "[red]--format gguf-ud requires --gguf-flavour [/]" + ) + raise typer.Exit(2) + + from soup_cli.utils.gguf_quant import ( + export_advanced_gguf, + is_advanced_gguf_format, + ) + + if not is_advanced_gguf_format(gguf_flavour): + console.print( + f"[red]Unknown gguf_flavour {gguf_flavour!r}. " + "See soup_cli.utils.gguf_quant.ALL_ADVANCED_GGUF_FORMATS.[/]" + ) + raise typer.Exit(2) + + # Calibration data path (UD / IQ require it; Apple/ARM Q4_0_4_4 doesn't). + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + if calibration_data is not None: + try: + enforce_under_cwd_and_no_symlink( + calibration_data, "calibration_data", + ) + except (TypeError, ValueError) as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(2) + if not Path(calibration_data).is_file(): + console.print( + f"[red]Calibration data file not found: {calibration_data}[/]" + ) + raise typer.Exit(2) + + if output is None: + output_path = ( + model_path.parent / f"{model_path.name}.{gguf_flavour}.gguf" + ) + else: + output_path = Path(output) + + # Pre-merge LoRA adapter if needed + adapter_config_path = model_path / "adapter_config.json" + merge_dir = None + if adapter_config_path.exists(): + console.print("[yellow]LoRA adapter detected — merging first...[/]") + base_model = base or _detect_base_model(adapter_config_path) + if not base_model: + console.print( + "[red]Cannot detect base model. Pass --base.[/]" + ) + raise typer.Exit(2) + merge_dir = model_path.parent / f".soup_merge_tmp_{model_path.name}" + _merge_adapter( + str(model_path), base_model, str(merge_dir), trust_remote_code, + ) + source_model_dir = merge_dir + else: + source_model_dir = model_path + + llama_dir = _find_llama_cpp(llama_cpp_path) + + console.print(Panel( + f"Model: [bold]{source_model_dir}[/]\n" + f"Flavour: [bold]{gguf_flavour}[/]\n" + f"Calib: [bold]{calibration_data or '(none — Apple/ARM)'}[/]\n" + f"Output: [bold]{output_path}[/]", + title="Advanced GGUF Export", + )) + + try: + export_advanced_gguf( + model_dir=str(source_model_dir), + output_path=str(output_path), + flavour=gguf_flavour, + calibration_data=calibration_data, + llama_cpp_dir=str(llama_dir), + ) + except (FileNotFoundError, RuntimeError, ValueError) as exc: + console.print(f"[red]Advanced GGUF export failed: {exc}[/]") + raise typer.Exit(1) + finally: + if merge_dir and merge_dir.exists(): + shutil.rmtree(merge_dir, ignore_errors=True) + + console.print(Panel( + f"Output: [bold]{output_path}[/]\n" + f"Flavour: [bold]{gguf_flavour}[/]", + title="[bold green]Advanced GGUF Export Complete[/]", + )) diff --git a/soup_cli/commands/merge.py b/soup_cli/commands/merge.py index 04eb480..30e1b60 100644 --- a/soup_cli/commands/merge.py +++ b/soup_cli/commands/merge.py @@ -43,8 +43,35 @@ def merge( "Default deny (v0.36.0). Only enable if you trust the source." ), ), + save_format: str = typer.Option( + "fp16", + "--save-format", + help=( + "Merged-checkpoint save format. fp16 (default) writes a " + "standard fp16 merge. 4bit / 4bit_forced write a single " + "BNB-4bit-quantized merge without the dequant-merge-requant " + "cycle (v0.53.1 #142)." + ), + ), ): """Merge a LoRA adapter with its base model into a full model.""" + # v0.53.1 #142 — validate save_format up front + from soup_cli.utils.save_formats import validate_merge_save_format + try: + save_format_canonical = validate_merge_save_format(save_format) + except (TypeError, ValueError) as exc: + console.print(f"[red]{exc}[/]") + raise typer.Exit(2) + + # v0.53.1 — early cwd containment on --output (security review M4). + # Mirrors v0.20.0 / v0.40.2 policy: containment check fires at the CLI + # boundary, not deferred to the deeper helper. + from soup_cli.utils.paths import is_under_cwd as _is_under_cwd + if not _is_under_cwd(output): + console.print( + f"[red]--output {output!r} must stay under cwd[/]" + ) + raise typer.Exit(2) adapter_path = Path(adapter) # --- Validate adapter --- @@ -128,15 +155,50 @@ def merge( console.print("[dim]Merging weights...[/]") model = model.merge_and_unload() - console.print(f"[dim]Saving merged model to {output_path}...[/]") - output_path.mkdir(parents=True, exist_ok=True) - model.save_pretrained(str(output_path)) + if save_format_canonical == "fp16": + console.print(f"[dim]Saving merged model to {output_path}...[/]") + output_path.mkdir(parents=True, exist_ok=True) + model.save_pretrained(str(output_path)) - console.print("[dim]Saving tokenizer...[/]") - tokenizer = AutoTokenizer.from_pretrained( - str(adapter_path), trust_remote_code=trc - ) - tokenizer.save_pretrained(str(output_path)) + console.print("[dim]Saving tokenizer...[/]") + tokenizer = AutoTokenizer.from_pretrained( + str(adapter_path), trust_remote_code=trc + ) + tokenizer.save_pretrained(str(output_path)) + else: + # v0.53.1 #142 — 4bit / 4bit_forced merged checkpoint. + # Two-stage: first write an fp16 merge to a tempdir, then + # reload with BNB-4bit config and save to output. + import tempfile + + from soup_cli.utils.save_formats import merge_4bit + + with tempfile.TemporaryDirectory( + prefix=".soup_4bit_merge_", dir=str(Path.cwd()), + ) as staged: + staged_path = Path(staged) + console.print( + f"[dim]Staging fp16 merge in {staged_path.name}...[/]" + ) + model.save_pretrained(str(staged_path)) + tokenizer = AutoTokenizer.from_pretrained( + str(adapter_path), trust_remote_code=trc + ) + tokenizer.save_pretrained(str(staged_path)) + + # Free the in-memory fp16 model before reloading 4bit + del model + console.print( + f"[dim]Re-loading + saving BNB-4bit merge " + f"({save_format_canonical}) to {output_path}...[/]" + ) + merge_4bit( + merged_dir=str(staged_path), + output_dir=str(output_path), + forced=(save_format_canonical == "4bit_forced"), + dtype="bfloat16" if dtype == "bfloat16" else "float16", + trust_remote_code=trc, + ) except ImportError as exc: console.print(f"[red]Missing dependency: {exc}[/]") diff --git a/soup_cli/utils/deploy_measure.py b/soup_cli/utils/deploy_measure.py new file mode 100644 index 0000000..19b1893 --- /dev/null +++ b/soup_cli/utils/deploy_measure.py @@ -0,0 +1,362 @@ +"""v0.53.1 #109 — soup deploy autopilot --measure helper. + +Live Quant-Lobotomy measurement for each candidate quant in a deploy profile. +Wraps v0.26.0 :mod:`soup_cli.eval.quant_check` with disk-cache so that +repeated invocations on the same (base, profile, eval-tasks) tuple short- +circuit. Soft-fallback policy: when no candidate clears ``OK``, the +highest-delta candidate (least negative drop) is selected. + +The actual model loading + generation is the caller's responsibility — this +module is pure-Python and takes opaque ``Callable[[str], str]`` generators +so it stays testable without a GPU. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import stat +import tempfile +from dataclasses import asdict, dataclass +from typing import TYPE_CHECKING, Callable, Optional, Sequence + +if TYPE_CHECKING: + from rich.table import Table + +# Mirrors v0.26.0 Part D thresholds (verdict OK = drop < 2%, MINOR = drop < +# 5%, MAJOR otherwise). Exposed for tests + consistency. +DEFAULT_MINOR_THRESHOLD: float = 0.02 +DEFAULT_MAJOR_THRESHOLD: float = 0.05 + +_MAX_CACHE_BYTES: int = 1 * 1024 * 1024 # 1 MB cap on cache file +_MAX_CANDIDATES: int = 32 + +# Test + advanced-operator escape hatch: when set on this module, the deploy +# CLI uses these callables instead of the v0.46.1 model-loading generators. +# Documented as v0.53.1 deferral — see the v0.53.1 Known Limitations entry +# in plan.md. NOT a public API; the live transformers / vLLM generators land +# alongside v0.53.2. +_DEPLOY_MEASURE_BEFORE_GEN: Optional[Callable[[str], str]] = None +_DEPLOY_MEASURE_AFTER_FACTORY: Optional[ + Callable[[str], Callable[[str], str]] +] = None + + +@dataclass(frozen=True) +class MeasureResult: + """Outcome of a single candidate measurement.""" + + candidate: str # e.g. "4bit" / "gptq" / "awq" + before: float # baseline (unquantized) score in [0, 1] + after: float # quantized score + delta: float # after - before (negative = worse) + verdict: str # "OK" | "MINOR" | "MAJOR" + + +def compute_cache_key(*, base_sha: str, profile_name: str, tasks_sha: str) -> str: + """Build a deterministic cache key from the input tuple. + + Callers should pass FULL SHA-256 hex strings (64 chars) for the two + digest inputs. The orchestrator in :mod:`soup_cli.commands.deploy` + currently truncates ``base_sha`` to 16 hex chars at the call site — + that 16-char truncation is acceptable for caching purposes (collision + probability ≈ 1 in 2³² across ~4 billion cache entries) but is the + operator-facing policy, not a property enforced here. Future callers + that truncate further should expect collisions in proportion. + """ + for name, value in ( + ("base_sha", base_sha), + ("profile_name", profile_name), + ("tasks_sha", tasks_sha), + ): + if isinstance(value, bool): + raise TypeError(f"{name} must not be bool") + if not isinstance(value, str): + raise TypeError( + f"{name} must be str, got {type(value).__name__}" + ) + if not value: + raise ValueError(f"{name} must be non-empty") + if "\x00" in value: + raise ValueError(f"{name} must not contain null bytes") + hasher = hashlib.sha256() + hasher.update(base_sha.encode("utf-8")) + hasher.update(b"\x1f") + hasher.update(profile_name.encode("utf-8")) + hasher.update(b"\x1f") + hasher.update(tasks_sha.encode("utf-8")) + return hasher.hexdigest()[:32] + + +def sha_of_file(path: str) -> str: + """SHA-256 of a file, used to fingerprint the eval tasks JSONL.""" + if not isinstance(path, str): + raise TypeError(f"path must be str, got {type(path).__name__}") + if "\x00" in path: + raise ValueError("path must not contain null bytes") + if not os.path.isfile(path): + raise FileNotFoundError(f"file not found: {os.path.basename(path)!r}") + hasher = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(64 * 1024), b""): + hasher.update(chunk) + return hasher.hexdigest() + + +def _default_cache_path() -> str: + """Return ``~/.soup/deploy_autopilot_cache.json`` (allowed override via env).""" + override = os.environ.get("SOUP_DEPLOY_AUTOPILOT_CACHE") + if override: + # Reject null-bytes + control chars before doing any path resolution. + if "\x00" in override or any(ord(ch) < 32 for ch in override): + override = None + if override: + # Honor override only if it's plausibly safe (under home / cwd / temp) + candidate = os.path.realpath(override) + for safe_root in ( + os.path.realpath(os.path.expanduser("~")), + os.path.realpath(os.getcwd()), + os.path.realpath(tempfile.gettempdir()), + ): + try: + if ( + os.path.commonpath([candidate, safe_root]) == safe_root + ): + return candidate + except ValueError: + continue + # Fall through to default if override is unsafe + return os.path.join( + os.path.expanduser("~"), ".soup", "deploy_autopilot_cache.json" + ) + + +def load_cache(path: Optional[str] = None) -> dict: + """Read the cache file. Returns ``{}`` if missing or malformed.""" + cache_path = path or _default_cache_path() + if not isinstance(cache_path, str): + return {} + if not os.path.isfile(cache_path): + return {} + # TOCTOU defence: reject a symlink at the cache target before open() + # (mirrors the existing guard in ``save_cache``). + try: + st = os.lstat(cache_path) + except OSError: + return {} + if stat.S_ISLNK(st.st_mode): + return {} + if st.st_size > _MAX_CACHE_BYTES: + return {} + try: + with open(cache_path, encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, ValueError, UnicodeDecodeError): + return {} + if not isinstance(data, dict): + return {} + return data + + +def save_cache(cache: dict, path: Optional[str] = None) -> None: + """Atomically write the cache to disk. Best-effort: silent on failure.""" + cache_path = path or _default_cache_path() + if not isinstance(cache_path, str): + return + dir_part = os.path.dirname(cache_path) + if dir_part: + try: + os.makedirs(dir_part, exist_ok=True) + except OSError: + return + # Reject symlink at target (TOCTOU defence) + if os.path.lexists(cache_path): + try: + st = os.lstat(cache_path) + except OSError: + return + if stat.S_ISLNK(st.st_mode): + return + serialised = json.dumps(cache, sort_keys=True, indent=2) + if len(serialised) > _MAX_CACHE_BYTES: + return + tmp_fd = None + try: + tmp_fd, tmp_name = tempfile.mkstemp( + prefix=".deploy_autopilot_cache_", suffix=".tmp", + dir=dir_part or None, + ) + try: + with os.fdopen(tmp_fd, "w", encoding="utf-8") as fh: + fh.write(serialised) + tmp_fd = None + os.replace(tmp_name, cache_path) + finally: + if tmp_fd is not None: + try: + os.close(tmp_fd) + except OSError: + pass + # Best-effort 0o600 on POSIX + try: + os.chmod(cache_path, 0o600) + except OSError: + pass + except OSError: + return + + +def _score_tasks( + tasks_file: str, generate_fn: Callable[[str], str], +) -> float: + """Average score across a JSONL task file (delegates to v0.25.0 eval).""" + from soup_cli.eval.custom import load_eval_tasks, score_task + + tasks = load_eval_tasks(tasks_file) + if not tasks: + return 0.0 + total = 0.0 + for task in tasks: + output = generate_fn(task.prompt) + total += float(score_task(task, output).score) + return total / len(tasks) + + +def measure_candidate( + *, + candidate: str, + tasks_file: str, + before_gen: Callable[[str], str], + after_gen: Callable[[str], str], +) -> MeasureResult: + """Score one quant candidate against the baseline. + + Returns a :class:`MeasureResult` with classify_delta verdict. + """ + if isinstance(candidate, bool) or not isinstance(candidate, str): + raise TypeError("candidate must be a non-bool str") + if not candidate: + raise ValueError("candidate must be non-empty") + if "\x00" in candidate: + raise ValueError("candidate must not contain null bytes") + + before = _score_tasks(tasks_file, before_gen) + after = _score_tasks(tasks_file, after_gen) + delta = after - before + if delta >= 0: + verdict = "OK" + else: + drop = -delta + if drop < DEFAULT_MINOR_THRESHOLD: + verdict = "OK" + elif drop < DEFAULT_MAJOR_THRESHOLD: + verdict = "MINOR" + else: + verdict = "MAJOR" + return MeasureResult( + candidate=candidate, before=before, after=after, + delta=delta, verdict=verdict, + ) + + +def pick_best(results: Sequence[MeasureResult]) -> Optional[MeasureResult]: + """Soft-fallback policy from v0.33.0 #54. + + Returns the first ``OK`` candidate by insertion order; if none clears OK, + returns the candidate with the highest ``delta`` (smallest drop relative + to its baseline — the v0.33.0 #54 design intent). Returns ``None`` only + for an empty sequence. + """ + if not results: + return None + for r in results: + if r.verdict == "OK": + return r + return max(results, key=lambda r: r.delta) + + +def run_measure( + *, + profile_name: str, + base_sha: str, + candidates: Sequence[str], + tasks_file: str, + before_gen: Callable[[str], str], + after_gen_factory: Callable[[str], Callable[[str], str]], + cache_path: Optional[str] = None, +) -> tuple[list[MeasureResult], bool]: + """Run measurement across every candidate quant for a deploy profile. + + ``after_gen_factory(candidate)`` returns a fresh generator for that + candidate (so the caller can lazy-load the quantized model per call). + + Returns ``(results, cache_hit)``. On a hit, ``results`` is loaded from + disk and no eval is run. + """ + if not isinstance(candidates, Sequence) or isinstance(candidates, (str, bytes)): + raise TypeError("candidates must be a sequence of strings") + if len(candidates) == 0: + raise ValueError("candidates must not be empty") + if len(candidates) > _MAX_CANDIDATES: + raise ValueError( + f"too many candidates ({len(candidates)}; cap {_MAX_CANDIDATES})" + ) + + tasks_sha = sha_of_file(tasks_file) + key = compute_cache_key( + base_sha=base_sha, profile_name=profile_name, tasks_sha=tasks_sha, + ) + + cache = load_cache(cache_path) + cached = cache.get(key) + if isinstance(cached, dict): + rows = cached.get("rows") + if isinstance(rows, list): + try: + hit = [MeasureResult(**r) for r in rows] + if all(isinstance(r, MeasureResult) for r in hit): + return hit, True + except (TypeError, ValueError): + pass + + # Cache miss — run the eval loop + results: list[MeasureResult] = [] + for candidate in candidates: + after_gen = after_gen_factory(candidate) + result = measure_candidate( + candidate=candidate, tasks_file=tasks_file, + before_gen=before_gen, after_gen=after_gen, + ) + results.append(result) + + cache[key] = {"rows": [asdict(r) for r in results]} + save_cache(cache, cache_path) + return results, False + + +def render_measure_table(results: Sequence[MeasureResult]) -> "Table": + """Render a Rich table from a sequence of :class:`MeasureResult` rows.""" + from rich.markup import escape + from rich.table import Table + + table = Table(title="Deploy autopilot — measured candidates") + table.add_column("Candidate", style="cyan") + table.add_column("Before", justify="right") + table.add_column("After", justify="right") + table.add_column("Delta", justify="right") + table.add_column("Verdict") + for r in results: + verdict_styled = { + "OK": f"[green]{r.verdict}[/]", + "MINOR": f"[yellow]{r.verdict}[/]", + "MAJOR": f"[red]{r.verdict}[/]", + }.get(r.verdict, escape(r.verdict)) + table.add_row( + escape(r.candidate), + f"{r.before:.3f}", + f"{r.after:.3f}", + f"{r.delta:+.3f}", + verdict_styled, + ) + return table diff --git a/soup_cli/utils/gguf_quant.py b/soup_cli/utils/gguf_quant.py index 11b9e6d..7aa9314 100644 --- a/soup_cli/utils/gguf_quant.py +++ b/soup_cli/utils/gguf_quant.py @@ -11,9 +11,15 @@ to v0.53.1 (mirrors v0.50.0 stub-then-live pattern). from __future__ import annotations +import json +import os +import subprocess +import sys +import tempfile from dataclasses import dataclass +from pathlib import Path from types import MappingProxyType -from typing import Mapping +from typing import Mapping, Optional, cast # --- Part A: Unsloth Dynamic 2.0 GGUF ladder --------------------------------- UD_GGUF_FORMATS: frozenset[str] = frozenset({ @@ -280,13 +286,358 @@ def validate_calibration_data_path(path: object) -> str: return path -def export_advanced_gguf() -> None: - """Live UD/IQ/Apple-ARM GGUF export — deferred to v0.53.1. +# --- v0.53.1 #139 — Live llama.cpp imatrix + quantize wiring --------------- - Mirrors v0.52.0 ``export_bitnet_gguf`` stub-then-live pattern. +# Max 30 min per subprocess so we don't hang CI forever on a bad build. +_SUBPROC_TIMEOUT_SECONDS: int = 30 * 60 + + +def _safe_stderr(stderr: Optional[str], cap: int = 512) -> str: + """Truncate + Rich-markup-escape subprocess stderr before embedding it + in ``RuntimeError`` messages. + + Security review L4 — the llama-imatrix / llama-quantize binaries may + echo crafted input back in their stderr; without escape, characters + like ``[red]`` would inject Rich markup when the wrapping exception + is later printed via ``console.print``. """ - raise NotImplementedError( - "UD / IQ / Apple-ARM GGUF export live wiring deferred to v0.53.1. " - "Schema accepts every format in ALL_ADVANCED_GGUF_FORMATS but no " - "llama.cpp imatrix invocation is registered yet." + if not stderr: + return "" + from rich.markup import escape + + truncated = stderr[:cap] + return escape(truncated) + +# Quantize flavours that require an imatrix file (UD ladder + low-bit IQ family). +_REQUIRES_IMATRIX: frozenset[str] = ( + UD_GGUF_FORMATS + | frozenset({"IQ1_S", "IQ1_M", "IQ2_XXS", "IQ2_XS", "IQ2_S", "IQ2_M", + "IQ3_XXS", "IQ3_XS"}) +) + + +def _enforce_under_cwd_and_no_symlink(path: str, field: str) -> str: + """Re-export shared helper from :mod:`soup_cli.utils.paths`.""" + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + return enforce_under_cwd_and_no_symlink(path, field) + + +def _resolve_quantize_binary(llama_cpp_dir: str) -> Path: + """Locate the ``llama-quantize`` (or legacy ``quantize``) binary.""" + base = Path(llama_cpp_dir) + candidates = [ + base / "llama-quantize", + base / "llama-quantize.exe", + base / "build" / "bin" / "llama-quantize", + base / "build" / "bin" / "llama-quantize.exe", + base / "quantize", + base / "quantize.exe", + base / "build" / "bin" / "quantize", + base / "build" / "bin" / "quantize.exe", + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + raise FileNotFoundError( + f"llama-quantize binary not found under {llama_cpp_dir!r}. " + "Build llama.cpp with cmake first." ) + + +def _resolve_imatrix_binary(llama_cpp_dir: str) -> Path: + """Locate the ``llama-imatrix`` (or legacy ``imatrix``) binary.""" + base = Path(llama_cpp_dir) + candidates = [ + base / "llama-imatrix", + base / "llama-imatrix.exe", + base / "build" / "bin" / "llama-imatrix", + base / "build" / "bin" / "llama-imatrix.exe", + base / "imatrix", + base / "imatrix.exe", + base / "build" / "bin" / "imatrix", + base / "build" / "bin" / "imatrix.exe", + ] + for candidate in candidates: + if candidate.is_file(): + return candidate + raise FileNotFoundError( + f"llama-imatrix binary not found under {llama_cpp_dir!r}. " + "Build llama.cpp tools with `cmake --build . --target llama-imatrix` " + "(or `-DLLAMA_BUILD_TOOLS=ON` then `cmake --build .`)." + ) + + +def _prepare_calibration_text(calibration_data: str, staged_dir: Path) -> Path: + """Read JSONL ``{"text": "..."}`` rows and write a plain-text file. + + llama.cpp ``imatrix`` accepts a raw text file (one paragraph per line is + fine). We extract the ``text`` field from each JSONL row, dropping any + row without a string ``text``. + """ + src = Path(calibration_data) + if not src.is_file(): + raise FileNotFoundError( + f"calibration_data file not found: {os.path.basename(calibration_data)!r}" + ) + # Security review M3 — defend against TOCTOU swap between the CLI-level + # symlink check and this open(). Use O_NOFOLLOW on POSIX so a symlink + # placed between the two calls is rejected at the kernel level. + # On Windows there is no O_NOFOLLOW; the dispatch-time check from + # ``enforce_under_cwd_and_no_symlink`` is the portable backstop. + if hasattr(os, "O_NOFOLLOW"): + try: + fd = os.open(str(src), os.O_RDONLY | os.O_NOFOLLOW) + except OSError as exc: + raise ValueError( + "calibration_data became a symlink during the export " + "(TOCTOU defence): refusing to open." + ) from exc + os.close(fd) + out = staged_dir / "calib.txt" + line_count = 0 + total_bytes = 0 + max_per_line = 8192 + max_total_bytes = 50 * 1024 * 1024 # 50 MB cap on the rendered calib file + + def _sanitise(text: str) -> str: + # Strip null bytes + collapse newlines to spaces; cap per-line length. + sanitised = text.replace("\x00", "").replace("\n", " ") + return sanitised[:max_per_line] + + with open(src, encoding="utf-8") as fh_in, open( + out, "w", encoding="utf-8" + ) as fh_out: + for raw_line in fh_in: + line = raw_line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError: + # Treat as a raw text line — sanitise the same way. + emitted = _sanitise(line) + if emitted: + fh_out.write(emitted + "\n") + line_count += 1 + total_bytes += len(emitted) + 1 + continue + if isinstance(row, dict): + text = row.get("text") or row.get("prompt") or row.get("content") + elif isinstance(row, str): + text = row + else: + text = None + if isinstance(text, str) and text: + emitted = _sanitise(text) + if emitted: + fh_out.write(emitted + "\n") + line_count += 1 + total_bytes += len(emitted) + 1 + if line_count >= 4096 or total_bytes >= max_total_bytes: + # Safety cap — imatrix doesn't need more than a few thousand + break + if line_count == 0: + raise ValueError( + "calibration_data produced 0 usable rows; " + "expected JSONL with a 'text' field." + ) + return out + + +def _run_convert_to_f16( + llama_cpp_dir: str, model_dir: str, f16_out: str, +) -> None: + """Invoke ``convert_hf_to_gguf.py`` to produce an f16 GGUF.""" + script = Path(llama_cpp_dir) / "convert_hf_to_gguf.py" + if not script.is_file(): + raise FileNotFoundError( + f"convert_hf_to_gguf.py not found in {llama_cpp_dir!r}" + ) + # Security review M5 — defend against a crafted llama_cpp_dir whose + # ``convert_hf_to_gguf.py`` is a symlink escaping the directory. Resolve + # both paths to realpath and require the script to stay inside. + script_real = os.path.realpath(str(script)) + dir_real = os.path.realpath(str(llama_cpp_dir)) + try: + common = os.path.commonpath([script_real, dir_real]) + except ValueError: + common = "" + if common != dir_real: + raise FileNotFoundError( + "convert_hf_to_gguf.py is outside the llama.cpp dir " + "(symlink escape rejected)" + ) + + argv = [ + sys.executable, + script_real, + str(model_dir), + "--outfile", str(f16_out), + "--outtype", "f16", + ] + result = subprocess.run( # noqa: S603 — argv list, no shell + argv, shell=False, timeout=_SUBPROC_TIMEOUT_SECONDS, + capture_output=True, text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"convert_hf_to_gguf.py failed (rc={result.returncode}): " + f"{_safe_stderr(result.stderr)}" + ) + + +def _run_imatrix( + *, + llama_cpp_dir: str, + f16_path: str, + calib_data: str, + imatrix_out: str, +) -> None: + """Run llama.cpp ``imatrix`` to compute an importance matrix.""" + binary = _resolve_imatrix_binary(llama_cpp_dir) + argv = [ + str(binary), + "-m", str(f16_path), + "-f", str(calib_data), + "-o", str(imatrix_out), + "--chunks", "32", + ] + result = subprocess.run( # noqa: S603 — argv list, no shell + argv, shell=False, timeout=_SUBPROC_TIMEOUT_SECONDS, + capture_output=True, text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"llama-imatrix failed (rc={result.returncode}): " + f"{_safe_stderr(result.stderr)}" + ) + + +def _flavour_to_quantize_arg(flavour: str) -> str: + """Map a v0.53.0 flavour string to the llama.cpp ``quantize`` CLI arg. + + UD ladder strips the ``UD-`` prefix (llama.cpp doesn't know UD; the UD + flavour is the underlying type + imatrix calibration). IQ + Apple/ARM + pass through verbatim. + """ + if flavour.startswith("UD-"): + return flavour[len("UD-"):] + return flavour + + +def _run_quantize_binary( + *, + llama_cpp_dir: str, + f16_path: str, + output_path: str, + flavour: str, + imatrix_path: Optional[str] = None, +) -> None: + """Run llama.cpp ``quantize`` to write the final GGUF.""" + binary = _resolve_quantize_binary(llama_cpp_dir) + argv: list[str] = [str(binary)] + if imatrix_path is not None: + argv += ["--imatrix", str(imatrix_path)] + argv += [str(f16_path), str(output_path), _flavour_to_quantize_arg(flavour)] + result = subprocess.run( # noqa: S603 — argv list, no shell + argv, shell=False, timeout=_SUBPROC_TIMEOUT_SECONDS, + capture_output=True, text=True, + ) + if result.returncode != 0: + raise RuntimeError( + f"llama-quantize failed (rc={result.returncode}): " + f"{_safe_stderr(result.stderr)}" + ) + + +def export_advanced_gguf( + *, + model_dir: str, + output_path: str, + flavour: str, + calibration_data: Optional[str], + llama_cpp_dir: str, +) -> None: + """Export a HuggingFace model as a UD / IQ / Apple-ARM GGUF. + + Three-stage pipeline: + 1. ``convert_hf_to_gguf.py`` → ``f16.gguf`` + 2. If ``flavour`` needs an importance matrix + (UD ladder + low-bit IQ family): ``imatrix`` → ``imatrix.dat`` + 3. ``quantize`` (with ``--imatrix`` when present) → ``output_path`` + + All subprocess invocations use argv-list form (no shell). Per the + v0.53.0 ``validate_calibration_data_path`` docstring contract, this + dispatch-time helper applies cwd containment + symlink rejection. + """ + # Flavour validation + if not is_advanced_gguf_format(flavour): + raise ValueError( + f"Unknown gguf flavour {flavour!r}. " + "See ALL_ADVANCED_GGUF_FORMATS." + ) + + _enforce_under_cwd_and_no_symlink(model_dir, "model_dir") + _enforce_under_cwd_and_no_symlink(output_path, "output_path") + _enforce_under_cwd_and_no_symlink(llama_cpp_dir, "llama_cpp_dir") + + if not os.path.isdir(model_dir): + raise FileNotFoundError( + f"model_dir not a directory: {os.path.basename(model_dir)!r}" + ) + if not os.path.isdir(llama_cpp_dir): + raise FileNotFoundError( + f"llama_cpp_dir not a directory: " + f"{os.path.basename(llama_cpp_dir)!r}" + ) + + needs_imatrix = flavour in _REQUIRES_IMATRIX + if needs_imatrix and calibration_data is None: + raise ValueError( + f"flavour {flavour!r} requires --calibration-data . " + "UD ladder + low-bit IQ flavours need an importance matrix." + ) + if calibration_data is not None: + _enforce_under_cwd_and_no_symlink( + calibration_data, "calibration_data", + ) + + # Stage intermediate files inside a tempdir under cwd + with tempfile.TemporaryDirectory( + prefix=".soup_gguf_", dir=str(Path.cwd()), + ) as staged: + staged_path = Path(staged) + f16_path = staged_path / "model.f16.gguf" + + # 1. Convert HF → f16 GGUF + _run_convert_to_f16(llama_cpp_dir, model_dir, str(f16_path)) + + # 2. Compute importance matrix (imatrix) — only for UD ladder + low-bit IQ + imatrix_path: Optional[str] = None + if needs_imatrix: + # cast() rather than assert — survives `python -O` + calib_data_str = cast(str, calibration_data) + calib_txt = _prepare_calibration_text(calib_data_str, staged_path) + imatrix_path = str(staged_path / "imatrix.dat") + _run_imatrix( + llama_cpp_dir=llama_cpp_dir, + f16_path=str(f16_path), + calib_data=str(calib_txt), + imatrix_out=imatrix_path, + ) + + # 3. Final quantize + _run_quantize_binary( + llama_cpp_dir=llama_cpp_dir, + f16_path=str(f16_path), + output_path=output_path, + flavour=flavour, + imatrix_path=imatrix_path, + ) + + # Ensure the writer produced the file + if not os.path.isfile(output_path): + raise RuntimeError( + f"llama-quantize did not produce {os.path.basename(output_path)!r}" + ) diff --git a/soup_cli/utils/paths.py b/soup_cli/utils/paths.py index b9d6b0a..0f56625 100644 --- a/soup_cli/utils/paths.py +++ b/soup_cli/utils/paths.py @@ -13,6 +13,7 @@ in a single module guarantees a single behaviour across the CLI. from __future__ import annotations import os +import stat from pathlib import Path from typing import Union @@ -36,3 +37,35 @@ def is_under(path: Union[str, Path], base: Union[str, Path]) -> bool: def is_under_cwd(path: Union[str, Path]) -> bool: """Whether ``path`` is inside the current working directory.""" return is_under(path, Path.cwd()) + + +def enforce_under_cwd_and_no_symlink(path: str, field: str) -> str: + """Apply cwd containment + ``os.lstat + S_ISLNK`` rejection (TOCTOU defence). + + Shared helper for v0.53.1 export / merge / advanced-GGUF dispatch. + Mirrors v0.33.0 #22 / v0.43.0 Part C / v0.46.0 Part A / v0.47.0 TOCTOU + policy: rejects symlinks at the target path before any open/write so a + pre-placed symlink cannot redirect a write to ``/etc/cron.d``. + """ + if not isinstance(path, str): + raise TypeError(f"{field} must be str, got {type(path).__name__}") + if not path: + raise ValueError(f"{field} must be non-empty") + if "\x00" in path: + raise ValueError(f"{field} must not contain null bytes") + if not is_under_cwd(path): + raise ValueError( + f"{field} {os.path.basename(path)!r} must stay under cwd" + ) + if os.path.lexists(path): + try: + st = os.lstat(path) + except OSError as exc: + raise ValueError( + f"{field} unreadable: {type(exc).__name__}" + ) from exc + if stat.S_ISLNK(st.st_mode): + raise ValueError( + f"{field} must not be a symlink (TOCTOU defence)" + ) + return path diff --git a/soup_cli/utils/save_formats.py b/soup_cli/utils/save_formats.py index 788734d..4920255 100644 --- a/soup_cli/utils/save_formats.py +++ b/soup_cli/utils/save_formats.py @@ -1,4 +1,4 @@ -"""v0.53.0 Part F — Advanced save / merge formats schema helpers. +"""v0.53.0 Part F + v0.53.1 #142 — Advanced save / merge formats. Two new save-format surfaces ship this release (schema-only): @@ -16,9 +16,10 @@ Live wiring deferred to v0.53.1. from __future__ import annotations +import os from dataclasses import dataclass from types import MappingProxyType -from typing import Mapping +from typing import Any, Mapping, Optional # --- Merge save formats ------------------------------------------------------ MERGE_SAVE_FORMATS: frozenset[str] = frozenset({ @@ -192,17 +193,228 @@ def validate_quant_config_path(path: object) -> str: return path -def merge_4bit() -> None: - """Live 4bit-merge wiring — deferred to v0.53.1.""" - raise NotImplementedError( - "soup merge --save-format 4bit live wiring deferred to v0.53.1. " - "Schema accepts the flag but no merged-4bit writer is registered yet." - ) +# --- v0.53.1 #142 — Live path-containment + symlink TOCTOU helpers --------- + +_MAX_QUANT_CONFIG_BYTES: int = 256 * 1024 -def export_torchao() -> None: - """Live TorchAO PTQ export — deferred to v0.53.1.""" - raise NotImplementedError( - "soup export --format torchao live wiring deferred to v0.53.1. " - "Schema accepts the format but no torchao.quantize_ call is wired." +def _enforce_under_cwd_and_no_symlink(path: str, field: str) -> str: + """Re-export the shared helper from :mod:`soup_cli.utils.paths`. + + Kept as a module-level alias so external callers (and the v0.53.1 CLI + dispatch path) can keep their existing imports. + """ + from soup_cli.utils.paths import enforce_under_cwd_and_no_symlink + + return enforce_under_cwd_and_no_symlink(path, field) + + +def load_quant_config(path: object) -> Mapping[str, Any]: + """Load + validate a TorchAO ``--quant-config`` YAML file. + + Enforces (per v0.53.0 docstring contract): + * Shape validation via :func:`validate_quant_config_path` + * ``os.path.realpath`` + ``os.path.commonpath`` cwd containment + * ``os.lstat + stat.S_ISLNK`` rejection (TOCTOU) + * Extension allowlist (``.yaml`` / ``.yml``) + * ``yaml.safe_load`` only + * 256 KB size cap + """ + import yaml + + validated_shape = validate_quant_config_path(path) + _enforce_under_cwd_and_no_symlink(validated_shape, "quant_config") + lower = validated_shape.lower() + if not (lower.endswith(".yaml") or lower.endswith(".yml")): + raise ValueError( + f"quant_config extension must be .yaml or .yml: " + f"{os.path.basename(validated_shape)!r}" + ) + if not os.path.isfile(validated_shape): + raise FileNotFoundError( + f"quant_config not found: {os.path.basename(validated_shape)!r}" + ) + size = os.path.getsize(validated_shape) + if size > _MAX_QUANT_CONFIG_BYTES: + raise ValueError( + f"quant_config too large ({size} bytes; " + f"cap {_MAX_QUANT_CONFIG_BYTES})" + ) + with open(validated_shape, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + if data is None: + return {} + if not isinstance(data, dict): + raise ValueError( + f"quant_config must be a YAML mapping, got {type(data).__name__}" + ) + return data + + +def merge_4bit( + *, + merged_dir: str, + output_dir: str, + forced: bool = False, + dtype: str = "bfloat16", + trust_remote_code: bool = False, +) -> None: + """Write a single BNB-4bit-quantized merged checkpoint. + + Unsloth ``merged_4bit`` / ``4bit_forced`` recipe — no + dequant → merge → requant cycle. ``forced=True`` quantizes ALL linear + layers including embeddings; default ``False`` follows BNB's default + skip-modules behaviour. + """ + if not isinstance(forced, bool): + raise TypeError(f"forced must be bool, got {type(forced).__name__}") + if not isinstance(dtype, str): + raise TypeError(f"dtype must be str, got {type(dtype).__name__}") + if dtype not in {"float16", "bfloat16", "float32"}: + raise ValueError( + f"dtype {dtype!r} invalid; expected float16 / bfloat16 / float32" + ) + + _enforce_under_cwd_and_no_symlink(merged_dir, "merged_dir") + _enforce_under_cwd_and_no_symlink(output_dir, "output_dir") + + if not os.path.isdir(merged_dir): + raise FileNotFoundError( + f"merged_dir not a directory: {os.path.basename(merged_dir)!r}" + ) + + import torch + from transformers import ( # type: ignore[import-not-found] + AutoModelForCausalLM, + AutoTokenizer, + BitsAndBytesConfig, ) + + dtype_map = { + "float16": torch.float16, + "bfloat16": torch.bfloat16, + "float32": torch.float32, + } + + bnb_kwargs: dict[str, Any] = { + "load_in_4bit": True, + "bnb_4bit_quant_type": "nf4", + "bnb_4bit_compute_dtype": dtype_map[dtype], + "bnb_4bit_use_double_quant": True, + } + if forced: + # ``forced`` => no skip-modules, so every Linear (incl. lm_head) is + # 4-bit quantized. BNB 4-bit uses ``bnb_4bit_skip_modules`` (the + # legacy 8-bit name was ``llm_int8_skip_modules`` — only emit it + # when the installed BNB exposes the 8-bit kwarg as a fallback). + bnb_kwargs["bnb_4bit_skip_modules"] = [] + bnb_config = BitsAndBytesConfig(**bnb_kwargs) + + os.makedirs(output_dir, exist_ok=True) + + model = AutoModelForCausalLM.from_pretrained( + merged_dir, + quantization_config=bnb_config, + trust_remote_code=trust_remote_code, + device_map="auto" if torch.cuda.is_available() else "cpu", + ) + model.save_pretrained(output_dir) + tokenizer = AutoTokenizer.from_pretrained( + merged_dir, trust_remote_code=trust_remote_code + ) + tokenizer.save_pretrained(output_dir) + + +def export_torchao( + *, + model_dir: str, + output_dir: str, + scheme: str, + quant_config_data: Optional[Mapping[str, Any]] = None, + trust_remote_code: bool = False, +) -> None: + """Run ``torchao.quantize_(scheme)`` + ``save_pretrained``. + + Schemes (CASE-SENSITIVE) from :data:`TORCHAO_PTQ_SCHEMES`: + ``Int4WeightOnly`` / ``Int8DynActInt4`` / ``Float8DynActFloat8`` / ``NVFP4``. + """ + canonical_scheme = validate_torchao_scheme(scheme) + _enforce_under_cwd_and_no_symlink(model_dir, "model_dir") + _enforce_under_cwd_and_no_symlink(output_dir, "output_dir") + if not os.path.isdir(model_dir): + raise FileNotFoundError( + f"model_dir not a directory: {os.path.basename(model_dir)!r}" + ) + if quant_config_data is not None and not isinstance(quant_config_data, Mapping): + raise TypeError( + f"quant_config_data must be Mapping, got {type(quant_config_data).__name__}" + ) + + from torchao import quantization as ao_q # type: ignore[import-not-found] + from transformers import ( # type: ignore[import-not-found] + AutoModelForCausalLM, + AutoTokenizer, + ) + + scheme_factory_map = { + "Int4WeightOnly": "Int4WeightOnlyConfig", + "Int8DynActInt4": "Int8DynActInt4Config", + "Float8DynActFloat8": "Float8DynActFloat8Config", + "NVFP4": "NVFP4Config", + } + factory_name = scheme_factory_map[canonical_scheme] + if not hasattr(ao_q, factory_name): + raise RuntimeError( + f"torchao does not expose {factory_name}; " + f"upgrade torchao or pick a different scheme." + ) + + # Build the config from quant_config_data if provided; else defaults. + # Apply a per-scheme closed key allowlist to defeat kwarg injection + # (security review H1). + scheme_kwarg_allowlist: dict[str, frozenset[str]] = { + "Int4WeightOnly": frozenset({"group_size", "inner_k_tiles"}), + "Int8DynActInt4": frozenset({"group_size"}), + "Float8DynActFloat8": frozenset(), + "NVFP4": frozenset(), + } + allowed = scheme_kwarg_allowlist.get(canonical_scheme, frozenset()) + raw_kwargs = dict(quant_config_data or {}) + raw_kwargs.pop("scheme", None) + bad_keys = [ + k for k in raw_kwargs + if (not isinstance(k, str)) or k.startswith("__") or k not in allowed + ] + if bad_keys: + allowed_str = ( + ", ".join(sorted(allowed)) + if allowed + else "(none — scheme takes no extra args)" + ) + raise ValueError( + f"quant_config keys not allowed for scheme {canonical_scheme}: " + f"{bad_keys}. Allowed: {allowed_str}" + ) + config_obj = getattr(ao_q, factory_name)(**raw_kwargs) + + os.makedirs(output_dir, exist_ok=True) + + model = AutoModelForCausalLM.from_pretrained( + model_dir, trust_remote_code=trust_remote_code, + ) + # torchao quantize in-place. Some torchao versions ship `quantize_` at + # top level; others under quantization. Try both. + try: + import torchao # type: ignore[import-not-found] + quantize_fn = getattr(torchao, "quantize_", None) or getattr(ao_q, "quantize_") + except (ImportError, AttributeError) as exc: + raise RuntimeError( + f"torchao.quantize_ entry point not found: {type(exc).__name__}" + ) from exc + + quantize_fn(model, config_obj) + model.save_pretrained(output_dir) + tokenizer = AutoTokenizer.from_pretrained( + model_dir, trust_remote_code=trust_remote_code + ) + tokenizer.save_pretrained(output_dir) diff --git a/tests/qa/v053_qa.md b/tests/qa/v053_qa.md new file mode 100644 index 0000000..67ba1e8 --- /dev/null +++ b/tests/qa/v053_qa.md @@ -0,0 +1,114 @@ +# v0.53.x QA log + +Manual smoke-test results for paths CI cannot reach (no GPU on CI runners, +no llama.cpp build). Each entry records command + observed result + date. + +## v0.53.1 + +### #70 — GGUF export (q4_0 / q4_k_m / q5_k_m / q8_0 / f16 / f32) + +**Status:** PENDING — requires a GPU box + built llama.cpp. + +Planned smoke (run on a 4 GB+ VRAM box with TinyLlama-1.1B-LoRA in `./adapter`): + +```bash +soup merge -a ./adapter -o ./merged +for q in q4_0 q4_k_m q5_k_m q8_0 f16 f32; do + soup export --model ./merged --format gguf --quant "$q" \ + --output "./out/tinyllama.${q}.gguf" +done +``` + +Acceptance criteria: +- All six files exist with non-zero size. +- File sizes increase monotonically: q4_0 < q4_k_m < q5_k_m < q8_0 < f16 < f32. +- `llama-cli` can load each file and emit at least one token. + +When run, record sizes + first-token-time below. + +### #72 — AWQ + GPTQ export (`--bits 4`) + +**Status:** PENDING — requires CUDA box with `autoawq` + `auto-gptq` extras. + +Planned smoke: + +```bash +soup export --model ./merged --format awq --bits 4 --output ./out/awq +soup export --model ./merged --format gptq --bits 4 --output ./out/gptq +``` + +Acceptance: both output directories contain a `config.json` and weight +shards; loading each via `transformers.AutoModelForCausalLM` returns a +quantized model whose `state_dict` reports the expected dtype. + +### #82 — Autopilot pre-quantized detection + +**Status:** PASS — automated regression coverage in `tests/test_v0531_82.py`. + +Notes: +- `detect_prequantized_format("TheBloke/Llama-2-7B-Chat-GPTQ")` → `"gptq"`. +- `detect_prequantized_format("clean/name", {"quantization_config": {"quant_method": "awq"}})` → `"awq"`. +- `decide_quantization(model_params_b=7.0, vram_gb=80.0, prequantized="gptq")` → `"gptq"` (takes precedence over the VRAM heuristic). + +### #109 — `soup deploy autopilot --measure` + +**Status:** PASS (orchestrator + cache); LIVE-EVAL PENDING (needs GPU + model loaders). + +Coverage: +- Cache key + sha-of-file: `tests/test_v0531_109.py::TestComputeCacheKey`. +- Measurement loop + cache round-trip: `tests/test_v0531_109.py::TestRunMeasure`. +- CLI plumbing with injected generators: `TestDeployAutopilotMeasureCLI`. + +The injected-generator escape hatch (`_DEPLOY_MEASURE_BEFORE_GEN`, +`_DEPLOY_MEASURE_AFTER_FACTORY` module attrs on +`soup_cli.utils.deploy_measure`) is the production hook for live model +loading in v0.53.x. Real transformers / vLLM generators land alongside +v0.53.2 deferred trainer wiring. + +### #139 — Advanced GGUF export via llama.cpp imatrix + +**Status:** PASS (orchestrator + subprocess argv shape + UD prefix +stripping); LIVE-RUN PENDING (needs llama.cpp build). + +Coverage: +- `_run_imatrix` + `_run_quantize_binary` argv lists (no shell) verified. +- UD-prefix-strip: `UD-Q4_K_XL` → llama.cpp arg `Q4_K_XL`. +- Calibration JSONL → plain-text conversion + 4096-row safety cap. +- Apple/ARM flavours (Q4_0_4_4 etc.) skip the imatrix stage; UD ladder + + low-bit IQ family require `--calibration-data`. + +Planned manual smoke when a llama.cpp build is available: + +```bash +soup export --model ./merged --format gguf-ud \ + --gguf-flavour UD-Q4_K_XL \ + --calibration-data ./calib.jsonl \ + --llama-cpp ~/.soup/llama.cpp \ + --output ./out/tinyllama.UD-Q4_K_XL.gguf +``` + +### #142 — Merge `--save-format 4bit` + export `--format torchao` + +**Status:** PASS (orchestrator + validators + path TOCTOU); LIVE-EVAL +PENDING (needs CUDA + bitsandbytes / torchao). + +Coverage: +- `merge_4bit` happy path with mocked transformers + BNB. +- `export_torchao` happy path with mocked torchao + transformers. +- `--save-format weird` rejected at CLI dispatch with exit code 2. +- `--format torchao` without `--quant-config` rejected. +- `--quant-config` outside cwd rejected. +- `load_quant_config` enforces yaml.safe_load + 256 KB cap + + extension allowlist + symlink rejection (POSIX). + +Planned manual smoke when a CUDA box is available: + +```bash +soup merge -a ./adapter -o ./merged_4bit --save-format 4bit +# Expect ~70-90 MB checkpoint vs ~2.2 GB fp16 for TinyLlama +cat > q.yaml <<'EOF' +scheme: Int4WeightOnly +EOF +soup export --model ./merged --format torchao --quant-config ./q.yaml \ + --output ./out/torchao +``` diff --git a/tests/test_awq_gptq_export.py b/tests/test_awq_gptq_export.py index 3afc9f8..62794ba 100644 --- a/tests/test_awq_gptq_export.py +++ b/tests/test_awq_gptq_export.py @@ -39,12 +39,16 @@ class TestExportFormatsExtended: assert "gptq" in SUPPORTED_FORMATS def test_format_count(self): - """v0.52.0 — 5 live formats + 2 BitNet stubs (bitnet, tq1_0).""" - assert len(SUPPORTED_FORMATS) == 7 + """v0.53.1 — 7 prior formats + torchao + gguf-ud = 9.""" + assert len(SUPPORTED_FORMATS) == 9 def test_all_formats_present(self): - """All seven formats should be present (v0.52.0 + bitnet/tq1_0 stubs).""" - expected = {"gguf", "onnx", "tensorrt", "awq", "gptq", "bitnet", "tq1_0"} + """v0.53.1 — adds torchao + gguf-ud to the v0.52.0 baseline.""" + expected = { + "gguf", "onnx", "tensorrt", "awq", "gptq", + "bitnet", "tq1_0", + "torchao", "gguf-ud", + } assert set(SUPPORTED_FORMATS) == expected diff --git a/tests/test_onnx_tensorrt_export.py b/tests/test_onnx_tensorrt_export.py index a30974a..8b57958 100644 --- a/tests/test_onnx_tensorrt_export.py +++ b/tests/test_onnx_tensorrt_export.py @@ -21,10 +21,12 @@ class TestExportFormats: assert "tensorrt" in SUPPORTED_FORMATS def test_format_count(self): - """v0.52.0 — 5 live formats (gguf/onnx/tensorrt/awq/gptq) + 2 BitNet stubs.""" - assert len(SUPPORTED_FORMATS) == 7 + """v0.53.1 — 7 prior formats + torchao + gguf-ud = 9.""" + assert len(SUPPORTED_FORMATS) == 9 assert "bitnet" in SUPPORTED_FORMATS assert "tq1_0" in SUPPORTED_FORMATS + assert "torchao" in SUPPORTED_FORMATS + assert "gguf-ud" in SUPPORTED_FORMATS # ─── ONNX Export CLI Tests ────────────────────────────────────────────── diff --git a/tests/test_v0530.py b/tests/test_v0530.py index b5d11af..02d422d 100644 --- a/tests/test_v0530.py +++ b/tests/test_v0530.py @@ -103,11 +103,18 @@ class TestUDGGUF: with pytest.raises(exc): validate_calibration_data_path(bad) - def test_export_deferred(self): + def test_export_now_live(self): + """v0.53.1 #139 — live wiring landed; stub is gone. + + ``export_advanced_gguf`` now requires keyword-only args. Calling + without args raises ``TypeError`` (not ``NotImplementedError``) + which is exactly the regression we want as proof the live wiring + is in place. + """ from soup_cli.utils.gguf_quant import export_advanced_gguf - with pytest.raises(NotImplementedError, match="v0.53.1"): - export_advanced_gguf() + with pytest.raises(TypeError): + export_advanced_gguf() # type: ignore[call-arg] # --------------------------------------------------------------------------- @@ -833,17 +840,19 @@ class TestSaveFormats: assert isinstance(_MERGE_METADATA, MappingProxyType) assert isinstance(_TORCHAO_METADATA, MappingProxyType) - def test_merge_4bit_deferred(self): + def test_merge_4bit_now_live(self): + """v0.53.1 #142 — live wiring landed; signature now requires kwargs.""" from soup_cli.utils.save_formats import merge_4bit - with pytest.raises(NotImplementedError, match="v0.53.1"): - merge_4bit() + with pytest.raises(TypeError): + merge_4bit() # type: ignore[call-arg] - def test_export_torchao_deferred(self): + def test_export_torchao_now_live(self): + """v0.53.1 #142 — live wiring landed; signature now requires kwargs.""" from soup_cli.utils.save_formats import export_torchao - with pytest.raises(NotImplementedError, match="v0.53.1"): - export_torchao() + with pytest.raises(TypeError): + export_torchao() # type: ignore[call-arg] # --------------------------------------------------------------------------- diff --git a/tests/test_v0531_109.py b/tests/test_v0531_109.py new file mode 100644 index 0000000..f0d9617 --- /dev/null +++ b/tests/test_v0531_109.py @@ -0,0 +1,513 @@ +"""v0.53.1 #109 — soup deploy autopilot --measure live wiring.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + +# --- compute_cache_key ------------------------------------------------------ + + +class TestComputeCacheKey: + def test_basic(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + key = compute_cache_key( + base_sha="abc123", profile_name="rtx-4090-24gb", + tasks_sha="def456", + ) + assert isinstance(key, str) + assert len(key) == 32 + + def test_deterministic(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + k1 = compute_cache_key( + base_sha="x", profile_name="p", tasks_sha="t", + ) + k2 = compute_cache_key( + base_sha="x", profile_name="p", tasks_sha="t", + ) + assert k1 == k2 + + def test_diff_on_each_arg(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + base = compute_cache_key(base_sha="a", profile_name="b", tasks_sha="c") + for changed in ( + compute_cache_key(base_sha="A", profile_name="b", tasks_sha="c"), + compute_cache_key(base_sha="a", profile_name="B", tasks_sha="c"), + compute_cache_key(base_sha="a", profile_name="b", tasks_sha="C"), + ): + assert changed != base + + def test_bool_rejected(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + with pytest.raises(TypeError): + compute_cache_key(base_sha=True, profile_name="x", tasks_sha="y") + + def test_null_byte_rejected(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + with pytest.raises(ValueError): + compute_cache_key( + base_sha="ev\x00il", profile_name="x", tasks_sha="y", + ) + + def test_empty_rejected(self): + from soup_cli.utils.deploy_measure import compute_cache_key + + with pytest.raises(ValueError): + compute_cache_key(base_sha="", profile_name="x", tasks_sha="y") + + +# --- sha_of_file ------------------------------------------------------------ + + +class TestShaOfFile: + def test_basic(self, tmp_path): + from soup_cli.utils.deploy_measure import sha_of_file + + f = tmp_path / "x.txt" + f.write_bytes(b"hello world") + h = sha_of_file(str(f)) + assert isinstance(h, str) + assert len(h) == 64 # full sha256 hex + + def test_missing_file(self, tmp_path): + from soup_cli.utils.deploy_measure import sha_of_file + + with pytest.raises(FileNotFoundError): + sha_of_file(str(tmp_path / "missing")) + + def test_null_byte_rejected(self): + from soup_cli.utils.deploy_measure import sha_of_file + + with pytest.raises(ValueError): + sha_of_file("ev\x00il") + + +# --- measure_candidate ------------------------------------------------------ + + +def _write_tasks(tmp_path: Path) -> Path: + """Write a 2-row JSONL eval task file.""" + f = tmp_path / "tasks.jsonl" + f.write_text( + '{"prompt": "say hello", "expected": "hello", "scoring": "exact"}\n' + '{"prompt": "say world", "expected": "world", "scoring": "exact"}\n', + encoding="utf-8", + ) + return f + + +class TestMeasureCandidate: + def test_ok_when_after_matches(self, tmp_path): + from soup_cli.utils.deploy_measure import measure_candidate + + tasks = _write_tasks(tmp_path) + def gen(p): + if "hello" in p: + return "hello" + return "world" + result = measure_candidate( + candidate="gptq", tasks_file=str(tasks), + before_gen=gen, after_gen=gen, + ) + assert result.candidate == "gptq" + assert result.verdict == "OK" + assert result.delta == 0.0 + + def test_minor_band_at_3pct_drop(self, tmp_path): + """L1: explicitly exercise the MINOR verdict band (2% < drop < 5%).""" + from soup_cli.utils.deploy_measure import MeasureResult, measure_candidate + + # Build a 100-task fixture so we can hit a 3% drop + big = tmp_path / "tasks_big.jsonl" + big.write_text( + "\n".join( + '{"prompt": "p%d", "expected": "ok", "scoring": "exact"}' % i + for i in range(100) + ), + encoding="utf-8", + ) + + def before(p): + return "ok" + + # After: miss exactly 3 out of 100 → drop=0.03 → MINOR band + miss_set = {"p7", "p23", "p64"} + + def after(p): + return "WRONG" if p in miss_set else "ok" + + r = measure_candidate( + candidate="awq", tasks_file=str(big), + before_gen=before, after_gen=after, + ) + assert isinstance(r, MeasureResult) + assert r.verdict == "MINOR" + assert 0.02 <= -r.delta < 0.05 + + def test_minor_drop(self, tmp_path): + from soup_cli.utils.deploy_measure import measure_candidate + + tasks = _write_tasks(tmp_path) + def before(p): + return "hello" if "hello" in p else "world" + # Always wrong → score 0.0; before score 1.0 → drop 1.0 → MAJOR + def after(p): + return "WRONG" + r = measure_candidate( + candidate="awq", tasks_file=str(tasks), + before_gen=before, after_gen=after, + ) + assert r.verdict == "MAJOR" + assert r.delta < 0 + + def test_invalid_candidate(self, tmp_path): + from soup_cli.utils.deploy_measure import measure_candidate + + tasks = _write_tasks(tmp_path) + with pytest.raises(TypeError): + measure_candidate( + candidate=True, # type: ignore[arg-type] + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen=lambda p: "", + ) + + def test_empty_candidate(self, tmp_path): + from soup_cli.utils.deploy_measure import measure_candidate + + tasks = _write_tasks(tmp_path) + with pytest.raises(ValueError): + measure_candidate( + candidate="", + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen=lambda p: "", + ) + + def test_null_byte_candidate(self, tmp_path): + from soup_cli.utils.deploy_measure import measure_candidate + + tasks = _write_tasks(tmp_path) + with pytest.raises(ValueError): + measure_candidate( + candidate="ev\x00il", + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen=lambda p: "", + ) + + +# --- pick_best -------------------------------------------------------------- + + +class TestPickBest: + def test_empty_returns_none(self): + from soup_cli.utils.deploy_measure import pick_best + + assert pick_best([]) is None + + def test_first_ok_wins(self): + from soup_cli.utils.deploy_measure import MeasureResult, pick_best + + rows = [ + MeasureResult("a", 0.8, 0.79, -0.01, "OK"), + MeasureResult("b", 0.8, 0.78, -0.02, "MINOR"), + ] + assert pick_best(rows).candidate == "a" + + def test_no_ok_picks_highest_after(self): + from soup_cli.utils.deploy_measure import MeasureResult, pick_best + + rows = [ + MeasureResult("a", 0.8, 0.5, -0.3, "MAJOR"), + MeasureResult("b", 0.8, 0.6, -0.2, "MAJOR"), + MeasureResult("c", 0.8, 0.55, -0.25, "MAJOR"), + ] + assert pick_best(rows).candidate == "b" + + +# --- cache load/save round-trip --------------------------------------------- + + +class TestCacheRoundtrip: + def test_save_then_load(self, tmp_path): + from soup_cli.utils.deploy_measure import load_cache, save_cache + + cache_path = tmp_path / "cache.json" + payload = {"abc123": {"rows": [{"candidate": "gptq", + "before": 0.8, "after": 0.79, + "delta": -0.01, "verdict": "OK"}]}} + save_cache(payload, str(cache_path)) + loaded = load_cache(str(cache_path)) + assert loaded == payload + + def test_load_missing_returns_empty(self, tmp_path): + from soup_cli.utils.deploy_measure import load_cache + + assert load_cache(str(tmp_path / "missing.json")) == {} + + def test_load_malformed_returns_empty(self, tmp_path): + from soup_cli.utils.deploy_measure import load_cache + + bad = tmp_path / "bad.json" + bad.write_text("not json {{{", encoding="utf-8") + assert load_cache(str(bad)) == {} + + +# --- run_measure (full loop) ------------------------------------------------ + + +class TestRunMeasure: + def test_first_run_misses_then_hits(self, tmp_path): + from soup_cli.utils.deploy_measure import run_measure + + tasks = _write_tasks(tmp_path) + cache_path = tmp_path / "cache.json" + + def before(p): + return "hello" if "hello" in p else "world" + + def after_factory(candidate): + def gen(p): + # awq matches; gptq always wrong (MAJOR drop) + if candidate == "awq": + return "hello" if "hello" in p else "world" + return "WRONG" + return gen + + results1, hit1 = run_measure( + profile_name="rtx-4090-24gb", + base_sha="basetestsha", + candidates=("awq", "gptq"), + tasks_file=str(tasks), + before_gen=before, + after_gen_factory=after_factory, + cache_path=str(cache_path), + ) + assert hit1 is False + assert [r.candidate for r in results1] == ["awq", "gptq"] + assert results1[0].verdict == "OK" + assert results1[1].verdict == "MAJOR" + + # Second invocation must hit cache and skip after_factory entirely + called = {"count": 0} + + def boom_factory(candidate): + called["count"] += 1 + return lambda p: "should not be called" + + results2, hit2 = run_measure( + profile_name="rtx-4090-24gb", + base_sha="basetestsha", + candidates=("awq", "gptq"), + tasks_file=str(tasks), + before_gen=before, + after_gen_factory=boom_factory, + cache_path=str(cache_path), + ) + assert hit2 is True + assert called["count"] == 0 + assert [r.candidate for r in results2] == ["awq", "gptq"] + + def test_candidates_empty_rejected(self, tmp_path): + from soup_cli.utils.deploy_measure import run_measure + + tasks = _write_tasks(tmp_path) + with pytest.raises(ValueError): + run_measure( + profile_name="p", base_sha="b", + candidates=(), + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen_factory=lambda c: (lambda p: ""), + cache_path=str(tmp_path / "cache.json"), + ) + + def test_candidates_string_rejected(self, tmp_path): + from soup_cli.utils.deploy_measure import run_measure + + tasks = _write_tasks(tmp_path) + with pytest.raises(TypeError): + run_measure( + profile_name="p", base_sha="b", + candidates="awq", # type: ignore[arg-type] + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen_factory=lambda c: (lambda p: ""), + cache_path=str(tmp_path / "cache.json"), + ) + + +# --- CLI plumbing ----------------------------------------------------------- + + +class TestDeployAutopilotMeasureCLI: + def test_help_lists_measure_flag(self): + import typer + from typer.testing import CliRunner + + from soup_cli.commands.deploy import autopilot + + app = typer.Typer() + app.command()(autopilot) + runner = CliRunner() + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "--measure" in result.output + assert "--tasks" in result.output + + def test_measure_without_tasks_rejected(self, tmp_path, monkeypatch): + import typer + from typer.testing import CliRunner + + from soup_cli.commands.deploy import autopilot + + monkeypatch.chdir(tmp_path) + app = typer.Typer() + app.command()(autopilot) + runner = CliRunner() + result = runner.invoke( + app, + ["--target", "rtx-4090-24gb", "--base", "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "--measure"], + ) + assert result.exit_code != 0 + assert "--tasks" in result.output + + def test_measure_with_injected_generators(self, tmp_path, monkeypatch): + """End-to-end with injected generators bypassing real model loading.""" + import typer + from typer.testing import CliRunner + + from soup_cli.commands.deploy import autopilot + from soup_cli.utils import deploy_measure as _dm + + monkeypatch.chdir(tmp_path) + tasks = _write_tasks(tmp_path) + + # Inject generators + def before(p): + return "hello" if "hello" in p else "world" + + def after_factory(candidate): + return lambda p: ("hello" if "hello" in p else "world") + + monkeypatch.setattr( + _dm, "_DEPLOY_MEASURE_BEFORE_GEN", before, raising=False + ) + monkeypatch.setattr( + _dm, "_DEPLOY_MEASURE_AFTER_FACTORY", after_factory, raising=False + ) + # Redirect cache to tmp + monkeypatch.setenv( + "SOUP_DEPLOY_AUTOPILOT_CACHE", + str(tmp_path / "cache.json"), + ) + + app = typer.Typer() + app.command()(autopilot) + runner = CliRunner() + result = runner.invoke( + app, + [ + "--target", "rtx-4090-24gb", + "--base", "TinyLlama/TinyLlama-1.1B-Chat-v1.0", + "--recipe-out", str(tmp_path / "recipe.yaml"), + "--script-out", str(tmp_path / "deploy.sh"), + "--measure", + "--tasks", str(tasks), + "--measure-candidates", "awq,gptq", + ], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + # L4: assert the specific verdict-recommendation line + that the + # measured candidate name appears in the table. + assert "Recommended" in result.output + assert "awq" in result.output + + +# --- M3: _MAX_CANDIDATES upper bound --------------------------------------- + + +class TestMaxCandidatesCap: + def test_too_many_candidates_rejected(self, tmp_path): + from soup_cli.utils.deploy_measure import run_measure + + tasks = _write_tasks(tmp_path) + with pytest.raises(ValueError, match="too many candidates"): + run_measure( + profile_name="p", base_sha="b", + candidates=tuple(f"q{i}" for i in range(33)), + tasks_file=str(tasks), + before_gen=lambda p: "", + after_gen_factory=lambda c: (lambda p: ""), + cache_path=str(tmp_path / "cache.json"), + ) + + +# --- M4: cache symlink TOCTOU rejection ------------------------------------ + + +class TestCacheSymlinkRejection: + @pytest.mark.skipif( + os.name == "nt", reason="symlink rejection POSIX-only" + ) + def test_load_cache_rejects_symlink_target(self, tmp_path): + from soup_cli.utils.deploy_measure import load_cache + + real = tmp_path / "real_cache.json" + real.write_text('{"k": {"rows": []}}', encoding="utf-8") + link = tmp_path / "link_cache.json" + link.symlink_to(real) + # load_cache must refuse to follow the symlink — returns {} + assert load_cache(str(link)) == {} + + @pytest.mark.skipif( + os.name == "nt", reason="symlink rejection POSIX-only" + ) + def test_save_cache_refuses_symlink_target(self, tmp_path): + from soup_cli.utils.deploy_measure import save_cache + + real = tmp_path / "real_target.json" + real.write_text("{}", encoding="utf-8") + link = tmp_path / "link.json" + link.symlink_to(real) + # save_cache silently refuses on a pre-placed symlink — no exception, + # but the underlying real file must NOT be overwritten. + original = real.read_text(encoding="utf-8") + save_cache({"k": {"rows": []}}, str(link)) + assert real.read_text(encoding="utf-8") == original + + +# --- H3: render_measure_table markup-escape regression --------------------- + + +class TestRenderMeasureTableEscape: + def test_candidate_with_markup_metacharacters_escaped(self): + from io import StringIO + + from rich.console import Console + + from soup_cli.utils.deploy_measure import ( + MeasureResult, + render_measure_table, + ) + + rows = [MeasureResult("[red]evil[/]", 0.8, 0.79, -0.01, "OK")] + table = render_measure_table(rows) + buf = StringIO() + Console(file=buf, force_terminal=False, no_color=True, width=200).print( + table + ) + # The raw bracketed text must appear (escaped); the colour markup + # must NOT have been interpreted as Rich styling. + output = buf.getvalue() + assert "[red]evil[/]" in output diff --git a/tests/test_v0531_139.py b/tests/test_v0531_139.py new file mode 100644 index 0000000..df59d28 --- /dev/null +++ b/tests/test_v0531_139.py @@ -0,0 +1,365 @@ +"""v0.53.1 #139 — export_advanced_gguf live wiring tests. + +We mock subprocess invocations so tests run without a real llama.cpp build. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest + + +class TestExportAdvancedGguf: + def test_not_implemented_error_gone(self): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + # Old stub had no args. New live function should accept kwargs. + # Calling without args should now raise TypeError, not + # NotImplementedError. + with pytest.raises(TypeError): + export_advanced_gguf() # type: ignore[call-arg] + + def test_unknown_flavour_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + with pytest.raises(ValueError): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path / "out.gguf"), + flavour="EvilQ", + calibration_data=None, + llama_cpp_dir=str(tmp_path / "llama"), + ) + + def test_outside_cwd_model_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside_model_gguf" + outside.mkdir(exist_ok=True) + with pytest.raises(ValueError): + export_advanced_gguf( + model_dir=str(outside), + output_path=str(tmp_path / "out.gguf"), + flavour="UD-Q4_K_XL", + calibration_data=None, + llama_cpp_dir=str(tmp_path / "llama"), + ) + + def test_outside_cwd_output_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + with pytest.raises(ValueError): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path.parent / "out.gguf"), + flavour="UD-Q4_K_XL", + calibration_data=None, + llama_cpp_dir=str(tmp_path / "llama"), + ) + + def test_ud_requires_calibration(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + llama = tmp_path / "llama" + llama.mkdir() + # UD ladder requires calibration data + with pytest.raises(ValueError, match="calibration"): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path / "out.gguf"), + flavour="UD-Q4_K_XL", + calibration_data=None, + llama_cpp_dir=str(llama), + ) + + def test_apple_arm_no_calibration_ok(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + # Write a real safetensors-ish marker so the model dir looks usable + (model / "config.json").write_text("{}", encoding="utf-8") + llama = tmp_path / "llama" + llama.mkdir() + # Convert script presence + (llama / "convert_hf_to_gguf.py").write_text("# fake", encoding="utf-8") + out_path = tmp_path / "out.gguf" + + def fake_quant(**kwargs): + # Simulate llama-quantize writing the output file + Path(kwargs["output_path"]).write_bytes(b"FAKEGGUF") + + with patch("soup_cli.utils.gguf_quant._run_convert_to_f16") as mock_conv, \ + patch("soup_cli.utils.gguf_quant._run_quantize_binary", + side_effect=fake_quant) as mock_quant: + mock_conv.return_value = None + export_advanced_gguf( + model_dir=str(model), + output_path=str(out_path), + flavour="Q4_0_4_4", + calibration_data=None, + llama_cpp_dir=str(llama), + ) + mock_quant.assert_called_once() + # No imatrix call for Apple/ARM + assert mock_quant.call_args.kwargs.get("imatrix_path") is None + assert out_path.is_file() + + def test_ud_with_calibration_invokes_imatrix(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + (model / "config.json").write_text("{}", encoding="utf-8") + calib = tmp_path / "calib.jsonl" + calib.write_text( + '{"text": "hello world"}\n{"text": "another sample"}\n', + encoding="utf-8", + ) + llama = tmp_path / "llama" + llama.mkdir() + (llama / "convert_hf_to_gguf.py").write_text("# fake", encoding="utf-8") + + out_path = tmp_path / "out.gguf" + + def fake_quant(**kwargs): + Path(kwargs["output_path"]).write_bytes(b"FAKEGGUF") + + with patch("soup_cli.utils.gguf_quant._run_convert_to_f16") as mock_conv, \ + patch("soup_cli.utils.gguf_quant._run_imatrix") as mock_imat, \ + patch("soup_cli.utils.gguf_quant._run_quantize_binary", + side_effect=fake_quant) as mock_quant: + mock_conv.return_value = None + mock_imat.return_value = None + export_advanced_gguf( + model_dir=str(model), + output_path=str(out_path), + flavour="UD-Q4_K_XL", + calibration_data=str(calib), + llama_cpp_dir=str(llama), + ) + mock_imat.assert_called_once() + mock_quant.assert_called_once() + assert out_path.is_file() + + def test_missing_llama_cpp_dir(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + # L3: narrowed from (FileNotFoundError, ValueError) — impl raises + # ValueError from `_enforce_under_cwd_and_no_symlink` (the dir + # doesn't exist on disk yet) OR FileNotFoundError. Accept both but + # keep the union tight (no RuntimeError). + with pytest.raises((FileNotFoundError, ValueError)): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path / "out.gguf"), + flavour="Q4_0_4_4", + calibration_data=None, + llama_cpp_dir=str(tmp_path / "no_llama"), + ) + + def test_missing_convert_script(self, tmp_path, monkeypatch): + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + llama = tmp_path / "llama" + llama.mkdir() # no convert_hf_to_gguf.py + with pytest.raises((FileNotFoundError, ValueError, RuntimeError)): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path / "out.gguf"), + flavour="Q4_0_4_4", + calibration_data=None, + llama_cpp_dir=str(llama), + ) + + def test_calibration_symlink_rejected(self, tmp_path, monkeypatch): + if sys.platform == "win32": + pytest.skip("symlink rejection POSIX-only") + from soup_cli.utils.gguf_quant import export_advanced_gguf + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + real = tmp_path / "real_calib.jsonl" + real.write_text('{"text":"x"}\n', encoding="utf-8") + link = tmp_path / "link_calib.jsonl" + link.symlink_to(real) + llama = tmp_path / "llama" + llama.mkdir() + with pytest.raises(ValueError, match="symlink"): + export_advanced_gguf( + model_dir=str(model), + output_path=str(tmp_path / "out.gguf"), + flavour="UD-Q4_K_XL", + calibration_data=str(link), + llama_cpp_dir=str(llama), + ) + + def test_run_imatrix_argv_shape(self, tmp_path): + """Verify _run_imatrix builds a list-args subprocess call (no shell).""" + from soup_cli.utils.gguf_quant import _run_imatrix + + # Drop a fake binary so the resolver finds it + fake_bin = tmp_path / "llama-imatrix" + fake_bin.write_bytes(b"#!/bin/sh\nexit 0\n") + try: + os.chmod(fake_bin, 0o755) + except OSError: + pass + + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + _run_imatrix( + llama_cpp_dir=str(tmp_path), + f16_path=str(tmp_path / "f16.gguf"), + calib_data=str(tmp_path / "calib.txt"), + imatrix_out=str(tmp_path / "imatrix.dat"), + ) + assert mock_run.called + args, kwargs = mock_run.call_args + # First positional must be a list (no shell=True) + assert isinstance(args[0], list) + assert kwargs.get("shell") is not True + # All argv elements must be strings + for arg in args[0]: + assert isinstance(arg, str) + + def test_run_quantize_argv_shape(self, tmp_path): + from soup_cli.utils.gguf_quant import _run_quantize_binary + + fake_bin = tmp_path / "llama-quantize" + fake_bin.write_bytes(b"#!/bin/sh\nexit 0\n") + try: + os.chmod(fake_bin, 0o755) + except OSError: + pass + + with patch("subprocess.run") as mock_run: + mock_run.return_value = MagicMock(returncode=0) + _run_quantize_binary( + llama_cpp_dir=str(tmp_path), + f16_path=str(tmp_path / "f16.gguf"), + output_path=str(tmp_path / "out.gguf"), + flavour="UD-Q4_K_XL", + imatrix_path=str(tmp_path / "imatrix.dat"), + ) + args, kwargs = mock_run.call_args + assert isinstance(args[0], list) + assert kwargs.get("shell") is not True + # UD- prefix must be stripped before passing to llama-quantize + assert "Q4_K_XL" in args[0] + assert "UD-Q4_K_XL" not in args[0] + + +# --- H2 (TDD review): _prepare_calibration_text direct coverage ---------- + + +class TestPrepareCalibrationText: + def test_jsonl_with_text_field(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + src.write_text( + '{"text": "hello world"}\n{"text": "foo bar"}\n', + encoding="utf-8", + ) + out = _prepare_calibration_text(str(src), tmp_path) + content = out.read_text(encoding="utf-8") + assert "hello world" in content + assert "foo bar" in content + + def test_jsonl_with_prompt_alias(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + src.write_text('{"prompt": "via prompt"}\n', encoding="utf-8") + out = _prepare_calibration_text(str(src), tmp_path) + assert "via prompt" in out.read_text(encoding="utf-8") + + def test_jsonl_with_content_alias(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + src.write_text('{"content": "via content"}\n', encoding="utf-8") + out = _prepare_calibration_text(str(src), tmp_path) + assert "via content" in out.read_text(encoding="utf-8") + + def test_null_bytes_stripped(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + # JSON-escape the null byte; json.loads turns \u0000 into a real NUL + src.write_text( + '{"text": "ev\\u0000il"}\n', encoding="utf-8" + ) + out = _prepare_calibration_text(str(src), tmp_path) + content = out.read_text(encoding="utf-8") + assert "\x00" not in content + assert "evil" in content # null byte stripped, neighbours preserved + + def test_newlines_collapsed_to_spaces(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + src.write_text('{"text": "line1\\nline2"}\n', encoding="utf-8") + out = _prepare_calibration_text(str(src), tmp_path) + content = out.read_text(encoding="utf-8") + # Each row should be a single line, so we should see "line1 line2" + # on one row + the trailing newline from the writer + lines = [ln for ln in content.splitlines() if ln] + assert len(lines) == 1 + assert "line1 line2" in lines[0] + + def test_raw_text_fallback(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + # Input must NOT be named `calib.txt` because the helper writes + # its output to `/calib.txt`. Use a different name. + src = tmp_path / "raw_input.txt" + src.write_text( + "this is not json\nand neither is this\n", encoding="utf-8" + ) + out = _prepare_calibration_text(str(src), tmp_path) + content = out.read_text(encoding="utf-8") + assert "this is not json" in content + assert "neither is this" in content + + def test_zero_usable_rows_raises(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + src = tmp_path / "calib.jsonl" + # All rows are JSON but lack a usable text field + src.write_text( + '{"unrelated": 1}\n{"other_field": "x"}\n', + encoding="utf-8", + ) + with pytest.raises(ValueError, match="0 usable rows"): + _prepare_calibration_text(str(src), tmp_path) + + def test_missing_calib_file_raises(self, tmp_path): + from soup_cli.utils.gguf_quant import _prepare_calibration_text + + with pytest.raises(FileNotFoundError): + _prepare_calibration_text(str(tmp_path / "nope.jsonl"), tmp_path) diff --git a/tests/test_v0531_142.py b/tests/test_v0531_142.py new file mode 100644 index 0000000..e51aac4 --- /dev/null +++ b/tests/test_v0531_142.py @@ -0,0 +1,529 @@ +"""v0.53.1 #142 — merge --save-format + export --format torchao live wiring. + +Tests cover: +* ``merge_4bit`` validators + happy path with mocked transformers / BNB +* ``export_torchao`` validators + happy path with mocked torchao +* CLI plumbing for ``soup merge --save-format`` (4bit / 4bit_forced / fp16) +* CLI plumbing for ``soup export --format torchao --quant-config `` +* Path containment + symlink TOCTOU rejection at dispatch time +""" + +from __future__ import annotations + +import sys +from unittest.mock import MagicMock, patch + +import pytest +import typer +from typer.testing import CliRunner + +runner = CliRunner() + + +# --- merge_4bit live wiring ------------------------------------------------- + + +class TestMerge4bitWiring: + def test_imports(self): + from soup_cli.utils.save_formats import merge_4bit + + assert callable(merge_4bit) + + def test_no_longer_raises_not_implemented(self, tmp_path): + from soup_cli.utils.save_formats import merge_4bit + + # The live wiring lands in v0.53.1; calling without args used to + # raise NotImplementedError. Now it accepts named args and runs + # the path-validation path before raising on missing model dir. + with pytest.raises((TypeError, ValueError, FileNotFoundError)): + # Missing source dir → FileNotFoundError or ValueError + merge_4bit( + merged_dir=str(tmp_path / "missing"), + output_dir=str(tmp_path / "out"), + forced=False, + ) + + def test_rejects_outside_cwd_source(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import merge_4bit + + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside" + outside.mkdir(exist_ok=True) + with pytest.raises(ValueError, match="under cwd"): + merge_4bit( + merged_dir=str(outside), + output_dir=str(tmp_path / "out"), + forced=False, + ) + + def test_rejects_outside_cwd_output(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import merge_4bit + + monkeypatch.chdir(tmp_path) + src = tmp_path / "src" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="under cwd"): + merge_4bit( + merged_dir=str(src), + output_dir=str(tmp_path.parent / "out"), + forced=False, + ) + + def test_rejects_symlink_output(self, tmp_path, monkeypatch): + if sys.platform == "win32": + pytest.skip("symlink rejection POSIX-only") + from soup_cli.utils.save_formats import merge_4bit + + monkeypatch.chdir(tmp_path) + src = tmp_path / "src" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + evil_target = tmp_path / "evil_target" + evil_target.mkdir() + link = tmp_path / "out_link" + link.symlink_to(evil_target) + with pytest.raises(ValueError, match="symlink"): + merge_4bit( + merged_dir=str(src), + output_dir=str(link), + forced=False, + ) + + def test_non_bool_forced_rejected(self, tmp_path, monkeypatch): + """forced must be a real bool (project bool-before-int policy). + + Renamed from ``test_bool_forced_rejected`` to match the actual + behaviour: the guard is ``if not isinstance(forced, bool)``, so + any non-bool value (including ``"yes"`` or ``1``) is rejected, + while ``True`` / ``False`` pass through. + """ + from soup_cli.utils.save_formats import merge_4bit + + monkeypatch.chdir(tmp_path) + src = tmp_path / "src" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + with pytest.raises(TypeError): + merge_4bit( + merged_dir=str(src), + output_dir=str(tmp_path / "out"), + forced="yes", # type: ignore[arg-type] + ) + with pytest.raises(TypeError): + merge_4bit( + merged_dir=str(src), + output_dir=str(tmp_path / "out"), + forced=1, # type: ignore[arg-type] + ) + + def test_happy_path_with_mocks(self, tmp_path, monkeypatch): + from soup_cli.utils import save_formats + + monkeypatch.chdir(tmp_path) + src = tmp_path / "merged" + src.mkdir() + (src / "config.json").write_text( + '{"model_type": "llama"}', encoding="utf-8" + ) + + # Patch the from_pretrained class methods on the real transformers + # module — this avoids the `patch.dict(sys.modules, ...)` approach + # which leaks state into later test files that import real torch. + fake_model = MagicMock() + fake_tokenizer = MagicMock() + + import transformers # noqa: F401 — needed before patching attrs + out_dir = tmp_path / "out_4bit" + with patch( + "transformers.AutoModelForCausalLM.from_pretrained", + return_value=fake_model, + ), patch( + "transformers.AutoTokenizer.from_pretrained", + return_value=fake_tokenizer, + ), patch( + "transformers.BitsAndBytesConfig", + MagicMock(return_value=MagicMock()), + ): + save_formats.merge_4bit( + merged_dir=str(src), + output_dir=str(out_dir), + forced=False, + ) + + fake_model.save_pretrained.assert_called_once_with(str(out_dir)) + fake_tokenizer.save_pretrained.assert_called_once_with(str(out_dir)) + + +# --- export_torchao live wiring --------------------------------------------- + + +class TestExportTorchAOWiring: + def test_imports(self): + from soup_cli.utils.save_formats import export_torchao + + assert callable(export_torchao) + + def test_invalid_scheme_rejected(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import export_torchao + + monkeypatch.chdir(tmp_path) + src = tmp_path / "src" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="not supported"): + export_torchao( + model_dir=str(src), + output_dir=str(tmp_path / "out"), + scheme="EvilScheme", + ) + + def test_rejects_outside_cwd_model(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import export_torchao + + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside_torchao" + outside.mkdir(exist_ok=True) + with pytest.raises(ValueError, match="under cwd"): + export_torchao( + model_dir=str(outside), + output_dir=str(tmp_path / "out"), + scheme="Int4WeightOnly", + ) + + def test_rejects_outside_cwd_output(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import export_torchao + + monkeypatch.chdir(tmp_path) + src = tmp_path / "src" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="under cwd"): + export_torchao( + model_dir=str(src), + output_dir=str(tmp_path.parent / "out"), + scheme="Int4WeightOnly", + ) + + def test_happy_path_with_mocks(self, tmp_path, monkeypatch): + from soup_cli.utils import save_formats + + monkeypatch.chdir(tmp_path) + src = tmp_path / "model" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + + fake_model = MagicMock() + fake_tokenizer = MagicMock() + + # Build a minimal in-process torchao stand-in. We do swap sys.modules + # for torchao + torchao.quantization (real torchao isn't installed in + # CI), but we patch transformers attrs directly to avoid the torch + # reload issue that breaks downstream tests. + fake_torchao = MagicMock() + fake_torchao.quantization.Int4WeightOnlyConfig.return_value = MagicMock() + fake_torchao.quantize_ = MagicMock() + + original_torchao = sys.modules.get("torchao") + original_torchao_q = sys.modules.get("torchao.quantization") + sys.modules["torchao"] = fake_torchao + sys.modules["torchao.quantization"] = fake_torchao.quantization + try: + with patch( + "transformers.AutoModelForCausalLM.from_pretrained", + return_value=fake_model, + ), patch( + "transformers.AutoTokenizer.from_pretrained", + return_value=fake_tokenizer, + ): + out_dir = tmp_path / "out_torchao" + save_formats.export_torchao( + model_dir=str(src), + output_dir=str(out_dir), + scheme="Int4WeightOnly", + ) + finally: + if original_torchao is None: + sys.modules.pop("torchao", None) + else: + sys.modules["torchao"] = original_torchao + if original_torchao_q is None: + sys.modules.pop("torchao.quantization", None) + else: + sys.modules["torchao.quantization"] = original_torchao_q + + fake_model.save_pretrained.assert_called_once() + + +# --- CLI plumbing for `soup merge --save-format` ---------------------------- + + +class TestMergeSaveFormatCLI: + def test_save_format_help_lists_flag(self): + from soup_cli.commands.merge import merge + + app = typer.Typer() + app.command()(merge) + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "--save-format" in result.output + + def test_invalid_save_format(self, tmp_path, monkeypatch): + from soup_cli.commands.merge import merge + + monkeypatch.chdir(tmp_path) + adapter = tmp_path / "adapter" + adapter.mkdir() + (adapter / "adapter_config.json").write_text( + '{"base_model_name_or_path": "some/base"}', encoding="utf-8" + ) + + app = typer.Typer() + app.command()(merge) + result = runner.invoke( + app, + [ + "--adapter", str(adapter), + "--save-format", "weird", + "--output", str(tmp_path / "out"), + ], + ) + assert result.exit_code != 0 + assert "save_format" in result.output or "save-format" in result.output + + +# --- CLI plumbing for `soup export --format torchao` ------------------------ + + +class TestExportTorchaoCLI: + def test_torchao_in_supported_formats(self): + from soup_cli.commands import export as export_mod + + assert "torchao" in export_mod.SUPPORTED_FORMATS + + def test_torchao_help_lists_quant_config(self): + from soup_cli.commands.export import export + + app = typer.Typer() + app.command()(export) + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "--quant-config" in result.output + + def test_torchao_requires_quant_config(self, tmp_path, monkeypatch): + from soup_cli.commands.export import export + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + (model / "config.json").write_text("{}", encoding="utf-8") + + app = typer.Typer() + app.command()(export) + result = runner.invoke( + app, + [ + "--model", str(model), + "--format", "torchao", + "--output", str(tmp_path / "out"), + ], + ) + assert result.exit_code != 0 + assert "--quant-config" in result.output or "quant_config" in result.output + + def test_torchao_quant_config_outside_cwd_rejected(self, tmp_path, monkeypatch): + from soup_cli.commands.export import export + + monkeypatch.chdir(tmp_path) + model = tmp_path / "model" + model.mkdir() + (model / "config.json").write_text("{}", encoding="utf-8") + + outside_yaml = tmp_path.parent / "evil_quant.yaml" + outside_yaml.write_text("scheme: Int4WeightOnly\n", encoding="utf-8") + + app = typer.Typer() + app.command()(export) + result = runner.invoke( + app, + [ + "--model", str(model), + "--format", "torchao", + "--quant-config", str(outside_yaml), + "--output", str(tmp_path / "out"), + ], + ) + assert result.exit_code != 0 + assert "under cwd" in result.output or "cwd" in result.output.lower() + + +# --- Path-containment helpers exposed by save_formats ----------------------- + + +class TestValidateQuantConfigPath: + def test_existing_shape_validators_still_work(self): + from soup_cli.utils.save_formats import validate_quant_config_path + + assert validate_quant_config_path("config.yaml") == "config.yaml" + + def test_null_byte_rejected(self): + from soup_cli.utils.save_formats import validate_quant_config_path + + with pytest.raises(ValueError): + validate_quant_config_path("ev\x00il.yaml") + + def test_load_quant_config_yaml_happy(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + yaml_path = tmp_path / "q.yaml" + yaml_path.write_text("scheme: Int4WeightOnly\n", encoding="utf-8") + data = load_quant_config(str(yaml_path)) + assert data == {"scheme": "Int4WeightOnly"} + + def test_load_quant_config_yaml_outside_cwd(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside.yaml" + outside.write_text("scheme: Int4WeightOnly\n", encoding="utf-8") + with pytest.raises(ValueError, match="under cwd"): + load_quant_config(str(outside)) + + def test_load_quant_config_yaml_symlink(self, tmp_path, monkeypatch): + if sys.platform == "win32": + pytest.skip("symlink rejection POSIX-only") + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + real = tmp_path / "real.yaml" + real.write_text("scheme: Int4WeightOnly\n", encoding="utf-8") + link = tmp_path / "link.yaml" + link.symlink_to(real) + with pytest.raises(ValueError, match="symlink"): + load_quant_config(str(link)) + + def test_load_quant_config_yaml_size_cap(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + big = tmp_path / "big.yaml" + # 300 KB blob — exceeds 256 KB cap + big.write_text("scheme: Int4WeightOnly\n" + ("x" * (300 * 1024)), encoding="utf-8") + with pytest.raises(ValueError, match="too large"): + load_quant_config(str(big)) + + def test_load_quant_config_yaml_invalid_extension(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + bad = tmp_path / "config.txt" + bad.write_text("scheme: Int4WeightOnly\n", encoding="utf-8") + with pytest.raises(ValueError, match="extension"): + load_quant_config(str(bad)) + + def test_load_quant_config_yaml_missing(self, tmp_path, monkeypatch): + from soup_cli.utils.save_formats import load_quant_config + + monkeypatch.chdir(tmp_path) + with pytest.raises(FileNotFoundError): + load_quant_config(str(tmp_path / "missing.yaml")) + + +# --- H1 (TDD review): torchao kwarg allowlist ------------------------------ + + +class TestTorchAOKwargAllowlist: + def _setup(self, tmp_path): + src = tmp_path / "model" + src.mkdir() + (src / "config.json").write_text("{}", encoding="utf-8") + return src + + def _run(self, src, tmp_path, scheme, quant_config_data): + from soup_cli.utils import save_formats + + fake_model = MagicMock() + fake_tokenizer = MagicMock() + fake_torchao = MagicMock() + fake_torchao.quantization.Int4WeightOnlyConfig.return_value = MagicMock() + fake_torchao.quantization.NVFP4Config.return_value = MagicMock() + fake_torchao.quantize_ = MagicMock() + + original_torchao = sys.modules.get("torchao") + original_torchao_q = sys.modules.get("torchao.quantization") + sys.modules["torchao"] = fake_torchao + sys.modules["torchao.quantization"] = fake_torchao.quantization + try: + with patch( + "transformers.AutoModelForCausalLM.from_pretrained", + return_value=fake_model, + ), patch( + "transformers.AutoTokenizer.from_pretrained", + return_value=fake_tokenizer, + ): + save_formats.export_torchao( + model_dir=str(src), + output_dir=str(tmp_path / "out"), + scheme=scheme, + quant_config_data=quant_config_data, + ) + finally: + if original_torchao is None: + sys.modules.pop("torchao", None) + else: + sys.modules["torchao"] = original_torchao + if original_torchao_q is None: + sys.modules.pop("torchao.quantization", None) + else: + sys.modules["torchao.quantization"] = original_torchao_q + + def test_dunder_key_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + src = self._setup(tmp_path) + with pytest.raises(ValueError, match="not allowed"): + self._run(src, tmp_path, "Int4WeightOnly", {"__class__": "evil"}) + + def test_unknown_key_on_int4_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + src = self._setup(tmp_path) + with pytest.raises(ValueError, match="not allowed"): + self._run(src, tmp_path, "Int4WeightOnly", {"unknown_key": 1}) + + def test_allowed_int4_group_size(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + src = self._setup(tmp_path) + # group_size is on the Int4 allowlist — should not raise + self._run(src, tmp_path, "Int4WeightOnly", {"group_size": 32}) + + def test_nvfp4_rejects_any_extra_kwargs(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + src = self._setup(tmp_path) + with pytest.raises(ValueError, match="not allowed"): + self._run(src, tmp_path, "NVFP4", {"group_size": 32}) + + +# --- M2 (TDD review): config.json symlink TOCTOU guard --------------------- + + +class TestDetectPrequantizedSymlinkRejection: + @pytest.mark.skipif( + sys.platform == "win32", reason="symlink rejection POSIX-only" + ) + def test_config_json_symlink_returns_none(self, tmp_path, monkeypatch): + """Security regression — `config.json` as a symlink is refused.""" + from soup_cli.autopilot.decisions import detect_prequantized_format_from_path + + monkeypatch.chdir(tmp_path) + model_dir = tmp_path / "model" + model_dir.mkdir() + real_config = tmp_path / "real_config.json" + real_config.write_text( + '{"quantization_config": {"quant_method": "gptq"}}', + encoding="utf-8", + ) + link = model_dir / "config.json" + link.symlink_to(real_config) + # Symlink config.json → soft-probe returns None instead of reading + assert detect_prequantized_format_from_path("./model") is None + diff --git a/tests/test_v0531_82.py b/tests/test_v0531_82.py new file mode 100644 index 0000000..ca4ce36 --- /dev/null +++ b/tests/test_v0531_82.py @@ -0,0 +1,296 @@ +"""v0.53.1 #82 — Autopilot pre-quantized base detection. + +Tests for ``detect_prequantized_format`` + ``decide_quantization`` accepting an +optional pre-quantized hint so a base like ``TheBloke/Llama-2-7B-Chat-GPTQ`` is +recommended ``gptq`` instead of ``4bit``-on-top-of-already-quantized. +""" + +from __future__ import annotations + +import json + +import pytest + +# --- detect_prequantized_format --------------------------------------------- + + +class TestDetectPrequantizedFormat: + def test_none_for_clean_name(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("meta-llama/Llama-3.1-8B") is None + + def test_gptq_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert ( + detect_prequantized_format("TheBloke/Llama-2-7B-Chat-GPTQ") == "gptq" + ) + + def test_gptq_lowercase(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("some-org/llama-7b-gptq") == "gptq" + + def test_awq_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("TheBloke/Mistral-7B-AWQ") == "awq" + + def test_hqq_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + result = detect_prequantized_format("mobiuslabsgmbh/Llama-3.1-8B-HQQ-4bit") + assert result == "hqq:4bit" + + def test_hqq_explicit_bits(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert ( + detect_prequantized_format("some-org/model-HQQ-2bit") == "hqq:2bit" + ) + + def test_aqlm_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("ISTA-DASLab/Llama-3-8B-AQLM") == "aqlm" + + def test_eetq_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("some-org/model-EETQ") == "eetq" + + def test_fp8_name_match(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert ( + detect_prequantized_format("neuralmagic/Meta-Llama-3-8B-FP8") == "fp8" + ) + + def test_config_quantization_method(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "gptq", "bits": 4}} + assert detect_prequantized_format("clean/name", cfg) == "gptq" + + def test_config_overrides_clean_name(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "awq"}} + assert detect_prequantized_format("meta/clean-llama", cfg) == "awq" + + def test_config_hqq_with_bits(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "hqq", "bits": 2}} + assert detect_prequantized_format("clean/name", cfg) == "hqq:2bit" + + def test_config_unknown_method(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "weirdq"}} + assert detect_prequantized_format("clean/name", cfg) is None + + def test_config_non_dict_quantization_config(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": "gptq"} # malformed + assert detect_prequantized_format("clean/name", cfg) is None + + def test_config_non_dict_root(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("clean/name", "not-a-dict") is None + + def test_empty_name_raises(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + with pytest.raises(ValueError): + detect_prequantized_format("") + + def test_null_byte_name_raises(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + with pytest.raises(ValueError): + detect_prequantized_format("evil\x00name") + + def test_non_string_name_raises(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + with pytest.raises(TypeError): + detect_prequantized_format(123) + + def test_bool_name_raises(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + with pytest.raises(TypeError): + detect_prequantized_format(True) + + def test_word_boundary_no_substring(self): + from soup_cli.autopilot.decisions import detect_prequantized_format + + # 'agptqa' should NOT match — word boundary + assert detect_prequantized_format("some-org/agptqa-model") is None + + def test_config_probe_path(self, tmp_path, monkeypatch): + from soup_cli.autopilot.decisions import detect_prequantized_format_from_path + + monkeypatch.chdir(tmp_path) + config_dir = tmp_path / "model" + config_dir.mkdir() + cfg_file = config_dir / "config.json" + cfg_file.write_text(json.dumps({ + "quantization_config": {"quant_method": "gptq", "bits": 4}, + }), encoding="utf-8") + + assert detect_prequantized_format_from_path("./model") == "gptq" + + def test_config_probe_path_missing(self, tmp_path, monkeypatch): + from soup_cli.autopilot.decisions import detect_prequantized_format_from_path + + monkeypatch.chdir(tmp_path) + assert detect_prequantized_format_from_path("./nope") is None + + def test_config_probe_malformed_json(self, tmp_path, monkeypatch): + from soup_cli.autopilot.decisions import detect_prequantized_format_from_path + + monkeypatch.chdir(tmp_path) + config_dir = tmp_path / "model" + config_dir.mkdir() + (config_dir / "config.json").write_text("{not json", encoding="utf-8") + + # Should not raise; returns None + assert detect_prequantized_format_from_path("./model") is None + + def test_config_probe_path_outside_cwd_returns_none( + self, tmp_path, monkeypatch, + ): + """Security review H2 — out-of-cwd model_dir silently falls through.""" + from soup_cli.autopilot.decisions import detect_prequantized_format_from_path + + monkeypatch.chdir(tmp_path) + outside = tmp_path.parent / "outside_probe" + outside.mkdir(exist_ok=True) + (outside / "config.json").write_text( + '{"quantization_config": {"quant_method": "gptq"}}', + encoding="utf-8", + ) + # Out-of-cwd path: soft-probe returns None (no read attempted) + assert detect_prequantized_format_from_path(str(outside)) is None + + +# --- decide_quantization with prequantized hint ----------------------------- + + +class TestDecideQuantizationPrequantized: + def test_prequantized_hint_returned(self): + from soup_cli.autopilot.decisions import decide_quantization + + # Even with plenty VRAM, prequantized hint takes precedence + result = decide_quantization( + model_params_b=7.0, vram_gb=80.0, prequantized="gptq", + ) + assert result == "gptq" + + def test_prequantized_awq(self): + from soup_cli.autopilot.decisions import decide_quantization + + assert ( + decide_quantization( + model_params_b=7.0, vram_gb=24.0, prequantized="awq", + ) + == "awq" + ) + + def test_prequantized_hqq(self): + from soup_cli.autopilot.decisions import decide_quantization + + assert ( + decide_quantization( + model_params_b=7.0, vram_gb=24.0, prequantized="hqq:4bit", + ) + == "hqq:4bit" + ) + + def test_no_prequantized_falls_through_to_vram_logic(self): + from soup_cli.autopilot.decisions import decide_quantization + + # Same as legacy behaviour when prequantized=None + assert ( + decide_quantization(model_params_b=7.0, vram_gb=80.0) + == "none" + ) + + def test_invalid_prequantized_raises(self): + from soup_cli.autopilot.decisions import decide_quantization + + with pytest.raises(ValueError): + decide_quantization( + model_params_b=7.0, vram_gb=24.0, prequantized="evilq", + ) + + def test_prequantized_bool_rejected(self): + from soup_cli.autopilot.decisions import decide_quantization + + with pytest.raises(TypeError): + decide_quantization( + model_params_b=7.0, vram_gb=24.0, prequantized=True, + ) + + def test_prequantized_null_byte_rejected(self): + from soup_cli.autopilot.decisions import decide_quantization + + with pytest.raises(ValueError): + decide_quantization( + model_params_b=7.0, vram_gb=24.0, prequantized="ev\x00il", + ) + + def test_prequantized_none_legacy(self): + from soup_cli.autopilot.decisions import decide_quantization + + # Explicit None == no hint == legacy behaviour + assert ( + decide_quantization( + model_params_b=15.0, vram_gb=24.0, prequantized=None, + ) + == "4bit" + ) + + def test_mxfp4_name_match(self): + """L2: mxfp4 word-boundary regex coverage.""" + from soup_cli.autopilot.decisions import detect_prequantized_format + + assert detect_prequantized_format("some-org/model-MXFP4") == "mxfp4" + assert detect_prequantized_format("some-org/notmxfp4good") is None + + def test_bnb_4bit_alias_via_config(self): + """L5: config quant_method=bitsandbytes_4bit aliases to '4bit'.""" + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "bitsandbytes_4bit"}} + assert detect_prequantized_format("clean/name", cfg) == "4bit" + + def test_bnb_8bit_alias_via_config(self): + """L5: config quant_method=bnb_8bit aliases to '8bit'.""" + from soup_cli.autopilot.decisions import detect_prequantized_format + + cfg = {"quantization_config": {"quant_method": "bnb_8bit"}} + assert detect_prequantized_format("clean/name", cfg) == "8bit" + + def test_decide_quantization_accepts_4bit_alias(self): + """L5: ``prequantized='4bit'`` short-circuits VRAM heuristic.""" + from soup_cli.autopilot.decisions import decide_quantization + + # Even with plenty of VRAM, '4bit' wins + assert ( + decide_quantization( + model_params_b=7.0, vram_gb=80.0, prequantized="4bit", + ) + == "4bit" + ) + assert ( + decide_quantization( + model_params_b=7.0, vram_gb=80.0, prequantized="8bit", + ) + == "8bit" + )