diff --git a/soup_cli/commands/deploy.py b/soup_cli/commands/deploy.py index 254fb01..f7621d2 100644 --- a/soup_cli/commands/deploy.py +++ b/soup_cli/commands/deploy.py @@ -559,6 +559,122 @@ def hf_space( ) +@app.command(name="autopilot") +def autopilot( + target: Optional[str] = typer.Option( + None, + "--target", + "-t", + help="Profile name (e.g. mac-m3, rtx-4090-24gb, ollama-local).", + ), + base: str = typer.Option( + "meta-llama/Llama-3.2-1B", + "--base", + "-b", + help="Base model HF repo id or local path to embed in the recipe.", + ), + recipe_out: str = typer.Option( + "deploy_autopilot.yaml", + "--recipe-out", + help="Where to write the rendered soup.yaml recipe (under cwd).", + ), + script_out: str = typer.Option( + "deploy_autopilot.sh", + "--script-out", + help="Where to write the planned deploy shell script (under cwd).", + ), + output_dir: str = typer.Option( + "./output", + "--output-dir", + help="Recipe's training output directory.", + ), + list_targets: bool = typer.Option( + False, + "--list", + "-l", + help="List all known deploy profiles.", + ), +): + """Pick PEFT + quant + spec-decoding combo for a hardware target. + + Writes a ready-to-train ``soup.yaml`` recipe and a planned deploy + shell script. Live Quant-Lobotomy measurement deferred to v0.46.1. + """ + from soup_cli.utils.deploy_autopilot import ( + get_profile, + list_profiles, + write_deploy_script, + write_recipe, + ) + + if list_targets: + table = Table(title="Deploy Autopilot Profiles") + table.add_column("Name", style="bold cyan") + table.add_column("Runtime", style="magenta") + table.add_column("Quant", style="green") + table.add_column("PEFT", style="yellow") + table.add_column("Description") + for profile in list_profiles().values(): + table.add_row( + profile.name, + profile.runtime, + profile.quant, + profile.peft, + profile.description, + ) + console.print(table) + raise typer.Exit(0) + + if not target: + console.print("[red]--target is required (or use --list).[/]") + raise typer.Exit(2) + + try: + profile = get_profile(target) + except (KeyError, ValueError, TypeError) as exc: + from rich.markup import escape + + console.print(f"[red]Unknown profile:[/] {escape(str(exc))}") + console.print( + "[dim]Run [bold]soup deploy autopilot --list[/] to see options.[/]" + ) + raise typer.Exit(2) from exc + + try: + recipe_path = write_recipe( + profile, base=base, output_dir=output_dir, recipe_path=recipe_out + ) + script_path = write_deploy_script( + profile, model_path=output_dir, script_path=script_out + ) + except (ValueError, TypeError) as exc: + from rich.markup import escape + + console.print(f"[red]Autopilot failed:[/] {escape(str(exc))}") + raise typer.Exit(1) from exc + + from rich.markup import escape as _escape + + console.print( + Panel( + f"Target: [bold]{_escape(profile.name)}[/]\n" + f"Runtime: [bold]{_escape(profile.runtime)}[/]\n" + f"Quant: [bold]{_escape(profile.quant)}[/]\n" + f"PEFT: [bold]{_escape(profile.peft)}[/]\n" + f"Spec dec: [bold]{profile.spec_decoding}[/]\n" + f"Recipe: [bold]{_escape(recipe_path)}[/]\n" + f"Script: [bold]{_escape(script_path)}[/]", + title="[bold green]Deploy Autopilot[/]", + ) + ) + if profile.notes: + console.print(f"[dim]Notes: {_escape(profile.notes)}[/]") + console.print( + "[yellow]Note:[/] Live Quant-Lobotomy auto-measure deferred to v0.46.1; " + "this release writes the canonical combo + recipe." + ) + + def _auto_detect_template() -> Optional[str]: """Try to infer chat template from soup.yaml in cwd.""" from soup_cli.utils.ollama import infer_chat_template diff --git a/soup_cli/utils/deploy_autopilot.py b/soup_cli/utils/deploy_autopilot.py new file mode 100644 index 0000000..2ef1f99 --- /dev/null +++ b/soup_cli/utils/deploy_autopilot.py @@ -0,0 +1,394 @@ +"""v0.46.0 Part A — On-Device Deploy Autopilot. + +Closed allowlist of deploy-target profiles. Each profile maps a hardware / +runtime target (mac-m3, rtx-4090-24gb, iphone-16, ollama-local, ...) to a +PEFT + quantisation + speculative-decoding combo plus an output recipe and +a deploy shell script. + +Live wiring into the v0.26.0 Quant-Lobotomy Checker (so the autopilot +actually *measures* OK/MINOR/MAJOR before picking a quant) is deferred to +v0.46.1; this release ships the schema + the canonical combo table + the +recipe-yaml / deploy-script writers so users have a reproducible artifact +the moment they run ``soup deploy autopilot --target mac-m3``. + +The catalog is frozen at import time (``MappingProxyType``); name lookup is +case-insensitive over a strict kebab-case regex (matches v0.45.0 Part A +``register_plugin`` policy). +""" + +from __future__ import annotations + +import os +import re +import shlex +from dataclasses import dataclass +from types import MappingProxyType +from typing import Mapping, Optional, Tuple + +from soup_cli.utils.paths import is_under_cwd + +_PROFILE_NAME_RE = re.compile(r"^[a-z0-9][a-z0-9\-]{0,31}$") +_MAX_BASE_LEN = 200 +_MAX_OUTPUT_PATH_LEN = 4096 + +# Allowlist for the quant field — matches v0.38.0 / v0.40.5 Quant Menu names +# plus the lightweight ``none`` / ``4bit`` / ``8bit`` legacy values. We keep +# this small on purpose: a profile that picks an unknown quant is a bug, not +# a free-form string. +_ALLOWED_QUANT = frozenset( + {"none", "4bit", "8bit", "gptq", "awq", "fp8", "mxfp4", "hqq:4bit", "hqq:8bit"} +) +_ALLOWED_PEFT = frozenset({"lora", "dora", "qlora", "full"}) +_ALLOWED_RUNTIME = frozenset( + {"transformers", "vllm", "sglang", "mlx", "ollama", "lm-studio", "executorch"} +) + + +@dataclass(frozen=True) +class DeployProfile: + """One canonical deploy-target combo entry.""" + + name: str + description: str + runtime: str + quant: str + peft: str + spec_decoding: bool + recommended_max_length: int + notes: str + + +def _make( + name: str, + description: str, + runtime: str, + quant: str, + peft: str, + spec_decoding: bool, + recommended_max_length: int, + notes: str = "", +) -> DeployProfile: + if not isinstance(name, str) or not _PROFILE_NAME_RE.match(name): + raise ValueError( + "profile name must be kebab-case ([a-z0-9][a-z0-9-]{0,31})" + ) + if runtime not in _ALLOWED_RUNTIME: + raise ValueError(f"runtime {runtime!r} not in allowlist") + if quant not in _ALLOWED_QUANT: + raise ValueError(f"quant {quant!r} not in allowlist") + if peft not in _ALLOWED_PEFT: + raise ValueError(f"peft {peft!r} not in allowlist") + if not isinstance(spec_decoding, bool): + raise TypeError("spec_decoding must be bool") + if isinstance(recommended_max_length, bool) or not isinstance( + recommended_max_length, int + ): + raise TypeError("recommended_max_length must be int (not bool)") + if not (64 <= recommended_max_length <= 1_048_576): + raise ValueError("recommended_max_length must be in [64, 1048576]") + for label, txt in (("description", description), ("notes", notes)): + if not isinstance(txt, str) or "\x00" in txt: + raise ValueError(f"{label} must be NUL-free string") + if len(txt) > 512: + raise ValueError(f"{label} exceeds 512 chars") + return DeployProfile( + name=name, + description=description, + runtime=runtime, + quant=quant, + peft=peft, + spec_decoding=spec_decoding, + recommended_max_length=recommended_max_length, + notes=notes, + ) + + +_BUILTIN: Mapping[str, DeployProfile] = MappingProxyType( + { + "mac-m3": _make( + "mac-m3", "Apple Silicon M3 / M3 Pro / M3 Max — MLX inference", + "mlx", "4bit", "lora", False, 8192, + "MLX backend handles quantisation; LoRA stays as adapter", + ), + "mac-m4-pro": _make( + "mac-m4-pro", "Apple Silicon M4 / M4 Pro — MLX inference", + "mlx", "4bit", "lora", False, 16384, + "Higher memory band; doubles default context window", + ), + "rtx-3060-12gb": _make( + "rtx-3060-12gb", "Consumer NVIDIA 12GB — 4bit + speculative decoding", + "transformers", "4bit", "qlora", True, 4096, + ), + "rtx-4090-24gb": _make( + "rtx-4090-24gb", "Consumer NVIDIA 24GB — AWQ + vLLM", + "vllm", "awq", "lora", True, 8192, + ), + "iphone-16": _make( + "iphone-16", "iPhone 16 / 16 Pro — ExecuTorch on-device", + "executorch", "4bit", "qlora", False, 2048, + "Plan-only; ExecuTorch packaging lands in v0.54.0 Part D", + ), + "pixel-9": _make( + "pixel-9", "Pixel 9 — ExecuTorch / AICore on-device", + "executorch", "4bit", "qlora", False, 2048, + "Plan-only; export pipeline lands in v0.54.0", + ), + "ollama-local": _make( + "ollama-local", "Local Ollama / llama.cpp via GGUF", + "ollama", "4bit", "lora", False, 4096, + ), + "lm-studio": _make( + "lm-studio", "LM Studio desktop app — GGUF model", + "lm-studio", "4bit", "lora", False, 4096, + ), + "runpod-a100": _make( + "runpod-a100", "RunPod A100 40GB — bf16 vLLM with speculative", + "vllm", "none", "lora", True, 32768, + ), + "hf-jobs-h100": _make( + "hf-jobs-h100", "HF Jobs H100 80GB — FP8 + vLLM prefix cache", + "vllm", "fp8", "lora", True, 65536, + ), + } +) + + +def list_profiles() -> Mapping[str, DeployProfile]: + """Return an immutable view of the deploy profile registry.""" + return _BUILTIN + + +def get_profile(name: str) -> DeployProfile: + """Return the profile for ``name`` (case-insensitive).""" + if not isinstance(name, str): + raise TypeError("name must be a string") + canonical = name.strip().lower() + if not canonical or "\x00" in canonical: + raise ValueError("name must be a non-empty NUL-free string") + if canonical not in _BUILTIN: + raise KeyError(canonical) + return _BUILTIN[canonical] + + +def has_profile(name: str) -> bool: + """True when ``name`` resolves to a known profile.""" + if not isinstance(name, str): + return False + return name.strip().lower() in _BUILTIN + + +def _validate_base(base: str) -> str: + if not isinstance(base, str): + raise TypeError("base must be a string") + if not base or "\x00" in base or "\n" in base or "\r" in base: + raise ValueError("base must be a non-empty single-line NUL-free string") + if len(base) > _MAX_BASE_LEN: + raise ValueError(f"base exceeds {_MAX_BASE_LEN} chars") + return base + + +def render_recipe_yaml(profile: DeployProfile, base: str, output_dir: str) -> str: + """Render a ready-to-train ``soup.yaml`` for ``profile`` + ``base``. + + Returns the YAML *text*; callers handle write-out with their own + containment checks. The output_dir is validated as a non-empty + NUL/newline-free string but is NOT path-realpath'd here so the recipe + can be re-used from a different working dir later. + """ + if not isinstance(profile, DeployProfile): + raise TypeError("profile must be a DeployProfile") + base = _validate_base(base) + if not isinstance(output_dir, str): + raise TypeError("output_dir must be a string") + if ( + not output_dir + or "\x00" in output_dir + or "\n" in output_dir + or "\r" in output_dir + ): + raise ValueError("output_dir must be a non-empty single-line NUL-free string") + if len(output_dir) > _MAX_OUTPUT_PATH_LEN: + raise ValueError(f"output_dir exceeds {_MAX_OUTPUT_PATH_LEN} chars") + peft_section = "" + if profile.peft in ("lora", "qlora", "dora"): + peft_section = ( + " lora:\n" + " r: 16\n" + " alpha: 32\n" + " dropout: 0.05\n" + ) + if profile.peft == "dora": + peft_section += " use_dora: true\n" + lines = [ + f"# Soup deploy autopilot — profile: {profile.name}", + f"# {profile.description}", + f"base: {base}", + "task: sft", + f"backend: {'mlx' if profile.runtime == 'mlx' else 'transformers'}", + "data:", + " train: ./data.jsonl", + " format: auto", + f" max_length: {profile.recommended_max_length}", + "training:", + " epochs: 3", + " lr: 2.0e-5", + " batch_size: auto", + f" quantization: {profile.quant}", + ] + if peft_section: + lines.append(peft_section.rstrip("\n")) + lines.append(f"output: {output_dir}") + return "\n".join(lines) + "\n" + + +def render_deploy_script(profile: DeployProfile, model_path: str) -> str: + """Render the shell script that takes the trained model to the target. + + The script is intentionally a stub: it prints the planned ``soup serve`` + or ``soup deploy`` command rather than executing it, so the user keeps + the final approval gate. ``model_path`` is shell-escaped via + ``shlex.quote`` so a crafted path cannot inject shell syntax. + """ + if not isinstance(profile, DeployProfile): + raise TypeError("profile must be a DeployProfile") + if not isinstance(model_path, str): + raise TypeError("model_path must be a string") + if ( + not model_path + or "\x00" in model_path + or "\n" in model_path + or "\r" in model_path + ): + raise ValueError("model_path must be non-empty single-line NUL-free string") + if len(model_path) > _MAX_OUTPUT_PATH_LEN: + raise ValueError(f"model_path exceeds {_MAX_OUTPUT_PATH_LEN} chars") + quoted = shlex.quote(model_path) + header = ( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + f"# Soup deploy autopilot — profile: {profile.name}\n" + f"# {profile.description}\n" + f"MODEL={quoted}\n" + ) + if profile.runtime == "ollama": + body = ( + 'NAME="soup-${USER:-local}-$(basename \"$MODEL\")"\n' + 'echo "Planned: soup deploy ollama --model $MODEL --name $NAME"\n' + ) + elif profile.runtime == "lm-studio": + body = 'echo "Planned: copy GGUF to ~/.cache/lm-studio/models/ then load in UI"\n' + elif profile.runtime == "executorch": + body = ( + 'echo "Plan-only: ExecuTorch packaging not yet implemented (v0.54.0)"\n' + 'echo "Target: $MODEL"\n' + ) + elif profile.runtime == "mlx": + body = 'echo "Planned: soup serve --backend mlx --model $MODEL"\n' + else: + spec_flag = " --auto-spec" if profile.spec_decoding else "" + body = ( + 'echo "Planned: soup serve --backend ' + f'{profile.runtime} --model $MODEL{spec_flag}"\n' + ) + return header + body + + +def _reject_symlink_target(path: str, label: str) -> None: + """Reject if ``path`` already exists and is a symlink (TOCTOU defence). + + Matches v0.33.0 #22 / v0.43.0 Part C / v0.44.0 Part B policy. + """ + import stat as _stat + + try: + st = os.lstat(path) + except FileNotFoundError: + return + if _stat.S_ISLNK(st.st_mode): + raise ValueError( + f"{label} must not be a symlink: {os.path.basename(path)}" + ) + + +def write_recipe( + profile: DeployProfile, + base: str, + output_dir: str, + recipe_path: str, +) -> str: + """Write the recipe YAML under cwd; returns the realpath written.""" + if not isinstance(recipe_path, str): + raise TypeError("recipe_path must be a string") + if not recipe_path or "\x00" in recipe_path: + raise ValueError("recipe_path must be non-empty NUL-free string") + if len(recipe_path) > _MAX_OUTPUT_PATH_LEN: + raise ValueError(f"recipe_path exceeds {_MAX_OUTPUT_PATH_LEN} chars") + if not is_under_cwd(recipe_path): + raise ValueError( + f"recipe_path must stay under cwd: {os.path.basename(recipe_path)}" + ) + _reject_symlink_target(recipe_path, "recipe_path") + text = render_recipe_yaml(profile, base=base, output_dir=output_dir) + real = os.path.realpath(recipe_path) + os.makedirs(os.path.dirname(real) or ".", exist_ok=True) + with open(real, "w", encoding="utf-8") as fh: + fh.write(text) + return real + + +def write_deploy_script( + profile: DeployProfile, model_path: str, script_path: str +) -> str: + """Write the deploy bash script under cwd; returns the realpath written.""" + if not isinstance(script_path, str): + raise TypeError("script_path must be a string") + if not script_path or "\x00" in script_path: + raise ValueError("script_path must be non-empty NUL-free string") + if len(script_path) > _MAX_OUTPUT_PATH_LEN: + raise ValueError(f"script_path exceeds {_MAX_OUTPUT_PATH_LEN} chars") + if not is_under_cwd(script_path): + raise ValueError( + f"script_path must stay under cwd: {os.path.basename(script_path)}" + ) + _reject_symlink_target(script_path, "script_path") + text = render_deploy_script(profile, model_path=model_path) + real = os.path.realpath(script_path) + os.makedirs(os.path.dirname(real) or ".", exist_ok=True) + with open(real, "w", encoding="utf-8") as fh: + fh.write(text) + if os.name != "nt": + try: + os.chmod(real, 0o755) + except OSError: + pass + return real + + +def autopilot_artifacts( + profile_name: str, + base: str, + output_dir: str, + model_path: Optional[str] = None, +) -> Tuple[str, str]: + """Return (recipe_yaml_text, deploy_script_text) for a profile. + + Helper for callers that want both artifacts in-memory without writing + them out. ``model_path`` defaults to ``output_dir`` when omitted. + """ + profile = get_profile(profile_name) + recipe = render_recipe_yaml(profile, base=base, output_dir=output_dir) + script = render_deploy_script(profile, model_path=model_path or output_dir) + return recipe, script + + +__all__ = [ + "DeployProfile", + "list_profiles", + "get_profile", + "has_profile", + "render_recipe_yaml", + "render_deploy_script", + "write_recipe", + "write_deploy_script", + "autopilot_artifacts", +] diff --git a/tests/test_v0460_part_a.py b/tests/test_v0460_part_a.py new file mode 100644 index 0000000..98ed46a --- /dev/null +++ b/tests/test_v0460_part_a.py @@ -0,0 +1,572 @@ +"""v0.46.0 Part A — On-Device Deploy Autopilot tests.""" + +from __future__ import annotations + +import os +import stat as _stat +import sys +from types import MappingProxyType + +import pytest +from typer.testing import CliRunner + +from soup_cli.utils.deploy_autopilot import ( + DeployProfile, + autopilot_artifacts, + get_profile, + has_profile, + list_profiles, + render_deploy_script, + render_recipe_yaml, + write_deploy_script, + write_recipe, +) + +runner = CliRunner() + +# --------------------------------------------------------------------------- +# Catalog +# --------------------------------------------------------------------------- + + +def test_catalog_is_mapping_proxy(): + profiles = list_profiles() + assert isinstance(profiles, MappingProxyType) + with pytest.raises(TypeError): + profiles["x"] = "y" # type: ignore[index] + + +def test_catalog_has_all_10_documented_profiles(): + names = set(list_profiles().keys()) + expected = { + "mac-m3", "mac-m4-pro", "rtx-3060-12gb", "rtx-4090-24gb", + "iphone-16", "pixel-9", "ollama-local", "lm-studio", + "runpod-a100", "hf-jobs-h100", + } + assert expected.issubset(names) + + +def test_get_profile_known(): + profile = get_profile("mac-m3") + assert isinstance(profile, DeployProfile) + assert profile.name == "mac-m3" + assert profile.runtime == "mlx" + + +def test_get_profile_case_insensitive(): + assert get_profile("MAC-M3").name == "mac-m3" + assert get_profile(" Mac-M3 ").name == "mac-m3" + + +def test_get_profile_unknown_raises_keyerror(): + with pytest.raises(KeyError): + get_profile("unknown-target") + + +def test_get_profile_empty_rejected(): + with pytest.raises(ValueError, match="non-empty"): + get_profile("") + + +def test_get_profile_null_byte_rejected(): + with pytest.raises(ValueError, match="NUL"): + get_profile("mac-m3\x00") + + +def test_get_profile_non_string_rejected(): + with pytest.raises(TypeError): + get_profile(123) # type: ignore[arg-type] + + +def test_has_profile_truthy(): + assert has_profile("mac-m3") is True + assert has_profile("unknown-target") is False + assert has_profile(None) is False # type: ignore[arg-type] + assert has_profile(b"mac-m3") is False # type: ignore[arg-type] + + +def test_deploy_profile_is_frozen(): + import dataclasses + + profile = get_profile("mac-m3") + with pytest.raises(dataclasses.FrozenInstanceError): + profile.name = "x" # type: ignore[misc] + + +# --------------------------------------------------------------------------- +# render_recipe_yaml +# --------------------------------------------------------------------------- + + +def test_render_recipe_includes_base_and_quant(): + profile = get_profile("rtx-4090-24gb") + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "base: meta-llama/Llama-3.2-1B" in yaml_text + assert "quantization: awq" in yaml_text + assert "output: ./out" in yaml_text + + +def test_render_recipe_mlx_backend_for_mlx_runtime(): + profile = get_profile("mac-m3") + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "backend: mlx" in yaml_text + + +def test_render_recipe_transformers_backend_default(): + profile = get_profile("rtx-3060-12gb") + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "backend: transformers" in yaml_text + + +def test_render_recipe_lora_section_present_for_lora_peft(): + profile = get_profile("rtx-4090-24gb") + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "lora:" in yaml_text + assert "r: 16" in yaml_text + + +def test_render_recipe_dora_flag_present(): + # No built-in profile uses dora, but the helper should emit it correctly + # when given a synthetic profile. Validate by constructing manually. + profile = DeployProfile( + name="dora-test", description="x", runtime="transformers", + quant="4bit", peft="dora", spec_decoding=False, + recommended_max_length=2048, notes="", + ) + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "use_dora: true" in yaml_text + + +def test_render_recipe_full_peft_omits_lora_section(): + profile = DeployProfile( + name="full-test", description="x", runtime="transformers", + quant="none", peft="full", spec_decoding=False, + recommended_max_length=2048, notes="", + ) + yaml_text = render_recipe_yaml(profile, base="meta-llama/Llama-3.2-1B", + output_dir="./out") + assert "lora:" not in yaml_text + + +def test_render_recipe_base_validation(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="non-empty"): + render_recipe_yaml(profile, base="", output_dir="./out") + with pytest.raises(ValueError, match="NUL"): + render_recipe_yaml(profile, base="evil\x00", output_dir="./out") + with pytest.raises(ValueError): + render_recipe_yaml(profile, base="a\nb", output_dir="./out") + + +def test_render_recipe_base_too_long(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="exceeds"): + render_recipe_yaml(profile, base="x" * 201, output_dir="./out") + + +def test_render_recipe_base_non_string(): + profile = get_profile("mac-m3") + with pytest.raises(TypeError): + render_recipe_yaml(profile, base=123, output_dir="./out") # type: ignore[arg-type] + + +def test_render_recipe_output_dir_validation(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + render_recipe_yaml(profile, base="m/r", output_dir="") + with pytest.raises(ValueError): + render_recipe_yaml(profile, base="m/r", output_dir="x\x00") + with pytest.raises(ValueError): + render_recipe_yaml(profile, base="m/r", output_dir="a\nb") + + +def test_render_recipe_profile_type_check(): + with pytest.raises(TypeError): + render_recipe_yaml("not-a-profile", base="m/r", # type: ignore[arg-type] + output_dir="./o") + + +def test_render_recipe_max_length_used(): + profile = get_profile("hf-jobs-h100") + yaml_text = render_recipe_yaml(profile, base="m/r", output_dir="./out") + assert "max_length: 65536" in yaml_text + + +# --------------------------------------------------------------------------- +# render_deploy_script +# --------------------------------------------------------------------------- + + +def test_deploy_script_for_ollama_uses_planned_command(): + profile = get_profile("ollama-local") + script = render_deploy_script(profile, model_path="./out") + assert "soup deploy ollama" in script + + +def test_deploy_script_for_vllm_uses_serve(): + profile = get_profile("rtx-4090-24gb") + script = render_deploy_script(profile, model_path="./out") + assert "soup serve --backend vllm" in script + assert "--auto-spec" in script # spec_decoding=True + + +def test_deploy_script_no_spec_flag_when_disabled(): + profile = get_profile("ollama-local") + script = render_deploy_script(profile, model_path="./out") + assert "--auto-spec" not in script + + +def test_deploy_script_for_mlx(): + profile = get_profile("mac-m3") + script = render_deploy_script(profile, model_path="./out") + assert "soup serve --backend mlx" in script + + +def test_deploy_script_for_lm_studio(): + profile = get_profile("lm-studio") + script = render_deploy_script(profile, model_path="./out") + assert "lm-studio" in script + + +def test_deploy_script_for_executorch(): + profile = get_profile("iphone-16") + script = render_deploy_script(profile, model_path="./out") + assert "ExecuTorch" in script or "executorch" in script.lower() + + +def test_deploy_script_quotes_model_path(): + profile = get_profile("mac-m3") + # Path with spaces — shlex.quote must be applied + script = render_deploy_script(profile, model_path="./path with spaces") + assert "'./path with spaces'" in script + + +def test_deploy_script_rejects_newline_path(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + render_deploy_script(profile, model_path="evil\necho") + + +def test_deploy_script_rejects_null_byte_path(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + render_deploy_script(profile, model_path="evil\x00") + + +def test_deploy_script_rejects_empty_path(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + render_deploy_script(profile, model_path="") + + +def test_deploy_script_rejects_oversize_path(): + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + render_deploy_script(profile, model_path="x" * 4097) + + +def test_deploy_script_non_string_path(): + profile = get_profile("mac-m3") + with pytest.raises(TypeError): + render_deploy_script(profile, model_path=123) # type: ignore[arg-type] + + +def test_deploy_script_profile_type_check(): + with pytest.raises(TypeError): + render_deploy_script("not-a-profile", model_path="./out") # type: ignore[arg-type] + + +def test_deploy_script_has_shebang_and_set_e(): + profile = get_profile("mac-m3") + script = render_deploy_script(profile, model_path="./out") + assert script.startswith("#!/usr/bin/env bash") + assert "set -euo pipefail" in script + + +# --------------------------------------------------------------------------- +# write_recipe / write_deploy_script (cwd containment) +# --------------------------------------------------------------------------- + + +def test_write_recipe_under_cwd_succeeds(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + out = write_recipe(profile, base="m/r", output_dir="./out", + recipe_path="recipe.yaml") + assert os.path.exists(out) + with open(out, encoding="utf-8") as fh: + assert "base: m/r" in fh.read() + + +def test_write_recipe_outside_cwd_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + abs_outside = str(tmp_path.parent / "evil.yaml") + with pytest.raises(ValueError, match="must stay under cwd"): + write_recipe(profile, base="m/r", output_dir="./out", + recipe_path=abs_outside) + + +def test_write_recipe_null_byte_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="NUL"): + write_recipe(profile, base="m/r", output_dir="./out", + recipe_path="recipe\x00.yaml") + + +def test_write_recipe_non_string_path(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + with pytest.raises(TypeError): + write_recipe(profile, base="m/r", output_dir="./out", + recipe_path=123) # type: ignore[arg-type] + + +def test_write_deploy_script_under_cwd_succeeds(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + out = write_deploy_script(profile, model_path="./out", + script_path="deploy.sh") + assert os.path.exists(out) + if os.name != "nt": + mode = os.stat(out).st_mode + assert mode & _stat.S_IXUSR + + +def test_write_deploy_script_outside_cwd_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + abs_outside = str(tmp_path.parent / "evil.sh") + with pytest.raises(ValueError, match="must stay under cwd"): + write_deploy_script(profile, model_path="./out", + script_path=abs_outside) + + +def test_write_deploy_script_null_byte_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + with pytest.raises(ValueError): + write_deploy_script(profile, model_path="./out", + script_path="x\x00.sh") + + +# --------------------------------------------------------------------------- +# autopilot_artifacts helper +# --------------------------------------------------------------------------- + + +def test_autopilot_artifacts_returns_pair(): + recipe, script = autopilot_artifacts( + "rtx-4090-24gb", base="m/r", output_dir="./out", + ) + assert "base: m/r" in recipe + assert "soup serve" in script + + +def test_autopilot_artifacts_default_model_path(): + _, script = autopilot_artifacts( + "mac-m3", base="m/r", output_dir="./out", + ) + assert "./out" in script + + +def test_autopilot_artifacts_explicit_model_path(): + _, script = autopilot_artifacts( + "mac-m3", base="m/r", output_dir="./train", model_path="./serve", + ) + assert "./serve" in script + assert "./serve" in script + + +def test_autopilot_artifacts_unknown_profile_raises(): + with pytest.raises(KeyError): + autopilot_artifacts("nope", base="m/r", output_dir="./out") + + +# --------------------------------------------------------------------------- +# CLI smoke tests +# --------------------------------------------------------------------------- + + +def test_cli_autopilot_list(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands import deploy + + result = runner.invoke(deploy.app, ["autopilot", "--list"]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + # At least one known profile name appears + assert "mac-m3" in result.output + assert "rtx-4090-24gb" in result.output + + +def test_cli_autopilot_help(): + from soup_cli.commands import deploy + + result = runner.invoke(deploy.app, ["autopilot", "--help"]) + assert result.exit_code == 0, result.output + assert "Profile name" in result.output or "profile" in result.output.lower() + + +def test_cli_autopilot_writes_recipe_and_script(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands import deploy + + result = runner.invoke( + deploy.app, + ["autopilot", "--target", "mac-m3", "--base", "meta-llama/Llama-3.2-1B", + "--recipe-out", "recipe.yaml", "--script-out", "deploy.sh"], + ) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert (tmp_path / "recipe.yaml").exists() + assert (tmp_path / "deploy.sh").exists() + + +def test_cli_autopilot_unknown_target_exits_2(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands import deploy + + result = runner.invoke(deploy.app, ["autopilot", "--target", "nope"]) + assert result.exit_code == 2, result.output + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlink ACL on Windows") +def test_write_recipe_symlink_target_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + real = tmp_path / "real.yaml" + real.write_text("", encoding="utf-8") + link = tmp_path / "recipe.yaml" + try: + os.symlink(real, link) + except (OSError, NotImplementedError): + pytest.skip("symlink unavailable") + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="symlink"): + write_recipe(profile, base="m/r", output_dir="./out", + recipe_path="recipe.yaml") + + +@pytest.mark.skipif(sys.platform == "win32", reason="symlink ACL on Windows") +def test_write_deploy_script_symlink_target_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + real = tmp_path / "real.sh" + real.write_text("", encoding="utf-8") + link = tmp_path / "deploy.sh" + try: + os.symlink(real, link) + except (OSError, NotImplementedError): + pytest.skip("symlink unavailable") + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="symlink"): + write_deploy_script(profile, model_path="./out", + script_path="deploy.sh") + + +def test_write_recipe_path_too_long_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + profile = get_profile("mac-m3") + with pytest.raises(ValueError, match="exceeds"): + write_recipe(profile, base="m/r", output_dir="./out", + recipe_path="x" * 4097) + + +def test_cli_autopilot_outside_cwd_script_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands import deploy + + abs_outside = str(tmp_path.parent / "evil.sh") + result = runner.invoke( + deploy.app, + ["autopilot", "--target", "mac-m3", "--script-out", abs_outside], + ) + assert result.exit_code == 1, result.output + + +def test_cli_autopilot_outside_cwd_recipe_rejected(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + from soup_cli.commands import deploy + + abs_outside = str(tmp_path.parent / "evil.yaml") + result = runner.invoke( + deploy.app, + ["autopilot", "--target", "mac-m3", "--recipe-out", abs_outside], + ) + assert result.exit_code == 1, result.output + + +# --------------------------------------------------------------------------- +# Internal validators (via DeployProfile construction) +# --------------------------------------------------------------------------- + + +def test_make_helper_rejects_invalid_runtime(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="runtime"): + _make("x", "d", runtime="nope", quant="4bit", peft="lora", + spec_decoding=False, recommended_max_length=2048) + + +def test_make_helper_rejects_invalid_quant(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="quant"): + _make("x", "d", runtime="transformers", quant="nope", peft="lora", + spec_decoding=False, recommended_max_length=2048) + + +def test_make_helper_rejects_invalid_peft(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="peft"): + _make("x", "d", runtime="transformers", quant="4bit", peft="nope", + spec_decoding=False, recommended_max_length=2048) + + +def test_make_helper_rejects_bool_max_length(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(TypeError, match="bool"): + _make("x", "d", runtime="transformers", quant="4bit", peft="lora", + spec_decoding=False, recommended_max_length=True) + + +def test_make_helper_rejects_max_length_out_of_bounds(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="64"): + _make("x", "d", runtime="transformers", quant="4bit", peft="lora", + spec_decoding=False, recommended_max_length=32) + + +def test_make_helper_rejects_bad_name(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="kebab"): + _make("Bad Name", "d", runtime="transformers", quant="4bit", peft="lora", + spec_decoding=False, recommended_max_length=2048) + + +def test_make_helper_rejects_null_byte_description(): + from soup_cli.utils.deploy_autopilot import _make + + with pytest.raises(ValueError, match="NUL"): + _make("ok", "evil\x00", runtime="transformers", quant="4bit", peft="lora", + spec_decoding=False, recommended_max_length=2048) + + +def test_make_helper_via_known_failure_modes(): + # Internal _make is used by _BUILTIN at import time. If the catalog + # is loaded then all known profiles passed validation. Just assert the + # imports succeeded and the count matches. + assert "soup_cli.utils.deploy_autopilot" in sys.modules + + +def test_recommended_max_length_in_bounds(): + for profile in list_profiles().values(): + assert 64 <= profile.recommended_max_length <= 1_048_576