diff --git a/CLAUDE.md b/CLAUDE.md index c37a637..824708f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,14 @@ soup train --config soup.yaml **DeepSpeed:** `utils/deepspeed.py` provides ZeRO Stage 2/3 config templates. `commands/train.py` supports `--deepspeed zero2|zero3|zero2_offload|`. Trainers (SFT/DPO) pass `deepspeed` to HF TrainingArguments. Requires `pip install 'soup-cli[deepspeed]'`. +**Error handling:** `utils/errors.py` maps known exceptions (CUDA OOM, missing deps, connection errors, validation errors) to friendly 2-3 line messages with fix suggestions. `cli.py` wraps all commands in a try/except and uses `--verbose` flag for full tracebacks. + +**Doctor:** `commands/doctor.py` checks system info, GPU availability, and all dependency versions. Reports missing/outdated packages with fix suggestions. + +**Quickstart:** `commands/quickstart.py` runs a complete demo — creates 20-example alpaca dataset, TinyLlama config, and trains a LoRA adapter. Supports `--dry-run` to create files only. + +**Confirmation prompts:** `commands/train.py` and `commands/sweep.py` ask for confirmation before starting. Skip with `--yes` / `-y`. + ## Code Conventions - **Line length:** 100 chars (ruff enforced) @@ -99,7 +107,7 @@ soup train --config soup.yaml ## Tests -Test suite (~281 tests) lives in `tests/`: +Test suite (~321 tests) lives in `tests/`: | File | Covers | |---|---| @@ -128,3 +136,6 @@ Test suite (~281 tests) lives in `tests/`: | `test_sweep.py` | Sweep params parsing, combinations, nested config | | `test_diff.py` | Diff prompts collection, metrics, CLI | | `test_deepspeed.py` | DeepSpeed configs, multi-GPU detection, trainer integration | +| `test_errors.py` | Friendly error messages, --verbose flag, error mapping | +| `test_doctor.py` | `soup doctor` command, version checking, dependency table | +| `test_quickstart.py` | `soup quickstart` demo, data/config creation, --dry-run | diff --git a/README.md b/README.md index f40b3d7..37a200d 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ PyPI Python 3.9+ MIT License - Tests + Tests CI

@@ -324,6 +324,43 @@ soup train --config soup.yaml --deepspeed zero2_offload soup train --config soup.yaml --deepspeed ./my_ds_config.json ``` +## Quickstart Demo + +Run a complete demo in one command — creates sample data, config, and trains a tiny model: + +```bash +# Full demo (creates data + config + trains TinyLlama) +soup quickstart + +# Just create files without training +soup quickstart --dry-run + +# Skip confirmation +soup quickstart --yes +``` + +## Health Check + +Check your environment for compatibility issues: + +```bash +soup doctor +``` + +Shows: Python version, GPU availability, all dependency versions, and fix suggestions. + +## Error Handling + +Soup shows friendly error messages by default (2-3 lines with a fix suggestion). For full tracebacks: + +```bash +# Any command with --verbose +soup train --config soup.yaml --verbose + +# Global flag works with all commands +soup --verbose eval --model ./output --benchmarks mmlu +``` + ## Data Formats Soup supports these formats (auto-detected): @@ -429,6 +466,10 @@ soup eval --model ./output --benchmarks mmlu --run-id run_20260223_143052_a1b2 | Hyperparameter sweep (grid/random) | ✅ | | Model comparison (diff) | ✅ | | Multi-GPU / DeepSpeed | ✅ | +| Friendly error messages | ✅ | +| Health check (soup doctor) | ✅ | +| Quickstart demo | ✅ | +| Confirmation prompts | ✅ | | Web dashboard | 🔜 | | Cloud mode (BYOG) | 🔜 | @@ -459,7 +500,10 @@ soup sweep --config soup.yaml --param lr=... Hyperparameter search soup diff --model-a ./a --model-b ./b Compare two models soup data generate --prompt "..." --count 100 Generate synthetic data soup train --deepspeed zero2 Multi-GPU with DeepSpeed +soup doctor Check environment & dependencies +soup quickstart [--dry-run] Full demo: create data + config + train soup version Show version +soup --verbose Show full traceback on errors ``` ## Requirements @@ -478,7 +522,7 @@ pip install -e ".[dev]" # Lint ruff check soup_cli/ tests/ -# Run unit tests (fast, no GPU needed — 281 tests) +# Run unit tests (fast, no GPU needed — 321 tests) pytest tests/ -v # Run smoke tests (downloads tiny model, runs real training) diff --git a/pyproject.toml b/pyproject.toml index ca89867..23f8dde 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "soup-cli" -version = "0.3.0" +version = "0.3.1" description = "Fine-tune LLMs in one command. No SSH, no config hell." readme = "README.md" license = "MIT" @@ -48,7 +48,7 @@ generate = ["httpx>=0.24.0"] deepspeed = ["deepspeed>=0.12.0"] [project.scripts] -soup = "soup_cli.cli:app" +soup = "soup_cli.cli:run" [project.urls] Homepage = "https://github.com/MakazhanAlpamys/Soup" diff --git a/soup_cli/__init__.py b/soup_cli/__init__.py index a68b702..d4540e5 100644 --- a/soup_cli/__init__.py +++ b/soup_cli/__init__.py @@ -1,3 +1,3 @@ """Soup CLI — Fine-tune LLMs in one command.""" -__version__ = "0.3.0" +__version__ = "0.3.1" diff --git a/soup_cli/__main__.py b/soup_cli/__main__.py index 85c1a5d..475eac1 100644 --- a/soup_cli/__main__.py +++ b/soup_cli/__main__.py @@ -1,5 +1,5 @@ """Allow running as `python -m soup_cli`.""" -from soup_cli.cli import app +from soup_cli.cli import run -app() +run() diff --git a/soup_cli/cli.py b/soup_cli/cli.py index 342ea93..c3708ee 100644 --- a/soup_cli/cli.py +++ b/soup_cli/cli.py @@ -1,5 +1,7 @@ """Main CLI entry point — all commands registered here.""" +import sys + import typer from rich.console import Console @@ -19,9 +21,14 @@ from soup_cli.commands import ( sweep, train, ) +from soup_cli.commands import doctor as doctor_cmd +from soup_cli.commands import quickstart as quickstart_cmd console = Console() +# Global verbose flag — set via callback, read by error handler +_verbose = False + app = typer.Typer( name="soup", help="Fine-tune LLMs in one command. No SSH, no config hell.", @@ -45,6 +52,8 @@ app.command(name="eval")(eval.eval_model) app.command()(serve.serve) app.command()(sweep.sweep) app.command(name="diff")(diff.diff) +app.command()(doctor_cmd.doctor) +app.command()(quickstart_cmd.quickstart) # Register data generate as a subcommand of data data.app.command(name="generate")(generate.generate) @@ -57,6 +66,39 @@ def version(): @app.callback(invoke_without_command=True) -def main(ctx: typer.Context): +def main( + ctx: typer.Context, + verbose: bool = typer.Option( + False, + "--verbose", + "-V", + help="Show full traceback on errors", + ), +): """Soup — fine-tune LLMs in one command.""" - pass + global _verbose + _verbose = verbose + + +def run(): + """Entry point with friendly error handling.""" + try: + app() + except SystemExit: + raise + except typer.Exit: + raise + except KeyboardInterrupt: + console.print("\n[yellow]Interrupted.[/]") + sys.exit(130) + except Exception as exc: + from soup_cli.utils.errors import format_friendly_error + + format_friendly_error(exc, verbose=_verbose) + sys.exit(1) + + +# When invoked via `soup` entry point, use run() for error handling. +# When invoked via `python -m soup_cli`, __main__.py calls run() directly. +if __name__ == "__main__": + run() diff --git a/soup_cli/commands/doctor.py b/soup_cli/commands/doctor.py new file mode 100644 index 0000000..a6d3e06 --- /dev/null +++ b/soup_cli/commands/doctor.py @@ -0,0 +1,174 @@ +"""soup doctor — check dependency compatibility and system health.""" + +import platform +import sys + +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +console = Console() + +# Dependencies to check: (import_name, package_name, min_version, required) +DEPS = [ + ("torch", "torch", "2.0.0", True), + ("transformers", "transformers", "4.36.0", True), + ("peft", "peft", "0.7.0", True), + ("trl", "trl", "0.7.0", True), + ("datasets", "datasets", "2.14.0", True), + ("bitsandbytes", "bitsandbytes", "0.41.0", True), + ("accelerate", "accelerate", "0.25.0", True), + ("pydantic", "pydantic", "2.0.0", True), + ("typer", "typer", "0.9.0", True), + ("rich", "rich", "13.0.0", True), + ("yaml", "pyyaml", "6.0", True), + ("plotext", "plotext", "5.2.0", True), + # Optional + ("fastapi", "fastapi", "0.104.0", False), + ("uvicorn", "uvicorn", "0.24.0", False), + ("datasketch", "datasketch", "1.6.0", False), + ("lm_eval", "lm-eval", "0.4.0", False), + ("wandb", "wandb", "0.15.0", False), + ("deepspeed", "deepspeed", "0.12.0", False), + ("httpx", "httpx", "0.24.0", False), +] + + +def doctor(): + """Check system dependencies, GPU, and compatibility.""" + console.print("[bold]Soup Doctor[/] — checking your environment...\n") + + # System info + console.print( + Panel( + f"Python: [bold]{sys.version.split()[0]}[/]\n" + f"Platform: [bold]{platform.system()} {platform.release()}[/]\n" + f"Arch: [bold]{platform.machine()}[/]", + title="System", + ) + ) + + # GPU check + _check_gpu() + + # Dependencies table + table = Table(title="Dependencies") + table.add_column("Package", style="bold") + table.add_column("Required", justify="center") + table.add_column("Installed", justify="center") + table.add_column("Min Version") + table.add_column("Status") + + issues = [] + + for import_name, pkg_name, min_ver, required in DEPS: + try: + mod = __import__(import_name) + version = getattr(mod, "__version__", getattr(mod, "VERSION", "?")) + version_str = str(version) + + if _version_ok(version_str, min_ver): + status = "[green]OK[/]" + else: + status = f"[yellow]outdated (need >={min_ver})[/]" + issues.append(f"Upgrade {pkg_name}: pip install '{pkg_name}>={min_ver}'") + + table.add_row( + pkg_name, + "yes" if required else "optional", + version_str, + f">={min_ver}", + status, + ) + except ImportError: + if required: + status = "[red]MISSING[/]" + issues.append(f"Install {pkg_name}: pip install '{pkg_name}>={min_ver}'") + else: + status = "[dim]not installed[/]" + + table.add_row( + pkg_name, + "yes" if required else "optional", + "—", + f">={min_ver}", + status, + ) + + console.print(table) + + # Summary + if issues: + console.print(f"\n[yellow]Found {len(issues)} issue(s):[/]") + for issue in issues: + console.print(f" [red]>[/] {issue}") + console.print("\n[dim]Fix all: pip install -U " + " ".join( + f"'{pkg_name}>={min_ver}'" + for _, pkg_name, min_ver, required in DEPS + if required + ) + "[/]") + else: + console.print("\n[bold green]All checks passed![/] Your environment is ready.") + + +def _check_gpu(): + """Check GPU availability and display info.""" + try: + import torch + + if torch.cuda.is_available(): + gpu_count = torch.cuda.device_count() + gpus = [] + for idx in range(gpu_count): + name = torch.cuda.get_device_name(idx) + mem = torch.cuda.get_device_properties(idx) + total_gb = getattr(mem, "total_memory", getattr(mem, "total_mem", 0)) + total_gb = total_gb / (1024 ** 3) + gpus.append(f" GPU {idx}: [bold]{name}[/] ({total_gb:.1f} GB)") + gpu_info = "\n".join(gpus) + cuda_ver = torch.version.cuda or "N/A" + console.print( + Panel( + f"CUDA: [bold green]available[/] (v{cuda_ver})\n" + f"GPUs: [bold]{gpu_count}[/]\n{gpu_info}", + title="GPU", + ) + ) + elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available(): + console.print( + Panel( + "Backend: [bold green]MPS (Apple Silicon)[/]\n" + "Status: [bold green]available[/]", + title="GPU", + ) + ) + else: + console.print( + Panel( + "Backend: [bold yellow]CPU only[/]\n" + "Warning: Training will be slow without GPU.", + title="GPU", + ) + ) + except ImportError: + console.print( + Panel( + "Backend: [red]unknown (torch not installed)[/]", + title="GPU", + ) + ) + + +def _version_ok(installed: str, minimum: str) -> bool: + """Check if installed version meets minimum requirement.""" + try: + inst_parts = [int(x) for x in installed.split(".")[:3]] + min_parts = [int(x) for x in minimum.split(".")[:3]] + # Pad to same length + while len(inst_parts) < 3: + inst_parts.append(0) + while len(min_parts) < 3: + min_parts.append(0) + return inst_parts >= min_parts + except (ValueError, AttributeError): + return True # Can't parse, assume OK diff --git a/soup_cli/commands/quickstart.py b/soup_cli/commands/quickstart.py new file mode 100644 index 0000000..0cbd228 --- /dev/null +++ b/soup_cli/commands/quickstart.py @@ -0,0 +1,137 @@ +"""soup quickstart — one command for a complete demo (create data + config + train).""" + +import json +from pathlib import Path + +import typer +from rich.console import Console +from rich.panel import Panel + +console = Console() + +# Minimal demo dataset — 20 instruction-following examples +DEMO_DATA = [ + {"instruction": "What is machine learning?", "input": "", + "output": "Machine learning is a subset of AI where computers learn patterns from data."}, + {"instruction": "Explain what a neural network is.", "input": "", + "output": "A neural network is a computing system inspired by biological neural networks."}, + {"instruction": "What is Python?", "input": "", + "output": "Python is a high-level programming language known for its readability."}, + {"instruction": "Define overfitting.", "input": "", + "output": "Overfitting is when a model learns noise in training data instead of patterns."}, + {"instruction": "What is a GPU?", "input": "", + "output": "A GPU is a specialized processor designed for parallel computation."}, + {"instruction": "Explain LoRA.", "input": "", + "output": "LoRA (Low-Rank Adaptation) is a technique to fine-tune large models efficiently."}, + {"instruction": "What is tokenization?", "input": "", + "output": "Tokenization is the process of splitting text into smaller units called tokens."}, + {"instruction": "Define transfer learning.", "input": "", + "output": "Transfer learning uses a pre-trained model as a starting point for a new task."}, + {"instruction": "What is an epoch?", "input": "", + "output": "An epoch is one complete pass through the entire training dataset."}, + {"instruction": "Explain gradient descent.", "input": "", + "output": "Gradient descent is an optimization algorithm that minimizes loss iteratively."}, + {"instruction": "What is a loss function?", "input": "", + "output": "A loss function measures how far model predictions are from actual values."}, + {"instruction": "Define batch size.", "input": "", + "output": "Batch size is the number of training samples processed before updating weights."}, + {"instruction": "What is quantization?", "input": "", + "output": "Quantization reduces model precision (e.g., 32-bit to 4-bit) to save memory."}, + {"instruction": "Explain attention mechanism.", "input": "", + "output": "Attention lets models focus on relevant parts of input when generating output."}, + {"instruction": "What is fine-tuning?", "input": "", + "output": "Fine-tuning is training a pre-trained model on task-specific data."}, + {"instruction": "Define learning rate.", "input": "", + "output": "Learning rate controls how much model weights change during each training step."}, + {"instruction": "What is a transformer?", "input": "", + "output": "A transformer is a neural network architecture based on self-attention."}, + {"instruction": "Explain backpropagation.", "input": "", + "output": "Backpropagation computes gradients by propagating errors backward through layers."}, + {"instruction": "What is RLHF?", "input": "", + "output": "RLHF trains models using human feedback as a reward signal."}, + {"instruction": "Define inference.", "input": "", + "output": "Inference is using a trained model to make predictions on new data."}, +] + +DEMO_CONFIG = """# Soup Quickstart Config — auto-generated demo +base: TinyLlama/TinyLlama-1.1B-Chat-v1.0 + +task: sft + +data: + train: ./quickstart_data.jsonl + format: alpaca + val_split: 0.1 + +training: + epochs: 1 + lr: 2e-4 + batch_size: auto + lora: + r: 16 + alpha: 32 + quantization: "none" + +output: ./quickstart_output +""" + + +def quickstart( + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompt", + ), + dry_run: bool = typer.Option( + False, + "--dry-run", + help="Create data and config only, do not train", + ), +): + """Run a complete demo: create sample data, config, and train.""" + console.print( + Panel( + "This will:\n" + " 1. Create [bold]quickstart_data.jsonl[/] (20 examples)\n" + " 2. Create [bold]quickstart_soup.yaml[/] config\n" + " 3. Train a tiny LoRA adapter (~1 min on GPU)\n\n" + "Model: [bold]TinyLlama/TinyLlama-1.1B-Chat-v1.0[/]", + title="[bold]Soup Quickstart[/]", + ) + ) + + if not yes and not dry_run: + confirm = typer.confirm("Continue?", default=True) + if not confirm: + console.print("[yellow]Cancelled.[/]") + raise typer.Exit() + + # 1. Create demo data + data_path = Path("quickstart_data.jsonl") + if data_path.exists(): + console.print(f"[yellow]Data file already exists:[/] {data_path}") + else: + with open(data_path, "w", encoding="utf-8") as fh: + for entry in DEMO_DATA: + fh.write(json.dumps(entry, ensure_ascii=False) + "\n") + console.print(f"[green]Created:[/] {data_path} ({len(DEMO_DATA)} examples)") + + # 2. Create demo config + config_path = Path("quickstart_soup.yaml") + if config_path.exists(): + console.print(f"[yellow]Config file already exists:[/] {config_path}") + else: + config_path.write_text(DEMO_CONFIG, encoding="utf-8") + console.print(f"[green]Created:[/] {config_path}") + + if dry_run: + console.print("\n[yellow]Dry run — files created, skipping training.[/]") + console.print(f"To train: [bold]soup train --config {config_path}[/]") + raise typer.Exit() + + # 3. Train + console.print("\n[bold]Starting training...[/]\n") + from soup_cli.commands.train import train as train_cmd + + train_cmd(config=str(config_path), yes=True) diff --git a/soup_cli/commands/sweep.py b/soup_cli/commands/sweep.py index b47e886..4fad178 100644 --- a/soup_cli/commands/sweep.py +++ b/soup_cli/commands/sweep.py @@ -50,6 +50,12 @@ def sweep( "--dry-run", help="Show planned runs without executing", ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompt", + ), ): """Run hyperparameter sweep: grid or random search over training parameters.""" config_path = Path(config) @@ -96,6 +102,11 @@ def sweep( console.print("[yellow]Dry run — no training will be executed.[/]") raise typer.Exit() + if not yes: + if not typer.confirm(f"Start {len(combinations)} training run(s)?", default=True): + console.print("[yellow]Cancelled.[/]") + raise typer.Exit() + # Execute sweep base_cfg = load_config(config_path) results = [] diff --git a/soup_cli/commands/train.py b/soup_cli/commands/train.py index 340d021..5ca9178 100644 --- a/soup_cli/commands/train.py +++ b/soup_cli/commands/train.py @@ -49,6 +49,12 @@ def train( "--deepspeed", help="Enable DeepSpeed: zero2, zero3, zero2_offload, or path to config JSON", ), + yes: bool = typer.Option( + False, + "--yes", + "-y", + help="Skip confirmation prompt", + ), ): """Start training from a soup.yaml config.""" config_path = Path(config) @@ -106,6 +112,11 @@ def train( ) ) + if not dry_run and not yes: + if not typer.confirm("Start training?", default=True): + console.print("[yellow]Cancelled.[/]") + raise typer.Exit() + if dry_run: console.print("[yellow]Dry run — validating data...[/]") dataset = load_dataset(cfg.data) diff --git a/soup_cli/utils/errors.py b/soup_cli/utils/errors.py new file mode 100644 index 0000000..f1de85e --- /dev/null +++ b/soup_cli/utils/errors.py @@ -0,0 +1,160 @@ +"""Friendly error handling — maps raw exceptions to actionable messages.""" + +import traceback + +from rich.console import Console +from rich.panel import Panel + +console = Console(stderr=True) + +# Map known error patterns to (short message, fix suggestion) +ERROR_MAP = [ + # CUDA OOM + ( + "CUDA out of memory", + "GPU ran out of memory during training.", + "Try: reduce batch_size, use quantization: 4bit, or use a smaller model.", + ), + ( + "OutOfMemoryError", + "GPU ran out of memory.", + "Try: reduce batch_size, use quantization: 4bit, or use a smaller model.", + ), + # Missing optional deps + ( + "No module named 'fastapi'", + "FastAPI is not installed (needed for soup serve).", + "Run: pip install 'soup-cli\\[serve]'", + ), + ( + "No module named 'uvicorn'", + "Uvicorn is not installed (needed for soup serve).", + "Run: pip install 'soup-cli\\[serve]'", + ), + ( + "No module named 'datasketch'", + "Datasketch is not installed (needed for dedup).", + "Run: pip install 'soup-cli\\[data]'", + ), + ( + "No module named 'lm_eval'", + "lm-evaluation-harness is not installed (needed for eval).", + "Run: pip install 'soup-cli\\[eval]'", + ), + ( + "No module named 'wandb'", + "Weights & Biases is not installed.", + "Run: pip install wandb", + ), + ( + "No module named 'deepspeed'", + "DeepSpeed is not installed.", + "Run: pip install 'soup-cli\\[deepspeed]'", + ), + ( + "No module named 'httpx'", + "httpx is not installed (needed for data generate).", + "Run: pip install 'soup-cli\\[generate]'", + ), + # Peft / transformers incompatibility + ( + "No module named 'peft'", + "PEFT is not installed.", + "Run: pip install peft>=0.7.0", + ), + ( + "No module named 'trl'", + "TRL is not installed.", + "Run: pip install trl>=0.7.0", + ), + ( + "No module named 'bitsandbytes'", + "BitsAndBytes is not installed (needed for quantization).", + "Run: pip install bitsandbytes>=0.41.0", + ), + # Connection errors + ( + "ConnectionError", + "Network connection failed.", + "Check your internet connection. If downloading from HuggingFace, check HF_TOKEN.", + ), + ( + "HTTPError", + "HTTP request failed.", + "Check your internet connection and API keys (OPENAI_API_KEY, HF_TOKEN).", + ), + ( + "ConnectTimeout", + "Connection timed out.", + "Check your internet connection and try again.", + ), + # File not found + ( + "No such file or directory", + None, # Will use the original message + "Check the file path. Run 'soup init' to create a config.", + ), + # YAML errors + ( + "yaml.scanner.ScannerError", + "Invalid YAML syntax in config file.", + "Check your soup.yaml for syntax errors (indentation, colons, quotes).", + ), + # Pydantic validation + ( + "validation error", + "Config validation failed.", + "Check your soup.yaml values. Run 'soup init' to generate a valid config.", + ), + # Auth errors + ( + "401", + "Authentication failed.", + "Check your API key or token (HF_TOKEN, OPENAI_API_KEY, WANDB_API_KEY).", + ), + ( + "403", + "Access denied.", + "Check your permissions. Some models require accepting a license on HuggingFace.", + ), +] + + +def format_friendly_error(exc: Exception, verbose: bool = False) -> None: + """Display a friendly error message for the given exception. + + In normal mode: 2-3 lines with error + fix suggestion. + In verbose mode: full traceback. + """ + exc_str = str(exc) + exc_type = type(exc).__name__ + + # Search for known error patterns + for pattern, short_msg, fix in ERROR_MAP: + if pattern in exc_str or pattern in exc_type: + error_msg = short_msg or exc_str + console.print(f"\n[bold red]Error:[/] {error_msg}") + console.print(f"[green]Fix:[/] {fix}") + if verbose: + console.print() + console.print( + Panel( + traceback.format_exc(), + title="[dim]Full Traceback[/]", + border_style="dim", + ) + ) + return + + # Unknown error — show type + message + console.print(f"\n[bold red]Error:[/] {exc_type}: {exc_str}") + console.print("[dim]Run with --verbose for the full traceback.[/]") + if verbose: + console.print() + console.print( + Panel( + traceback.format_exc(), + title="[dim]Full Traceback[/]", + border_style="dim", + ) + ) diff --git a/tests/test_doctor.py b/tests/test_doctor.py new file mode 100644 index 0000000..cd53def --- /dev/null +++ b/tests/test_doctor.py @@ -0,0 +1,121 @@ +"""Tests for soup doctor command.""" + +from unittest.mock import patch + +from typer.testing import CliRunner + +from soup_cli.cli import app +from soup_cli.commands.doctor import _version_ok + +runner = CliRunner() + + +# --- _version_ok tests --- + + +def test_version_ok_exact(): + assert _version_ok("2.0.0", "2.0.0") is True + + +def test_version_ok_higher(): + assert _version_ok("2.1.0", "2.0.0") is True + + +def test_version_ok_lower(): + assert _version_ok("1.9.0", "2.0.0") is False + + +def test_version_ok_patch(): + assert _version_ok("2.0.1", "2.0.0") is True + + +def test_version_ok_major_higher(): + assert _version_ok("3.0.0", "2.0.0") is True + + +def test_version_ok_two_part(): + assert _version_ok("6.0", "6.0") is True + + +def test_version_ok_unparseable(): + """Unparseable versions should return True (assume OK).""" + assert _version_ok("unknown", "2.0.0") is True + + +def test_version_ok_dev_suffix(): + """Version with dev suffix (can't fully parse).""" + assert _version_ok("2.1.0.dev0", "2.0.0") is True + + +# --- doctor CLI tests --- + + +def test_doctor_runs(): + """soup doctor runs without crashing.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "Soup Doctor" in result.output + + +def test_doctor_shows_system_info(): + """soup doctor shows system info panel.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "Python" in result.output + assert "Platform" in result.output + + +def test_doctor_shows_dependencies(): + """soup doctor shows dependency table.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "Dependencies" in result.output + assert "Package" in result.output + + +def test_doctor_shows_gpu_section(): + """soup doctor shows GPU section.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "GPU" in result.output + + +def test_doctor_checks_torch(): + """soup doctor checks for torch.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "torch" in result.output + + +def test_doctor_checks_pydantic(): + """soup doctor checks for pydantic.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "pydantic" in result.output + + +def test_doctor_checks_optional_deps(): + """soup doctor shows optional deps.""" + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "optional" in result.output + + +def test_doctor_missing_dep(): + """soup doctor reports missing required dep.""" + with patch("soup_cli.commands.doctor.DEPS", [ + ("nonexistent_fake_pkg_xyz", "nonexistent-pkg", "1.0.0", True), + ]): + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + assert "MISSING" in result.output + + +def test_doctor_outdated_dep(): + """soup doctor reports outdated dep.""" + with patch("soup_cli.commands.doctor.DEPS", [ + ("sys", "sys", "999.0.0", True), # sys has no __version__ but import won't fail + ]): + result = runner.invoke(app, ["doctor"]) + assert result.exit_code == 0 + # Either outdated or OK (depends on version attr presence) diff --git a/tests/test_errors.py b/tests/test_errors.py new file mode 100644 index 0000000..8fdc54a --- /dev/null +++ b/tests/test_errors.py @@ -0,0 +1,186 @@ +"""Tests for friendly error handling and --verbose flag.""" + +from io import StringIO +from unittest.mock import patch + +from rich.console import Console +from typer.testing import CliRunner + +from soup_cli.cli import app +from soup_cli.utils.errors import format_friendly_error + +runner = CliRunner() + + +# --- format_friendly_error tests --- + + +def test_cuda_oom_error(): + """CUDA OOM gets a friendly message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = RuntimeError("CUDA out of memory. Tried to allocate 2.00 GiB") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "GPU ran out of memory" in output + assert "batch_size" in output + + +def test_missing_fastapi_error(): + """Missing fastapi gives install hint.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ModuleNotFoundError("No module named 'fastapi'") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "soup-cli[serve]" in output + + +def test_missing_datasketch_error(): + """Missing datasketch gives install hint.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ModuleNotFoundError("No module named 'datasketch'") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "soup-cli[data]" in output + + +def test_missing_wandb_error(): + """Missing wandb gives install hint.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ModuleNotFoundError("No module named 'wandb'") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "pip install wandb" in output + + +def test_missing_deepspeed_error(): + """Missing deepspeed gives install hint.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ModuleNotFoundError("No module named 'deepspeed'") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "soup-cli[deepspeed]" in output + + +def test_connection_error(): + """Connection error gets friendly message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ConnectionError("Failed to connect") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "Network connection failed" in output + + +def test_unknown_error_shows_type(): + """Unknown errors show the exception type and message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = ZeroDivisionError("division by zero") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "ZeroDivisionError" in output + assert "division by zero" in output + assert "--verbose" in output + + +def test_verbose_shows_traceback(): + """Verbose mode shows full traceback.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + try: + raise RuntimeError("CUDA out of memory. Tried to allocate") + except RuntimeError as exc: + format_friendly_error(exc, verbose=True) + output = buf.getvalue() + assert "GPU ran out of memory" in output + assert "Traceback" in output or "RuntimeError" in output + + +def test_verbose_unknown_error(): + """Verbose mode for unknown errors shows traceback.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + try: + raise ValueError("something weird") + except ValueError as exc: + format_friendly_error(exc, verbose=True) + output = buf.getvalue() + assert "ValueError" in output + assert "something weird" in output + + +def test_yaml_error(): + """YAML syntax error gets friendly message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = Exception("yaml.scanner.ScannerError: mapping values are not allowed here") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "YAML syntax" in output + + +def test_validation_error(): + """Pydantic validation error gets friendly message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = Exception("2 validation error for SoupConfig") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "Config validation failed" in output + + +def test_auth_401_error(): + """401 error gets auth hint.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = Exception("401 Client Error: Unauthorized") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "Authentication failed" in output + + +def test_file_not_found_error(): + """File not found gets friendly message.""" + buf = StringIO() + test_console = Console(file=buf, stderr=False) + with patch("soup_cli.utils.errors.console", test_console): + exc = FileNotFoundError("No such file or directory: 'model.bin'") + format_friendly_error(exc, verbose=False) + output = buf.getvalue() + assert "No such file or directory" in output + assert "soup init" in output + + +# --- CLI --verbose flag tests --- + + +def test_verbose_flag_in_help(): + """--verbose flag is shown in help.""" + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "--verbose" in result.output + + +def test_help_shows_doctor_and_quickstart(): + """New commands are visible in help.""" + result = runner.invoke(app, ["--help"]) + assert result.exit_code == 0 + assert "doctor" in result.output + assert "quickstart" in result.output diff --git a/tests/test_quickstart.py b/tests/test_quickstart.py new file mode 100644 index 0000000..35e7a75 --- /dev/null +++ b/tests/test_quickstart.py @@ -0,0 +1,103 @@ +"""Tests for soup quickstart command.""" + +import json + +from typer.testing import CliRunner + +from soup_cli.cli import app +from soup_cli.commands.quickstart import DEMO_CONFIG, DEMO_DATA + +runner = CliRunner() + + +def test_quickstart_help(): + """soup quickstart --help works.""" + result = runner.invoke(app, ["quickstart", "--help"]) + assert result.exit_code == 0 + assert "demo" in result.output.lower() or "quickstart" in result.output.lower() + + +def test_quickstart_dry_run(tmp_path, monkeypatch): + """soup quickstart --dry-run creates files but does not train.""" + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["quickstart", "--dry-run"]) + assert result.exit_code == 0 + assert "Dry run" in result.output + + # Check files were created + data_path = tmp_path / "quickstart_data.jsonl" + config_path = tmp_path / "quickstart_soup.yaml" + assert data_path.exists() + assert config_path.exists() + + # Validate data + with open(data_path) as fh: + lines = fh.readlines() + assert len(lines) == len(DEMO_DATA) + first = json.loads(lines[0]) + assert "instruction" in first + assert "output" in first + + +def test_quickstart_dry_run_existing_files(tmp_path, monkeypatch): + """soup quickstart --dry-run does not overwrite existing files.""" + monkeypatch.chdir(tmp_path) + + # Create existing files + data_path = tmp_path / "quickstart_data.jsonl" + data_path.write_text("existing data\n") + config_path = tmp_path / "quickstart_soup.yaml" + config_path.write_text("existing config\n") + + result = runner.invoke(app, ["quickstart", "--dry-run"]) + assert result.exit_code == 0 + assert "already exists" in result.output + + # Files should NOT be overwritten + assert data_path.read_text() == "existing data\n" + assert config_path.read_text() == "existing config\n" + + +def test_quickstart_cancel(tmp_path, monkeypatch): + """soup quickstart can be cancelled.""" + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["quickstart"], input="n\n") + assert result.exit_code == 0 + assert "Cancelled" in result.output + + +def test_demo_data_valid(): + """Demo data is valid alpaca format.""" + for entry in DEMO_DATA: + assert "instruction" in entry + assert "output" in entry + assert "input" in entry + assert len(entry["instruction"]) > 0 + assert len(entry["output"]) > 0 + + +def test_demo_data_count(): + """Demo data has 20 examples.""" + assert len(DEMO_DATA) == 20 + + +def test_demo_config_valid(): + """Demo config is valid YAML.""" + import yaml + + config = yaml.safe_load(DEMO_CONFIG) + assert config["base"] == "TinyLlama/TinyLlama-1.1B-Chat-v1.0" + assert config["task"] == "sft" + assert config["data"]["train"] == "./quickstart_data.jsonl" + assert config["data"]["format"] == "alpaca" + assert config["training"]["epochs"] == 1 + + +def test_quickstart_yes_dry_run(tmp_path, monkeypatch): + """soup quickstart --yes --dry-run skips confirmation.""" + monkeypatch.chdir(tmp_path) + result = runner.invoke(app, ["quickstart", "--yes", "--dry-run"]) + assert result.exit_code == 0 + assert "Dry run" in result.output + assert (tmp_path / "quickstart_data.jsonl").exists() + assert (tmp_path / "quickstart_soup.yaml").exists()