"""Tests for soup recipes — ready-made configs for popular models.""" import pytest from typer.testing import CliRunner from soup_cli.cli import app runner = CliRunner() # --------------------------------------------------------------------------- # Recipe catalog tests # --------------------------------------------------------------------------- class TestRecipeCatalog: """Tests for recipe catalog and search.""" def test_list_recipes(self): """list_recipes returns all recipes.""" from soup_cli.recipes.catalog import RECIPES, list_recipes recipes = list_recipes() assert len(recipes) > 0 assert len(recipes) == len(RECIPES) def test_get_recipe_exists(self): """get_recipe returns a known recipe.""" from soup_cli.recipes.catalog import get_recipe recipe = get_recipe("llama3.1-8b-sft") assert recipe is not None assert recipe.model == "meta-llama/Llama-3.1-8B-Instruct" assert recipe.task == "sft" def test_get_recipe_not_exists(self): """get_recipe returns None for unknown recipe.""" from soup_cli.recipes.catalog import get_recipe recipe = get_recipe("nonexistent-recipe") assert recipe is None def test_search_by_task(self): """search_recipes filters by task.""" from soup_cli.recipes.catalog import search_recipes results = search_recipes(task="grpo") assert len(results) > 0 assert all(r.task == "grpo" for r in results) def test_search_by_keyword(self): """search_recipes matches by keyword.""" from soup_cli.recipes.catalog import search_recipes results = search_recipes(query="reasoning") assert len(results) > 0 def test_search_by_size(self): """search_recipes filters by model size.""" from soup_cli.recipes.catalog import search_recipes results = search_recipes(size="7b") assert len(results) > 0 for recipe in results: assert "7b" in recipe.size.lower() or "7b" in recipe.model.lower() def test_search_no_results(self): """search_recipes returns empty list for no matches.""" from soup_cli.recipes.catalog import search_recipes results = search_recipes(query="zzzznonexistent") assert results == [] def test_all_recipes_valid_yaml(self): """All recipes contain valid YAML that loads as SoupConfig.""" from soup_cli.config.loader import load_config_from_string from soup_cli.recipes.catalog import RECIPES for name, recipe in RECIPES.items(): try: config = load_config_from_string(recipe.yaml_str) assert config.base is not None, f"Recipe {name} has no base model" except Exception as exc: pytest.fail(f"Recipe '{name}' is invalid: {exc}") def test_recipe_has_required_fields(self): """All recipes have model, task, size, tags, yaml_str.""" from soup_cli.recipes.catalog import RECIPES for name, recipe in RECIPES.items(): assert recipe.model, f"Recipe {name} missing model" assert recipe.task, f"Recipe {name} missing task" assert recipe.size, f"Recipe {name} missing size" assert recipe.yaml_str, f"Recipe {name} missing yaml_str" def test_recipe_tasks_match_yaml(self): """Recipe.task matches the task in the YAML content.""" import yaml from soup_cli.recipes.catalog import RECIPES for name, recipe in RECIPES.items(): parsed = yaml.safe_load(recipe.yaml_str) yaml_task = parsed.get("task", "sft") assert recipe.task == yaml_task, ( f"Recipe '{name}' task mismatch: meta={recipe.task}, yaml={yaml_task}" ) # --------------------------------------------------------------------------- # CLI tests # --------------------------------------------------------------------------- class TestRecipesCLI: """CLI tests for soup recipes command.""" def test_list_command(self): """soup recipes list shows all recipes.""" result = runner.invoke(app, ["recipes", "list"]) assert result.exit_code == 0 assert "llama3.1-8b-sft" in result.output def test_list_default(self): """soup recipes (no subcommand) shows help (exit 0 or 2 depending on Typer version).""" result = runner.invoke(app, ["recipes"]) assert result.exit_code in (0, 2) def test_show_recipe(self): """soup recipes show prints YAML.""" result = runner.invoke(app, ["recipes", "show", "llama3.1-8b-sft"]) assert result.exit_code == 0 assert "meta-llama" in result.output def test_show_unknown_recipe(self): """soup recipes show shows error.""" result = runner.invoke(app, ["recipes", "show", "nonexistent"]) assert result.exit_code != 0 def test_use_recipe(self, tmp_path, monkeypatch): """soup recipes use writes soup.yaml.""" monkeypatch.chdir(tmp_path) result = runner.invoke(app, [ "recipes", "use", "llama3.1-8b-sft", "--yes", ]) assert result.exit_code == 0 out_path = tmp_path / "soup.yaml" assert out_path.exists() content = out_path.read_text(encoding="utf-8") assert "meta-llama/Llama-3.1-8B-Instruct" in content def test_use_custom_output(self, tmp_path, monkeypatch): """soup recipes use -o custom.yaml writes to custom path.""" monkeypatch.chdir(tmp_path) result = runner.invoke(app, [ "recipes", "use", "qwen2.5-7b-sft", "-o", "my_config.yaml", "--yes", ]) assert result.exit_code == 0 assert (tmp_path / "my_config.yaml").exists() def test_use_overwrite_confirmation(self, tmp_path, monkeypatch): """Existing file asks for confirmation.""" monkeypatch.chdir(tmp_path) out_file = tmp_path / "soup.yaml" out_file.write_text("existing content", encoding="utf-8") # Deny confirmation runner.invoke(app, [ "recipes", "use", "llama3.1-8b-sft", ], input="n\n") assert out_file.read_text(encoding="utf-8") == "existing content" def test_search_command(self): """soup recipes search shows results.""" result = runner.invoke(app, ["recipes", "search", "reasoning"]) assert result.exit_code == 0 def test_search_by_task_flag(self): """soup recipes search --task grpo shows GRPO recipes.""" result = runner.invoke(app, ["recipes", "search", "--task", "grpo"]) assert result.exit_code == 0 assert "grpo" in result.output.lower() def test_search_by_size_flag(self): """soup recipes search --size 7b shows 7B recipes.""" result = runner.invoke(app, ["recipes", "search", "--size", "7b"]) assert result.exit_code == 0 def test_use_output_path_traversal(self, tmp_path, monkeypatch): """Output path traversal is blocked.""" monkeypatch.chdir(tmp_path) result = runner.invoke(app, [ "recipes", "use", "llama3.1-8b-sft", "-o", "../../../tmp/evil.yaml", "--yes", ]) assert result.exit_code != 0 def test_use_unknown_recipe(self, tmp_path, monkeypatch): """soup recipes use shows error.""" monkeypatch.chdir(tmp_path) result = runner.invoke(app, [ "recipes", "use", "nonexistent-recipe", "--yes", ]) assert result.exit_code != 0 def test_help(self): """soup recipes --help shows usage.""" result = runner.invoke(app, ["recipes", "--help"]) assert result.exit_code == 0 assert "recipes" in result.output.lower() # --------------------------------------------------------------------------- # Part A: v0.25.0 new model recipes (Llama 4, Qwen 3, Gemma 3, DeepSeek V3) # --------------------------------------------------------------------------- class TestV025NewRecipes: """Tests for the 9 new recipes added in v0.25.0.""" EXPECTED = [ ("llama4-scout-17b-sft", "sft", "meta-llama/Llama-4-Scout-17B-16E-Instruct"), ("llama4-scout-17b-dpo", "dpo", "meta-llama/Llama-4-Scout-17B-16E-Instruct"), ("llama4-scout-17b-grpo", "grpo", "meta-llama/Llama-4-Scout-17B-16E-Instruct"), ("qwen3-14b-sft", "sft", "Qwen/Qwen3-14B"), ("qwen3-32b-sft", "sft", "Qwen/Qwen3-32B"), ("qwen3-8b-grpo", "grpo", "Qwen/Qwen3-8B"), ("gemma3-12b-sft", "sft", "google/gemma-3-12b-it"), ("gemma3-27b-dpo", "dpo", "google/gemma-3-27b-it"), ("deepseek-v3-7b-sft", "sft", "deepseek-ai/DeepSeek-V3-0324"), ] def test_all_new_recipes_registered(self): """All 9 new recipes are in the catalog.""" from soup_cli.recipes.catalog import RECIPES for name, _task, _model in self.EXPECTED: assert name in RECIPES, f"Missing recipe: {name}" def test_new_recipes_have_correct_task_and_model(self): """Each new recipe has the expected task and model.""" from soup_cli.recipes.catalog import get_recipe for name, task, model in self.EXPECTED: recipe = get_recipe(name) assert recipe is not None assert recipe.task == task assert recipe.model == model def test_new_recipes_load_as_soupconfig(self): """All new recipes produce a valid SoupConfig.""" from soup_cli.config.loader import load_config_from_string from soup_cli.recipes.catalog import get_recipe for name, _task, _model in self.EXPECTED: recipe = get_recipe(name) cfg = load_config_from_string(recipe.yaml_str) assert cfg.base == recipe.model assert cfg.task == recipe.task def test_catalog_size_is_112(self): """Total catalog size — grew with each release. v0.25.0 shipped 43 recipes (29 + 9 Part A + 2 Part B tools + 3 Part E MLX). v0.27.0 added 3 multi-GPU recipes -> 46. v0.31.0 added 34 (vision/audio/reasoning/edge/domain/multimodal) -> 80. v0.51.0 added 26 (model catalog expansion) -> 106. v0.52.0 added 6 (5 TTS + Falcon-E BitNet) -> 112. """ from soup_cli.recipes.catalog import RECIPES assert len(RECIPES) == 112 def test_new_recipes_searchable(self): """Search returns the new recipes via keyword/task filter.""" from soup_cli.recipes.catalog import search_recipes llama4_results = search_recipes(query="Llama-4") llama4_names = {r.model for r in llama4_results} assert "meta-llama/Llama-4-Scout-17B-16E-Instruct" in llama4_names qwen3_sft = search_recipes(query="qwen3", task="sft") assert any("Qwen/Qwen3-14B" == r.model for r in qwen3_sft) assert any("Qwen/Qwen3-32B" == r.model for r in qwen3_sft) def test_deepseek_v3_uses_moe_lora(self): """deepseek-v3-7b-sft recipe enables moe_lora.""" from soup_cli.config.loader import load_config_from_string from soup_cli.recipes.catalog import get_recipe recipe = get_recipe("deepseek-v3-7b-sft") cfg = load_config_from_string(recipe.yaml_str) assert cfg.training.moe_lora is True