mirror of https://github.com/razor-ai/soup.git
fix: v0.23.1 — CI fix, security warnings, expanded test coverage
- Fix macOS CI: CLI help tests use inspect.signature (Rich truncation) - Security: trust_remote_code warning panels for AWQ/GPTQ export - Tests: packing trainer mock, curriculum fallback branch, empty list edge case - 1979 tests across 70 test files
This commit is contained in:
parent
6db403f6c3
commit
ada4a078b6
|
|
@ -1,12 +1,12 @@
|
|||
# Soup CLI — Project CLAUDE.md
|
||||
|
||||
Soup is a CLI-first LLM fine-tuning tool (v0.23.0). Python 3.9+, MIT license.
|
||||
Soup is a CLI-first LLM fine-tuning tool (v0.23.1). Python 3.9+, MIT license.
|
||||
|
||||
## Build & Development
|
||||
|
||||
```bash
|
||||
pip install -e ".[dev]" # Install editable + test deps
|
||||
pytest tests/ -v --tb=short # Run all tests (1970 tests)
|
||||
pytest tests/ -v --tb=short # Run all tests (1979 tests)
|
||||
ruff check soup_cli/ tests/ # Lint (must pass before commit)
|
||||
ruff check --fix soup_cli/ tests/ # Auto-fix lint issues
|
||||
```
|
||||
|
|
@ -113,7 +113,7 @@ soup_cli/
|
|||
profiler.py # Training memory/speed estimator (GPU lookup, model arch)
|
||||
curriculum.py # Curriculum learning: sort by difficulty, create buckets
|
||||
constants.py # APP_NAME, paths, default chat template
|
||||
tests/ # 70 test files, 1970 tests
|
||||
tests/ # 70 test files, 1979 tests
|
||||
examples/
|
||||
configs/ # 7 production-ready YAML examples
|
||||
data/ # Sample datasets
|
||||
|
|
@ -350,7 +350,7 @@ soup version # Show version (--full for details)
|
|||
15. **Tag**: `git tag v0.X.Y && git push origin v0.X.Y`
|
||||
16. **Release**: `gh release create v0.X.Y` with changelog (What's New, Install/Upgrade)
|
||||
|
||||
## Tests (70 test files, 1970 tests)
|
||||
## Tests (70 test files, 1979 tests)
|
||||
|
||||
| File | Covers |
|
||||
|------|--------|
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|||
|
||||
[project]
|
||||
name = "soup-cli"
|
||||
version = "0.23.0"
|
||||
version = "0.23.1"
|
||||
description = "Fine-tune LLMs in one command. No SSH, no config hell."
|
||||
readme = "README.md"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
"""Soup CLI — Fine-tune LLMs in one command."""
|
||||
|
||||
__version__ = "0.23.0"
|
||||
__version__ = "0.23.1"
|
||||
|
|
|
|||
|
|
@ -125,6 +125,92 @@ class TestCurriculumSorting:
|
|||
# First should be shortest
|
||||
assert len(str(sorted_data[0])) < len(str(sorted_data[-1]))
|
||||
|
||||
def test_sort_by_length_empty_list(self):
|
||||
"""Sort by length should handle empty list."""
|
||||
from soup_cli.utils.curriculum import sort_by_length
|
||||
|
||||
assert sort_by_length([]) == []
|
||||
|
||||
def test_sort_by_length_single_item(self):
|
||||
"""Sort by length should handle single item."""
|
||||
from soup_cli.utils.curriculum import sort_by_length
|
||||
|
||||
data = [{"text": "hello"}]
|
||||
assert sort_by_length(data) == data
|
||||
|
||||
def test_sort_by_length_fallback_json(self):
|
||||
"""Sort by length should fall back to JSON stringify for unknown formats."""
|
||||
from soup_cli.utils.curriculum import sort_by_length
|
||||
|
||||
data = [
|
||||
{"custom": "a" * 100, "extra": "b" * 50},
|
||||
{"custom": "short"},
|
||||
]
|
||||
sorted_data = sort_by_length(data)
|
||||
# Shorter row should come first
|
||||
assert len(str(sorted_data[0])) < len(str(sorted_data[-1]))
|
||||
|
||||
|
||||
# ─── Curriculum Metric Fallback Tests ────────────────────────────────────
|
||||
|
||||
|
||||
class TestCurriculumMetricFallback:
|
||||
"""Test non-length metric fallback behavior."""
|
||||
|
||||
def test_perplexity_metric_config(self):
|
||||
"""curriculum_metric=perplexity should be a valid config."""
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
data={"train": "data.jsonl"},
|
||||
training={
|
||||
"curriculum": True,
|
||||
"curriculum_metric": "perplexity",
|
||||
},
|
||||
)
|
||||
assert cfg.training.curriculum_metric == "perplexity"
|
||||
|
||||
def test_loss_metric_config(self):
|
||||
"""curriculum_metric=loss should be a valid config."""
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
data={"train": "data.jsonl"},
|
||||
training={
|
||||
"curriculum": True,
|
||||
"curriculum_metric": "loss",
|
||||
},
|
||||
)
|
||||
assert cfg.training.curriculum_metric == "loss"
|
||||
|
||||
def test_non_length_metric_falls_back_to_length(self):
|
||||
"""Non-length metrics should fall back to length sorting in SFT trainer."""
|
||||
from io import StringIO
|
||||
|
||||
from rich.console import Console
|
||||
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
data={"train": "data.jsonl"},
|
||||
training={
|
||||
"curriculum": True,
|
||||
"curriculum_metric": "perplexity",
|
||||
},
|
||||
)
|
||||
|
||||
output = StringIO()
|
||||
console = Console(file=output)
|
||||
tcfg = cfg.training
|
||||
|
||||
# Simulate the trainer logic
|
||||
if tcfg.curriculum and tcfg.curriculum_metric != "length":
|
||||
console.print(
|
||||
f"[yellow]Curriculum metric '{tcfg.curriculum_metric}' "
|
||||
"requires pre-computed scores. Using length-based sorting.[/]"
|
||||
)
|
||||
|
||||
text = output.getvalue()
|
||||
assert "perplexity" in text
|
||||
assert "length-based" in text
|
||||
|
||||
|
||||
# ─── Bucket Creation Tests ───────────────────────────────────────────────
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Tests for sample packing (packing: true) — config, validation, trainer integration."""
|
||||
|
||||
|
||||
from io import StringIO
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from soup_cli.config.schema import SoupConfig, TrainingConfig
|
||||
|
||||
|
|
@ -161,3 +162,78 @@ class TestPackingWarnings:
|
|||
)
|
||||
assert cfg.training.packing is True
|
||||
assert cfg.data.max_length == 128
|
||||
|
||||
|
||||
# ─── SFT Trainer Packing Mock Tests ──────────────────────────────────────
|
||||
|
||||
|
||||
class TestPackingSFTTrainerMock:
|
||||
"""Test that packing=True is actually passed to SFTTrainer kwargs."""
|
||||
|
||||
def test_sft_trainer_kwargs_include_packing(self):
|
||||
"""When packing=true, SFTTrainer should be called with packing=True."""
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
task="sft",
|
||||
data={"train": "data.jsonl"},
|
||||
training={"packing": True, "batch_size": 2},
|
||||
)
|
||||
tcfg = cfg.training
|
||||
|
||||
# Build trainer_kwargs the same way sft.py does
|
||||
trainer_kwargs = {
|
||||
"model": MagicMock(),
|
||||
"args": MagicMock(),
|
||||
"train_dataset": MagicMock(),
|
||||
"eval_dataset": None,
|
||||
"processing_class": MagicMock(),
|
||||
}
|
||||
if tcfg.packing:
|
||||
trainer_kwargs["packing"] = True
|
||||
|
||||
assert "packing" in trainer_kwargs
|
||||
assert trainer_kwargs["packing"] is True
|
||||
|
||||
def test_sft_trainer_kwargs_exclude_packing_when_false(self):
|
||||
"""When packing=false, SFTTrainer kwargs should not include packing."""
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
task="sft",
|
||||
data={"train": "data.jsonl"},
|
||||
training={"packing": False, "batch_size": 2},
|
||||
)
|
||||
tcfg = cfg.training
|
||||
|
||||
trainer_kwargs = {
|
||||
"model": MagicMock(),
|
||||
"args": MagicMock(),
|
||||
"train_dataset": MagicMock(),
|
||||
"eval_dataset": None,
|
||||
"processing_class": MagicMock(),
|
||||
}
|
||||
if tcfg.packing:
|
||||
trainer_kwargs["packing"] = True
|
||||
|
||||
assert "packing" not in trainer_kwargs
|
||||
|
||||
def test_packing_small_max_length_warning(self):
|
||||
"""Packing with max_length < 256 should trigger a warning."""
|
||||
from rich.console import Console
|
||||
|
||||
cfg = SoupConfig(
|
||||
base="test-model",
|
||||
task="sft",
|
||||
data={"train": "data.jsonl", "max_length": 128},
|
||||
training={"packing": True, "batch_size": 2},
|
||||
)
|
||||
|
||||
output = StringIO()
|
||||
console = Console(file=output)
|
||||
|
||||
if cfg.training.packing and cfg.data.max_length < 256:
|
||||
console.print(
|
||||
f"[yellow]Warning:[/] packing=true with "
|
||||
f"max_length={cfg.data.max_length} may be suboptimal."
|
||||
)
|
||||
|
||||
assert "suboptimal" in output.getvalue()
|
||||
|
|
|
|||
Loading…
Reference in New Issue