RomanNum3ral
/
soup
mirror of https://github.com/razor-ai/soup.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
soup/tests/test_migrate.py

1376 lines
47 KiB
Python
Raw Blame History

"""Tests for soup migrate — config import from LLaMA-Factory, Axolotl, Unsloth."""
import json
from pathlib import Path
import pytest
from typer.testing import CliRunner
from soup_cli.cli import app
runner = CliRunner()
# ---------------------------------------------------------------------------
# Fixtures — sample configs
# ---------------------------------------------------------------------------
LLAMA_FACTORY_SFT = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: sft
finetuning_type: lora
lora_rank: 16
lora_alpha: 32
lora_dropout: 0.05
lora_target: all
dataset: alpaca_en
template: llama3
cutoff_len: 2048
per_device_train_batch_size: 4
gradient_accumulation_steps: 4
num_train_epochs: 3
learning_rate: 2e-4
lr_scheduler_type: cosine
warmup_ratio: 0.03
output_dir: ./output
quantization_bit: 4
bf16: true
"""
LLAMA_FACTORY_DPO = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: dpo
finetuning_type: lora
lora_rank: 16
lora_alpha: 32
dataset: dpo_data
template: llama3
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 1
learning_rate: 5e-6
output_dir: ./output_dpo
quantization_bit: 4
pref_beta: 0.1
"""
LLAMA_FACTORY_KTO = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: kto
finetuning_type: lora
lora_rank: 16
dataset: kto_data
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 3
learning_rate: 1e-5
output_dir: ./output_kto
pref_beta: 0.1
"""
LLAMA_FACTORY_PPO = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: ppo
finetuning_type: lora
lora_rank: 16
dataset: ppo_prompts
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 1
learning_rate: 1e-6
output_dir: ./output_ppo
reward_model: ./reward_model
"""
LLAMA_FACTORY_PRETRAIN = """\
model_name_or_path: meta-llama/Llama-3.1-8B
stage: pt
finetuning_type: lora
lora_rank: 32
dataset: corpus
cutoff_len: 4096
per_device_train_batch_size: 2
num_train_epochs: 1
learning_rate: 1e-5
output_dir: ./output_pretrain
"""
LLAMA_FACTORY_RM = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: rm
finetuning_type: lora
lora_rank: 16
dataset: rm_data
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 1
learning_rate: 1e-5
output_dir: ./output_rm
"""
LLAMA_FACTORY_ORPO = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: dpo
pref_loss: orpo
finetuning_type: lora
lora_rank: 16
dataset: pref_data
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 3
learning_rate: 1e-5
output_dir: ./output_orpo
pref_beta: 0.1
"""
LLAMA_FACTORY_SIMPO = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: dpo
pref_loss: simpo
finetuning_type: lora
lora_rank: 16
dataset: pref_data
cutoff_len: 2048
per_device_train_batch_size: 2
num_train_epochs: 3
learning_rate: 1e-5
output_dir: ./output_simpo
"""
LLAMA_FACTORY_NEFTUNE = """\
model_name_or_path: meta-llama/Llama-3.1-8B-Instruct
stage: sft
finetuning_type: lora
lora_rank: 16
dataset: alpaca_en
cutoff_len: 2048
per_device_train_batch_size: 4
num_train_epochs: 3
learning_rate: 2e-4
output_dir: ./output
neftune_noise_alpha: 5.0
use_dora: true
loraplus_lr_ratio: 16.0
"""
AXOLOTL_SFT = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train.jsonl
type: alpaca
sequence_len: 2048
adapter: lora
lora_r: 16
lora_alpha: 32
lora_dropout: 0.05
lora_target_modules:
- q_proj
- v_proj
micro_batch_size: 4
gradient_accumulation_steps: 4
num_epochs: 3
learning_rate: 2e-4
optimizer: adamw_torch
lr_scheduler: cosine
warmup_ratio: 0.03
output_dir: ./output
"""
AXOLOTL_DPO = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/dpo_train.jsonl
type: sharegpt
sequence_len: 2048
adapter: qlora
lora_r: 16
lora_alpha: 32
micro_batch_size: 2
num_epochs: 1
learning_rate: 5e-6
output_dir: ./output_dpo
rl: dpo
"""
AXOLOTL_GRPO = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/reasoning.jsonl
type: sharegpt
sequence_len: 4096
adapter: lora
lora_r: 16
lora_alpha: 32
micro_batch_size: 2
num_epochs: 3
learning_rate: 1e-5
output_dir: ./output_grpo
rl: grpo
flash_attention: true
"""
AXOLOTL_MULTI_DATASET = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train1.jsonl
type: alpaca
- path: ./data/train2.jsonl
type: sharegpt
sequence_len: 2048
adapter: lora
lora_r: 16
micro_batch_size: 4
num_epochs: 3
learning_rate: 2e-4
output_dir: ./output
"""
AXOLOTL_SAMPLE_PACKING = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train.jsonl
type: alpaca
sequence_len: 2048
adapter: lora
lora_r: 16
micro_batch_size: 4
num_epochs: 3
learning_rate: 2e-4
output_dir: ./output
sample_packing: true
"""
AXOLOTL_LOAD_IN_4BIT = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train.jsonl
type: alpaca
sequence_len: 2048
adapter: lora
lora_r: 16
micro_batch_size: 4
num_epochs: 3
learning_rate: 2e-4
output_dir: ./output
load_in_4bit: true
"""
AXOLOTL_LORA_TARGET_LINEAR = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train.jsonl
type: alpaca
sequence_len: 2048
adapter: lora
lora_r: 16
lora_target_linear: true
micro_batch_size: 4
num_epochs: 3
learning_rate: 2e-4
output_dir: ./output
"""
AXOLOTL_ADAMW_8BIT = """\
base_model: meta-llama/Llama-3.1-8B-Instruct
datasets:
- path: ./data/train.jsonl
type: alpaca
sequence_len: 2048
adapter: lora
lora_r: 16
micro_batch_size: 4
num_epochs: 3
learning_rate: 2e-4
optimizer: adamw_8bit
output_dir: ./output
"""
UNSLOTH_SFT_NOTEBOOK = {
"cells": [
{
"cell_type": "code",
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='meta-llama/Llama-3.1-8B-Instruct',\n",
" max_seq_length=2048,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r=16,\n",
" lora_alpha=32,\n",
" lora_dropout=0.05,\n",
" target_modules=['q_proj', 'v_proj', 'k_proj', 'o_proj'],\n",
" use_dora=False,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import SFTTrainer\n",
"from transformers import TrainingArguments\n",
"trainer = SFTTrainer(\n",
" model=model,\n",
" train_dataset=dataset,\n",
" args=TrainingArguments(\n",
" per_device_train_batch_size=4,\n",
" num_train_epochs=3,\n",
" learning_rate=2e-4,\n",
" optim='adamw_torch',\n",
" lr_scheduler_type='cosine',\n",
" output_dir='./output',\n",
" ),\n",
")\n",
],
},
],
}
UNSLOTH_DPO_NOTEBOOK = {
"cells": [
{
"cell_type": "code",
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='meta-llama/Llama-3.1-8B-Instruct',\n",
" max_seq_length=2048,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r=16,\n",
" lora_alpha=32,\n",
" lora_dropout=0.0,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import DPOTrainer, DPOConfig\n",
"trainer = DPOTrainer(\n",
" model=model,\n",
" train_dataset=dataset,\n",
" args=DPOConfig(\n",
" per_device_train_batch_size=2,\n",
" num_train_epochs=1,\n",
" learning_rate=5e-6,\n",
" beta=0.1,\n",
" output_dir='./output_dpo',\n",
" ),\n",
")\n",
],
},
],
}
UNSLOTH_GRPO_NOTEBOOK = {
"cells": [
{
"cell_type": "code",
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='deepseek-ai/DeepSeek-R1-Distill-Llama-8B',\n",
" max_seq_length=4096,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r=32,\n",
" lora_alpha=64,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import GRPOTrainer, GRPOConfig\n",
"trainer = GRPOTrainer(\n",
" model=model,\n",
" train_dataset=dataset,\n",
" args=GRPOConfig(\n",
" per_device_train_batch_size=2,\n",
" num_train_epochs=3,\n",
" learning_rate=1e-5,\n",
" output_dir='./output_grpo',\n",
" ),\n",
")\n",
],
},
],
}
UNSLOTH_RSLORA_NOTEBOOK = {
"cells": [
{
"cell_type": "code",
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='meta-llama/Llama-3.1-8B-Instruct',\n",
" max_seq_length=2048,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r=16,\n",
" lora_alpha=32,\n",
" use_rslora=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import SFTTrainer\n",
"from transformers import TrainingArguments\n",
"trainer = SFTTrainer(\n",
" model=model,\n",
" train_dataset=dataset,\n",
" args=TrainingArguments(\n",
" per_device_train_batch_size=4,\n",
" num_train_epochs=3,\n",
" learning_rate=2e-4,\n",
" output_dir='./output',\n",
" ),\n",
")\n",
],
},
],
}
UNSLOTH_PACKING_NOTEBOOK = {
"cells": [
{
"cell_type": "code",
"source": [
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='meta-llama/Llama-3.1-8B-Instruct',\n",
" max_seq_length=2048,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(\n",
" model,\n",
" r=16,\n",
" lora_alpha=32,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import SFTTrainer\n",
"from transformers import TrainingArguments\n",
"trainer = SFTTrainer(\n",
" model=model,\n",
" train_dataset=dataset,\n",
" packing=True,\n",
" args=TrainingArguments(\n",
" per_device_train_batch_size=4,\n",
" num_train_epochs=3,\n",
" learning_rate=2e-4,\n",
" output_dir='./output',\n",
" ),\n",
")\n",
],
},
],
}
# ---------------------------------------------------------------------------
# LLaMA-Factory migration tests
# ---------------------------------------------------------------------------
class TestLlamaFactoryMigration:
"""LLaMA-Factory → Soup config migration."""
def test_sft_basic(self, tmp_path):
"""SFT config maps correctly."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["base"] == "meta-llama/Llama-3.1-8B-Instruct"
assert result["task"] == "sft"
assert result["training"]["lora"]["r"] == 16
assert result["training"]["lora"]["alpha"] == 32
assert result["training"]["lora"]["dropout"] == 0.05
assert result["training"]["quantization"] == "4bit"
assert result["training"]["batch_size"] == 4
assert result["training"]["epochs"] == 3
assert result["training"]["lr"] == 2e-4
assert result["training"]["scheduler"] == "cosine"
assert result["training"]["warmup_ratio"] == 0.03
assert result["training"]["gradient_accumulation_steps"] == 4
assert result["data"]["max_length"] == 2048
assert result["output"] == "./output"
def test_sft_lora_target_all(self, tmp_path):
"""lora_target: all → auto (Soup auto-detects)."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["training"]["lora"]["target_modules"] == "auto"
def test_dpo_config(self, tmp_path):
"""DPO stage maps to task: dpo with dpo_beta."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_DPO, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "dpo"
assert result["training"]["dpo_beta"] == 0.1
assert result["data"]["format"] == "dpo"
def test_kto_config(self, tmp_path):
"""KTO stage maps to task: kto with kto_beta."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_KTO, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "kto"
assert result["training"]["kto_beta"] == 0.1
assert result["data"]["format"] == "kto"
def test_ppo_config(self, tmp_path):
"""PPO stage maps to task: ppo with reward_model."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_PPO, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "ppo"
assert result["training"]["reward_model"] == "./reward_model"
def test_pretrain_config(self, tmp_path):
"""pt stage maps to task: pretrain, format: plaintext."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_PRETRAIN, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "pretrain"
assert result["data"]["format"] == "plaintext"
def test_reward_model_config(self, tmp_path):
"""rm stage maps to task: reward_model."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_RM, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "reward_model"
assert result["data"]["format"] == "dpo"
def test_orpo_via_pref_loss(self, tmp_path):
"""stage: dpo + pref_loss: orpo → task: orpo."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_ORPO, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "orpo"
assert result["training"]["orpo_beta"] == 0.1
def test_simpo_via_pref_loss(self, tmp_path):
"""stage: dpo + pref_loss: simpo → task: simpo."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SIMPO, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["task"] == "simpo"
def test_neftune_and_dora(self, tmp_path):
"""NEFTune, DoRA, LoRA+ are mapped."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_NEFTUNE, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert result["training"]["lora"]["use_dora"] is True
assert result["training"]["loraplus_lr_ratio"] == 16.0
def test_dataset_warning(self, tmp_path):
"""Dataset name (not path) triggers a warning."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert any("dataset" in w.lower() for w in result.get("_warnings", []))
def test_empty_config(self, tmp_path):
"""Empty YAML raises ValueError."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("", encoding="utf-8")
with pytest.raises(ValueError, match="empty"):
migrate_llamafactory(cfg_file)
def test_missing_model(self, tmp_path):
"""Config without model_name_or_path raises ValueError."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("stage: sft\n", encoding="utf-8")
with pytest.raises(ValueError, match="model_name_or_path"):
migrate_llamafactory(cfg_file)
def test_full_finetuning_no_lora(self, tmp_path):
"""finetuning_type: full → no lora section."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(
"model_name_or_path: meta-llama/Llama-3.1-8B\n"
"stage: sft\n"
"finetuning_type: full\n"
"dataset: data\n"
"num_train_epochs: 1\n"
"output_dir: ./output\n",
encoding="utf-8",
)
result = migrate_llamafactory(cfg_file)
assert "lora" not in result.get("training", {})
def test_quantization_8bit(self, tmp_path):
"""quantization_bit: 8 → quantization: 8bit."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(
"model_name_or_path: meta-llama/Llama-3.1-8B\n"
"stage: sft\n"
"finetuning_type: lora\n"
"lora_rank: 16\n"
"dataset: data\n"
"num_train_epochs: 1\n"
"output_dir: ./output\n"
"quantization_bit: 8\n",
encoding="utf-8",
)
result = migrate_llamafactory(cfg_file)
assert result["training"]["quantization"] == "8bit"
def test_round_trip_valid_config(self, tmp_path):
"""Migrated config loads as valid SoupConfig."""
from soup_cli.config.loader import load_config_from_string
from soup_cli.migrate.common import config_to_yaml
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
yaml_str = config_to_yaml(result)
soup_config = load_config_from_string(yaml_str)
assert soup_config.base == "meta-llama/Llama-3.1-8B-Instruct"
assert soup_config.task == "sft"
# ---------------------------------------------------------------------------
# Axolotl migration tests
# ---------------------------------------------------------------------------
class TestAxolotlMigration:
"""Axolotl → Soup config migration."""
def test_sft_basic(self, tmp_path):
"""SFT config maps correctly."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_SFT, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["base"] == "meta-llama/Llama-3.1-8B-Instruct"
assert result["task"] == "sft"
assert result["training"]["lora"]["r"] == 16
assert result["training"]["lora"]["alpha"] == 32
assert result["training"]["lora"]["dropout"] == 0.05
assert result["training"]["lora"]["target_modules"] == ["q_proj", "v_proj"]
assert result["training"]["batch_size"] == 4
assert result["training"]["epochs"] == 3
assert result["training"]["lr"] == 2e-4
assert result["training"]["optimizer"] == "adamw_torch"
assert result["training"]["scheduler"] == "cosine"
assert result["data"]["train"] == "./data/train.jsonl"
assert result["data"]["format"] == "alpaca"
assert result["data"]["max_length"] == 2048
assert result["output"] == "./output"
def test_dpo_with_qlora(self, tmp_path):
"""rl: dpo → task: dpo, adapter: qlora → quantization: 4bit."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_DPO, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["task"] == "dpo"
assert result["training"]["quantization"] == "4bit"
assert result["data"]["format"] == "dpo"
def test_grpo_with_flash_attn(self, tmp_path):
"""rl: grpo → task: grpo, flash_attention → use_flash_attn."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_GRPO, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["task"] == "grpo"
assert result["training"]["use_flash_attn"] is True
def test_multi_dataset_warning(self, tmp_path):
"""Multiple datasets triggers a warning (Soup uses single dataset)."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_MULTI_DATASET, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["data"]["train"] == "./data/train1.jsonl"
assert any("multiple datasets" in w.lower() for w in result.get("_warnings", []))
def test_sample_packing_warning(self, tmp_path):
"""sample_packing generates unsupported feature warning."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_SAMPLE_PACKING, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert any("sample_packing" in w.lower() for w in result.get("_warnings", []))
def test_load_in_4bit(self, tmp_path):
"""load_in_4bit: true → quantization: 4bit."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_LOAD_IN_4BIT, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["training"]["quantization"] == "4bit"
def test_lora_target_linear(self, tmp_path):
"""lora_target_linear: true → target_modules: auto."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_LORA_TARGET_LINEAR, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["training"]["lora"]["target_modules"] == "auto"
def test_adamw_8bit_mapping(self, tmp_path):
"""adamw_8bit → adamw_bnb_8bit."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_ADAMW_8BIT, encoding="utf-8")
result = migrate_axolotl(cfg_file)
assert result["training"]["optimizer"] == "adamw_bnb_8bit"
def test_empty_config(self, tmp_path):
"""Empty YAML raises ValueError."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("", encoding="utf-8")
with pytest.raises(ValueError, match="empty"):
migrate_axolotl(cfg_file)
def test_missing_base_model(self, tmp_path):
"""Config without base_model raises ValueError."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("num_epochs: 3\n", encoding="utf-8")
with pytest.raises(ValueError, match="base_model"):
migrate_axolotl(cfg_file)
def test_round_trip_valid_config(self, tmp_path):
"""Migrated config loads as valid SoupConfig."""
from soup_cli.config.loader import load_config_from_string
from soup_cli.migrate.axolotl import migrate_axolotl
from soup_cli.migrate.common import config_to_yaml
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_SFT, encoding="utf-8")
result = migrate_axolotl(cfg_file)
yaml_str = config_to_yaml(result)
soup_config = load_config_from_string(yaml_str)
assert soup_config.base == "meta-llama/Llama-3.1-8B-Instruct"
# ---------------------------------------------------------------------------
# Unsloth migration tests
# ---------------------------------------------------------------------------
class TestUnslothMigration:
"""Unsloth notebook → Soup config migration."""
def test_sft_notebook(self, tmp_path):
"""SFT notebook extracted correctly."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "finetune.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_SFT_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
assert result["base"] == "meta-llama/Llama-3.1-8B-Instruct"
assert result["task"] == "sft"
assert result["training"]["lora"]["r"] == 16
assert result["training"]["lora"]["alpha"] == 32
assert result["training"]["lora"]["dropout"] == 0.05
assert result["training"]["lora"]["target_modules"] == [
"q_proj", "v_proj", "k_proj", "o_proj"
]
assert result["training"]["quantization"] == "4bit"
assert result["training"]["batch_size"] == 4
assert result["training"]["epochs"] == 3
assert result["training"]["lr"] == 2e-4
assert result["training"]["optimizer"] == "adamw_torch"
assert result["training"]["scheduler"] == "cosine"
assert result["data"]["max_length"] == 2048
assert result["output"] == "./output"
def test_dpo_notebook(self, tmp_path):
"""DPO notebook extracted correctly."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "dpo.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_DPO_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
assert result["task"] == "dpo"
assert result["training"]["dpo_beta"] == 0.1
assert result["training"]["batch_size"] == 2
assert result["data"]["format"] == "dpo"
def test_grpo_notebook(self, tmp_path):
"""GRPO notebook extracted correctly."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "grpo.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_GRPO_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
assert result["task"] == "grpo"
assert result["base"] == "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
assert result["training"]["lora"]["r"] == 32
assert result["data"]["max_length"] == 4096
def test_rslora_propagated(self, tmp_path):
"""use_rslora=True is propagated to lora config."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "rslora.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_RSLORA_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
assert result["training"]["lora"]["use_rslora"] is True
def test_packing_warning(self, tmp_path):
"""packing=True generates unsupported warning."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "packing.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_PACKING_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
assert any("packing" in w.lower() for w in result.get("_warnings", []))
def test_empty_notebook(self, tmp_path):
"""Notebook with no code cells raises ValueError."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "empty.ipynb"
nb_file.write_text(json.dumps({"cells": []}), encoding="utf-8")
with pytest.raises(ValueError, match="(?i)no .* found"):
migrate_unsloth(nb_file)
def test_invalid_json(self, tmp_path):
"""Non-JSON file raises ValueError."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "bad.ipynb"
nb_file.write_text("not json", encoding="utf-8")
with pytest.raises(ValueError, match="JSON"):
migrate_unsloth(nb_file)
def test_round_trip_valid_config(self, tmp_path):
"""Migrated config loads as valid SoupConfig."""
from soup_cli.config.loader import load_config_from_string
from soup_cli.migrate.common import config_to_yaml
from soup_cli.migrate.unsloth import migrate_unsloth
nb_file = tmp_path / "finetune.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_SFT_NOTEBOOK), encoding="utf-8")
result = migrate_unsloth(nb_file)
yaml_str = config_to_yaml(result)
soup_config = load_config_from_string(yaml_str)
assert soup_config.base == "meta-llama/Llama-3.1-8B-Instruct"
# ---------------------------------------------------------------------------
# Common utilities tests
# ---------------------------------------------------------------------------
class TestCommon:
"""Common migration utilities."""
def test_config_to_yaml_basic(self):
"""config_to_yaml generates valid YAML."""
from soup_cli.migrate.common import config_to_yaml
config = {
"base": "meta-llama/Llama-3.1-8B-Instruct",
"task": "sft",
"data": {"train": "./data.jsonl", "format": "auto", "max_length": 2048},
"training": {
"epochs": 3,
"lr": 2e-4,
"lora": {"r": 16, "alpha": 32},
},
"output": "./output",
}
yaml_str = config_to_yaml(config)
assert "base:" in yaml_str
assert "meta-llama/Llama-3.1-8B-Instruct" in yaml_str
assert "task:" in yaml_str
assert "sft" in yaml_str
def test_config_to_yaml_strips_warnings(self):
"""_warnings key is stripped from YAML output."""
from soup_cli.migrate.common import config_to_yaml
config = {
"base": "model",
"task": "sft",
"data": {"train": "./data.jsonl"},
"output": "./output",
"_warnings": ["some warning"],
}
yaml_str = config_to_yaml(config)
assert "_warnings" not in yaml_str
def test_validate_input_path(self, tmp_path, monkeypatch):
"""Input path must exist and be under cwd."""
from soup_cli.migrate.common import validate_input_path
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("test", encoding="utf-8")
# Valid path
validated = validate_input_path(cfg_file)
assert validated.exists()
def test_validate_input_path_traversal(self, tmp_path, monkeypatch):
"""Path traversal outside cwd is blocked."""
from soup_cli.migrate.common import validate_input_path
monkeypatch.chdir(tmp_path)
with pytest.raises(ValueError, match="outside"):
validate_input_path(Path("/etc/passwd"))
def test_validate_input_path_not_exists(self, tmp_path, monkeypatch):
"""Non-existent file raises ValueError."""
from soup_cli.migrate.common import validate_input_path
monkeypatch.chdir(tmp_path)
with pytest.raises(ValueError, match="not found"):
validate_input_path(tmp_path / "nonexistent.yaml")
def test_validate_output_path(self, tmp_path, monkeypatch):
"""Output path must be under cwd."""
from soup_cli.migrate.common import validate_output_path
monkeypatch.chdir(tmp_path)
validated = validate_output_path(Path("soup.yaml"))
assert str(validated).endswith("soup.yaml")
def test_validate_output_path_traversal(self, tmp_path, monkeypatch):
"""Output path traversal outside cwd is blocked."""
from soup_cli.migrate.common import validate_output_path
monkeypatch.chdir(tmp_path)
with pytest.raises(ValueError, match="outside"):
validate_output_path(Path("/tmp/evil.yaml"))
# ---------------------------------------------------------------------------
# CLI integration tests
# ---------------------------------------------------------------------------
class TestMigrateCLI:
"""CLI tests for soup migrate command."""
def test_help(self):
"""soup migrate --help shows usage."""
result = runner.invoke(app, ["migrate", "--help"])
assert result.exit_code == 0
assert "migrate" in result.output.lower()
def test_llamafactory_sft(self, tmp_path, monkeypatch):
"""Full CLI: soup migrate --from llamafactory config.yaml."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
str(cfg_file),
"--output", "soup.yaml",
"--yes",
])
assert result.exit_code == 0
out_path = tmp_path / "soup.yaml"
assert out_path.exists()
content = out_path.read_text(encoding="utf-8")
assert "meta-llama/Llama-3.1-8B-Instruct" in content
def test_axolotl_sft(self, tmp_path, monkeypatch):
"""Full CLI: soup migrate --from axolotl config.yaml."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(AXOLOTL_SFT, encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "axolotl",
str(cfg_file),
"--output", "soup.yaml",
"--yes",
])
assert result.exit_code == 0
out_path = tmp_path / "soup.yaml"
assert out_path.exists()
def test_unsloth_notebook(self, tmp_path, monkeypatch):
"""Full CLI: soup migrate --from unsloth finetune.ipynb."""
monkeypatch.chdir(tmp_path)
nb_file = tmp_path / "finetune.ipynb"
nb_file.write_text(json.dumps(UNSLOTH_SFT_NOTEBOOK), encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "unsloth",
str(nb_file),
"--output", "soup.yaml",
"--yes",
])
assert result.exit_code == 0
out_path = tmp_path / "soup.yaml"
assert out_path.exists()
def test_dry_run(self, tmp_path, monkeypatch):
"""--dry-run prints config without writing file."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
str(cfg_file),
"--dry-run",
])
assert result.exit_code == 0
out_path = tmp_path / "soup.yaml"
assert not out_path.exists()
assert "meta-llama" in result.output
def test_invalid_source(self, tmp_path, monkeypatch):
"""Unknown --from value shows error."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text("test: true\n", encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "invalid_tool",
str(cfg_file),
])
assert result.exit_code != 0
def test_file_not_found(self, tmp_path, monkeypatch):
"""Non-existent input file shows error."""
monkeypatch.chdir(tmp_path)
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
"nonexistent.yaml",
])
assert result.exit_code != 0
def test_overwrite_confirmation(self, tmp_path, monkeypatch):
"""Existing soup.yaml asks for confirmation."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
out_file = tmp_path / "soup.yaml"
out_file.write_text("existing content", encoding="utf-8")
# Without --yes, deny confirmation
runner.invoke(app, [
"migrate",
"--from", "llamafactory",
str(cfg_file),
"--output", "soup.yaml",
], input="n\n")
# File should be unchanged
assert out_file.read_text(encoding="utf-8") == "existing content"
def test_warnings_shown(self, tmp_path, monkeypatch):
"""Warnings from migration are displayed."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
str(cfg_file),
"--dry-run",
])
assert result.exit_code == 0
# LF SFT has dataset name warning
assert "warning" in result.output.lower() or "Warning" in result.output
# ---------------------------------------------------------------------------
# Security tests
# ---------------------------------------------------------------------------
class TestMigrateSecurity:
"""Security tests for soup migrate."""
def test_input_path_traversal_cli(self, tmp_path, monkeypatch):
"""Path traversal in input file is blocked."""
monkeypatch.chdir(tmp_path)
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
"../../../etc/passwd",
])
assert result.exit_code != 0
def test_output_path_traversal_cli(self, tmp_path, monkeypatch):
"""Path traversal in output path is blocked."""
monkeypatch.chdir(tmp_path)
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_SFT, encoding="utf-8")
result = runner.invoke(app, [
"migrate",
"--from", "llamafactory",
str(cfg_file),
"--output", "../../../tmp/evil.yaml",
"--yes",
])
assert result.exit_code != 0
def test_unsloth_no_exec(self, tmp_path):
"""Unsloth migration uses AST only, never exec/eval."""
from soup_cli.migrate.unsloth import migrate_unsloth
# A notebook with dangerous code — should parse but not execute
dangerous_nb = {
"cells": [
{
"cell_type": "code",
"source": [
"import os; os.system('echo pwned')\n",
"from unsloth import FastLanguageModel\n",
"model, tokenizer = FastLanguageModel.from_pretrained(\n",
" model_name='meta-llama/Llama-3.1-8B',\n",
" max_seq_length=2048,\n",
" load_in_4bit=True,\n",
")\n",
],
},
{
"cell_type": "code",
"source": [
"model = FastLanguageModel.get_peft_model(model, r=16)\n",
],
},
{
"cell_type": "code",
"source": [
"from trl import SFTTrainer\n",
"from transformers import TrainingArguments\n",
"trainer = SFTTrainer(\n",
" model=model,\n",
" args=TrainingArguments(\n",
" per_device_train_batch_size=4,\n",
" num_train_epochs=1,\n",
" output_dir='./output',\n",
" ),\n",
")\n",
],
},
],
}
nb_file = tmp_path / "dangerous.ipynb"
nb_file.write_text(json.dumps(dangerous_nb), encoding="utf-8")
# Should extract config without executing os.system
result = migrate_unsloth(nb_file)
assert result["base"] == "meta-llama/Llama-3.1-8B"
# ---------------------------------------------------------------------------
# Additional coverage tests (TDD review gaps)
# ---------------------------------------------------------------------------
class TestMigrateTDDGaps:
"""Tests for coverage gaps identified in TDD review."""
def test_oversized_input_file(self, tmp_path, monkeypatch):
"""Input file larger than MAX_CONFIG_FILE_SIZE is rejected."""
from soup_cli.migrate.common import validate_input_path
monkeypatch.chdir(tmp_path)
big_file = tmp_path / "big.yaml"
big_file.write_text("x" * 100, encoding="utf-8")
# Temporarily lower the limit to test the check
import soup_cli.migrate.common as common_mod
original = common_mod.MAX_CONFIG_FILE_SIZE
common_mod.MAX_CONFIG_FILE_SIZE = 50 # 50 bytes
try:
with pytest.raises(ValueError, match="too large"):
validate_input_path(big_file)
finally:
common_mod.MAX_CONFIG_FILE_SIZE = original
def test_llamafactory_freeze_warning(self, tmp_path):
"""finetuning_type: freeze → warning + fallback to LoRA."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(
"model_name_or_path: meta-llama/Llama-3.1-8B\n"
"stage: sft\n"
"finetuning_type: freeze\n"
"lora_rank: 16\n"
"dataset: data\n"
"num_train_epochs: 1\n"
"output_dir: ./output\n",
encoding="utf-8",
)
result = migrate_llamafactory(cfg_file)
# Should fallback to LoRA and warn
assert "lora" in result.get("training", {})
assert any("freeze" in w.lower() for w in result.get("_warnings", []))
def test_axolotl_load_in_8bit(self, tmp_path):
"""load_in_8bit: true → quantization: 8bit."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(
"base_model: meta-llama/Llama-3.1-8B\n"
"datasets:\n"
" - path: ./data/train.jsonl\n"
" type: alpaca\n"
"sequence_len: 2048\n"
"adapter: lora\n"
"lora_r: 16\n"
"micro_batch_size: 4\n"
"num_epochs: 3\n"
"learning_rate: 2e-4\n"
"output_dir: ./output\n"
"load_in_8bit: true\n",
encoding="utf-8",
)
result = migrate_axolotl(cfg_file)
assert result["training"]["quantization"] == "8bit"
def test_axolotl_gdpo_alias(self, tmp_path):
"""rl: gdpo → task: grpo."""
from soup_cli.migrate.axolotl import migrate_axolotl
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(
"base_model: meta-llama/Llama-3.1-8B\n"
"datasets:\n"
" - path: ./data/train.jsonl\n"
" type: sharegpt\n"
"sequence_len: 2048\n"
"adapter: lora\n"
"lora_r: 16\n"
"micro_batch_size: 2\n"
"num_epochs: 3\n"
"learning_rate: 1e-5\n"
"output_dir: ./output\n"
"rl: gdpo\n",
encoding="utf-8",
)
result = migrate_axolotl(cfg_file)
assert result["task"] == "grpo"
def test_unsloth_markdown_only_notebook(self, tmp_path):
"""Notebook with only markdown cells (no code) raises ValueError."""
from soup_cli.migrate.unsloth import migrate_unsloth
nb = {
"cells": [
{"cell_type": "markdown", "source": ["# Title"]},
{"cell_type": "markdown", "source": ["Some text"]},
],
}
nb_file = tmp_path / "markdown_only.ipynb"
nb_file.write_text(json.dumps(nb), encoding="utf-8")
with pytest.raises(ValueError, match="(?i)no .* found"):
migrate_unsloth(nb_file)
def test_config_to_yaml_strips_all_private_keys(self):
"""config_to_yaml strips all underscore-prefixed keys."""
from soup_cli.migrate.common import config_to_yaml
config = {
"base": "model",
"task": "sft",
"data": {"train": "./data.jsonl"},
"output": "./output",
"_warnings": ["w1"],
"_internal": "private",
"_debug": True,
}
yaml_str = config_to_yaml(config)
assert "_warnings" not in yaml_str
assert "_internal" not in yaml_str
assert "_debug" not in yaml_str
assert "base:" in yaml_str
def test_llamafactory_neftune_warning_only(self, tmp_path):
"""NEFTune in LF config produces warning (not auto-migrated to config)."""
from soup_cli.migrate.llamafactory import migrate_llamafactory
cfg_file = tmp_path / "config.yaml"
cfg_file.write_text(LLAMA_FACTORY_NEFTUNE, encoding="utf-8")
result = migrate_llamafactory(cfg_file)
assert any(
"neftune" in w.lower() for w in result.get("_warnings", [])
)
Reference in New Issue View Git Blame Copy Permalink