mirror of https://github.com/razor-ai/soup.git
538 lines
18 KiB
Python
538 lines
18 KiB
Python
"""Tests for v0.61.0 Part A — `soup unlearn` trainer (NPO / SimNPO / RMU).
|
|
|
|
Schema-only release: tests cover the new ``task='unlearn'`` Literal entry,
|
|
the ``unlearn_method`` field, the ``forget_set`` / ``retain_set`` data
|
|
schema, and the ``UnlearnTrainerWrapper`` stub. Live wiring is deferred
|
|
to v0.61.1 (mirrors v0.50.0 / v0.52.0 / v0.53.0 stub-then-live pattern).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import dataclasses
|
|
|
|
import pytest
|
|
|
|
# ---------- Module surface ----------
|
|
|
|
|
|
class TestModuleSurface:
|
|
def test_imports(self):
|
|
from soup_cli.utils.unlearning import (
|
|
SUPPORTED_UNLEARN_METHODS,
|
|
UnlearnMethodSpec,
|
|
apply_unlearn_loss,
|
|
build_unlearn_trainer,
|
|
get_unlearn_method_spec,
|
|
validate_unlearn_method,
|
|
)
|
|
assert callable(validate_unlearn_method)
|
|
assert callable(get_unlearn_method_spec)
|
|
assert callable(apply_unlearn_loss)
|
|
assert callable(build_unlearn_trainer)
|
|
assert dataclasses.is_dataclass(UnlearnMethodSpec)
|
|
assert isinstance(SUPPORTED_UNLEARN_METHODS, frozenset)
|
|
|
|
def test_supported_methods_exact(self):
|
|
from soup_cli.utils.unlearning import SUPPORTED_UNLEARN_METHODS
|
|
|
|
assert SUPPORTED_UNLEARN_METHODS == frozenset({"npo", "simnpo", "rmu"})
|
|
|
|
def test_metadata_mapping_proxy(self):
|
|
from types import MappingProxyType
|
|
|
|
from soup_cli.utils.unlearning import _UNLEARN_METHOD_METADATA # type: ignore
|
|
|
|
assert isinstance(_UNLEARN_METHOD_METADATA, MappingProxyType)
|
|
|
|
|
|
# ---------- validate_unlearn_method ----------
|
|
|
|
|
|
class TestValidateUnlearnMethod:
|
|
def test_happy_path(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
for name in ("npo", "simnpo", "rmu"):
|
|
assert validate_unlearn_method(name) == name
|
|
|
|
def test_case_insensitive(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
assert validate_unlearn_method("NPO") == "npo"
|
|
assert validate_unlearn_method("SimNPO") == "simnpo"
|
|
assert validate_unlearn_method("RMU") == "rmu"
|
|
|
|
def test_unknown_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(ValueError, match="unknown unlearn method"):
|
|
validate_unlearn_method("dpo")
|
|
|
|
def test_bool_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_unlearn_method(True)
|
|
|
|
def test_non_string_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_unlearn_method(123)
|
|
|
|
def test_empty_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_unlearn_method("")
|
|
|
|
def test_null_byte_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(ValueError, match="null"):
|
|
validate_unlearn_method("npo\x00")
|
|
|
|
def test_oversize_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_method
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_unlearn_method("a" * 100)
|
|
|
|
|
|
# ---------- get_unlearn_method_spec ----------
|
|
|
|
|
|
class TestGetUnlearnMethodSpec:
|
|
def test_happy_path(self):
|
|
from soup_cli.utils.unlearning import get_unlearn_method_spec
|
|
|
|
spec = get_unlearn_method_spec("npo")
|
|
assert spec.name == "npo"
|
|
assert isinstance(spec.description, str) and spec.description
|
|
assert spec.live_wired is False # deferred to v0.61.1
|
|
|
|
def test_frozen(self):
|
|
from soup_cli.utils.unlearning import get_unlearn_method_spec
|
|
|
|
spec = get_unlearn_method_spec("npo")
|
|
with pytest.raises(dataclasses.FrozenInstanceError):
|
|
spec.name = "other" # type: ignore
|
|
|
|
def test_unknown_raises(self):
|
|
from soup_cli.utils.unlearning import get_unlearn_method_spec
|
|
|
|
with pytest.raises(ValueError):
|
|
get_unlearn_method_spec("zzz")
|
|
|
|
def test_all_three_have_specs(self):
|
|
from soup_cli.utils.unlearning import get_unlearn_method_spec
|
|
|
|
for name in ("npo", "simnpo", "rmu"):
|
|
spec = get_unlearn_method_spec(name)
|
|
assert spec.name == name
|
|
|
|
|
|
# ---------- apply_unlearn_loss (stub) ----------
|
|
|
|
|
|
class TestApplyUnlearnLoss:
|
|
def test_unknown_rejected_before_notimplemented(self):
|
|
from soup_cli.utils.unlearning import apply_unlearn_loss
|
|
|
|
with pytest.raises(ValueError):
|
|
apply_unlearn_loss("zzz")
|
|
|
|
def test_returns_live_kernel(self):
|
|
# v0.71.9 #193 — apply_unlearn_loss now returns the live kernel.
|
|
from soup_cli.utils import unlearn_kernels
|
|
from soup_cli.utils.unlearning import apply_unlearn_loss
|
|
|
|
assert apply_unlearn_loss("npo") is unlearn_kernels.npo_loss
|
|
assert apply_unlearn_loss("simnpo") is unlearn_kernels.simnpo_loss
|
|
assert apply_unlearn_loss("rmu") is unlearn_kernels.rmu_loss
|
|
|
|
|
|
# ---------- build_unlearn_trainer (stub) ----------
|
|
|
|
|
|
class TestBuildUnlearnTrainer:
|
|
def _make_cfg(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
return SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"forget_set": "f.jsonl",
|
|
"retain_set": "r.jsonl",
|
|
},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
|
|
def test_returns_wrapper_instance(self):
|
|
from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
|
|
from soup_cli.utils.unlearning import build_unlearn_trainer
|
|
|
|
cfg = self._make_cfg()
|
|
wrapper = build_unlearn_trainer(cfg)
|
|
assert isinstance(wrapper, UnlearnTrainerWrapper)
|
|
|
|
def test_kwargs_signature_allows_known(self, monkeypatch):
|
|
from soup_cli.utils.unlearning import build_unlearn_trainer
|
|
|
|
cfg = self._make_cfg()
|
|
# Forward-compat kwargs accepted at construction time. v0.71.9 #193:
|
|
# setup() is now LIVE — it loads a model. Prove that by monkeypatching
|
|
# the loader to a sentinel and asserting setup() invokes it.
|
|
wrapper = build_unlearn_trainer(cfg, device="cpu", trust_remote_code=False)
|
|
import soup_cli.utils.live_eval as live_eval
|
|
|
|
def _boom(*a, **k):
|
|
raise RuntimeError("LOAD_CALLED")
|
|
|
|
monkeypatch.setattr(live_eval, "load_model_and_tokenizer", _boom)
|
|
with pytest.raises(RuntimeError, match="LOAD_CALLED"):
|
|
wrapper.setup()
|
|
|
|
def test_invalid_config_rejected(self):
|
|
from soup_cli.utils.unlearning import build_unlearn_trainer
|
|
|
|
class _Cfg:
|
|
pass
|
|
|
|
with pytest.raises(AttributeError, match="SoupConfig"):
|
|
build_unlearn_trainer(_Cfg())
|
|
|
|
|
|
# ---------- validate_unlearn_compat ----------
|
|
|
|
|
|
class TestValidateUnlearnCompat:
|
|
def test_happy_path(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
# Should not raise
|
|
validate_unlearn_compat(task="unlearn", backend="transformers")
|
|
|
|
def test_mlx_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(ValueError, match="mlx"):
|
|
validate_unlearn_compat(task="unlearn", backend="mlx")
|
|
|
|
def test_wrong_task_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_unlearn_compat(task="sft", backend="transformers")
|
|
|
|
def test_bool_task_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_unlearn_compat(task=True, backend="transformers") # type: ignore
|
|
|
|
def test_bool_backend_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(TypeError):
|
|
validate_unlearn_compat(task="unlearn", backend=True) # type: ignore
|
|
|
|
def test_empty_task_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(ValueError):
|
|
validate_unlearn_compat(task="", backend="transformers")
|
|
|
|
def test_null_byte_rejected(self):
|
|
from soup_cli.utils.unlearning import validate_unlearn_compat
|
|
|
|
with pytest.raises(ValueError, match="null"):
|
|
validate_unlearn_compat(task="unlearn", backend="transformers\x00")
|
|
|
|
|
|
# ---------- Schema integration: task='unlearn' ----------
|
|
|
|
|
|
class TestSchemaUnlearnTask:
|
|
def test_unlearn_task_accepted(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"format": "auto",
|
|
"forget_set": "forget.jsonl",
|
|
"retain_set": "retain.jsonl",
|
|
},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
assert cfg.task == "unlearn"
|
|
assert cfg.training.unlearn_method == "npo"
|
|
|
|
def test_unlearn_method_default_none_for_non_unlearn(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="sft",
|
|
data={"train": "test.jsonl"},
|
|
)
|
|
assert cfg.training.unlearn_method is None
|
|
|
|
def test_unlearn_method_unknown_rejected(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={"train": "test.jsonl", "forget_set": "f.jsonl"},
|
|
training={"unlearn_method": "zzz"},
|
|
)
|
|
|
|
def test_unlearn_method_case_insensitive(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"forget_set": "f.jsonl",
|
|
"retain_set": "r.jsonl",
|
|
},
|
|
training={"unlearn_method": "NPO"},
|
|
)
|
|
assert cfg.training.unlearn_method == "npo"
|
|
|
|
def test_mlx_rejected(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
with pytest.raises(ValidationError, match="mlx"):
|
|
SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
backend="mlx",
|
|
data={"train": "test.jsonl", "forget_set": "f.jsonl"},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
|
|
def test_unlearn_method_outside_unlearn_task_rejected(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
with pytest.raises(ValidationError, match="unlearn"):
|
|
SoupConfig(
|
|
base="test-model",
|
|
task="sft",
|
|
data={"train": "test.jsonl"},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
|
|
def test_forget_set_field(self):
|
|
from soup_cli.config.schema import DataConfig
|
|
|
|
data = DataConfig(train="test.jsonl", forget_set="forget.jsonl")
|
|
assert data.forget_set == "forget.jsonl"
|
|
|
|
def test_retain_set_field(self):
|
|
from soup_cli.config.schema import DataConfig
|
|
|
|
data = DataConfig(train="test.jsonl", retain_set="retain.jsonl")
|
|
assert data.retain_set == "retain.jsonl"
|
|
|
|
def test_forget_set_null_byte_rejected(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import DataConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
DataConfig(train="test.jsonl", forget_set="f\x00.jsonl")
|
|
|
|
def test_retain_set_oversize_rejected(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import DataConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
DataConfig(train="test.jsonl", retain_set="x" * 5000)
|
|
|
|
def test_unlearn_requires_forget_set(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
with pytest.raises(ValidationError, match="forget_set"):
|
|
SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={"train": "test.jsonl"},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
|
|
def test_unlearn_alpha_field(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
tcfg = TrainingConfig(unlearn_method="npo", unlearn_alpha=0.5)
|
|
assert tcfg.unlearn_alpha == 0.5
|
|
|
|
def test_unlearn_alpha_bounds(self):
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(unlearn_method="npo", unlearn_alpha=-0.1)
|
|
|
|
with pytest.raises(ValidationError):
|
|
TrainingConfig(unlearn_method="npo", unlearn_alpha=11.0)
|
|
|
|
def test_unlearn_alpha_bool_rejected(self):
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
# Bool raises TypeError from the validator; Pydantic v2 does not
|
|
# wrap TypeError in ValidationError (only ValueError /
|
|
# AssertionError / PydanticCustomError).
|
|
with pytest.raises(TypeError, match="bool"):
|
|
TrainingConfig(unlearn_method="npo", unlearn_alpha=True)
|
|
|
|
def test_unlearn_alpha_without_method_rejected(self):
|
|
"""Review L10 — `unlearn_alpha` without `unlearn_method` is a footgun."""
|
|
from pydantic import ValidationError
|
|
|
|
from soup_cli.config.schema import SoupConfig
|
|
|
|
with pytest.raises(ValidationError, match="unlearn_method"):
|
|
SoupConfig(
|
|
base="test-model",
|
|
task="sft",
|
|
data={"train": "test.jsonl"},
|
|
training={"unlearn_alpha": 0.5},
|
|
)
|
|
|
|
def test_unlearn_alpha_boundary_zero_accepted(self):
|
|
"""Review L3 — exact lower boundary (0.0)."""
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
tcfg = TrainingConfig(unlearn_method="npo", unlearn_alpha=0.0)
|
|
assert tcfg.unlearn_alpha == 0.0
|
|
|
|
def test_unlearn_alpha_boundary_ten_accepted(self):
|
|
"""Review L3 — exact upper boundary (10.0)."""
|
|
from soup_cli.config.schema import TrainingConfig
|
|
|
|
tcfg = TrainingConfig(unlearn_method="npo", unlearn_alpha=10.0)
|
|
assert tcfg.unlearn_alpha == 10.0
|
|
|
|
|
|
# ---------- Source-grep regression guards (review L7) ----------
|
|
|
|
|
|
class TestSourceWiring:
|
|
def test_cli_registers_edit_typer(self):
|
|
"""cli.py must add `_edit_cmd.app` under name='edit'."""
|
|
from pathlib import Path
|
|
|
|
cli_src = Path(__file__).resolve().parent.parent / "src" / "soup_cli" / "cli.py"
|
|
text = cli_src.read_text(encoding="utf-8")
|
|
assert "_edit_cmd" in text
|
|
assert 'name="edit"' in text
|
|
|
|
def test_eval_registers_v0610(self):
|
|
"""eval.py must call register(app, console) from _eval_v0610."""
|
|
from pathlib import Path
|
|
|
|
eval_src = (
|
|
Path(__file__).resolve().parent.parent
|
|
/ "src" / "soup_cli" / "commands" / "eval.py"
|
|
)
|
|
text = eval_src.read_text(encoding="utf-8")
|
|
assert "_register_v0610" in text
|
|
|
|
def test_pyproject_includes_unlearning_fixtures(self):
|
|
"""pyproject.toml artifacts list must include unlearning fixtures."""
|
|
from pathlib import Path
|
|
|
|
py = Path(__file__).resolve().parent.parent / "pyproject.toml"
|
|
text = py.read_text(encoding="utf-8")
|
|
assert "unlearning/*.jsonl" in text
|
|
|
|
|
|
# ---------- UnlearnTrainerWrapper ----------
|
|
|
|
|
|
class TestUnlearnTrainerWrapper:
|
|
def test_import(self):
|
|
from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
|
|
|
|
assert UnlearnTrainerWrapper is not None
|
|
|
|
def test_train_before_setup_raises(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"forget_set": "f.jsonl",
|
|
"retain_set": "r.jsonl",
|
|
},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
wrapper = UnlearnTrainerWrapper(cfg)
|
|
with pytest.raises(RuntimeError, match="setup"):
|
|
wrapper.train()
|
|
|
|
def test_setup_loads_model(self, monkeypatch):
|
|
# v0.71.9 #193 — setup() is live; it loads a model + tokenizer.
|
|
from soup_cli.config.schema import SoupConfig
|
|
from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"forget_set": "f.jsonl",
|
|
"retain_set": "r.jsonl",
|
|
},
|
|
training={"unlearn_method": "npo"},
|
|
)
|
|
wrapper = UnlearnTrainerWrapper(cfg)
|
|
import soup_cli.utils.live_eval as live_eval
|
|
|
|
def _boom(*a, **k):
|
|
raise RuntimeError("LOAD_CALLED")
|
|
|
|
monkeypatch.setattr(live_eval, "load_model_and_tokenizer", _boom)
|
|
with pytest.raises(RuntimeError, match="LOAD_CALLED"):
|
|
wrapper.setup()
|
|
|
|
def test_method_attribute(self):
|
|
from soup_cli.config.schema import SoupConfig
|
|
from soup_cli.trainer.unlearn import UnlearnTrainerWrapper
|
|
|
|
cfg = SoupConfig(
|
|
base="test-model",
|
|
task="unlearn",
|
|
data={
|
|
"train": "test.jsonl",
|
|
"forget_set": "f.jsonl",
|
|
"retain_set": "r.jsonl",
|
|
},
|
|
training={"unlearn_method": "simnpo"},
|
|
)
|
|
wrapper = UnlearnTrainerWrapper(cfg)
|
|
assert wrapper.method == "simnpo"
|