feat(train): --reward-hack-detector / --reward-hack-halt CLI flags (v0.71.26)

Folds in the doc-vs-reality cleanup the plan flagged: the docs referenced
soup train --reward-hack-detector / --reward-hack-halt but they were config-only.
Add both as CLI passthroughs mirroring --reward-hack-mitigation (validate value,
set cfg.training field, accelerate re-exec passthrough) so the docs are true and
the reward-hack CLI is consistent. +4 tests (test_v07126: 180 -> 184; full suite
-> 14788).
This commit is contained in:
Alpamys 2026-07-01 17:02:37 +05:00
parent dca58c4107
commit 2ffc3743ae
3 changed files with 89 additions and 1 deletions

View File

@ -120,7 +120,7 @@ src/soup_cli/
templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0)
ui/ - Web UI (FastAPI + HTML/JS SPA)
tests/ - Test suite (298 files, 14784 tests)
tests/ - Test suite (298 files, 14788 tests)
examples/ - Real-world config examples and datasets
```

View File

@ -167,6 +167,22 @@ def train(
"training.echo_trap_enabled=true on grpo/ppo."
),
),
reward_hack_detector: str = typer.Option(
None,
"--reward-hack-detector",
help=(
"Reward-hacking detector for GRPO/PPO: info_rm | rm_ensemble. "
"Overrides training.reward_hack_detector. (v0.71.26)"
),
),
reward_hack_halt: bool = typer.Option(
False,
"--reward-hack-halt",
help=(
"Auto-halt training on a HACK verdict. Requires "
"--reward-hack-detector (or training.reward_hack_detector). (v0.71.26)"
),
),
reward_hack_mitigation: str = typer.Option(
None,
"--reward-hack-mitigation",
@ -353,6 +369,25 @@ def train(
cfg.training.echo_trap_tokenizer_aware = True
console.print("[green]Echo-trap tokenizer-aware scoring enabled[/]")
# --- Reward-hack detector / halt shortcut (v0.71.26) ---
if reward_hack_detector is not None:
if reward_hack_detector not in ("info_rm", "rm_ensemble"):
console.print(
"[red]--reward-hack-detector must be info_rm or rm_ensemble[/]"
)
raise typer.Exit(1)
cfg.training.reward_hack_detector = reward_hack_detector
console.print(f"[green]Reward-hack detector:[/] {reward_hack_detector}")
if reward_hack_halt:
if cfg.training.reward_hack_detector is None:
console.print(
"[red]--reward-hack-halt requires --reward-hack-detector "
"(or training.reward_hack_detector)[/]"
)
raise typer.Exit(1)
cfg.training.reward_hack_halt = True
console.print("[green]Reward-hack auto-halt enabled[/]")
# --- Reward-hack mitigation shortcut (v0.71.26) ---
if reward_hack_mitigation is not None:
valid_modes = ("off", "log_only", "kl_control", "pid_lagrangian")
@ -650,6 +685,12 @@ def train(
script_args.append("--tensorboard")
if echo_trap_tokenizer_aware:
script_args.append("--echo-trap-tokenizer-aware")
if reward_hack_detector is not None:
script_args.extend(
["--reward-hack-detector", reward_hack_detector]
)
if reward_hack_halt:
script_args.append("--reward-hack-halt")
if reward_hack_mitigation is not None:
script_args.extend(
["--reward-hack-mitigation", reward_hack_mitigation]

View File

@ -1047,6 +1047,53 @@ class TestRewardHackMitigationCli:
assert "--reward-hack-mitigation" in src
assert "cfg.training.reward_hack_mitigation" in src
def test_help_shows_detector_and_halt_flags(self):
runner, app = self._runner()
result = runner.invoke(app, ["train", "--help"])
assert result.exit_code == 0, result.output
assert "--reward-hack-detector" in result.output
assert "--reward-hack-halt" in result.output
def test_bad_detector_value_rejected(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
(tmp_path / "cfg.yaml").write_text(
"base: HuggingFaceTB/SmolLM2-135M\n"
"task: grpo\n"
"data:\n train: ./train.jsonl\n format: chatml\n"
"training:\n reward_fn: accuracy\n"
)
runner, app = self._runner()
result = runner.invoke(
app,
["train", "--config", "cfg.yaml", "--reward-hack-detector", "bogus", "--yes"],
)
assert result.exit_code == 1
assert "info_rm" in result.output
def test_halt_flag_without_detector_rejected(self, tmp_path, monkeypatch):
monkeypatch.chdir(tmp_path)
(tmp_path / "cfg.yaml").write_text(
"base: HuggingFaceTB/SmolLM2-135M\n"
"task: grpo\n"
"data:\n train: ./train.jsonl\n format: chatml\n"
"training:\n reward_fn: accuracy\n"
)
runner, app = self._runner()
result = runner.invoke(
app, ["train", "--config", "cfg.yaml", "--reward-hack-halt", "--yes"]
)
assert result.exit_code == 1
assert "reward-hack-detector" in result.output
def test_detector_reexec_passthrough_present(self):
import inspect
from soup_cli.commands import train as train_mod
src = inspect.getsource(train_mod)
assert "--reward-hack-detector" in src
assert "cfg.training.reward_hack_detector = reward_hack_detector" in src
# =====================================================================
# Part C / Stage 2 — schema fields (PID + rollback) (Task C1)