mirror of https://github.com/razor-ai/soup.git
feat(train): --reward-hack-detector / --reward-hack-halt CLI flags (v0.71.26)
Folds in the doc-vs-reality cleanup the plan flagged: the docs referenced soup train --reward-hack-detector / --reward-hack-halt but they were config-only. Add both as CLI passthroughs mirroring --reward-hack-mitigation (validate value, set cfg.training field, accelerate re-exec passthrough) so the docs are true and the reward-hack CLI is consistent. +4 tests (test_v07126: 180 -> 184; full suite -> 14788).
This commit is contained in:
parent
dca58c4107
commit
2ffc3743ae
|
|
@ -120,7 +120,7 @@ src/soup_cli/
|
|||
templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0)
|
||||
ui/ - Web UI (FastAPI + HTML/JS SPA)
|
||||
|
||||
tests/ - Test suite (298 files, 14784 tests)
|
||||
tests/ - Test suite (298 files, 14788 tests)
|
||||
examples/ - Real-world config examples and datasets
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -167,6 +167,22 @@ def train(
|
|||
"training.echo_trap_enabled=true on grpo/ppo."
|
||||
),
|
||||
),
|
||||
reward_hack_detector: str = typer.Option(
|
||||
None,
|
||||
"--reward-hack-detector",
|
||||
help=(
|
||||
"Reward-hacking detector for GRPO/PPO: info_rm | rm_ensemble. "
|
||||
"Overrides training.reward_hack_detector. (v0.71.26)"
|
||||
),
|
||||
),
|
||||
reward_hack_halt: bool = typer.Option(
|
||||
False,
|
||||
"--reward-hack-halt",
|
||||
help=(
|
||||
"Auto-halt training on a HACK verdict. Requires "
|
||||
"--reward-hack-detector (or training.reward_hack_detector). (v0.71.26)"
|
||||
),
|
||||
),
|
||||
reward_hack_mitigation: str = typer.Option(
|
||||
None,
|
||||
"--reward-hack-mitigation",
|
||||
|
|
@ -353,6 +369,25 @@ def train(
|
|||
cfg.training.echo_trap_tokenizer_aware = True
|
||||
console.print("[green]Echo-trap tokenizer-aware scoring enabled[/]")
|
||||
|
||||
# --- Reward-hack detector / halt shortcut (v0.71.26) ---
|
||||
if reward_hack_detector is not None:
|
||||
if reward_hack_detector not in ("info_rm", "rm_ensemble"):
|
||||
console.print(
|
||||
"[red]--reward-hack-detector must be info_rm or rm_ensemble[/]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
cfg.training.reward_hack_detector = reward_hack_detector
|
||||
console.print(f"[green]Reward-hack detector:[/] {reward_hack_detector}")
|
||||
if reward_hack_halt:
|
||||
if cfg.training.reward_hack_detector is None:
|
||||
console.print(
|
||||
"[red]--reward-hack-halt requires --reward-hack-detector "
|
||||
"(or training.reward_hack_detector)[/]"
|
||||
)
|
||||
raise typer.Exit(1)
|
||||
cfg.training.reward_hack_halt = True
|
||||
console.print("[green]Reward-hack auto-halt enabled[/]")
|
||||
|
||||
# --- Reward-hack mitigation shortcut (v0.71.26) ---
|
||||
if reward_hack_mitigation is not None:
|
||||
valid_modes = ("off", "log_only", "kl_control", "pid_lagrangian")
|
||||
|
|
@ -650,6 +685,12 @@ def train(
|
|||
script_args.append("--tensorboard")
|
||||
if echo_trap_tokenizer_aware:
|
||||
script_args.append("--echo-trap-tokenizer-aware")
|
||||
if reward_hack_detector is not None:
|
||||
script_args.extend(
|
||||
["--reward-hack-detector", reward_hack_detector]
|
||||
)
|
||||
if reward_hack_halt:
|
||||
script_args.append("--reward-hack-halt")
|
||||
if reward_hack_mitigation is not None:
|
||||
script_args.extend(
|
||||
["--reward-hack-mitigation", reward_hack_mitigation]
|
||||
|
|
|
|||
|
|
@ -1047,6 +1047,53 @@ class TestRewardHackMitigationCli:
|
|||
assert "--reward-hack-mitigation" in src
|
||||
assert "cfg.training.reward_hack_mitigation" in src
|
||||
|
||||
def test_help_shows_detector_and_halt_flags(self):
|
||||
runner, app = self._runner()
|
||||
result = runner.invoke(app, ["train", "--help"])
|
||||
assert result.exit_code == 0, result.output
|
||||
assert "--reward-hack-detector" in result.output
|
||||
assert "--reward-hack-halt" in result.output
|
||||
|
||||
def test_bad_detector_value_rejected(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "cfg.yaml").write_text(
|
||||
"base: HuggingFaceTB/SmolLM2-135M\n"
|
||||
"task: grpo\n"
|
||||
"data:\n train: ./train.jsonl\n format: chatml\n"
|
||||
"training:\n reward_fn: accuracy\n"
|
||||
)
|
||||
runner, app = self._runner()
|
||||
result = runner.invoke(
|
||||
app,
|
||||
["train", "--config", "cfg.yaml", "--reward-hack-detector", "bogus", "--yes"],
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "info_rm" in result.output
|
||||
|
||||
def test_halt_flag_without_detector_rejected(self, tmp_path, monkeypatch):
|
||||
monkeypatch.chdir(tmp_path)
|
||||
(tmp_path / "cfg.yaml").write_text(
|
||||
"base: HuggingFaceTB/SmolLM2-135M\n"
|
||||
"task: grpo\n"
|
||||
"data:\n train: ./train.jsonl\n format: chatml\n"
|
||||
"training:\n reward_fn: accuracy\n"
|
||||
)
|
||||
runner, app = self._runner()
|
||||
result = runner.invoke(
|
||||
app, ["train", "--config", "cfg.yaml", "--reward-hack-halt", "--yes"]
|
||||
)
|
||||
assert result.exit_code == 1
|
||||
assert "reward-hack-detector" in result.output
|
||||
|
||||
def test_detector_reexec_passthrough_present(self):
|
||||
import inspect
|
||||
|
||||
from soup_cli.commands import train as train_mod
|
||||
|
||||
src = inspect.getsource(train_mod)
|
||||
assert "--reward-hack-detector" in src
|
||||
assert "cfg.training.reward_hack_detector = reward_hack_detector" in src
|
||||
|
||||
|
||||
# =====================================================================
|
||||
# Part C / Stage 2 — schema fields (PID + rollback) (Task C1)
|
||||
|
|
|
|||
Loading…
Reference in New Issue