diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9bd6904..432fded 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -120,7 +120,7 @@ src/soup_cli/ templates/ - 17 built-in soup.yaml templates (YAML + manifest.json) with load_template loader (v0.39.0, +bco v0.40.0) ui/ - Web UI (FastAPI + HTML/JS SPA) -tests/ - Test suite (298 files, 14784 tests) +tests/ - Test suite (298 files, 14788 tests) examples/ - Real-world config examples and datasets ``` diff --git a/src/soup_cli/commands/train.py b/src/soup_cli/commands/train.py index 5e1b0e9..4d380ce 100644 --- a/src/soup_cli/commands/train.py +++ b/src/soup_cli/commands/train.py @@ -167,6 +167,22 @@ def train( "training.echo_trap_enabled=true on grpo/ppo." ), ), + reward_hack_detector: str = typer.Option( + None, + "--reward-hack-detector", + help=( + "Reward-hacking detector for GRPO/PPO: info_rm | rm_ensemble. " + "Overrides training.reward_hack_detector. (v0.71.26)" + ), + ), + reward_hack_halt: bool = typer.Option( + False, + "--reward-hack-halt", + help=( + "Auto-halt training on a HACK verdict. Requires " + "--reward-hack-detector (or training.reward_hack_detector). (v0.71.26)" + ), + ), reward_hack_mitigation: str = typer.Option( None, "--reward-hack-mitigation", @@ -353,6 +369,25 @@ def train( cfg.training.echo_trap_tokenizer_aware = True console.print("[green]Echo-trap tokenizer-aware scoring enabled[/]") + # --- Reward-hack detector / halt shortcut (v0.71.26) --- + if reward_hack_detector is not None: + if reward_hack_detector not in ("info_rm", "rm_ensemble"): + console.print( + "[red]--reward-hack-detector must be info_rm or rm_ensemble[/]" + ) + raise typer.Exit(1) + cfg.training.reward_hack_detector = reward_hack_detector + console.print(f"[green]Reward-hack detector:[/] {reward_hack_detector}") + if reward_hack_halt: + if cfg.training.reward_hack_detector is None: + console.print( + "[red]--reward-hack-halt requires --reward-hack-detector " + "(or training.reward_hack_detector)[/]" + ) + raise typer.Exit(1) + cfg.training.reward_hack_halt = True + console.print("[green]Reward-hack auto-halt enabled[/]") + # --- Reward-hack mitigation shortcut (v0.71.26) --- if reward_hack_mitigation is not None: valid_modes = ("off", "log_only", "kl_control", "pid_lagrangian") @@ -650,6 +685,12 @@ def train( script_args.append("--tensorboard") if echo_trap_tokenizer_aware: script_args.append("--echo-trap-tokenizer-aware") + if reward_hack_detector is not None: + script_args.extend( + ["--reward-hack-detector", reward_hack_detector] + ) + if reward_hack_halt: + script_args.append("--reward-hack-halt") if reward_hack_mitigation is not None: script_args.extend( ["--reward-hack-mitigation", reward_hack_mitigation] diff --git a/tests/test_v07126.py b/tests/test_v07126.py index 8ad6c89..0a24b0a 100644 --- a/tests/test_v07126.py +++ b/tests/test_v07126.py @@ -1047,6 +1047,53 @@ class TestRewardHackMitigationCli: assert "--reward-hack-mitigation" in src assert "cfg.training.reward_hack_mitigation" in src + def test_help_shows_detector_and_halt_flags(self): + runner, app = self._runner() + result = runner.invoke(app, ["train", "--help"]) + assert result.exit_code == 0, result.output + assert "--reward-hack-detector" in result.output + assert "--reward-hack-halt" in result.output + + def test_bad_detector_value_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "cfg.yaml").write_text( + "base: HuggingFaceTB/SmolLM2-135M\n" + "task: grpo\n" + "data:\n train: ./train.jsonl\n format: chatml\n" + "training:\n reward_fn: accuracy\n" + ) + runner, app = self._runner() + result = runner.invoke( + app, + ["train", "--config", "cfg.yaml", "--reward-hack-detector", "bogus", "--yes"], + ) + assert result.exit_code == 1 + assert "info_rm" in result.output + + def test_halt_flag_without_detector_rejected(self, tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "cfg.yaml").write_text( + "base: HuggingFaceTB/SmolLM2-135M\n" + "task: grpo\n" + "data:\n train: ./train.jsonl\n format: chatml\n" + "training:\n reward_fn: accuracy\n" + ) + runner, app = self._runner() + result = runner.invoke( + app, ["train", "--config", "cfg.yaml", "--reward-hack-halt", "--yes"] + ) + assert result.exit_code == 1 + assert "reward-hack-detector" in result.output + + def test_detector_reexec_passthrough_present(self): + import inspect + + from soup_cli.commands import train as train_mod + + src = inspect.getsource(train_mod) + assert "--reward-hack-detector" in src + assert "cfg.training.reward_hack_detector = reward_hack_detector" in src + # ===================================================================== # Part C / Stage 2 — schema fields (PID + rollback) (Task C1)