diff --git a/src/soup_cli/commands/ship.py b/src/soup_cli/commands/ship.py index e3b4437..89fef50 100644 --- a/src/soup_cli/commands/ship.py +++ b/src/soup_cli/commands/ship.py @@ -286,6 +286,32 @@ def _leg1_judge( return build_task_win("judge_score", _score(base_gen), _score(tuned_gen)) +def _leg1_pairwise( + base_gen: Callable[[str], str], + tuned_gen: Callable[[str], str], + task_eval: str, + judge_model: str, +) -> TaskWin: + """Leg-1 via a true pairwise judge win-rate (#284). + + For each task prompt, generate a base and a tuned response and ask the judge + which is better (swap-debiased). The tuned win-rate becomes leg 1, framed as + ``TaskWin(base=0.5 coin-flip, tuned=win-rate)`` so ``won <=> win-rate > 0.5``. + """ + from soup_cli.eval.custom import load_eval_tasks + from soup_cli.eval.gate import _parse_judge_url + from soup_cli.eval.judge import JudgeEvaluator, pairwise_winrate + + tasks = load_eval_tasks(task_eval) + if not tasks: + raise ValueError(f"task-eval file {task_eval!r} has no tasks") + provider, model, api_base = _parse_judge_url(judge_model) + evaluator = JudgeEvaluator(provider=provider, model=model, api_base=api_base) + pairs = [(t.prompt, base_gen(t.prompt), tuned_gen(t.prompt)) for t in tasks] + winrate = pairwise_winrate(pairs, evaluator) + return build_task_win("pairwise", 0.5, winrate) + + def _mini_score(gen: Callable[[str], str], benchmark: str) -> float: from soup_cli.eval.forgetting import ForgettingDetector @@ -483,6 +509,11 @@ def _verdict_live( _fail("--task-mode judge_score needs --judge-model ", 2) _validate_judge_model_url(judge_model) task_win = _leg1_judge(base_gen, tuned_gen, task_eval, judge_model) + elif task_mode == "pairwise": + if not judge_model: + _fail("--task-mode pairwise needs --judge-model ", 2) + _validate_judge_model_url(judge_model) + task_win = _leg1_pairwise(base_gen, tuned_gen, task_eval, judge_model) else: task_win = _leg1_metric(base_gen, tuned_gen, base, tuned_id, task_eval) base_scores, tuned_scores = _leg2_scores( @@ -546,7 +577,7 @@ def ship( task_mode: str = typer.Option( "metric", "--task-mode", - help="Leg-1 mode: metric | judge_score (pairwise: later release).", + help="Leg-1 mode: metric | judge_score | pairwise (judge win-rate).", ), judge_model: Optional[str] = typer.Option( None, "--judge-model", help="Judge model URL for --task-mode judge_score." diff --git a/src/soup_cli/eval/judge.py b/src/soup_cli/eval/judge.py index 5118ab2..89d3203 100644 --- a/src/soup_cli/eval/judge.py +++ b/src/soup_cli/eval/judge.py @@ -294,6 +294,18 @@ class JudgeEvaluator: results.compute() return results + def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int: + """One pairwise A/B judgment -> 0 (A) / 1 (B) / -1 (tie / parse fail).""" + judge_prompt = _PAIRWISE_INSTRUCTIONS.format( + prompt=prompt, resp_a=resp_a, resp_b=resp_b + ) + try: + reply = self._call_llm(judge_prompt) + except Exception as exc: # noqa: BLE001 — network/parse variety -> tie + logger.debug("pairwise judge call failed: %s", exc) + return -1 + return _parse_pairwise(reply) + def _call_llm(self, prompt: str) -> str: """Call the judge LLM. Uses OpenAI-compatible API for all providers.""" import httpx @@ -324,3 +336,136 @@ class JudgeEvaluator: data = resp.json() return data["choices"][0]["message"]["content"] + + +# --------------------------------------------------------------------------- +# Pairwise judging (v0.71.31) — shared by online-DPO + `soup ship --task-mode +# pairwise`. `compare_pair` (above) issues one A/B judgment; the free functions +# add swap-debiasing and a win-rate reduction. +# --------------------------------------------------------------------------- + +_PAIRWISE_INSTRUCTIONS = ( + "You are comparing two AI responses to the same prompt.\n\n" + "## Prompt\n{prompt}\n\n" + "## Response A\n{resp_a}\n\n" + "## Response B\n{resp_b}\n\n" + "## Task\nWhich response is better overall (helpfulness, accuracy, " + "safety)? Reply with a JSON object: {{\"winner\": \"A\"}} or " + "{{\"winner\": \"B\"}}. Return ONLY the JSON object." +) + + +def _parse_pairwise(text: str) -> int: + """Parse a judge reply into 0 (A) / 1 (B) / -1 (tie or unparseable).""" + match = re.search(r'\{[^{}]*\}', text or "", re.DOTALL) + if match: + try: + data = json.loads(match.group()) + winner = str(data.get("winner", "")).strip().upper() + if winner == "A": + return 0 + if winner == "B": + return 1 + except (json.JSONDecodeError, TypeError, ValueError): + pass + # Fallback: a bare "A" / "B" token. + stripped = (text or "").strip().upper() + if stripped.startswith("A") and not stripped.startswith("B"): + return 0 + if stripped.startswith("B"): + return 1 + return -1 + + +def pairwise_compare( + prompt: str, + resp_a: str, + resp_b: str, + evaluator: object, + *, + swap: bool = True, +) -> int: + """Return 0 (A preferred), 1 (B preferred), or -1 (tie / disagreement). + + When ``swap`` is True the pair is judged in BOTH orders (A,B and B,A) and a + winner is returned only if the two runs agree — the standard defence against + a judge's positional bias. Disagreement -> -1 (tie). + """ + first = evaluator.compare_pair(prompt, resp_a, resp_b) + if not swap: + return first + swapped = evaluator.compare_pair(prompt, resp_b, resp_a) + # Translate the swapped verdict back into A/B space: 0 -> B(1), 1 -> A(0). + if swapped == 0: + second = 1 + elif swapped == 1: + second = 0 + else: + second = -1 + if first == -1 and second == -1: + return -1 + if first == -1: + return second + if second == -1: + return first + return first if first == second else -1 + + +def pairwise_winrate(pairs: list, evaluator: object) -> float: + """Tuned win-rate in [0, 1] over ``(prompt, base_resp, tuned_resp)`` triples. + + Base is compared as A, tuned as B. A tuned win (verdict 1) scores 1.0, a tie + (-1) scores 0.5, a loss (0) scores 0.0. Empty input -> 0.5 (no evidence). + """ + if not pairs: + return 0.5 + total = 0.0 + for prompt, base_resp, tuned_resp in pairs: + verdict = pairwise_compare(prompt, base_resp, tuned_resp, evaluator, swap=True) + if verdict == 1: + total += 1.0 + elif verdict == -1: + total += 0.5 + return total / len(pairs) + + +def _base_pairwise_judge_cls(): + """Lazily import TRL's ``BasePairwiseJudge`` with a friendly error.""" + try: + from trl import BasePairwiseJudge + except ImportError as exc: # pragma: no cover — trl ships in [train]/[dev] + raise ImportError( + "SoupPairwiseJudge needs trl>=0.19 (pip install 'soup-cli[train]')" + ) from exc + return BasePairwiseJudge + + +def make_soup_pairwise_judge(evaluator: "JudgeEvaluator"): + """Build a TRL ``BasePairwiseJudge`` bound to a Soup ``JudgeEvaluator``. + + Factory (not a module-level subclass) so ``eval/judge.py`` stays importable + without trl for the pure ``pairwise_*`` functions. ``judge`` returns, per + prompt, the index of the best completion (0/1), or ``-1`` on tie/failure — + the exact ``BasePairwiseJudge`` contract (TRL treats -1 as a dropped sample). + """ + base_cls = _base_pairwise_judge_cls() + + class _SoupPairwiseJudge(base_cls): + def __init__(self, ev): + self.evaluator = ev + + def judge(self, prompts, completions, shuffle_order: bool = True): + out = [] + for prompt, pair in zip(prompts, completions): + if not isinstance(pair, (list, tuple)) or len(pair) != 2: + out.append(-1) + continue + out.append( + pairwise_compare( + prompt, pair[0], pair[1], self.evaluator, + swap=shuffle_order, + ) + ) + return out + + return _SoupPairwiseJudge(evaluator) diff --git a/src/soup_cli/utils/ship_verdict.py b/src/soup_cli/utils/ship_verdict.py index c04877c..bd2abe1 100644 --- a/src/soup_cli/utils/ship_verdict.py +++ b/src/soup_cli/utils/ship_verdict.py @@ -46,11 +46,10 @@ from soup_cli import __version__ # Public constants # --------------------------------------------------------------------------- -# Leg-1 task-win modes. ``pairwise`` (true judge win-rate) is reserved for a -# later release — it is in the enum so the engine stays pluggable, but is not -# in SUPPORTED_TASK_MODES yet (the CLI rejects it for now). +# Leg-1 task-win modes. ``pairwise`` (true judge win-rate) landed in v0.71.31: +# a ``TaskWin(base=0.5 coin-flip, tuned=win-rate)`` where ``won = tuned > 0.5``. TASK_MODES: Tuple[str, ...] = ("metric", "judge_score", "pairwise") -SUPPORTED_TASK_MODES: Tuple[str, ...] = ("metric", "judge_score") +SUPPORTED_TASK_MODES: Tuple[str, ...] = ("metric", "judge_score", "pairwise") DECISION_SHIP = "SHIP" DECISION_DONT_SHIP = "DON'T SHIP" diff --git a/tests/test_v07125.py b/tests/test_v07125.py index 14f34ff..456ed29 100644 --- a/tests/test_v07125.py +++ b/tests/test_v07125.py @@ -106,9 +106,9 @@ class TestBuildTaskWin: assert set(SUPPORTED_TASK_MODES).issubset(set(TASK_MODES)) assert "metric" in SUPPORTED_TASK_MODES assert "judge_score" in SUPPORTED_TASK_MODES - # pairwise is reserved for a later release — in the enum, not supported. + # pairwise became supported in v0.71.31 (#284). assert "pairwise" in TASK_MODES - assert "pairwise" not in SUPPORTED_TASK_MODES + assert "pairwise" in SUPPORTED_TASK_MODES # --------------------------------------------------------------------------- @@ -575,7 +575,10 @@ class TestShipCliEvidence: assert res.exit_code == 2, (res.output, repr(res.exception)) assert "threshold" in res.output.lower() - def test_pairwise_mode_rejected_for_now(self): + def test_pairwise_mode_now_supported(self): + # v0.71.31 (#284): --task-mode pairwise is no longer rejected. The + # evidence path reads mode from the file (metric here -> SHIP), so the + # flag is simply accepted (not an exit-2 "later release" refusal). from soup_cli.commands import ship as ship_cmd with runner.isolated_filesystem(): @@ -584,8 +587,8 @@ class TestShipCliEvidence: ship_cmd.app, ["--evidence", "ev.json", "--task-mode", "pairwise"], ) - assert res.exit_code == 2, (res.output, repr(res.exception)) - assert "pairwise" in res.output.lower() + assert res.exit_code == 0, (res.output, repr(res.exception)) + assert "later release" not in res.output.lower() def test_evidence_outside_cwd_rejected(self): from soup_cli.commands import ship as ship_cmd diff --git a/tests/test_v07131.py b/tests/test_v07131.py new file mode 100644 index 0000000..1c53443 --- /dev/null +++ b/tests/test_v07131.py @@ -0,0 +1,222 @@ +"""v0.71.31 — Judge-in-the-loop suite. + +Covers the shared pairwise-judge layer (``eval/judge.pairwise_compare`` / +``pairwise_winrate`` / ``make_soup_pairwise_judge``), ``soup ship --task-mode +pairwise`` (#284), ``task='online_dpo'`` (schema + trainer + routing), +``soup data best-of-n``, and ``soup data evolve``. +""" + +from __future__ import annotations + + +# --------------------------------------------------------------------------- +# Shared test doubles +# --------------------------------------------------------------------------- + + +class _FakeJudge: + """Deterministic pairwise judge: prefers the LONGER response (A vs B).""" + + def __init__(self, rubric=None): + self.rubric = rubric or {"scale": {"min": 1, "max": 5}, "criteria": []} + + def compare_pair(self, prompt, resp_a, resp_b): + if len(resp_a) == len(resp_b): + return -1 + return 0 if len(resp_a) > len(resp_b) else 1 + + +class _PosBias: + """A biased judge that ALWAYS says the first response is best.""" + + def compare_pair(self, prompt, resp_a, resp_b): + return 0 + + +# --------------------------------------------------------------------------- +# Task 1 — pairwise_compare / pairwise_winrate +# --------------------------------------------------------------------------- + + +class TestPairwiseCompare: + def test_a_preferred(self): + from soup_cli.eval.judge import pairwise_compare + + assert pairwise_compare("p", "longer response", "short", _FakeJudge(), swap=True) == 0 + + def test_b_preferred(self): + from soup_cli.eval.judge import pairwise_compare + + assert pairwise_compare("p", "short", "longer response", _FakeJudge(), swap=True) == 1 + + def test_tie_on_equal(self): + from soup_cli.eval.judge import pairwise_compare + + assert pairwise_compare("p", "aaaa", "bbbb", _FakeJudge(), swap=True) == -1 + + def test_swap_debias_disagreement_is_tie(self): + from soup_cli.eval.judge import pairwise_compare + + # A judge that ALWAYS says "first is best" disagrees under swap -> tie. + assert pairwise_compare("p", "x", "y", _PosBias(), swap=True) == -1 + + def test_no_swap_uses_single_call(self): + from soup_cli.eval.judge import pairwise_compare + + assert pairwise_compare("p", "x", "y", _PosBias(), swap=False) == 0 + + +class TestPairwiseWinrate: + def test_tuned_always_wins(self): + from soup_cli.eval.judge import pairwise_winrate + + # base short, tuned long -> _FakeJudge prefers tuned every time -> 1.0 + pairs = [("p", "s", "longer"), ("q", "s", "longer")] + assert pairwise_winrate(pairs, _FakeJudge()) == 1.0 + + def test_all_ties_is_half(self): + from soup_cli.eval.judge import pairwise_winrate + + pairs = [("p", "aaa", "bbb")] # equal length -> tie -> 0.5 + assert pairwise_winrate(pairs, _FakeJudge()) == 0.5 + + def test_empty_pairs_is_half(self): + from soup_cli.eval.judge import pairwise_winrate + + assert pairwise_winrate([], _FakeJudge()) == 0.5 + + def test_mixed_winrate(self): + from soup_cli.eval.judge import pairwise_winrate + + # tuned wins (long), tuned loses (short), tie (equal) -> (1 + 0 + 0.5)/3 + pairs = [("a", "s", "longer"), ("b", "longer", "s"), ("c", "xx", "yy")] + assert pairwise_winrate(pairs, _FakeJudge()) == (1.0 + 0.0 + 0.5) / 3 + + +# --------------------------------------------------------------------------- +# Task 2 — make_soup_pairwise_judge (TRL BasePairwiseJudge adapter) +# --------------------------------------------------------------------------- + + +class TestSoupPairwiseJudge: + def test_judge_returns_best_index(self): + from soup_cli.eval.judge import make_soup_pairwise_judge + + j = make_soup_pairwise_judge(_FakeJudge()) # prefers longer + # prompt0: [short, long] -> B(1); prompt1: [long, short] -> A(0) + out = j.judge(["p0", "p1"], [["s", "longer"], ["longer", "s"]]) + assert out == [1, 0] + + def test_judge_tie_returns_minus_one(self): + from soup_cli.eval.judge import make_soup_pairwise_judge + + j = make_soup_pairwise_judge(_FakeJudge()) + assert j.judge(["p"], [["aaaa", "bbbb"]]) == [-1] + + def test_shuffle_order_false_no_swap(self): + from soup_cli.eval.judge import make_soup_pairwise_judge + + j = make_soup_pairwise_judge(_PosBias()) + assert j.judge(["p"], [["x", "y"]], shuffle_order=False) == [0] + + def test_malformed_pair_returns_minus_one(self): + from soup_cli.eval.judge import make_soup_pairwise_judge + + j = make_soup_pairwise_judge(_FakeJudge()) + assert j.judge(["p"], [["only-one"]]) == [-1] + + def test_is_trl_base_pairwise_judge(self): + from trl import BasePairwiseJudge + + from soup_cli.eval.judge import make_soup_pairwise_judge + + assert isinstance(make_soup_pairwise_judge(_FakeJudge()), BasePairwiseJudge) + + +# --------------------------------------------------------------------------- +# Task 3 — soup ship --task-mode pairwise (#284) +# --------------------------------------------------------------------------- + + +class _Task: + def __init__(self, prompt): + self.prompt = prompt + self.category = "default" + + +class TestShipPairwise: + def test_pairwise_in_supported(self): + from soup_cli.utils.ship_verdict import SUPPORTED_TASK_MODES + + assert "pairwise" in SUPPORTED_TASK_MODES + + def test_leg1_pairwise_builds_taskwin(self, monkeypatch): + from soup_cli.commands import ship as ship_cmd + + monkeypatch.setattr( + "soup_cli.eval.custom.load_eval_tasks", + lambda path: [_Task("q1"), _Task("q2")], + ) + monkeypatch.setattr( + "soup_cli.eval.gate._parse_judge_url", + lambda url: ("ollama", "m", None), + ) + monkeypatch.setattr( + "soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _FakeJudge() + ) + # base_gen short, tuned_gen long; _FakeJudge prefers long -> winrate 1.0 + tw = ship_cmd._leg1_pairwise( + lambda p: "s", lambda p: "longer", "x.jsonl", "ollama://m" + ) + assert tw.mode == "pairwise" + assert tw.base == 0.5 + assert tw.tuned == 1.0 + assert tw.won is True + + def test_evidence_pairwise_accepted(self): + import json + import os + + from typer.testing import CliRunner + + from soup_cli.commands.ship import app + + path = os.path.join(os.getcwd(), "_ev_pairwise_v07131.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump( + { + "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.7}, + "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}}, + }, + fh, + ) + try: + result = CliRunner().invoke(app, ["--evidence", path]) + assert result.exit_code == 0, (result.output, repr(result.exception)) + assert "SHIP" in result.output + finally: + os.remove(path) + + def test_evidence_pairwise_tie_is_dont_ship(self): + import json + import os + + from typer.testing import CliRunner + + from soup_cli.commands.ship import app + + path = os.path.join(os.getcwd(), "_ev_pairwise_tie_v07131.json") + with open(path, "w", encoding="utf-8") as fh: + json.dump( + { + "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.5}, + "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}}, + }, + fh, + ) + try: + result = CliRunner().invoke(app, ["--evidence", path]) + assert result.exit_code == 2, (result.output, repr(result.exception)) + assert "DON'T SHIP" in result.output + finally: + os.remove(path)