feat(ship): pairwise judge win-rate leg + shared pairwise judge helpers (#284)

Add eval/judge.pairwise_compare (swap-debiased) / pairwise_winrate /
make_soup_pairwise_judge (TRL BasePairwiseJudge adapter over the httpx
JudgeEvaluator), enable 'pairwise' in ship_verdict.SUPPORTED_TASK_MODES,
and wire commands/ship._leg1_pairwise + --task-mode pairwise routing.

src/soup_cli/commands/ship.py

@@ -286,6 +286,32 @@ def _leg1_judge(
     return build_task_win("judge_score", _score(base_gen), _score(tuned_gen))
 
 
+def _leg1_pairwise(
+    base_gen: Callable[[str], str],
+    tuned_gen: Callable[[str], str],
+    task_eval: str,
+    judge_model: str,
+) -> TaskWin:
+    """Leg-1 via a true pairwise judge win-rate (#284).
+
+    For each task prompt, generate a base and a tuned response and ask the judge
+    which is better (swap-debiased). The tuned win-rate becomes leg 1, framed as
+    ``TaskWin(base=0.5 coin-flip, tuned=win-rate)`` so ``won <=> win-rate > 0.5``.
+    """
+    from soup_cli.eval.custom import load_eval_tasks
+    from soup_cli.eval.gate import _parse_judge_url
+    from soup_cli.eval.judge import JudgeEvaluator, pairwise_winrate
+
+    tasks = load_eval_tasks(task_eval)
+    if not tasks:
+        raise ValueError(f"task-eval file {task_eval!r} has no tasks")
+    provider, model, api_base = _parse_judge_url(judge_model)
+    evaluator = JudgeEvaluator(provider=provider, model=model, api_base=api_base)
+    pairs = [(t.prompt, base_gen(t.prompt), tuned_gen(t.prompt)) for t in tasks]
+    winrate = pairwise_winrate(pairs, evaluator)
+    return build_task_win("pairwise", 0.5, winrate)
+
+
 def _mini_score(gen: Callable[[str], str], benchmark: str) -> float:
     from soup_cli.eval.forgetting import ForgettingDetector
 
@@ -483,6 +509,11 @@ def _verdict_live(
                 _fail("--task-mode judge_score needs --judge-model <url>", 2)
             _validate_judge_model_url(judge_model)
             task_win = _leg1_judge(base_gen, tuned_gen, task_eval, judge_model)
+        elif task_mode == "pairwise":
+            if not judge_model:
+                _fail("--task-mode pairwise needs --judge-model <url>", 2)
+            _validate_judge_model_url(judge_model)
+            task_win = _leg1_pairwise(base_gen, tuned_gen, task_eval, judge_model)
         else:
             task_win = _leg1_metric(base_gen, tuned_gen, base, tuned_id, task_eval)
         base_scores, tuned_scores = _leg2_scores(
@@ -546,7 +577,7 @@ def ship(
     task_mode: str = typer.Option(
         "metric",
         "--task-mode",
-        help="Leg-1 mode: metric | judge_score (pairwise: later release).",
+        help="Leg-1 mode: metric | judge_score | pairwise (judge win-rate).",
     ),
     judge_model: Optional[str] = typer.Option(
         None, "--judge-model", help="Judge model URL for --task-mode judge_score."

src/soup_cli/eval/judge.py

@@ -294,6 +294,18 @@ class JudgeEvaluator:
         results.compute()
         return results
 
+    def compare_pair(self, prompt: str, resp_a: str, resp_b: str) -> int:
+        """One pairwise A/B judgment -> 0 (A) / 1 (B) / -1 (tie / parse fail)."""
+        judge_prompt = _PAIRWISE_INSTRUCTIONS.format(
+            prompt=prompt, resp_a=resp_a, resp_b=resp_b
+        )
+        try:
+            reply = self._call_llm(judge_prompt)
+        except Exception as exc:  # noqa: BLE001 — network/parse variety -> tie
+            logger.debug("pairwise judge call failed: %s", exc)
+            return -1
+        return _parse_pairwise(reply)
+
     def _call_llm(self, prompt: str) -> str:
         """Call the judge LLM. Uses OpenAI-compatible API for all providers."""
         import httpx
@@ -324,3 +336,136 @@ class JudgeEvaluator:
 
         data = resp.json()
         return data["choices"][0]["message"]["content"]
+
+
+# ---------------------------------------------------------------------------
+# Pairwise judging (v0.71.31) — shared by online-DPO + `soup ship --task-mode
+# pairwise`. `compare_pair` (above) issues one A/B judgment; the free functions
+# add swap-debiasing and a win-rate reduction.
+# ---------------------------------------------------------------------------
+
+_PAIRWISE_INSTRUCTIONS = (
+    "You are comparing two AI responses to the same prompt.\n\n"
+    "## Prompt\n{prompt}\n\n"
+    "## Response A\n{resp_a}\n\n"
+    "## Response B\n{resp_b}\n\n"
+    "## Task\nWhich response is better overall (helpfulness, accuracy, "
+    "safety)? Reply with a JSON object: {{\"winner\": \"A\"}} or "
+    "{{\"winner\": \"B\"}}. Return ONLY the JSON object."
+)
+
+
+def _parse_pairwise(text: str) -> int:
+    """Parse a judge reply into 0 (A) / 1 (B) / -1 (tie or unparseable)."""
+    match = re.search(r'\{[^{}]*\}', text or "", re.DOTALL)
+    if match:
+        try:
+            data = json.loads(match.group())
+            winner = str(data.get("winner", "")).strip().upper()
+            if winner == "A":
+                return 0
+            if winner == "B":
+                return 1
+        except (json.JSONDecodeError, TypeError, ValueError):
+            pass
+    # Fallback: a bare "A" / "B" token.
+    stripped = (text or "").strip().upper()
+    if stripped.startswith("A") and not stripped.startswith("B"):
+        return 0
+    if stripped.startswith("B"):
+        return 1
+    return -1
+
+
+def pairwise_compare(
+    prompt: str,
+    resp_a: str,
+    resp_b: str,
+    evaluator: object,
+    *,
+    swap: bool = True,
+) -> int:
+    """Return 0 (A preferred), 1 (B preferred), or -1 (tie / disagreement).
+
+    When ``swap`` is True the pair is judged in BOTH orders (A,B and B,A) and a
+    winner is returned only if the two runs agree — the standard defence against
+    a judge's positional bias. Disagreement -> -1 (tie).
+    """
+    first = evaluator.compare_pair(prompt, resp_a, resp_b)
+    if not swap:
+        return first
+    swapped = evaluator.compare_pair(prompt, resp_b, resp_a)
+    # Translate the swapped verdict back into A/B space: 0 -> B(1), 1 -> A(0).
+    if swapped == 0:
+        second = 1
+    elif swapped == 1:
+        second = 0
+    else:
+        second = -1
+    if first == -1 and second == -1:
+        return -1
+    if first == -1:
+        return second
+    if second == -1:
+        return first
+    return first if first == second else -1
+
+
+def pairwise_winrate(pairs: list, evaluator: object) -> float:
+    """Tuned win-rate in [0, 1] over ``(prompt, base_resp, tuned_resp)`` triples.
+
+    Base is compared as A, tuned as B. A tuned win (verdict 1) scores 1.0, a tie
+    (-1) scores 0.5, a loss (0) scores 0.0. Empty input -> 0.5 (no evidence).
+    """
+    if not pairs:
+        return 0.5
+    total = 0.0
+    for prompt, base_resp, tuned_resp in pairs:
+        verdict = pairwise_compare(prompt, base_resp, tuned_resp, evaluator, swap=True)
+        if verdict == 1:
+            total += 1.0
+        elif verdict == -1:
+            total += 0.5
+    return total / len(pairs)
+
+
+def _base_pairwise_judge_cls():
+    """Lazily import TRL's ``BasePairwiseJudge`` with a friendly error."""
+    try:
+        from trl import BasePairwiseJudge
+    except ImportError as exc:  # pragma: no cover — trl ships in [train]/[dev]
+        raise ImportError(
+            "SoupPairwiseJudge needs trl>=0.19 (pip install 'soup-cli[train]')"
+        ) from exc
+    return BasePairwiseJudge
+
+
+def make_soup_pairwise_judge(evaluator: "JudgeEvaluator"):
+    """Build a TRL ``BasePairwiseJudge`` bound to a Soup ``JudgeEvaluator``.
+
+    Factory (not a module-level subclass) so ``eval/judge.py`` stays importable
+    without trl for the pure ``pairwise_*`` functions. ``judge`` returns, per
+    prompt, the index of the best completion (0/1), or ``-1`` on tie/failure —
+    the exact ``BasePairwiseJudge`` contract (TRL treats -1 as a dropped sample).
+    """
+    base_cls = _base_pairwise_judge_cls()
+
+    class _SoupPairwiseJudge(base_cls):
+        def __init__(self, ev):
+            self.evaluator = ev
+
+        def judge(self, prompts, completions, shuffle_order: bool = True):
+            out = []
+            for prompt, pair in zip(prompts, completions):
+                if not isinstance(pair, (list, tuple)) or len(pair) != 2:
+                    out.append(-1)
+                    continue
+                out.append(
+                    pairwise_compare(
+                        prompt, pair[0], pair[1], self.evaluator,
+                        swap=shuffle_order,
+                    )
+                )
+            return out
+
+    return _SoupPairwiseJudge(evaluator)

src/soup_cli/utils/ship_verdict.py

@@ -46,11 +46,10 @@ from soup_cli import __version__
 # Public constants
 # ---------------------------------------------------------------------------
 
-# Leg-1 task-win modes. ``pairwise`` (true judge win-rate) is reserved for a
-# later release — it is in the enum so the engine stays pluggable, but is not
-# in SUPPORTED_TASK_MODES yet (the CLI rejects it for now).
+# Leg-1 task-win modes. ``pairwise`` (true judge win-rate) landed in v0.71.31:
+# a ``TaskWin(base=0.5 coin-flip, tuned=win-rate)`` where ``won = tuned > 0.5``.
 TASK_MODES: Tuple[str, ...] = ("metric", "judge_score", "pairwise")
-SUPPORTED_TASK_MODES: Tuple[str, ...] = ("metric", "judge_score")
+SUPPORTED_TASK_MODES: Tuple[str, ...] = ("metric", "judge_score", "pairwise")
 
 DECISION_SHIP = "SHIP"
 DECISION_DONT_SHIP = "DON'T SHIP"

tests/test_v07125.py

@@ -106,9 +106,9 @@ class TestBuildTaskWin:
         assert set(SUPPORTED_TASK_MODES).issubset(set(TASK_MODES))
         assert "metric" in SUPPORTED_TASK_MODES
         assert "judge_score" in SUPPORTED_TASK_MODES
-        # pairwise is reserved for a later release — in the enum, not supported.
+        # pairwise became supported in v0.71.31 (#284).
         assert "pairwise" in TASK_MODES
-        assert "pairwise" not in SUPPORTED_TASK_MODES
+        assert "pairwise" in SUPPORTED_TASK_MODES
 
 
 # ---------------------------------------------------------------------------
@@ -575,7 +575,10 @@ class TestShipCliEvidence:
             assert res.exit_code == 2, (res.output, repr(res.exception))
             assert "threshold" in res.output.lower()
 
-    def test_pairwise_mode_rejected_for_now(self):
+    def test_pairwise_mode_now_supported(self):
+        # v0.71.31 (#284): --task-mode pairwise is no longer rejected. The
+        # evidence path reads mode from the file (metric here -> SHIP), so the
+        # flag is simply accepted (not an exit-2 "later release" refusal).
         from soup_cli.commands import ship as ship_cmd
 
         with runner.isolated_filesystem():
@@ -584,8 +587,8 @@ class TestShipCliEvidence:
                 ship_cmd.app,
                 ["--evidence", "ev.json", "--task-mode", "pairwise"],
             )
-            assert res.exit_code == 2, (res.output, repr(res.exception))
-            assert "pairwise" in res.output.lower()
+            assert res.exit_code == 0, (res.output, repr(res.exception))
+            assert "later release" not in res.output.lower()
 
     def test_evidence_outside_cwd_rejected(self):
         from soup_cli.commands import ship as ship_cmd

tests/test_v07131.py

@@ -0,0 +1,222 @@
+"""v0.71.31 — Judge-in-the-loop suite.
+
+Covers the shared pairwise-judge layer (``eval/judge.pairwise_compare`` /
+``pairwise_winrate`` / ``make_soup_pairwise_judge``), ``soup ship --task-mode
+pairwise`` (#284), ``task='online_dpo'`` (schema + trainer + routing),
+``soup data best-of-n``, and ``soup data evolve``.
+"""
+
+from __future__ import annotations
+
+
+# ---------------------------------------------------------------------------
+# Shared test doubles
+# ---------------------------------------------------------------------------
+
+
+class _FakeJudge:
+    """Deterministic pairwise judge: prefers the LONGER response (A vs B)."""
+
+    def __init__(self, rubric=None):
+        self.rubric = rubric or {"scale": {"min": 1, "max": 5}, "criteria": []}
+
+    def compare_pair(self, prompt, resp_a, resp_b):
+        if len(resp_a) == len(resp_b):
+            return -1
+        return 0 if len(resp_a) > len(resp_b) else 1
+
+
+class _PosBias:
+    """A biased judge that ALWAYS says the first response is best."""
+
+    def compare_pair(self, prompt, resp_a, resp_b):
+        return 0
+
+
+# ---------------------------------------------------------------------------
+# Task 1 — pairwise_compare / pairwise_winrate
+# ---------------------------------------------------------------------------
+
+
+class TestPairwiseCompare:
+    def test_a_preferred(self):
+        from soup_cli.eval.judge import pairwise_compare
+
+        assert pairwise_compare("p", "longer response", "short", _FakeJudge(), swap=True) == 0
+
+    def test_b_preferred(self):
+        from soup_cli.eval.judge import pairwise_compare
+
+        assert pairwise_compare("p", "short", "longer response", _FakeJudge(), swap=True) == 1
+
+    def test_tie_on_equal(self):
+        from soup_cli.eval.judge import pairwise_compare
+
+        assert pairwise_compare("p", "aaaa", "bbbb", _FakeJudge(), swap=True) == -1
+
+    def test_swap_debias_disagreement_is_tie(self):
+        from soup_cli.eval.judge import pairwise_compare
+
+        # A judge that ALWAYS says "first is best" disagrees under swap -> tie.
+        assert pairwise_compare("p", "x", "y", _PosBias(), swap=True) == -1
+
+    def test_no_swap_uses_single_call(self):
+        from soup_cli.eval.judge import pairwise_compare
+
+        assert pairwise_compare("p", "x", "y", _PosBias(), swap=False) == 0
+
+
+class TestPairwiseWinrate:
+    def test_tuned_always_wins(self):
+        from soup_cli.eval.judge import pairwise_winrate
+
+        # base short, tuned long -> _FakeJudge prefers tuned every time -> 1.0
+        pairs = [("p", "s", "longer"), ("q", "s", "longer")]
+        assert pairwise_winrate(pairs, _FakeJudge()) == 1.0
+
+    def test_all_ties_is_half(self):
+        from soup_cli.eval.judge import pairwise_winrate
+
+        pairs = [("p", "aaa", "bbb")]  # equal length -> tie -> 0.5
+        assert pairwise_winrate(pairs, _FakeJudge()) == 0.5
+
+    def test_empty_pairs_is_half(self):
+        from soup_cli.eval.judge import pairwise_winrate
+
+        assert pairwise_winrate([], _FakeJudge()) == 0.5
+
+    def test_mixed_winrate(self):
+        from soup_cli.eval.judge import pairwise_winrate
+
+        # tuned wins (long), tuned loses (short), tie (equal) -> (1 + 0 + 0.5)/3
+        pairs = [("a", "s", "longer"), ("b", "longer", "s"), ("c", "xx", "yy")]
+        assert pairwise_winrate(pairs, _FakeJudge()) == (1.0 + 0.0 + 0.5) / 3
+
+
+# ---------------------------------------------------------------------------
+# Task 2 — make_soup_pairwise_judge (TRL BasePairwiseJudge adapter)
+# ---------------------------------------------------------------------------
+
+
+class TestSoupPairwiseJudge:
+    def test_judge_returns_best_index(self):
+        from soup_cli.eval.judge import make_soup_pairwise_judge
+
+        j = make_soup_pairwise_judge(_FakeJudge())  # prefers longer
+        # prompt0: [short, long] -> B(1); prompt1: [long, short] -> A(0)
+        out = j.judge(["p0", "p1"], [["s", "longer"], ["longer", "s"]])
+        assert out == [1, 0]
+
+    def test_judge_tie_returns_minus_one(self):
+        from soup_cli.eval.judge import make_soup_pairwise_judge
+
+        j = make_soup_pairwise_judge(_FakeJudge())
+        assert j.judge(["p"], [["aaaa", "bbbb"]]) == [-1]
+
+    def test_shuffle_order_false_no_swap(self):
+        from soup_cli.eval.judge import make_soup_pairwise_judge
+
+        j = make_soup_pairwise_judge(_PosBias())
+        assert j.judge(["p"], [["x", "y"]], shuffle_order=False) == [0]
+
+    def test_malformed_pair_returns_minus_one(self):
+        from soup_cli.eval.judge import make_soup_pairwise_judge
+
+        j = make_soup_pairwise_judge(_FakeJudge())
+        assert j.judge(["p"], [["only-one"]]) == [-1]
+
+    def test_is_trl_base_pairwise_judge(self):
+        from trl import BasePairwiseJudge
+
+        from soup_cli.eval.judge import make_soup_pairwise_judge
+
+        assert isinstance(make_soup_pairwise_judge(_FakeJudge()), BasePairwiseJudge)
+
+
+# ---------------------------------------------------------------------------
+# Task 3 — soup ship --task-mode pairwise (#284)
+# ---------------------------------------------------------------------------
+
+
+class _Task:
+    def __init__(self, prompt):
+        self.prompt = prompt
+        self.category = "default"
+
+
+class TestShipPairwise:
+    def test_pairwise_in_supported(self):
+        from soup_cli.utils.ship_verdict import SUPPORTED_TASK_MODES
+
+        assert "pairwise" in SUPPORTED_TASK_MODES
+
+    def test_leg1_pairwise_builds_taskwin(self, monkeypatch):
+        from soup_cli.commands import ship as ship_cmd
+
+        monkeypatch.setattr(
+            "soup_cli.eval.custom.load_eval_tasks",
+            lambda path: [_Task("q1"), _Task("q2")],
+        )
+        monkeypatch.setattr(
+            "soup_cli.eval.gate._parse_judge_url",
+            lambda url: ("ollama", "m", None),
+        )
+        monkeypatch.setattr(
+            "soup_cli.eval.judge.JudgeEvaluator", lambda **kw: _FakeJudge()
+        )
+        # base_gen short, tuned_gen long; _FakeJudge prefers long -> winrate 1.0
+        tw = ship_cmd._leg1_pairwise(
+            lambda p: "s", lambda p: "longer", "x.jsonl", "ollama://m"
+        )
+        assert tw.mode == "pairwise"
+        assert tw.base == 0.5
+        assert tw.tuned == 1.0
+        assert tw.won is True
+
+    def test_evidence_pairwise_accepted(self):
+        import json
+        import os
+
+        from typer.testing import CliRunner
+
+        from soup_cli.commands.ship import app
+
+        path = os.path.join(os.getcwd(), "_ev_pairwise_v07131.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(
+                {
+                    "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.7},
+                    "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
+                },
+                fh,
+            )
+        try:
+            result = CliRunner().invoke(app, ["--evidence", path])
+            assert result.exit_code == 0, (result.output, repr(result.exception))
+            assert "SHIP" in result.output
+        finally:
+            os.remove(path)
+
+    def test_evidence_pairwise_tie_is_dont_ship(self):
+        import json
+        import os
+
+        from typer.testing import CliRunner
+
+        from soup_cli.commands.ship import app
+
+        path = os.path.join(os.getcwd(), "_ev_pairwise_tie_v07131.json")
+        with open(path, "w", encoding="utf-8") as fh:
+            json.dump(
+                {
+                    "task": {"mode": "pairwise", "base": 0.5, "tuned": 0.5},
+                    "benchmarks": {"mini_mmlu": {"base": 0.8, "tuned": 0.8}},
+                },
+                fh,
+            )
+        try:
+            result = CliRunner().invoke(app, ["--evidence", path])
+            assert result.exit_code == 2, (result.output, repr(result.exception))
+            assert "DON'T SHIP" in result.output
+        finally:
+            os.remove(path)