From 3b752b7b60981e915bbeb4f4c67a18d1f380f6c1 Mon Sep 17 00:00:00 2001
From: Sanjay Santhanam <51058514+Sanjays2402@users.noreply.github.com>
Date: Fri, 17 Jul 2026 04:30:16 -0700
Subject: [PATCH] fix(eval): run built-in benchmark gate tasks (#315)

Use ForgettingDetector for benchmark tasks instead of probing a nonexistent helper. Cover successful scoring and unknown benchmark errors.
---
 src/soup_cli/eval/gate.py  | 13 +++++--------
 tests/test_part_a_wave1.py | 30 +++++++++++++++++-------------
 2 files changed, 22 insertions(+), 21 deletions(-)

diff --git a/src/soup_cli/eval/gate.py b/src/soup_cli/eval/gate.py
index a4e338f..4499c66 100644
--- a/src/soup_cli/eval/gate.py
+++ b/src/soup_cli/eval/gate.py
@@ -268,15 +268,12 @@ def _run_benchmark_task(
         raise ValueError(
             f"task '{task.name}' is type=benchmark but 'benchmark' is missing"
         )
-    from soup_cli.eval import forgetting
+    from soup_cli.eval.forgetting import ForgettingDetector
 
-    runner = getattr(forgetting, "run_mini_benchmark", None)
-    if runner is None:
-        raise RuntimeError(
-            "mini-benchmark runner unavailable - "
-            "install [eval] extras or update soup-cli"
-        )
-    score = runner(benchmark=task.benchmark, generate_fn=generate_fn)
+    score = ForgettingDetector(
+        generate_fn=generate_fn,
+        benchmark=task.benchmark,
+    ).run_baseline()
     return max(0.0, min(1.0, float(score)))
 
 
diff --git a/tests/test_part_a_wave1.py b/tests/test_part_a_wave1.py
index d81a2bf..0d05548 100644
--- a/tests/test_part_a_wave1.py
+++ b/tests/test_part_a_wave1.py
@@ -121,28 +121,32 @@ class TestRunGateErrorPropagation:
         assert row.error
         assert row.passed is False
 
-    def test_benchmark_task_unavailable(self, tmp_path, monkeypatch):
-        from soup_cli.eval import forgetting
+    def test_benchmark_task_runs_builtin_benchmark(self):
         from soup_cli.eval.gate import EvalSuite, GateTask, run_gate
 
-        # Strip the runner attr to force the RuntimeError branch
-        monkeypatch.setattr(
-            forgetting, "run_mini_benchmark", None, raising=False,
-        )
-        # Ensure attribute lookup returns None
-        if hasattr(forgetting, "run_mini_benchmark"):
-            monkeypatch.delattr(
-                forgetting, "run_mini_benchmark", raising=False,
-            )
+        suite = EvalSuite(suite="t", tasks=[GateTask(
+            type="benchmark", name="bench", threshold=0.2,
+            benchmark="mini_mmlu",
+        )])
+        result = run_gate(suite, generate_fn=lambda _p: "B")
+        row = result.task_results[0]
+
+        assert row.score == 0.4
+        assert row.error is None
+        assert row.passed is True
+
+    def test_benchmark_task_unknown_name_lists_options(self):
+        from soup_cli.eval.gate import EvalSuite, GateTask, run_gate
 
         suite = EvalSuite(suite="t", tasks=[GateTask(
             type="benchmark", name="bench", threshold=0.3,
-            benchmark="mini_mmlu",
+            benchmark="not_a_benchmark",
         )])
         result = run_gate(suite, generate_fn=lambda _p: "")
         row = result.task_results[0]
+
         assert row.score is None
-        assert row.error and "unavailable" in row.error
+        assert row.error and "Options: mini_mmlu" in row.error
         assert row.passed is False