fix(evals): honor maintained live campaign

.github/workflows/runner-protocol-live-evals.yml

@@ -14,7 +14,7 @@ on:
         type: string
         required: false
       rosters:
-        description: "Comma-separated live roster IDs/files, or all for the complete direct suite"
+        description: "Comma-separated live roster IDs/files, or all for the maintained enabled direct suite"
         type: string
         default: "all"
         required: false

packages/paperclip-runner/docs/runner-protocol-live-evals.md

@@ -25,8 +25,11 @@ from the default branch and provide:
 - `target_branch`: the Paperclip branch to build and test;
 - `evals_sha`: an exact 40-character commit from
   `paperclipai/paperclip-evals`;
-- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic
-  subset;
+- `rosters`: `all` for every enabled lane in the canonical
+  `live-direct-full.json` campaign, or a comma-separated diagnostic subset.
+  Disabled lanes remain available only through an explicit diagnostic
+  selection; `all` never spends against a lane that the eval program has
+  marked disabled;
 - `max_infrastructure_retries`: zero through three, applied only when an
   attempt explicitly reports a retryable infrastructure failure.
 

packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs

@@ -77,6 +77,37 @@ function parseRosterSelection(value) {
   return new Set(selected);
 }
 
+async function maintainedRosterSelection(programRoot) {
+  const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json");
+  let campaign;
+  try {
+    campaign = await loadObject(campaignPath);
+  } catch (error) {
+    if (error?.code === "ENOENT") return null;
+    throw error;
+  }
+  if (
+    campaign.schema !== "paperclip-runner/live-campaign/v1" ||
+    !Array.isArray(campaign.lanes)
+  ) {
+    throw new Error(`Unsupported live campaign schema in ${campaignPath}`);
+  }
+  const selected = campaign.lanes
+    .filter((lane) => lane.executionClass !== "disabled")
+    .map((lane) => {
+      const rosterPath = inside(
+        programRoot,
+        resolve(dirname(campaignPath), String(lane.roster ?? "")),
+        "Campaign roster",
+      );
+      return basename(rosterPath);
+    });
+  if (selected.length === 0 || new Set(selected).size !== selected.length) {
+    throw new Error("Maintained live campaign must contain unique enabled rosters");
+  }
+  return new Set(selected);
+}
+
 export async function buildProtocolEvalCatalog({
   evalsRoot,
   rosterSelection = "all",
@@ -94,7 +125,9 @@ export async function buildProtocolEvalCatalog({
   }
   const programRoot = resolve(evalsRoot, "evals/paperclip-runner");
   const rosterRoot = resolve(programRoot, "rosters");
-  const selected = parseRosterSelection(rosterSelection);
+  const requested = parseRosterSelection(rosterSelection);
+  const selected =
+    requested ?? (await maintainedRosterSelection(programRoot));
   const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true }))
     .filter(
       (entry) =>

packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs

@@ -26,6 +26,7 @@ async function fixture() {
     mkdir(join(program, "rosters"), { recursive: true }),
     mkdir(join(program, "configs"), { recursive: true }),
     mkdir(join(program, "cases"), { recursive: true }),
+    mkdir(join(program, "campaigns"), { recursive: true }),
   ]);
   const config = {
     schema: "paperclip-runner/eval-config/v1",
@@ -65,6 +66,19 @@ async function fixture() {
       join(program, "rosters/live-opencode-model.json"),
       JSON.stringify(roster),
     ),
+    writeFile(
+      join(program, "campaigns/live-direct-full.json"),
+      JSON.stringify({
+        schema: "paperclip-runner/live-campaign/v1",
+        lanes: [
+          {
+            id: "opencode-model",
+            executionClass: "default",
+            roster: "../rosters/live-opencode-model.json",
+          },
+        ],
+      }),
+    ),
   ]);
   return { root, program, config, evalCase, roster };
 }
@@ -119,6 +133,60 @@ test("catalogs roster plus case cells and emits bounded balanced shards", async
   );
 });
 
+test("all selects the maintained enabled campaign and explicit diagnostics can select disabled rosters", async () => {
+  const { root, program, config, evalCase } = await fixture();
+  const disabledRoster = {
+    schema: "paperclip-runner/live-roster/v1",
+    id: "protocol-live-disabled-model",
+    model: config.model,
+    config: "../configs/live-opencode-model.json",
+    cases: [evalCase.id],
+  };
+  await Promise.all([
+    writeFile(
+      join(program, "rosters/live-disabled-model.json"),
+      JSON.stringify(disabledRoster),
+    ),
+    writeFile(
+      join(program, "campaigns/live-direct-full.json"),
+      JSON.stringify({
+        schema: "paperclip-runner/live-campaign/v1",
+        lanes: [
+          {
+            id: "opencode-model",
+            executionClass: "default",
+            roster: "../rosters/live-opencode-model.json",
+          },
+          {
+            id: "disabled-model",
+            executionClass: "disabled",
+            roster: "../rosters/live-disabled-model.json",
+          },
+        ],
+      }),
+    ),
+  ]);
+
+  const maintained = await buildProtocolEvalCatalog({
+    evalsRoot: root,
+    campaignId: "gha-42-1",
+  });
+  assert.deepEqual(
+    maintained.rosters.map((roster) => roster.rosterId),
+    ["protocol-live-opencode-model"],
+  );
+
+  const diagnostic = await buildProtocolEvalCatalog({
+    evalsRoot: root,
+    campaignId: "gha-42-2",
+    rosterSelection: "protocol-live-disabled-model",
+  });
+  assert.deepEqual(
+    diagnostic.rosters.map((roster) => roster.rosterId),
+    ["protocol-live-disabled-model"],
+  );
+});
+
 test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => {
   const { root, config, evalCase } = await fixture();
   const catalog = await buildProtocolEvalCatalog({