From c3adbe751cae7253cb87ff155c448497dd5f3d4a Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 6 Sep 2026 12:09:53 -0500 Subject: [PATCH] fix(evals): honor maintained live campaign --- .../workflows/runner-protocol-live-evals.yml | 2 +- .../docs/runner-protocol-live-evals.md | 7 +- .../scripts/runner-protocol-eval-campaign.mjs | 35 +++++++++- .../runner-protocol-eval-campaign.test.mjs | 68 +++++++++++++++++++ 4 files changed, 108 insertions(+), 4 deletions(-) diff --git a/.github/workflows/runner-protocol-live-evals.yml b/.github/workflows/runner-protocol-live-evals.yml index f396e8a975..f659f5ec65 100644 --- a/.github/workflows/runner-protocol-live-evals.yml +++ b/.github/workflows/runner-protocol-live-evals.yml @@ -14,7 +14,7 @@ on: type: string required: false rosters: - description: "Comma-separated live roster IDs/files, or all for the complete direct suite" + description: "Comma-separated live roster IDs/files, or all for the maintained enabled direct suite" type: string default: "all" required: false diff --git a/packages/paperclip-runner/docs/runner-protocol-live-evals.md b/packages/paperclip-runner/docs/runner-protocol-live-evals.md index 62092030af..34ef5fb959 100644 --- a/packages/paperclip-runner/docs/runner-protocol-live-evals.md +++ b/packages/paperclip-runner/docs/runner-protocol-live-evals.md @@ -25,8 +25,11 @@ from the default branch and provide: - `target_branch`: the Paperclip branch to build and test; - `evals_sha`: an exact 40-character commit from `paperclipai/paperclip-evals`; -- `rosters`: `all` for the entire direct suite, or a comma-separated diagnostic - subset; +- `rosters`: `all` for every enabled lane in the canonical + `live-direct-full.json` campaign, or a comma-separated diagnostic subset. + Disabled lanes remain available only through an explicit diagnostic + selection; `all` never spends against a lane that the eval program has + marked disabled; - `max_infrastructure_retries`: zero through three, applied only when an attempt explicitly reports a retryable infrastructure failure. diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs index 7314b63748..360472ae32 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs @@ -77,6 +77,37 @@ function parseRosterSelection(value) { return new Set(selected); } +async function maintainedRosterSelection(programRoot) { + const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json"); + let campaign; + try { + campaign = await loadObject(campaignPath); + } catch (error) { + if (error?.code === "ENOENT") return null; + throw error; + } + if ( + campaign.schema !== "paperclip-runner/live-campaign/v1" || + !Array.isArray(campaign.lanes) + ) { + throw new Error(`Unsupported live campaign schema in ${campaignPath}`); + } + const selected = campaign.lanes + .filter((lane) => lane.executionClass !== "disabled") + .map((lane) => { + const rosterPath = inside( + programRoot, + resolve(dirname(campaignPath), String(lane.roster ?? "")), + "Campaign roster", + ); + return basename(rosterPath); + }); + if (selected.length === 0 || new Set(selected).size !== selected.length) { + throw new Error("Maintained live campaign must contain unique enabled rosters"); + } + return new Set(selected); +} + export async function buildProtocolEvalCatalog({ evalsRoot, rosterSelection = "all", @@ -94,7 +125,9 @@ export async function buildProtocolEvalCatalog({ } const programRoot = resolve(evalsRoot, "evals/paperclip-runner"); const rosterRoot = resolve(programRoot, "rosters"); - const selected = parseRosterSelection(rosterSelection); + const requested = parseRosterSelection(rosterSelection); + const selected = + requested ?? (await maintainedRosterSelection(programRoot)); const rosterFiles = (await readdir(rosterRoot, { withFileTypes: true })) .filter( (entry) => diff --git a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs index 474f89971b..511ed41c64 100644 --- a/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs +++ b/packages/paperclip-runner/scripts/runner-protocol-eval-campaign.test.mjs @@ -26,6 +26,7 @@ async function fixture() { mkdir(join(program, "rosters"), { recursive: true }), mkdir(join(program, "configs"), { recursive: true }), mkdir(join(program, "cases"), { recursive: true }), + mkdir(join(program, "campaigns"), { recursive: true }), ]); const config = { schema: "paperclip-runner/eval-config/v1", @@ -65,6 +66,19 @@ async function fixture() { join(program, "rosters/live-opencode-model.json"), JSON.stringify(roster), ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + ], + }), + ), ]); return { root, program, config, evalCase, roster }; } @@ -119,6 +133,60 @@ test("catalogs roster plus case cells and emits bounded balanced shards", async ); }); +test("all selects the maintained enabled campaign and explicit diagnostics can select disabled rosters", async () => { + const { root, program, config, evalCase } = await fixture(); + const disabledRoster = { + schema: "paperclip-runner/live-roster/v1", + id: "protocol-live-disabled-model", + model: config.model, + config: "../configs/live-opencode-model.json", + cases: [evalCase.id], + }; + await Promise.all([ + writeFile( + join(program, "rosters/live-disabled-model.json"), + JSON.stringify(disabledRoster), + ), + writeFile( + join(program, "campaigns/live-direct-full.json"), + JSON.stringify({ + schema: "paperclip-runner/live-campaign/v1", + lanes: [ + { + id: "opencode-model", + executionClass: "default", + roster: "../rosters/live-opencode-model.json", + }, + { + id: "disabled-model", + executionClass: "disabled", + roster: "../rosters/live-disabled-model.json", + }, + ], + }), + ), + ]); + + const maintained = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-1", + }); + assert.deepEqual( + maintained.rosters.map((roster) => roster.rosterId), + ["protocol-live-opencode-model"], + ); + + const diagnostic = await buildProtocolEvalCatalog({ + evalsRoot: root, + campaignId: "gha-42-2", + rosterSelection: "protocol-live-disabled-model", + }); + assert.deepEqual( + diagnostic.rosters.map((roster) => roster.rosterId), + ["protocol-live-disabled-model"], + ); +}); + test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => { const { root, config, evalCase } = await fixture(); const catalog = await buildProtocolEvalCatalog({