fix(evals): fail closed without maintained campaign
This commit is contained in:
parent
c3adbe751c
commit
653510aab4
|
|
@ -79,13 +79,7 @@ function parseRosterSelection(value) {
|
|||
|
||||
async function maintainedRosterSelection(programRoot) {
|
||||
const campaignPath = resolve(programRoot, "campaigns/live-direct-full.json");
|
||||
let campaign;
|
||||
try {
|
||||
campaign = await loadObject(campaignPath);
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return null;
|
||||
throw error;
|
||||
}
|
||||
const campaign = await loadObject(campaignPath);
|
||||
if (
|
||||
campaign.schema !== "paperclip-runner/live-campaign/v1" ||
|
||||
!Array.isArray(campaign.lanes)
|
||||
|
|
|
|||
|
|
@ -187,6 +187,29 @@ test("all selects the maintained enabled campaign and explicit diagnostics can s
|
|||
);
|
||||
});
|
||||
|
||||
test("all fails closed when the maintained campaign is missing", async () => {
|
||||
const { root, program } = await fixture();
|
||||
await rm(join(program, "campaigns/live-direct-full.json"));
|
||||
|
||||
await assert.rejects(
|
||||
buildProtocolEvalCatalog({
|
||||
evalsRoot: root,
|
||||
campaignId: "gha-42-1",
|
||||
}),
|
||||
/ENOENT/,
|
||||
);
|
||||
|
||||
const diagnostic = await buildProtocolEvalCatalog({
|
||||
evalsRoot: root,
|
||||
campaignId: "gha-42-2",
|
||||
rosterSelection: "protocol-live-opencode-model",
|
||||
});
|
||||
assert.deepEqual(
|
||||
diagnostic.rosters.map((roster) => roster.rosterId),
|
||||
["protocol-live-opencode-model"],
|
||||
);
|
||||
});
|
||||
|
||||
test("aggregates retained attempts and synthesizes missing cells as infrastructure", async () => {
|
||||
const { root, config, evalCase } = await fixture();
|
||||
const catalog = await buildProtocolEvalCatalog({
|
||||
|
|
|
|||
Loading…
Reference in New Issue