From 8d533f1a46b4eaee1627e6db2429873c6d8ab58f Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 08:52:30 -0700 Subject: [PATCH] evals: fix context-save-list test that was 0-for-26 ($5.28, zero passes) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Disposition for the eval store's only permanently-red test. Root cause: the hide-other-branches assertions scanned the FULL output surface (incl. bash tool_results), so any agent that ran ls on the checkpoints dir — the natural first step of a list flow — surfaced all three seeded filenames and failed, even when its user-facing listing filtered correctly. The test punished the agent for looking at the directory. The hide-assertions now scan the agent's FINAL TEXT (the listing the user sees); showsMain keeps the broad surface for its documented reason. Validated live in the final gate run rather than quarantined: the test guards real behavior (branch filtering) and the assertion was the bug. Co-Authored-By: Claude Fable 5 --- test/skill-e2e-context-skills.test.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/test/skill-e2e-context-skills.test.ts b/test/skill-e2e-context-skills.test.ts index 505307ca7..d0896cc0f 100644 --- a/test/skill-e2e-context-skills.test.ts +++ b/test/skill-e2e-context-skills.test.ts @@ -447,12 +447,18 @@ Do NOT use AskUserQuestion.`, // Broad surface: the list output may only appear in bash tool_result // entries (find output, file reads) rather than the agent's final text. const out = fullOutputSurface(result); - // Must show the main-branch save. Hide the other branches' saves. - // Match by filename timestamp (stable, unambiguous) plus a looser - // prose check. + // Must show the main-branch save. Match by filename timestamp (stable, + // unambiguous) plus a looser prose check. const showsMain = /20260101-120000|main-work/.test(out); - const hidesAlpha = !/20260202-120000/.test(out); - const hidesBeta = !/20260303-120000/.test(out); + // The hide-assertions scan the FINAL TEXT only. This test went 0-for-26 + // ($5.28 burned, zero passes) because they used the broad surface: any + // agent that ran `ls` on the checkpoints dir — the natural first step of + // a list flow — surfaced all three filenames in a tool_result and failed, + // even when its user-facing listing filtered correctly. What must hide + // the other branches is the LISTING the user sees, not the agent's eyes. + const finalText = result.output ?? ''; + const hidesAlpha = !/20260202-120000|LISTCURR_ALPHA_TOKEN/.test(finalText); + const hidesBeta = !/20260303-120000|LISTCURR_BETA_TOKEN/.test(finalText); const routed = skillCalls(result).includes('context-save'); const exitOk = ['success', 'error_max_turns'].includes(result.exitReason);