RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(evals): give the plan-ceo plan-mode smoke real budget headroom 

Measured 2026-08-11: a clean isolated pass took 295.7s against the 300s
inner budget (4s of margin) and the same test timed out at ~308s three
times under concurrent eval load — a budget-edge flake in the gate lane,
not a behavior regression (it passed isolated on both this branch and
main). Inner budget 300s -> 420s, outer bun timeout 360s -> 480s, and the
test file is now named in its own touchfiles dep list so the tier-alignment
invariant binds for it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan 2026-08-12 10:23:11 -07:00
parent a36925cab9
commit 6422c78f6f
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
2 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
test/helpers/touchfiles.ts
View File

@ -98,7 +98,7 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
// include question-tuning.ts and generate-ask-user-format.ts because the
// AUTO_DECIDE preamble injection lives there and changes can flip the
// regression test outcome between 'asked' and 'auto_decided'.
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-ceo-plan-mode.test.ts'],
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],

9
test/skill-e2e-plan-ceo-plan-mode.test.ts
View File

@ -47,7 +47,12 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
const obs = await runPlanSkillObservation({
skillName: 'plan-ceo-review',
inPlanMode: true,
timeoutMs: 300_000,
// 420s, not 300s: measured 2026-08-11, a clean isolated pass took
// 295.7s (80s on a quiet main run) — 4s under the old budget — and the
// same run timed out at ~308s three times under concurrent eval load.
// Same runner-contention class as review-dashboard-via/retro-base-
// branch; headroom instead of a budget-edge flake in the gate lane.
timeoutMs: 420_000,
env: { QUESTION_TUNING: 'false', EXPLAIN_LEVEL: 'default' },
});
@ -72,5 +77,5 @@ describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
);
}
assertReportAtBottomIfPlanWritten(obs);
}, 360_000);
}, 480_000);
});