mirror of https://github.com/garrytan/gstack.git
fix(evals): align plan-eng/design plan-mode + finding-floor smokes to their declared periodic tier
The #2077 demotion of these four stochastic tests to 'periodic' was inert: E2E_TIERS declared periodic but the files self-gated on EVALS_TIER === 'gate', so they kept running in the blocking gate lane and never in the weekly lane. Flip the four self-gates to 'periodic' (headers/describe labels updated), add a free static tier-alignment invariant test (dep-list filename mapping; unmapped self-gated files are reported, never silently skipped), and name the two plan-mode test files in their own touchfiles dep lists so the invariant binds for them. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
parent
94993f7401
commit
f349d7c894
|
|
@ -0,0 +1,73 @@
|
|||
/**
|
||||
* Tier-alignment invariant (free, static).
|
||||
*
|
||||
* Kills the "inert demotion" defect class: E2E_TIERS declares a test's tier,
|
||||
* but the paid test files also self-gate on `process.env.EVALS_TIER === '<tier>'`.
|
||||
* When the two disagree, the touchfiles declaration is dead metadata — the
|
||||
* #2077 demotion of the plan-mode/finding-floor smokes to 'periodic' was inert
|
||||
* for months because the files still gated on 'gate' and ran in the blocking
|
||||
* lane on every gate run.
|
||||
*
|
||||
* Mapping rule (test filenames do NOT map mechanically to tier keys): for each
|
||||
* `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the
|
||||
* E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If
|
||||
* found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files
|
||||
* not named in any dep list are REPORTED as unmapped (a nudge to add them to
|
||||
* their eval's dep list), never silently skipped.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { readdirSync, readFileSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
|
||||
|
||||
const TEST_DIR = import.meta.dir;
|
||||
const SELF_GATE_RE = /EVALS_TIER\s*===\s*'(gate|periodic)'/g;
|
||||
|
||||
describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
|
||||
const testFiles = readdirSync(TEST_DIR)
|
||||
.filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts'))
|
||||
.sort();
|
||||
|
||||
const allDeps: Record<string, string[]> = { ...E2E_TOUCHFILES, ...LLM_JUDGE_TOUCHFILES };
|
||||
|
||||
test('every self-gated test file named in a dep list matches its declared tier', () => {
|
||||
const misaligned: string[] = [];
|
||||
const unmapped: string[] = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
|
||||
const tiers = new Set<string>();
|
||||
for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
|
||||
if (tiers.size !== 1) continue; // no self-gate, or mixed-tier file — out of scope
|
||||
const selfTier = [...tiers][0];
|
||||
|
||||
const repoPath = `test/${file}`;
|
||||
const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath));
|
||||
if (owningKeys.length === 0) {
|
||||
unmapped.push(`${repoPath} (self-gates '${selfTier}')`);
|
||||
continue;
|
||||
}
|
||||
for (const k of owningKeys) {
|
||||
const declared = E2E_TIERS[k];
|
||||
if (declared && declared !== selfTier) {
|
||||
misaligned.push(
|
||||
`${repoPath}: self-gates on '${selfTier}' but E2E_TIERS['${k}'] declares '${declared}' — the declaration is inert`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reported, not asserted: these files' evals can't be tier-checked until
|
||||
// their dep lists name the test file. Add `test/<file>` to the eval's
|
||||
// touchfiles entry to bring them under the invariant.
|
||||
if (unmapped.length > 0) {
|
||||
console.warn(
|
||||
`[tier-alignment] ${unmapped.length} self-gated test file(s) not named in any touchfiles dep list:\n ` +
|
||||
unmapped.join('\n '),
|
||||
);
|
||||
}
|
||||
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
@ -99,8 +99,8 @@ export const E2E_TOUCHFILES: Record<string, string[]> = {
|
|||
// AUTO_DECIDE preamble injection lives there and changes can flip the
|
||||
// regression test outcome between 'asked' and 'auto_decided'.
|
||||
'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'],
|
||||
'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'],
|
||||
'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
'plan-mode-no-op': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'],
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* /plan-design-review AskUserQuestion floor regression (gate, paid, real-PTY).
|
||||
* /plan-design-review AskUserQuestion floor regression (periodic, paid, real-PTY).
|
||||
*
|
||||
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
|
||||
*/
|
||||
|
|
@ -8,10 +8,10 @@ import { describe, test } from 'bun:test';
|
|||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
|
||||
describeE2E('/plan-design-review AskUserQuestion floor (gate)', () => {
|
||||
describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
test(
|
||||
'seeded forcing finding causes the agent to fire at least one AskUserQuestion',
|
||||
async () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* plan-design-review plan-mode smoke (gate, paid, real-PTY).
|
||||
* plan-design-review plan-mode smoke (periodic, paid, real-PTY).
|
||||
*
|
||||
* See test/skill-e2e-plan-ceo-plan-mode.test.ts for the shared assertion
|
||||
* contract. Exercises the same contract against /plan-design-review.
|
||||
|
|
@ -15,10 +15,10 @@ import {
|
|||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
|
||||
describeE2E('plan-design-review plan-mode smoke (gate)', () => {
|
||||
describeE2E('plan-design-review plan-mode smoke (periodic)', () => {
|
||||
test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-design-review',
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* /plan-eng-review AskUserQuestion floor regression (gate, paid, real-PTY).
|
||||
* /plan-eng-review AskUserQuestion floor regression (periodic, paid, real-PTY).
|
||||
*
|
||||
* Catches the May 2026 transcript bug where /plan-eng-review wrote a
|
||||
* multi-section review plan to ~/.claude/plans/ and called ExitPlanMode
|
||||
|
|
@ -11,7 +11,7 @@
|
|||
* render. See claude-pty-runner.ts for why this is separate from the
|
||||
* runPlanSkillCounting harness used by periodic finding-count tests.
|
||||
*
|
||||
* Tier: gate. Budget: 10 min (early exit on success ~30-90s typical).
|
||||
* Tier: periodic. Budget: 10 min (early exit on success ~30-90s typical).
|
||||
* Cost: ~$0.50-$1.50 per run depending on early-exit timing.
|
||||
*/
|
||||
|
||||
|
|
@ -19,10 +19,10 @@ import { describe, test } from 'bun:test';
|
|||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
|
||||
describeE2E('/plan-eng-review AskUserQuestion floor (gate)', () => {
|
||||
describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
test(
|
||||
'seeded forcing finding causes the agent to fire at least one AskUserQuestion',
|
||||
async () => {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* plan-eng-review plan-mode smoke (gate, paid, real-PTY).
|
||||
* plan-eng-review plan-mode smoke (periodic, paid, real-PTY).
|
||||
*
|
||||
* See test/skill-e2e-plan-ceo-plan-mode.test.ts for the shared assertion
|
||||
* contract. This file exercises the same contract against /plan-eng-review.
|
||||
|
|
@ -12,7 +12,7 @@ import {
|
|||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
|
||||
// SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the
|
||||
|
|
@ -45,7 +45,7 @@ Ignore Bun's native --shard flag because we want full control.
|
|||
None planned — will add later.
|
||||
`;
|
||||
|
||||
describeE2E('plan-eng-review plan-mode smoke (gate)', () => {
|
||||
describeE2E('plan-eng-review plan-mode smoke (periodic)', () => {
|
||||
test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => {
|
||||
const obs = await runPlanSkillObservation({
|
||||
skillName: 'plan-eng-review',
|
||||
|
|
|
|||
Loading…
Reference in New Issue