diff --git a/test/e2e-tier-alignment.test.ts b/test/e2e-tier-alignment.test.ts new file mode 100644 index 000000000..681a8ce94 --- /dev/null +++ b/test/e2e-tier-alignment.test.ts @@ -0,0 +1,73 @@ +/** + * Tier-alignment invariant (free, static). + * + * Kills the "inert demotion" defect class: E2E_TIERS declares a test's tier, + * but the paid test files also self-gate on `process.env.EVALS_TIER === ''`. + * When the two disagree, the touchfiles declaration is dead metadata — the + * #2077 demotion of the plan-mode/finding-floor smokes to 'periodic' was inert + * for months because the files still gated on 'gate' and ran in the blocking + * lane on every gate run. + * + * Mapping rule (test filenames do NOT map mechanically to tier keys): for each + * `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the + * E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If + * found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files + * not named in any dep list are REPORTED as unmapped (a nudge to add them to + * their eval's dep list), never silently skipped. + */ + +import { describe, test, expect } from 'bun:test'; +import { readdirSync, readFileSync } from 'fs'; +import * as path from 'path'; +import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles'; + +const TEST_DIR = import.meta.dir; +const SELF_GATE_RE = /EVALS_TIER\s*===\s*'(gate|periodic)'/g; + +describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => { + const testFiles = readdirSync(TEST_DIR) + .filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts')) + .sort(); + + const allDeps: Record = { ...E2E_TOUCHFILES, ...LLM_JUDGE_TOUCHFILES }; + + test('every self-gated test file named in a dep list matches its declared tier', () => { + const misaligned: string[] = []; + const unmapped: string[] = []; + + for (const file of testFiles) { + const content = readFileSync(path.join(TEST_DIR, file), 'utf-8'); + const tiers = new Set(); + for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]); + if (tiers.size !== 1) continue; // no self-gate, or mixed-tier file — out of scope + const selfTier = [...tiers][0]; + + const repoPath = `test/${file}`; + const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath)); + if (owningKeys.length === 0) { + unmapped.push(`${repoPath} (self-gates '${selfTier}')`); + continue; + } + for (const k of owningKeys) { + const declared = E2E_TIERS[k]; + if (declared && declared !== selfTier) { + misaligned.push( + `${repoPath}: self-gates on '${selfTier}' but E2E_TIERS['${k}'] declares '${declared}' — the declaration is inert`, + ); + } + } + } + + // Reported, not asserted: these files' evals can't be tier-checked until + // their dep lists name the test file. Add `test/` to the eval's + // touchfiles entry to bring them under the invariant. + if (unmapped.length > 0) { + console.warn( + `[tier-alignment] ${unmapped.length} self-gated test file(s) not named in any touchfiles dep list:\n ` + + unmapped.join('\n '), + ); + } + + expect(misaligned).toEqual([]); + }); +}); diff --git a/test/helpers/touchfiles.ts b/test/helpers/touchfiles.ts index 33ba9ad97..3550f6ff3 100644 --- a/test/helpers/touchfiles.ts +++ b/test/helpers/touchfiles.ts @@ -99,8 +99,8 @@ export const E2E_TOUCHFILES: Record = { // AUTO_DECIDE preamble injection lives there and changes can flip the // regression test outcome between 'asked' and 'auto_decided'. 'plan-ceo-review-plan-mode': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'], - 'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'], - 'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'], + 'plan-eng-review-plan-mode': ['plan-eng-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-eng-plan-mode.test.ts'], + 'plan-design-review-plan-mode': ['plan-design-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts', 'test/skill-e2e-plan-design-plan-mode.test.ts'], 'plan-devex-review-plan-mode': ['plan-devex-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/question-tuning.ts', 'scripts/resolvers/preamble/generate-ask-user-format.ts', 'scripts/resolvers/preamble.ts', 'scripts/resolvers/review.ts', 'test/helpers/claude-pty-runner.ts'], 'plan-mode-no-op': ['plan-ceo-review/**', 'scripts/resolvers/preamble/generate-completion-status.ts', 'scripts/resolvers/preamble.ts', 'test/helpers/claude-pty-runner.ts'], diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index ea3d47046..c6a893f8a 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -1,5 +1,5 @@ /** - * /plan-design-review AskUserQuestion floor regression (gate, paid, real-PTY). + * /plan-design-review AskUserQuestion floor regression (periodic, paid, real-PTY). * * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. */ @@ -8,10 +8,10 @@ import { describe, test } from 'bun:test'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; +const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = shouldRun ? describe : describe.skip; -describeE2E('/plan-design-review AskUserQuestion floor (gate)', () => { +describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { test( 'seeded forcing finding causes the agent to fire at least one AskUserQuestion', async () => { diff --git a/test/skill-e2e-plan-design-plan-mode.test.ts b/test/skill-e2e-plan-design-plan-mode.test.ts index 80b982878..4fef6a19e 100644 --- a/test/skill-e2e-plan-design-plan-mode.test.ts +++ b/test/skill-e2e-plan-design-plan-mode.test.ts @@ -1,5 +1,5 @@ /** - * plan-design-review plan-mode smoke (gate, paid, real-PTY). + * plan-design-review plan-mode smoke (periodic, paid, real-PTY). * * See test/skill-e2e-plan-ceo-plan-mode.test.ts for the shared assertion * contract. Exercises the same contract against /plan-design-review. @@ -15,10 +15,10 @@ import { assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; +const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = shouldRun ? describe : describe.skip; -describeE2E('plan-design-review plan-mode smoke (gate)', () => { +describeE2E('plan-design-review plan-mode smoke (periodic)', () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-design-review', diff --git a/test/skill-e2e-plan-eng-finding-floor.test.ts b/test/skill-e2e-plan-eng-finding-floor.test.ts index bd35e1c12..56a31abb6 100644 --- a/test/skill-e2e-plan-eng-finding-floor.test.ts +++ b/test/skill-e2e-plan-eng-finding-floor.test.ts @@ -1,5 +1,5 @@ /** - * /plan-eng-review AskUserQuestion floor regression (gate, paid, real-PTY). + * /plan-eng-review AskUserQuestion floor regression (periodic, paid, real-PTY). * * Catches the May 2026 transcript bug where /plan-eng-review wrote a * multi-section review plan to ~/.claude/plans/ and called ExitPlanMode @@ -11,7 +11,7 @@ * render. See claude-pty-runner.ts for why this is separate from the * runPlanSkillCounting harness used by periodic finding-count tests. * - * Tier: gate. Budget: 10 min (early exit on success ~30-90s typical). + * Tier: periodic. Budget: 10 min (early exit on success ~30-90s typical). * Cost: ~$0.50-$1.50 per run depending on early-exit timing. */ @@ -19,10 +19,10 @@ import { describe, test } from 'bun:test'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; +const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = shouldRun ? describe : describe.skip; -describeE2E('/plan-eng-review AskUserQuestion floor (gate)', () => { +describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { test( 'seeded forcing finding causes the agent to fire at least one AskUserQuestion', async () => { diff --git a/test/skill-e2e-plan-eng-plan-mode.test.ts b/test/skill-e2e-plan-eng-plan-mode.test.ts index ec2adca44..2c6703e9f 100644 --- a/test/skill-e2e-plan-eng-plan-mode.test.ts +++ b/test/skill-e2e-plan-eng-plan-mode.test.ts @@ -1,5 +1,5 @@ /** - * plan-eng-review plan-mode smoke (gate, paid, real-PTY). + * plan-eng-review plan-mode smoke (periodic, paid, real-PTY). * * See test/skill-e2e-plan-ceo-plan-mode.test.ts for the shared assertion * contract. This file exercises the same contract against /plan-eng-review. @@ -12,7 +12,7 @@ import { assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; +const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; const describeE2E = shouldRun ? describe : describe.skip; // SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the @@ -45,7 +45,7 @@ Ignore Bun's native --shard flag because we want full control. None planned — will add later. `; -describeE2E('plan-eng-review plan-mode smoke (gate)', () => { +describeE2E('plan-eng-review plan-mode smoke (periodic)', () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { const obs = await runPlanSkillObservation({ skillName: 'plan-eng-review',