/** * Tier-alignment invariant (free, static). * * Kills the "inert demotion" defect class: E2E_TIERS declares a test's tier, * but the paid test files also self-gate on `process.env.EVALS_TIER === ''`. * When the two disagree, the touchfiles declaration is dead metadata — the * #2077 demotion of the plan-mode/finding-floor smokes to 'periodic' was inert * for months because the files still gated on 'gate' and ran in the blocking * lane on every gate run. * * Mapping rule (test filenames do NOT map mechanically to tier keys): for each * `test/skill-e2e-*.test.ts` with an EVALS_TIER self-gate, search the * E2E_TOUCHFILES / LLM_JUDGE_TOUCHFILES dep lists for the exact file path. If * found under key K, the file's self-gate tier must equal E2E_TIERS[K]. Files * not named in any dep list are REPORTED as unmapped (a nudge to add them to * their eval's dep list), never silently skipped. */ import { describe, test, expect } from 'bun:test'; import { readdirSync, readFileSync } from 'fs'; import * as path from 'path'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles'; const TEST_DIR = import.meta.dir; const SELF_GATE_RE = /EVALS_TIER\s*===\s*'(gate|periodic)'/g; describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => { const testFiles = readdirSync(TEST_DIR) .filter((f) => f.startsWith('skill-e2e-') && f.endsWith('.test.ts')) .sort(); const allDeps: Record = { ...E2E_TOUCHFILES, ...LLM_JUDGE_TOUCHFILES }; test('every self-gated test file named in a dep list matches its declared tier', () => { const misaligned: string[] = []; const unmapped: string[] = []; for (const file of testFiles) { const content = readFileSync(path.join(TEST_DIR, file), 'utf-8'); const tiers = new Set(); for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]); if (tiers.size !== 1) continue; // no self-gate, or mixed-tier file — out of scope const selfTier = [...tiers][0]; const repoPath = `test/${file}`; const owningKeys = Object.keys(allDeps).filter((k) => allDeps[k].includes(repoPath)); if (owningKeys.length === 0) { unmapped.push(`${repoPath} (self-gates '${selfTier}')`); continue; } for (const k of owningKeys) { const declared = E2E_TIERS[k]; if (declared && declared !== selfTier) { misaligned.push( `${repoPath}: self-gates on '${selfTier}' but E2E_TIERS['${k}'] declares '${declared}' — the declaration is inert`, ); } } } // Reported, not asserted: these files' evals can't be tier-checked until // their dep lists name the test file. Add `test/` to the eval's // touchfiles entry to bring them under the invariant. if (unmapped.length > 0) { console.warn( `[tier-alignment] ${unmapped.length} self-gated test file(s) not named in any touchfiles dep list:\n ` + unmapped.join('\n '), ); } expect(misaligned).toEqual([]); }); });