diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index fad74e868..b52f899dc 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -90,8 +90,11 @@ export interface TierClassification { * * Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly * the invisible-non-execution bug this runner exists to kill), so the only - * exclusion evidence accepted is an explicit whole-file `EVALS_TIER === ''` - * guard. Inferring a file's tier from which E2E_TIERS names appear in its source + * exclusion evidence accepted is an explicit whole-file tier guard: either the + * raw `EVALS_TIER === ''` predicate or the consolidated helper form + * `describeE2ETier('')` / `e2eTierEnabled('')` from + * test/helpers/e2e-gate.ts (same semantics, read from env at module load). + * Inferring a file's tier from which E2E_TIERS names appear in its source * is guesswork that silently drops real work: short keys like 'retro' match * unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and * carry no E2E_TIERS name at all. Everything without an explicit other-tier @@ -100,10 +103,11 @@ export interface TierClassification { export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification { const other: PaidTier = tier === 'gate' ? 'periodic' : 'gate'; const declares = (candidate: PaidTier) => - new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source); + new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source) || + new RegExp(`\\b(?:describeE2ETier|e2eTierEnabled)\\(\\s*['"\`]${candidate}['"\`]`).test(source); - if (declares(tier)) return { included: true, reason: `declares EVALS_TIER === '${tier}'` }; - if (declares(other)) return { included: false, reason: `declares EVALS_TIER === '${other}' only` }; + if (declares(tier)) return { included: true, reason: `declares tier '${tier}'` }; + if (declares(other)) return { included: false, reason: `declares tier '${other}' only` }; return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' }; } diff --git a/test/e2e-tier-alignment.test.ts b/test/e2e-tier-alignment.test.ts index 34451a0d4..7e201b09f 100644 --- a/test/e2e-tier-alignment.test.ts +++ b/test/e2e-tier-alignment.test.ts @@ -26,6 +26,12 @@ const TEST_DIR = import.meta.dir; // silently drop a file from the invariant (fail-open is the defect class // this test exists to kill). const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g; +// Consolidated gate helper (test/helpers/e2e-gate.ts). Both regexes stay +// active: migrated files self-gate via `describeE2ETier('')` (or the +// boolean form `e2eTierEnabled('')`), while stragglers still using the +// raw predicate are caught by SELF_GATE_RE above. The tier argument maps to +// the declared tier exactly like the raw predicate's tier literal did. +const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g; describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => { const testFiles = readdirSync(TEST_DIR) @@ -42,6 +48,7 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => const content = readFileSync(path.join(TEST_DIR, file), 'utf-8'); const tiers = new Set(); for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]); + for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]); const repoPath = `test/${file}`; if (tiers.size === 0) { // Every skill-e2e file is expected to self-gate; zero matches means diff --git a/test/helpers/e2e-gate.ts b/test/helpers/e2e-gate.ts new file mode 100644 index 000000000..5bd147e7d --- /dev/null +++ b/test/helpers/e2e-gate.ts @@ -0,0 +1,51 @@ +/** + * Whole-file E2E tier gate — the single definition of the + * `EVALS=1 && EVALS_TIER === ''` predicate that tier-gated paid test + * files used to copy-paste (~36 local copies before consolidation). + * + * This module MUST stay side-effect-free. It is imported at module scope by + * every tier-gated test file, including files the sharded paid runner + * (scripts/test-paid-shards.ts) spawns one-process-each — unlike + * test/helpers/e2e-helpers.ts, whose EVALS=1 module-scope work includes a + * ~30s `claude -p` connectivity ping, diff-based selection, and ~/.gstack + * pre-seeding. The only import allowed here is `bun:test`. + * test/helpers/e2e-gate.unit.test.ts enforces this with a source scan. + * + * Env is read at CALL time (the importing test file's module top-level), not + * captured at this module's load time, so the gate behaves identically under + * single-process `bun test` globs and the sharded runner's per-shard env. + * + * Static-grep consumers that must recognize the call shape + * `describeE2ETier('')` / `e2eTierEnabled('')` alongside the raw + * `EVALS_TIER === ''` predicate: + * - test/e2e-tier-alignment.test.ts (HELPER_GATE_RE) — tier-alignment invariant + * - scripts/test-paid-shards.ts classifyPaidTestFile — pre-spawn tier exclusion + */ + +import { describe } from 'bun:test'; + +export type E2ETier = 'gate' | 'periodic'; + +/** + * True when this process should run whole-file-gated paid tests of `tier`: + * EVALS=1 AND EVALS_TIER exactly equals the tier. + * + * Deliberate consequence: EVALS=1 with EVALS_TIER unset is false for BOTH + * tiers. Tierless runs (`test:evals` / `eval:bg` / `eval:bg:all`) skip every + * tier-gated file and rely on diff-based per-test selection instead — that is + * the long-standing behavior of the copy-pasted predicates, pinned by + * test/helpers/e2e-gate.unit.test.ts. + */ +export function e2eTierEnabled(tier: E2ETier): boolean { + return !!process.env.EVALS && process.env.EVALS_TIER === tier; +} + +/** + * `describe` when `e2eTierEnabled(tier)`, else `describe.skip`. + * + * Usage (module top-level of a tier-gated test file): + * const describeE2E = describeE2ETier('periodic'); + */ +export function describeE2ETier(tier: E2ETier): typeof describe | typeof describe.skip { + return e2eTierEnabled(tier) ? describe : describe.skip; +} diff --git a/test/helpers/e2e-gate.unit.test.ts b/test/helpers/e2e-gate.unit.test.ts new file mode 100644 index 000000000..418eb06be --- /dev/null +++ b/test/helpers/e2e-gate.unit.test.ts @@ -0,0 +1,103 @@ +/** + * Pins the consolidated E2E tier gate (test/helpers/e2e-gate.ts). + * + * Two invariants: + * 1. The env matrix — including the tierless-run trap: EVALS=1 with + * EVALS_TIER unset must SKIP both tiers (that is how `test:evals` / + * `eval:bg:all` have always treated whole-file tier gates; per-test + * diff selection covers those runs instead). + * 2. Module purity — e2e-gate.ts is imported at module scope by every + * tier-gated paid test file, one-process-each under the sharded + * runner. Its only import must be `bun:test` and it must contain no + * spawn/network/fs machinery (the reason it cannot live in + * e2e-helpers.ts, whose EVALS=1 module scope runs a ~30s claude ping). + */ + +import { describe, test, expect, beforeEach, afterEach } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import { describeE2ETier, e2eTierEnabled } from './e2e-gate'; + +const SAVED_EVALS = process.env.EVALS; +const SAVED_TIER = process.env.EVALS_TIER; + +function restoreEnv() { + if (SAVED_EVALS === undefined) delete process.env.EVALS; + else process.env.EVALS = SAVED_EVALS; + if (SAVED_TIER === undefined) delete process.env.EVALS_TIER; + else process.env.EVALS_TIER = SAVED_TIER; +} + +describe('e2e-gate: env matrix (read at call time)', () => { + beforeEach(() => { + delete process.env.EVALS; + delete process.env.EVALS_TIER; + }); + afterEach(restoreEnv); + + test('EVALS unset → skip, even when EVALS_TIER matches', () => { + process.env.EVALS_TIER = 'gate'; + expect(e2eTierEnabled('gate')).toBe(false); + expect(describeE2ETier('gate')).toBe(describe.skip); + expect(describeE2ETier('periodic')).toBe(describe.skip); + }); + + test('EVALS=1 + matching tier → run', () => { + process.env.EVALS = '1'; + process.env.EVALS_TIER = 'gate'; + expect(e2eTierEnabled('gate')).toBe(true); + expect(describeE2ETier('gate')).toBe(describe); + + process.env.EVALS_TIER = 'periodic'; + expect(e2eTierEnabled('periodic')).toBe(true); + expect(describeE2ETier('periodic')).toBe(describe); + }); + + test('EVALS=1 + other tier → skip', () => { + process.env.EVALS = '1'; + process.env.EVALS_TIER = 'periodic'; + expect(e2eTierEnabled('gate')).toBe(false); + expect(describeE2ETier('gate')).toBe(describe.skip); + + process.env.EVALS_TIER = 'gate'; + expect(e2eTierEnabled('periodic')).toBe(false); + expect(describeE2ETier('periodic')).toBe(describe.skip); + }); + + test('EVALS=1 + EVALS_TIER unset → skip both tiers (the tierless test:evals / eval:bg:all trap)', () => { + process.env.EVALS = '1'; + expect(e2eTierEnabled('gate')).toBe(false); + expect(e2eTierEnabled('periodic')).toBe(false); + expect(describeE2ETier('gate')).toBe(describe.skip); + expect(describeE2ETier('periodic')).toBe(describe.skip); + }); +}); + +describe('e2e-gate: module purity (side-effect-free import)', () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'e2e-gate.ts'), 'utf-8'); + + test('the only import specifier is bun:test', () => { + const specifiers = [...source.matchAll(/from\s+['"]([^'"]+)['"]/g)].map((m) => m[1]); + expect(specifiers.length).toBeGreaterThan(0); + expect(specifiers.filter((s) => s !== 'bun:test')).toEqual([]); + // No dynamic escape hatches either. + expect(source).not.toMatch(/\brequire\s*\(/); + expect(source).not.toMatch(/\bimport\s*\(/); + }); + + test('no spawn / network / fs machinery in the module body', () => { + // Strip comments so prose explaining WHY the module must stay pure + // (which legitimately names spawnSync etc.) doesn't trip the scan. + const code = source + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/\/\/[^\n]*/g, ''); + for (const banned of [ + 'spawnSync', 'spawn(', 'execSync', 'child_process', + 'Bun.spawn', 'Bun.file', 'Bun.write', + 'fetch(', 'WebSocket', 'XMLHttpRequest', + 'readFileSync', 'writeFileSync', 'mkdirSync', 'node:fs', "from 'fs'", + ]) { + expect(code.includes(banned), `e2e-gate.ts must not contain "${banned}"`).toBe(false); + } + }); +}); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 2f58bef6b..8d217725b 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -56,6 +56,20 @@ describe('tier classification', () => { expect(classifyPaidTestFile(periodicGuard, 'periodic').included).toBe(true); }); + test('recognizes the consolidated e2e-gate helper guard (both forms)', () => { + // The shape test/helpers/e2e-gate.ts consumers use after consolidation. + const helperGate = "const describeE2E = describeE2ETier('gate');"; + const helperPeriodic = "const describeE2E = describeE2ETier('periodic');"; + const boolPeriodic = "const shouldRun = CODEX_AVAILABLE && e2eTierEnabled('periodic');"; + + expect(classifyPaidTestFile(helperGate, 'gate').included).toBe(true); + expect(classifyPaidTestFile(helperGate, 'periodic').included).toBe(false); + expect(classifyPaidTestFile(helperPeriodic, 'periodic').included).toBe(true); + expect(classifyPaidTestFile(helperPeriodic, 'gate').included).toBe(false); + expect(classifyPaidTestFile(boolPeriodic, 'gate').included).toBe(false); + expect(classifyPaidTestFile(boolPeriodic, 'periodic').included).toBe(true); + }); + test('keeps files whose tier is decided per-test at runtime', () => { // Naming an E2E_TIERS key is not evidence — 'retro' appears in the // LLM-judge file, which test:gate does run.