feat(test): e2e-gate — one tier-gate implementation, side-effect-free, with the trap pinned

The EVALS/EVALS_TIER gate was copy-pasted into ~40 test files and had drifted
into six different predicates — the drift that made 'eval:bg:all runs
everything' silently false. test/helpers/e2e-gate.ts owns the semantics now:
describeE2ETier(tier) + e2eTierEnabled(tier), env read at call time, zero
side effects (the existing e2e-helpers module runs a ~30s claude ping at
import under EVALS=1, so the gate lives in its own module; purity is pinned
by tests that scan imports and comment-stripped source).

The unit matrix pins all four env combos — including EVALS=1 with EVALS_TIER
unset -> SKIP, the exact trap that made eval:bg:all a non-run. The
tier-alignment tripwire gains a second regex for the helper shape (old shape
still detected — stragglers can't hide), and the sharded paid runner's
PRE-SPAWN tier classifier learns the helper shape too: without that, every
gate-sharded run would have spawned all 28 periodic shards just to skip them,
each paying the e2e-helpers import ping (~15 min of dead wall clock in the
CI-blocking lane). Verified: gate runs exclude the 29 periodic files,
periodic excludes the 8 gate files — identical to pre-migration.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

scripts/test-paid-shards.ts

@@ -90,8 +90,11 @@ export interface TierClassification {
  *
  * Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly
  * the invisible-non-execution bug this runner exists to kill), so the only
- * exclusion evidence accepted is an explicit whole-file `EVALS_TIER === '<other>'`
- * guard. Inferring a file's tier from which E2E_TIERS names appear in its source
+ * exclusion evidence accepted is an explicit whole-file tier guard: either the
+ * raw `EVALS_TIER === '<other>'` predicate or the consolidated helper form
+ * `describeE2ETier('<other>')` / `e2eTierEnabled('<other>')` from
+ * test/helpers/e2e-gate.ts (same semantics, read from env at module load).
+ * Inferring a file's tier from which E2E_TIERS names appear in its source
  * is guesswork that silently drops real work: short keys like 'retro' match
  * unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and
  * carry no E2E_TIERS name at all. Everything without an explicit other-tier
@@ -100,10 +103,11 @@ export interface TierClassification {
 export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification {
   const other: PaidTier = tier === 'gate' ? 'periodic' : 'gate';
   const declares = (candidate: PaidTier) =>
-    new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source);
+    new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source) ||
+    new RegExp(`\\b(?:describeE2ETier|e2eTierEnabled)\\(\\s*['"\`]${candidate}['"\`]`).test(source);
 
-  if (declares(tier)) return { included: true, reason: `declares EVALS_TIER === '${tier}'` };
-  if (declares(other)) return { included: false, reason: `declares EVALS_TIER === '${other}' only` };
+  if (declares(tier)) return { included: true, reason: `declares tier '${tier}'` };
+  if (declares(other)) return { included: false, reason: `declares tier '${other}' only` };
   return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' };
 }
 

test/e2e-tier-alignment.test.ts

@@ -26,6 +26,12 @@ const TEST_DIR = import.meta.dir;
 // silently drop a file from the invariant (fail-open is the defect class
 // this test exists to kill).
 const SELF_GATE_RE = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/g;
+// Consolidated gate helper (test/helpers/e2e-gate.ts). Both regexes stay
+// active: migrated files self-gate via `describeE2ETier('<tier>')` (or the
+// boolean form `e2eTierEnabled('<tier>')`), while stragglers still using the
+// raw predicate are caught by SELF_GATE_RE above. The tier argument maps to
+// the declared tier exactly like the raw predicate's tier literal did.
+const HELPER_GATE_RE = /\b(?:describeE2ETier|e2eTierEnabled)\(\s*['"](gate|periodic)['"]/g;
 
 describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => {
   const testFiles = readdirSync(TEST_DIR)
@@ -42,6 +48,7 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
       const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
       const tiers = new Set<string>();
       for (const m of content.matchAll(SELF_GATE_RE)) tiers.add(m[1]);
+      for (const m of content.matchAll(HELPER_GATE_RE)) tiers.add(m[1]);
       const repoPath = `test/${file}`;
       if (tiers.size === 0) {
         // Every skill-e2e file is expected to self-gate; zero matches means

test/helpers/e2e-gate.ts

@@ -0,0 +1,51 @@
+/**
+ * Whole-file E2E tier gate — the single definition of the
+ * `EVALS=1 && EVALS_TIER === '<tier>'` predicate that tier-gated paid test
+ * files used to copy-paste (~36 local copies before consolidation).
+ *
+ * This module MUST stay side-effect-free. It is imported at module scope by
+ * every tier-gated test file, including files the sharded paid runner
+ * (scripts/test-paid-shards.ts) spawns one-process-each — unlike
+ * test/helpers/e2e-helpers.ts, whose EVALS=1 module-scope work includes a
+ * ~30s `claude -p` connectivity ping, diff-based selection, and ~/.gstack
+ * pre-seeding. The only import allowed here is `bun:test`.
+ * test/helpers/e2e-gate.unit.test.ts enforces this with a source scan.
+ *
+ * Env is read at CALL time (the importing test file's module top-level), not
+ * captured at this module's load time, so the gate behaves identically under
+ * single-process `bun test` globs and the sharded runner's per-shard env.
+ *
+ * Static-grep consumers that must recognize the call shape
+ * `describeE2ETier('<tier>')` / `e2eTierEnabled('<tier>')` alongside the raw
+ * `EVALS_TIER === '<tier>'` predicate:
+ *   - test/e2e-tier-alignment.test.ts (HELPER_GATE_RE) — tier-alignment invariant
+ *   - scripts/test-paid-shards.ts classifyPaidTestFile — pre-spawn tier exclusion
+ */
+
+import { describe } from 'bun:test';
+
+export type E2ETier = 'gate' | 'periodic';
+
+/**
+ * True when this process should run whole-file-gated paid tests of `tier`:
+ * EVALS=1 AND EVALS_TIER exactly equals the tier.
+ *
+ * Deliberate consequence: EVALS=1 with EVALS_TIER unset is false for BOTH
+ * tiers. Tierless runs (`test:evals` / `eval:bg` / `eval:bg:all`) skip every
+ * tier-gated file and rely on diff-based per-test selection instead — that is
+ * the long-standing behavior of the copy-pasted predicates, pinned by
+ * test/helpers/e2e-gate.unit.test.ts.
+ */
+export function e2eTierEnabled(tier: E2ETier): boolean {
+  return !!process.env.EVALS && process.env.EVALS_TIER === tier;
+}
+
+/**
+ * `describe` when `e2eTierEnabled(tier)`, else `describe.skip`.
+ *
+ * Usage (module top-level of a tier-gated test file):
+ *   const describeE2E = describeE2ETier('periodic');
+ */
+export function describeE2ETier(tier: E2ETier): typeof describe | typeof describe.skip {
+  return e2eTierEnabled(tier) ? describe : describe.skip;
+}

test/helpers/e2e-gate.unit.test.ts

@@ -0,0 +1,103 @@
+/**
+ * Pins the consolidated E2E tier gate (test/helpers/e2e-gate.ts).
+ *
+ * Two invariants:
+ *   1. The env matrix — including the tierless-run trap: EVALS=1 with
+ *      EVALS_TIER unset must SKIP both tiers (that is how `test:evals` /
+ *      `eval:bg:all` have always treated whole-file tier gates; per-test
+ *      diff selection covers those runs instead).
+ *   2. Module purity — e2e-gate.ts is imported at module scope by every
+ *      tier-gated paid test file, one-process-each under the sharded
+ *      runner. Its only import must be `bun:test` and it must contain no
+ *      spawn/network/fs machinery (the reason it cannot live in
+ *      e2e-helpers.ts, whose EVALS=1 module scope runs a ~30s claude ping).
+ */
+
+import { describe, test, expect, beforeEach, afterEach } from 'bun:test';
+import * as fs from 'fs';
+import * as path from 'path';
+import { describeE2ETier, e2eTierEnabled } from './e2e-gate';
+
+const SAVED_EVALS = process.env.EVALS;
+const SAVED_TIER = process.env.EVALS_TIER;
+
+function restoreEnv() {
+  if (SAVED_EVALS === undefined) delete process.env.EVALS;
+  else process.env.EVALS = SAVED_EVALS;
+  if (SAVED_TIER === undefined) delete process.env.EVALS_TIER;
+  else process.env.EVALS_TIER = SAVED_TIER;
+}
+
+describe('e2e-gate: env matrix (read at call time)', () => {
+  beforeEach(() => {
+    delete process.env.EVALS;
+    delete process.env.EVALS_TIER;
+  });
+  afterEach(restoreEnv);
+
+  test('EVALS unset → skip, even when EVALS_TIER matches', () => {
+    process.env.EVALS_TIER = 'gate';
+    expect(e2eTierEnabled('gate')).toBe(false);
+    expect(describeE2ETier('gate')).toBe(describe.skip);
+    expect(describeE2ETier('periodic')).toBe(describe.skip);
+  });
+
+  test('EVALS=1 + matching tier → run', () => {
+    process.env.EVALS = '1';
+    process.env.EVALS_TIER = 'gate';
+    expect(e2eTierEnabled('gate')).toBe(true);
+    expect(describeE2ETier('gate')).toBe(describe);
+
+    process.env.EVALS_TIER = 'periodic';
+    expect(e2eTierEnabled('periodic')).toBe(true);
+    expect(describeE2ETier('periodic')).toBe(describe);
+  });
+
+  test('EVALS=1 + other tier → skip', () => {
+    process.env.EVALS = '1';
+    process.env.EVALS_TIER = 'periodic';
+    expect(e2eTierEnabled('gate')).toBe(false);
+    expect(describeE2ETier('gate')).toBe(describe.skip);
+
+    process.env.EVALS_TIER = 'gate';
+    expect(e2eTierEnabled('periodic')).toBe(false);
+    expect(describeE2ETier('periodic')).toBe(describe.skip);
+  });
+
+  test('EVALS=1 + EVALS_TIER unset → skip both tiers (the tierless test:evals / eval:bg:all trap)', () => {
+    process.env.EVALS = '1';
+    expect(e2eTierEnabled('gate')).toBe(false);
+    expect(e2eTierEnabled('periodic')).toBe(false);
+    expect(describeE2ETier('gate')).toBe(describe.skip);
+    expect(describeE2ETier('periodic')).toBe(describe.skip);
+  });
+});
+
+describe('e2e-gate: module purity (side-effect-free import)', () => {
+  const source = fs.readFileSync(path.join(import.meta.dir, 'e2e-gate.ts'), 'utf-8');
+
+  test('the only import specifier is bun:test', () => {
+    const specifiers = [...source.matchAll(/from\s+['"]([^'"]+)['"]/g)].map((m) => m[1]);
+    expect(specifiers.length).toBeGreaterThan(0);
+    expect(specifiers.filter((s) => s !== 'bun:test')).toEqual([]);
+    // No dynamic escape hatches either.
+    expect(source).not.toMatch(/\brequire\s*\(/);
+    expect(source).not.toMatch(/\bimport\s*\(/);
+  });
+
+  test('no spawn / network / fs machinery in the module body', () => {
+    // Strip comments so prose explaining WHY the module must stay pure
+    // (which legitimately names spawnSync etc.) doesn't trip the scan.
+    const code = source
+      .replace(/\/\*[\s\S]*?\*\//g, '')
+      .replace(/\/\/[^\n]*/g, '');
+    for (const banned of [
+      'spawnSync', 'spawn(', 'execSync', 'child_process',
+      'Bun.spawn', 'Bun.file', 'Bun.write',
+      'fetch(', 'WebSocket', 'XMLHttpRequest',
+      'readFileSync', 'writeFileSync', 'mkdirSync', 'node:fs', "from 'fs'",
+    ]) {
+      expect(code.includes(banned), `e2e-gate.ts must not contain "${banned}"`).toBe(false);
+    }
+  });
+});

test/paid-shards.test.ts

@@ -56,6 +56,20 @@ describe('tier classification', () => {
     expect(classifyPaidTestFile(periodicGuard, 'periodic').included).toBe(true);
   });
 
+  test('recognizes the consolidated e2e-gate helper guard (both forms)', () => {
+    // The shape test/helpers/e2e-gate.ts consumers use after consolidation.
+    const helperGate = "const describeE2E = describeE2ETier('gate');";
+    const helperPeriodic = "const describeE2E = describeE2ETier('periodic');";
+    const boolPeriodic = "const shouldRun = CODEX_AVAILABLE && e2eTierEnabled('periodic');";
+
+    expect(classifyPaidTestFile(helperGate, 'gate').included).toBe(true);
+    expect(classifyPaidTestFile(helperGate, 'periodic').included).toBe(false);
+    expect(classifyPaidTestFile(helperPeriodic, 'periodic').included).toBe(true);
+    expect(classifyPaidTestFile(helperPeriodic, 'gate').included).toBe(false);
+    expect(classifyPaidTestFile(boolPeriodic, 'gate').included).toBe(false);
+    expect(classifyPaidTestFile(boolPeriodic, 'periodic').included).toBe(true);
+  });
+
   test('keeps files whose tier is decided per-test at runtime', () => {
     // Naming an E2E_TIERS key is not evidence — 'retro' appears in the
     // LLM-judge file, which test:gate does run.