From 290c71046b40230e16aaeee682402a9dac187b4a Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Fri, 14 Aug 2026 20:25:25 -0700 Subject: [PATCH] refactor(test): migrate the 36 tier-gated eval files to describeE2ETier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanical two-liner swap in 34 files (each keeping its declared tier — all 36 predicates verified against E2E_TIERS before migrating); the two files with compound gates (overlay-harness's EvalCollector feed, codex-e2e's CODEX_AVAILABLE) keep their extra conditions via e2eTierEnabled. Tier rationale comments preserved. codex-e2e/gemini-e2e/benchmark-providers keep their distinct stderr-message gate shapes by design. Co-Authored-By: Claude Fable 5 --- test/carve-section-loading.test.ts | 6 +++--- test/codex-e2e-recommendation-substance.test.ts | 4 ++-- ...skill-e2e-ask-user-question-format-compliance.test.ts | 6 +++--- test/skill-e2e-auq-consistency.test.ts | 6 +++--- test/skill-e2e-auq-matrix.test.ts | 6 +++--- test/skill-e2e-auq-verbose-vs-carved-ab.test.ts | 6 +++--- test/skill-e2e-auto-decide-preserved.test.ts | 6 +++--- test/skill-e2e-autoplan-chain.test.ts | 6 +++--- test/skill-e2e-brain-privacy-gate.test.ts | 6 +++--- test/skill-e2e-conductor-prose.test.ts | 6 +++--- test/skill-e2e-office-hours-auto-mode.test.ts | 6 +++--- test/skill-e2e-overlay-harness.test.ts | 9 ++++----- test/skill-e2e-plan-ceo-finding-count.test.ts | 6 +++--- test/skill-e2e-plan-ceo-finding-floor.test.ts | 6 +++--- test/skill-e2e-plan-ceo-mode-routing.test.ts | 6 +++--- test/skill-e2e-plan-ceo-plan-mode.test.ts | 6 +++--- test/skill-e2e-plan-ceo-review-section-loading.test.ts | 6 +++--- test/skill-e2e-plan-ceo-split-overflow.test.ts | 6 +++--- test/skill-e2e-plan-design-finding-count.test.ts | 6 +++--- test/skill-e2e-plan-design-finding-floor.test.ts | 6 +++--- test/skill-e2e-plan-design-plan-mode.test.ts | 6 +++--- test/skill-e2e-plan-design-with-ui.test.ts | 6 +++--- test/skill-e2e-plan-devex-finding-count.test.ts | 6 +++--- test/skill-e2e-plan-devex-finding-floor.test.ts | 6 +++--- test/skill-e2e-plan-devex-plan-mode.test.ts | 6 +++--- test/skill-e2e-plan-eng-finding-count.test.ts | 6 +++--- test/skill-e2e-plan-eng-finding-floor.test.ts | 6 +++--- test/skill-e2e-plan-eng-multi-finding-batching.test.ts | 6 +++--- test/skill-e2e-plan-eng-plan-mode.test.ts | 6 +++--- test/skill-e2e-plan-mode-no-op.test.ts | 6 +++--- test/skill-e2e-setup-gbrain-bad-token.test.ts | 6 +++--- test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts | 6 +++--- test/skill-e2e-setup-gbrain-remote.test.ts | 6 +++--- test/skill-e2e-ship-idempotency.test.ts | 6 +++--- test/skill-e2e-ship-section-loading.test.ts | 6 +++--- test/skill-e2e-spec-execute.test.ts | 6 +++--- 36 files changed, 108 insertions(+), 109 deletions(-) diff --git a/test/carve-section-loading.test.ts b/test/carve-section-loading.test.ts index 0aaea779f..075f2c79e 100644 --- a/test/carve-section-loading.test.ts +++ b/test/carve-section-loading.test.ts @@ -19,12 +19,12 @@ * Step-0 mode loop) and keep their dedicated tests; E1 asserts those exist. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture'; import { CARVE_GUARDS } from './helpers/carve-guards'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const runId = `carve-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; const only = process.env.GSTACK_CARVE_SKILL?.trim(); diff --git a/test/codex-e2e-recommendation-substance.test.ts b/test/codex-e2e-recommendation-substance.test.ts index 8d33c0311..b62c3daa5 100644 --- a/test/codex-e2e-recommendation-substance.test.ts +++ b/test/codex-e2e-recommendation-substance.test.ts @@ -22,6 +22,7 @@ */ import { describe, test, expect } from 'bun:test'; import * as path from 'node:path'; +import { e2eTierEnabled } from './helpers/e2e-gate'; import { runCodexSkill } from './helpers/codex-session-runner'; import { judgeRecommendation } from './helpers/llm-judge'; @@ -34,8 +35,7 @@ const CODEX_AVAILABLE = (() => { return false; } })(); -const shouldRun = - CODEX_AVAILABLE && !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; +const shouldRun = CODEX_AVAILABLE && e2eTierEnabled('periodic'); const describeCodex = shouldRun ? describe : describe.skip; // A small fixture with two real, comparable problems so a good recommendation diff --git a/test/skill-e2e-ask-user-question-format-compliance.test.ts b/test/skill-e2e-ask-user-question-format-compliance.test.ts index 787350fb5..2823febc3 100644 --- a/test/skill-e2e-ask-user-question-format-compliance.test.ts +++ b/test/skill-e2e-ask-user-question-format-compliance.test.ts @@ -22,7 +22,8 @@ * hide; the model's composed question is. Shares the engine with the periodic * A/B and matrix evals (test/helpers/auq-sdk-capture.ts). */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { setupPlanCeoDir, @@ -32,8 +33,7 @@ import { carvedSkill, } from './helpers/auq-sdk-capture'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); const runId = `auq-format-gate-${process.env.EVALS_RUN_ID ?? 'local'}`; describeE2E('AskUserQuestion format compliance (gate)', () => { diff --git a/test/skill-e2e-auq-consistency.test.ts b/test/skill-e2e-auq-consistency.test.ts index 9601981cf..ca1c9f93d 100644 --- a/test/skill-e2e-auq-consistency.test.ts +++ b/test/skill-e2e-auq-consistency.test.ts @@ -15,7 +15,8 @@ * Reports per-run scores so drift is visible even on a pass. Periodic tier * (N SDK runs, ~$0.50-1 each). */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { setupPlanCeoDir, @@ -25,8 +26,7 @@ import { } from './helpers/auq-sdk-capture'; import { judgeRecommendation } from './helpers/llm-judge'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3'); const runId = `auq-consistency-${process.env.EVALS_RUN_ID ?? 'local'}`; diff --git a/test/skill-e2e-auq-matrix.test.ts b/test/skill-e2e-auq-matrix.test.ts index e386e929b..80a3da7ff 100644 --- a/test/skill-e2e-auq-matrix.test.ts +++ b/test/skill-e2e-auq-matrix.test.ts @@ -22,7 +22,8 @@ * * Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso". */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { setupSkillDir, @@ -32,8 +33,7 @@ import { gradeAuqRecommendation, } from './helpers/auq-sdk-capture'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`; const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean); diff --git a/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts b/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts index dd55d9f51..a812785fa 100644 --- a/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts +++ b/test/skill-e2e-auq-verbose-vs-carved-ab.test.ts @@ -22,7 +22,8 @@ * carries the same {{PREAMBLE}} format spec + Step 0 prose as verbose, with * strictly less unrelated review-section text in context. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { setupPlanCeoDir, @@ -33,8 +34,7 @@ import { } from './helpers/auq-sdk-capture'; import { judgeRecommendation } from './helpers/llm-judge'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const runId = `auq-ab-${process.env.EVALS_RUN_ID ?? 'local'}`; async function grade(label: string, dir: string) { diff --git a/test/skill-e2e-auto-decide-preserved.test.ts b/test/skill-e2e-auto-decide-preserved.test.ts index 7e74d8438..5d3ec36d3 100644 --- a/test/skill-e2e-auto-decide-preserved.test.ts +++ b/test/skill-e2e-auto-decide-preserved.test.ts @@ -37,15 +37,15 @@ * practice but not the load-bearing behavior). */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { spawnSync } from 'child_process'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const ROOT = path.resolve(import.meta.dir, '..'); diff --git a/test/skill-e2e-autoplan-chain.test.ts b/test/skill-e2e-autoplan-chain.test.ts index 645e55bf9..8d580c02e 100644 --- a/test/skill-e2e-autoplan-chain.test.ts +++ b/test/skill-e2e-autoplan-chain.test.ts @@ -24,7 +24,8 @@ * Cost: ~$5-8/run, 10-15 min wall clock. Periodic — runs weekly. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -36,8 +37,7 @@ import { isNumberedOptionListVisible, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const ROOT = path.resolve(import.meta.dir, '..'); const UI_FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md'); diff --git a/test/skill-e2e-brain-privacy-gate.test.ts b/test/skill-e2e-brain-privacy-gate.test.ts index 27caf29c4..762b7bdc4 100644 --- a/test/skill-e2e-brain-privacy-gate.test.ts +++ b/test/skill-e2e-brain-privacy-gate.test.ts @@ -20,14 +20,14 @@ * prose contract this test locks in. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); describeE2E('gbrain-sync privacy gate fires once via preamble', () => { test('gstack skill preamble fires the 3-option AskUserQuestion when gbrain is detected', async () => { diff --git a/test/skill-e2e-conductor-prose.test.ts b/test/skill-e2e-conductor-prose.test.ts index a130dcab5..7b71677e4 100644 --- a/test/skill-e2e-conductor-prose.test.ts +++ b/test/skill-e2e-conductor-prose.test.ts @@ -20,11 +20,11 @@ * Periodic tier: model-behavior, non-deterministic. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const FLAWED_PLAN = `# Plan: add a "developer-friendly" pricing tier diff --git a/test/skill-e2e-office-hours-auto-mode.test.ts b/test/skill-e2e-office-hours-auto-mode.test.ts index 5e1a29489..49eb4f764 100644 --- a/test/skill-e2e-office-hours-auto-mode.test.ts +++ b/test/skill-e2e-office-hours-auto-mode.test.ts @@ -16,11 +16,11 @@ * distinct silencing mechanism; both share the same fix surface. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => { // Pass envelope is ['asked', 'plan_ready']; failure signals are diff --git a/test/skill-e2e-overlay-harness.test.ts b/test/skill-e2e-overlay-harness.test.ts index c00a27f60..85f689513 100644 --- a/test/skill-e2e-overlay-harness.test.ts +++ b/test/skill-e2e-overlay-harness.test.ts @@ -20,7 +20,8 @@ * Gated by EVALS=1 AND EVALS_TIER=periodic. Never runs under test:gate. */ -import { describe, test, expect, afterAll } from 'bun:test'; +import { test, expect, afterAll } from 'bun:test'; +import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; @@ -37,11 +38,9 @@ import { } from './fixtures/overlay-nudges'; import { readOverlay } from '../scripts/resolvers/model-overlay'; -const evalsEnabled = !!process.env.EVALS; -const periodicTier = process.env.EVALS_TIER === 'periodic'; -const shouldRun = evalsEnabled && periodicTier; +const shouldRun = e2eTierEnabled('periodic'); -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); // EvalCollector's tier must be 'e2e' | 'llm-judge' per its type signature. // The existing paid evals violate this by passing descriptive names like // 'e2e-opus-47' — a pre-existing pattern that only works because bun-test diff --git a/test/skill-e2e-plan-ceo-finding-count.test.ts b/test/skill-e2e-plan-ceo-finding-count.test.ts index 850c1a033..e299adc56 100644 --- a/test/skill-e2e-plan-ceo-finding-count.test.ts +++ b/test/skill-e2e-plan-ceo-finding-count.test.ts @@ -15,7 +15,8 @@ * test/helpers/claude-pty-runner.ts for runPlanSkillCounting internals. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -51,8 +52,7 @@ function pickSkipInterview(fp: AskUserQuestionFingerprint): number { return 1; } -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N_DISTINCT = 5; const FLOOR_DISTINCT = N_DISTINCT - 1; // 4 (D11) diff --git a/test/skill-e2e-plan-ceo-finding-floor.test.ts b/test/skill-e2e-plan-ceo-finding-floor.test.ts index ba3d7c968..e99da2c53 100644 --- a/test/skill-e2e-plan-ceo-finding-floor.test.ts +++ b/test/skill-e2e-plan-ceo-finding-floor.test.ts @@ -4,12 +4,12 @@ * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => { test( diff --git a/test/skill-e2e-plan-ceo-mode-routing.test.ts b/test/skill-e2e-plan-ceo-mode-routing.test.ts index 0772e7a11..8b54a4d3c 100644 --- a/test/skill-e2e-plan-ceo-mode-routing.test.ts +++ b/test/skill-e2e-plan-ceo-mode-routing.test.ts @@ -30,7 +30,8 @@ * SCOPE EXPANSION — "expansion" or "10x" or "delight" or "dream" */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { launchClaudePty, isNumberedOptionListVisible, @@ -43,8 +44,7 @@ import { type ClaudePtySession, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); interface ModeCase { mode: 'HOLD SCOPE' | 'SCOPE EXPANSION'; diff --git a/test/skill-e2e-plan-ceo-plan-mode.test.ts b/test/skill-e2e-plan-ceo-plan-mode.test.ts index 861605d88..0a358a0b1 100644 --- a/test/skill-e2e-plan-ceo-plan-mode.test.ts +++ b/test/skill-e2e-plan-ceo-plan-mode.test.ts @@ -33,14 +33,14 @@ * See test/helpers/claude-pty-runner.ts for runner internals. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); describeE2E('plan-ceo-review plan-mode smoke (gate)', () => { test('first terminal outcome is asked (Step 0 fires before any plan write)', async () => { diff --git a/test/skill-e2e-plan-ceo-review-section-loading.test.ts b/test/skill-e2e-plan-ceo-review-section-loading.test.ts index 6cac8960e..7074fc141 100644 --- a/test/skill-e2e-plan-ceo-review-section-loading.test.ts +++ b/test/skill-e2e-plan-ceo-review-section-loading.test.ts @@ -24,15 +24,15 @@ * ~$1-2/run. Periodic tier. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, skillFromWorktree, captureSectionReads, } from './helpers/auq-sdk-capture'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; // Sections every plan-ceo-review run must consult after Step 0. diff --git a/test/skill-e2e-plan-ceo-split-overflow.test.ts b/test/skill-e2e-plan-ceo-split-overflow.test.ts index 05f3c51f2..896c8c84b 100644 --- a/test/skill-e2e-plan-ceo-split-overflow.test.ts +++ b/test/skill-e2e-plan-ceo-split-overflow.test.ts @@ -32,7 +32,8 @@ * Sequential by default. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -40,8 +41,7 @@ import { } from './helpers/claude-pty-runner'; import { FORCING_SPLIT_OVERFLOW_CEO } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option diff --git a/test/skill-e2e-plan-design-finding-count.test.ts b/test/skill-e2e-plan-design-finding-count.test.ts index ef0d9b681..8793dca38 100644 --- a/test/skill-e2e-plan-design-finding-count.test.ts +++ b/test/skill-e2e-plan-design-finding-count.test.ts @@ -8,7 +8,8 @@ * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -16,8 +17,7 @@ import { assertReviewReportAtBottom, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; diff --git a/test/skill-e2e-plan-design-finding-floor.test.ts b/test/skill-e2e-plan-design-finding-floor.test.ts index c6a893f8a..dc556f33b 100644 --- a/test/skill-e2e-plan-design-finding-floor.test.ts +++ b/test/skill-e2e-plan-design-finding-floor.test.ts @@ -4,12 +4,12 @@ * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => { test( diff --git a/test/skill-e2e-plan-design-plan-mode.test.ts b/test/skill-e2e-plan-design-plan-mode.test.ts index d32e73515..7d2a373fc 100644 --- a/test/skill-e2e-plan-design-plan-mode.test.ts +++ b/test/skill-e2e-plan-design-plan-mode.test.ts @@ -9,14 +9,14 @@ * 'plan_ready' are valid pass outcomes. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); // UI-heavy seed with guaranteed design gaps (center-aligned everything, no // empty states, no responsive intent) so the review has real findings to diff --git a/test/skill-e2e-plan-design-with-ui.test.ts b/test/skill-e2e-plan-design-with-ui.test.ts index 904fe96ba..a9877922f 100644 --- a/test/skill-e2e-plan-design-with-ui.test.ts +++ b/test/skill-e2e-plan-design-with-ui.test.ts @@ -19,7 +19,8 @@ * contain "no UI scope". */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as path from 'path'; import { launchClaudePty, @@ -29,8 +30,7 @@ import { isPlanReadyVisible, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); const ROOT = path.resolve(import.meta.dir, '..'); const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md'); diff --git a/test/skill-e2e-plan-devex-finding-count.test.ts b/test/skill-e2e-plan-devex-finding-count.test.ts index e4b3f8e77..0f1eb8fb7 100644 --- a/test/skill-e2e-plan-devex-finding-count.test.ts +++ b/test/skill-e2e-plan-devex-finding-count.test.ts @@ -8,7 +8,8 @@ * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -16,8 +17,7 @@ import { assertReviewReportAtBottom, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; diff --git a/test/skill-e2e-plan-devex-finding-floor.test.ts b/test/skill-e2e-plan-devex-finding-floor.test.ts index e2e394fcf..e87d7e0ed 100644 --- a/test/skill-e2e-plan-devex-finding-floor.test.ts +++ b/test/skill-e2e-plan-devex-finding-floor.test.ts @@ -4,12 +4,12 @@ * See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => { test( diff --git a/test/skill-e2e-plan-devex-plan-mode.test.ts b/test/skill-e2e-plan-devex-plan-mode.test.ts index 959efce08..2f1c73a8e 100644 --- a/test/skill-e2e-plan-devex-plan-mode.test.ts +++ b/test/skill-e2e-plan-devex-plan-mode.test.ts @@ -5,15 +5,15 @@ * contract. Exercises the same contract against /plan-devex-review. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, planFileHasDecisionsSection, assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); describeE2E('plan-devex-review plan-mode smoke (gate)', () => { test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => { diff --git a/test/skill-e2e-plan-eng-finding-count.test.ts b/test/skill-e2e-plan-eng-finding-count.test.ts index 93b8ba687..257e579d9 100644 --- a/test/skill-e2e-plan-eng-finding-count.test.ts +++ b/test/skill-e2e-plan-eng-finding-count.test.ts @@ -8,7 +8,8 @@ * Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -16,8 +17,7 @@ import { assertReviewReportAtBottom, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N = 5; const FLOOR = N - 1; // 4 diff --git a/test/skill-e2e-plan-eng-finding-floor.test.ts b/test/skill-e2e-plan-eng-finding-floor.test.ts index 56a31abb6..f5b7cfeea 100644 --- a/test/skill-e2e-plan-eng-finding-floor.test.ts +++ b/test/skill-e2e-plan-eng-finding-floor.test.ts @@ -15,12 +15,12 @@ * Cost: ~$0.50-$1.50 per run depending on early-exit timing. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner'; import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => { test( diff --git a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts index 1eef660a9..bac69496a 100644 --- a/test/skill-e2e-plan-eng-multi-finding-batching.test.ts +++ b/test/skill-e2e-plan-eng-multi-finding-batching.test.ts @@ -24,7 +24,8 @@ * Tier: periodic (~25 min, ~$5/run). Sequential by default. */ -import { describe, test } from 'bun:test'; +import { test } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'node:fs'; import { runPlanSkillCounting, @@ -32,8 +33,7 @@ import { } from './helpers/claude-pty-runner'; import { FORCING_BATCHING_ENG } from './fixtures/forcing-finding-seeds'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const N = 4; const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding diff --git a/test/skill-e2e-plan-eng-plan-mode.test.ts b/test/skill-e2e-plan-eng-plan-mode.test.ts index c72a84ba9..c632abe6f 100644 --- a/test/skill-e2e-plan-eng-plan-mode.test.ts +++ b/test/skill-e2e-plan-eng-plan-mode.test.ts @@ -5,15 +5,15 @@ * contract. This file exercises the same contract against /plan-eng-review. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation, planFileHasDecisionsSection, assertReportAtBottomIfPlanWritten, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); // SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the // Step 0 complexity check to trigger. Passed via runPlanSkillObservation's diff --git a/test/skill-e2e-plan-mode-no-op.test.ts b/test/skill-e2e-plan-mode-no-op.test.ts index ae1311156..c69428129 100644 --- a/test/skill-e2e-plan-mode-no-op.test.ts +++ b/test/skill-e2e-plan-mode-no-op.test.ts @@ -30,11 +30,11 @@ * change (see 'plan-mode-no-op' in touchfiles.ts). */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { runPlanSkillObservation } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('gate'); const PLAN_MODE_REMINDER = 'Plan mode is active. The user indicated that they do not want you to execute yet'; diff --git a/test/skill-e2e-setup-gbrain-bad-token.test.ts b/test/skill-e2e-setup-gbrain-bad-token.test.ts index 84cb2ab42..f4ad75080 100644 --- a/test/skill-e2e-setup-gbrain-bad-token.test.ts +++ b/test/skill-e2e-setup-gbrain-bad-token.test.ts @@ -8,7 +8,8 @@ // // Cost: ~$0.30-$0.50 per run. Gate-tier (EVALS=1 EVALS_TIER=gate). -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -17,8 +18,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f // Periodic-tier (companion to skill-e2e-setup-gbrain-remote.test.ts). // Deterministic gate coverage lives in setup-gbrain-path4-structure.test.ts. -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); function startStub401(): Promise<{ url: string; close: () => Promise }> { return new Promise((resolve) => { diff --git a/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts b/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts index a78503c36..59275355b 100644 --- a/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts +++ b/test/skill-e2e-setup-gbrain-path4-local-pglite.test.ts @@ -18,7 +18,8 @@ // // Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic). -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -29,8 +30,7 @@ import { resolveClaudeBinary, } from './helpers/agent-sdk-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); /** * Minimal stub MCP server that returns success on initialize / tools/list. diff --git a/test/skill-e2e-setup-gbrain-remote.test.ts b/test/skill-e2e-setup-gbrain-remote.test.ts index 651317e58..ed2cf116b 100644 --- a/test/skill-e2e-setup-gbrain-remote.test.ts +++ b/test/skill-e2e-setup-gbrain-remote.test.ts @@ -11,7 +11,8 @@ // // See setup-gbrain/SKILL.md.tmpl Step 4 (Path 4) for the contract under test. -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as os from 'os'; import * as path from 'path'; @@ -22,8 +23,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f // non-deterministic (it sometimes skips Step 8 CLAUDE.md write, sometimes // shortcuts past the verify helper). The deterministic gate coverage for // Path 4 lives in test/setup-gbrain-path4-structure.test.ts (free, <200ms). -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); // Spin up a stub MCP server that responds to initialize + tools/list. function startStubMcpServer(opts: { failWithStatus?: number; failBody?: string } = {}): Promise<{ url: string; close: () => Promise }> { diff --git a/test/skill-e2e-ship-idempotency.test.ts b/test/skill-e2e-ship-idempotency.test.ts index e4388a7d0..322bf9f8f 100644 --- a/test/skill-e2e-ship-idempotency.test.ts +++ b/test/skill-e2e-ship-idempotency.test.ts @@ -30,7 +30,8 @@ * Cost: ~$2-4/run. Periodic tier — long, runs weekly. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; @@ -41,8 +42,7 @@ import { isNumberedOptionListVisible, } from './helpers/claude-pty-runner'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); interface ShipFixture { workTree: string; diff --git a/test/skill-e2e-ship-section-loading.test.ts b/test/skill-e2e-ship-section-loading.test.ts index 4e6a717cf..23d9db9e8 100644 --- a/test/skill-e2e-ship-section-loading.test.ts +++ b/test/skill-e2e-ship-section-loading.test.ts @@ -23,15 +23,15 @@ * Periodic tier. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import { setupSkillDir, skillFromWorktree, captureSectionReads, } from './helpers/auq-sdk-capture'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const runId = `ship-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`; // Sections every version-changing ship must consult. diff --git a/test/skill-e2e-spec-execute.test.ts b/test/skill-e2e-spec-execute.test.ts index 5937dd06a..4d99c957c 100644 --- a/test/skill-e2e-spec-execute.test.ts +++ b/test/skill-e2e-spec-execute.test.ts @@ -16,12 +16,12 @@ * minimum smoke that proves --execute end-to-end works. */ -import { describe, test, expect } from 'bun:test'; +import { test, expect } from 'bun:test'; +import { describeE2ETier } from './helpers/e2e-gate'; import * as fs from 'fs'; import * as path from 'path'; -const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic'; -const describeE2E = shouldRun ? describe : describe.skip; +const describeE2E = describeE2ETier('periodic'); const ROOT = path.resolve(import.meta.dir, '..');