mirror of https://github.com/garrytan/gstack.git
refactor(test): migrate the 36 tier-gated eval files to describeE2ETier
Mechanical two-liner swap in 34 files (each keeping its declared tier — all 36 predicates verified against E2E_TIERS before migrating); the two files with compound gates (overlay-harness's EvalCollector feed, codex-e2e's CODEX_AVAILABLE) keep their extra conditions via e2eTierEnabled. Tier rationale comments preserved. codex-e2e/gemini-e2e/benchmark-providers keep their distinct stderr-message gate shapes by design. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
e23a3ae3b1
commit
290c71046b
|
|
@ -19,12 +19,12 @@
|
|||
* Step-0 mode loop) and keep their dedicated tests; E1 asserts those exist.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { setupSkillDir, skillFromWorktree, captureSectionReads } from './helpers/auq-sdk-capture';
|
||||
import { CARVE_GUARDS } from './helpers/carve-guards';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `carve-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
const only = process.env.GSTACK_CARVE_SKILL?.trim();
|
||||
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@
|
|||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import * as path from 'node:path';
|
||||
import { e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import { runCodexSkill } from './helpers/codex-session-runner';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
|
||||
|
|
@ -34,8 +35,7 @@ const CODEX_AVAILABLE = (() => {
|
|||
return false;
|
||||
}
|
||||
})();
|
||||
const shouldRun =
|
||||
CODEX_AVAILABLE && !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const shouldRun = CODEX_AVAILABLE && e2eTierEnabled('periodic');
|
||||
const describeCodex = shouldRun ? describe : describe.skip;
|
||||
|
||||
// A small fixture with two real, comparable problems so a good recommendation
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
* hide; the model's composed question is. Shares the engine with the periodic
|
||||
* A/B and matrix evals (test/helpers/auq-sdk-capture.ts).
|
||||
*/
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
|
|
@ -32,8 +33,7 @@ import {
|
|||
carvedSkill,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
const runId = `auq-format-gate-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
describeE2E('AskUserQuestion format compliance (gate)', () => {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
* Reports per-run scores so drift is visible even on a pass. Periodic tier
|
||||
* (N SDK runs, ~$0.50-1 each).
|
||||
*/
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
|
|
@ -25,8 +26,7 @@ import {
|
|||
} from './helpers/auq-sdk-capture';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const N_RUNS = Number(process.env.AUQ_CONSISTENCY_RUNS ?? '3');
|
||||
const runId = `auq-consistency-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
*
|
||||
* Run a subset in the foreground with AUQ_MATRIX_ONLY="plan-eng-review,cso".
|
||||
*/
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
setupSkillDir,
|
||||
|
|
@ -32,8 +33,7 @@ import {
|
|||
gradeAuqRecommendation,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `auq-matrix-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
const ONLY = (process.env.AUQ_MATRIX_ONLY ?? '').split(',').map(s => s.trim()).filter(Boolean);
|
||||
|
||||
|
|
|
|||
|
|
@ -22,7 +22,8 @@
|
|||
* carries the same {{PREAMBLE}} format spec + Step 0 prose as verbose, with
|
||||
* strictly less unrelated review-section text in context.
|
||||
*/
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
setupPlanCeoDir,
|
||||
|
|
@ -33,8 +34,7 @@ import {
|
|||
} from './helpers/auq-sdk-capture';
|
||||
import { judgeRecommendation } from './helpers/llm-judge';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `auq-ab-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
async function grade(label: string, dir: string) {
|
||||
|
|
|
|||
|
|
@ -37,15 +37,15 @@
|
|||
* practice but not the load-bearing behavior).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { spawnSync } from 'child_process';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@
|
|||
* Cost: ~$5-8/run, 10-15 min wall clock. Periodic — runs weekly.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
|
@ -36,8 +37,7 @@ import {
|
|||
isNumberedOptionListVisible,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const UI_FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
|
||||
|
|
|
|||
|
|
@ -20,14 +20,14 @@
|
|||
* prose contract this test locks in.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } from './helpers/agent-sdk-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
describeE2E('gbrain-sync privacy gate fires once via preamble', () => {
|
||||
test('gstack skill preamble fires the 3-option AskUserQuestion when gbrain is detected', async () => {
|
||||
|
|
|
|||
|
|
@ -20,11 +20,11 @@
|
|||
* Periodic tier: model-behavior, non-deterministic.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const FLAWED_PLAN = `# Plan: add a "developer-friendly" pricing tier
|
||||
|
||||
|
|
|
|||
|
|
@ -16,11 +16,11 @@
|
|||
* distinct silencing mechanism; both share the same fix surface.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation, planFileHasDecisionsSection } from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('office-hours AskUserQuestion-blocked smoke (gate)', () => {
|
||||
// Pass envelope is ['asked', 'plan_ready']; failure signals are
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@
|
|||
* Gated by EVALS=1 AND EVALS_TIER=periodic. Never runs under test:gate.
|
||||
*/
|
||||
|
||||
import { describe, test, expect, afterAll } from 'bun:test';
|
||||
import { test, expect, afterAll } from 'bun:test';
|
||||
import { describeE2ETier, e2eTierEnabled } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
|
|
@ -37,11 +38,9 @@ import {
|
|||
} from './fixtures/overlay-nudges';
|
||||
import { readOverlay } from '../scripts/resolvers/model-overlay';
|
||||
|
||||
const evalsEnabled = !!process.env.EVALS;
|
||||
const periodicTier = process.env.EVALS_TIER === 'periodic';
|
||||
const shouldRun = evalsEnabled && periodicTier;
|
||||
const shouldRun = e2eTierEnabled('periodic');
|
||||
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
// EvalCollector's tier must be 'e2e' | 'llm-judge' per its type signature.
|
||||
// The existing paid evals violate this by passing descriptive names like
|
||||
// 'e2e-opus-47' — a pre-existing pattern that only works because bun-test
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@
|
|||
* test/helpers/claude-pty-runner.ts for runPlanSkillCounting internals.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -51,8 +52,7 @@ function pickSkipInterview(fp: AskUserQuestionFingerprint): number {
|
|||
return 1;
|
||||
}
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N_DISTINCT = 5;
|
||||
const FLOOR_DISTINCT = N_DISTINCT - 1; // 4 (D11)
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_CEO } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('/plan-ceo-review AskUserQuestion floor (gate)', () => {
|
||||
test(
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@
|
|||
* SCOPE EXPANSION — "expansion" or "10x" or "delight" or "dream"
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
launchClaudePty,
|
||||
isNumberedOptionListVisible,
|
||||
|
|
@ -43,8 +44,7 @@ import {
|
|||
type ClaudePtySession,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
interface ModeCase {
|
||||
mode: 'HOLD SCOPE' | 'SCOPE EXPANSION';
|
||||
|
|
|
|||
|
|
@ -33,14 +33,14 @@
|
|||
* See test/helpers/claude-pty-runner.ts for runner internals.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('plan-ceo-review plan-mode smoke (gate)', () => {
|
||||
test('first terminal outcome is asked (Step 0 fires before any plan write)', async () => {
|
||||
|
|
|
|||
|
|
@ -24,15 +24,15 @@
|
|||
* ~$1-2/run. Periodic tier.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
setupSkillDir,
|
||||
skillFromWorktree,
|
||||
captureSectionReads,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `plan-ceo-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
// Sections every plan-ceo-review run must consult after Step 0.
|
||||
|
|
|
|||
|
|
@ -32,7 +32,8 @@
|
|||
* Sequential by default.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -40,8 +41,7 @@ import {
|
|||
} from './helpers/claude-pty-runner';
|
||||
import { FORCING_SPLIT_OVERFLOW_CEO } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 5;
|
||||
const FLOOR = N - 1; // 4 — must fire at least one AUQ per non-dropped option
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -16,8 +17,7 @@ import {
|
|||
assertReviewReportAtBottom,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 5;
|
||||
const FLOOR = N - 1;
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DESIGN } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
describeE2E('/plan-design-review AskUserQuestion floor (periodic)', () => {
|
||||
test(
|
||||
|
|
|
|||
|
|
@ -9,14 +9,14 @@
|
|||
* 'plan_ready' are valid pass outcomes.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
// UI-heavy seed with guaranteed design gaps (center-aligned everything, no
|
||||
// empty states, no responsive intent) so the review has real findings to
|
||||
|
|
|
|||
|
|
@ -19,7 +19,8 @@
|
|||
* contain "no UI scope".
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as path from 'path';
|
||||
import {
|
||||
launchClaudePty,
|
||||
|
|
@ -29,8 +30,7 @@ import {
|
|||
isPlanReadyVisible,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const FIXTURE = path.join(ROOT, 'test', 'fixtures', 'plans', 'ui-heavy-feature.md');
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -16,8 +17,7 @@ import {
|
|||
assertReviewReportAtBottom,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 5;
|
||||
const FLOOR = N - 1;
|
||||
|
|
|
|||
|
|
@ -4,12 +4,12 @@
|
|||
* See test/skill-e2e-plan-eng-finding-floor.test.ts for the contract.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_DEVEX } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('/plan-devex-review AskUserQuestion floor (gate)', () => {
|
||||
test(
|
||||
|
|
|
|||
|
|
@ -5,15 +5,15 @@
|
|||
* contract. Exercises the same contract against /plan-devex-review.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
planFileHasDecisionsSection,
|
||||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
describeE2E('plan-devex-review plan-mode smoke (gate)', () => {
|
||||
test('reaches a terminal outcome (asked or plan_ready) without silent writes', async () => {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
* Tier: periodic (~25 min, ~$5/run). Sequential by default per plan §D15.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -16,8 +17,7 @@ import {
|
|||
assertReviewReportAtBottom,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 5;
|
||||
const FLOOR = N - 1; // 4
|
||||
|
|
|
|||
|
|
@ -15,12 +15,12 @@
|
|||
* Cost: ~$0.50-$1.50 per run depending on early-exit timing.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillFloorCheck } from './helpers/claude-pty-runner';
|
||||
import { FORCING_FLOOR_ENG } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
describeE2E('/plan-eng-review AskUserQuestion floor (periodic)', () => {
|
||||
test(
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@
|
|||
* Tier: periodic (~25 min, ~$5/run). Sequential by default.
|
||||
*/
|
||||
|
||||
import { describe, test } from 'bun:test';
|
||||
import { test } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'node:fs';
|
||||
import {
|
||||
runPlanSkillCounting,
|
||||
|
|
@ -32,8 +33,7 @@ import {
|
|||
} from './helpers/claude-pty-runner';
|
||||
import { FORCING_BATCHING_ENG } from './fixtures/forcing-finding-seeds';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const N = 4;
|
||||
const FLOOR = N - 1; // 3 — agent must fire at least one AUQ per non-batched finding
|
||||
|
|
|
|||
|
|
@ -5,15 +5,15 @@
|
|||
* contract. This file exercises the same contract against /plan-eng-review.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
runPlanSkillObservation,
|
||||
planFileHasDecisionsSection,
|
||||
assertReportAtBottomIfPlanWritten,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
// SEED_PLAN_FORCING_FINDINGS: 8+ files + custom-vs-builtin smell forces the
|
||||
// Step 0 complexity check to trigger. Passed via runPlanSkillObservation's
|
||||
|
|
|
|||
|
|
@ -30,11 +30,11 @@
|
|||
* change (see 'plan-mode-no-op' in touchfiles.ts).
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { runPlanSkillObservation } from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('gate');
|
||||
|
||||
const PLAN_MODE_REMINDER =
|
||||
'Plan mode is active. The user indicated that they do not want you to execute yet';
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@
|
|||
//
|
||||
// Cost: ~$0.30-$0.50 per run. Gate-tier (EVALS=1 EVALS_TIER=gate).
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
|
@ -17,8 +18,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f
|
|||
|
||||
// Periodic-tier (companion to skill-e2e-setup-gbrain-remote.test.ts).
|
||||
// Deterministic gate coverage lives in setup-gbrain-path4-structure.test.ts.
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
function startStub401(): Promise<{ url: string; close: () => Promise<void> }> {
|
||||
return new Promise((resolve) => {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,8 @@
|
|||
//
|
||||
// Cost: ~$0.50-$1.00 per run. Periodic-tier (EVALS=1 EVALS_TIER=periodic).
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
|
@ -29,8 +30,7 @@ import {
|
|||
resolveClaudeBinary,
|
||||
} from './helpers/agent-sdk-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
/**
|
||||
* Minimal stub MCP server that returns success on initialize / tools/list.
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@
|
|||
//
|
||||
// See setup-gbrain/SKILL.md.tmpl Step 4 (Path 4) for the contract under test.
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
|
|
@ -22,8 +23,7 @@ import { runAgentSdkTest, passThroughNonAskUserQuestion, resolveClaudeBinary } f
|
|||
// non-deterministic (it sometimes skips Step 8 CLAUDE.md write, sometimes
|
||||
// shortcuts past the verify helper). The deterministic gate coverage for
|
||||
// Path 4 lives in test/setup-gbrain-path4-structure.test.ts (free, <200ms).
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
// Spin up a stub MCP server that responds to initialize + tools/list.
|
||||
function startStubMcpServer(opts: { failWithStatus?: number; failBody?: string } = {}): Promise<{ url: string; close: () => Promise<void> }> {
|
||||
|
|
|
|||
|
|
@ -30,7 +30,8 @@
|
|||
* Cost: ~$2-4/run. Periodic tier — long, runs weekly.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import { spawnSync } from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
|
@ -41,8 +42,7 @@ import {
|
|||
isNumberedOptionListVisible,
|
||||
} from './helpers/claude-pty-runner';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
interface ShipFixture {
|
||||
workTree: string;
|
||||
|
|
|
|||
|
|
@ -23,15 +23,15 @@
|
|||
* Periodic tier.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import {
|
||||
setupSkillDir,
|
||||
skillFromWorktree,
|
||||
captureSectionReads,
|
||||
} from './helpers/auq-sdk-capture';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
const runId = `ship-section-loading-${process.env.EVALS_RUN_ID ?? 'local'}`;
|
||||
|
||||
// Sections every version-changing ship must consult.
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@
|
|||
* minimum smoke that proves --execute end-to-end works.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import { test, expect } from 'bun:test';
|
||||
import { describeE2ETier } from './helpers/e2e-gate';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
|
||||
const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';
|
||||
const describeE2E = shouldRun ? describe : describe.skip;
|
||||
const describeE2E = describeE2ETier('periodic');
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue