mirror of https://github.com/garrytan/gstack.git
feat(evals): sharded paid tier runner
scripts/test-paid-shards.ts runs the gate/periodic tier one Bun process per test file, with an EXTERNAL wall-clock timeout that SIGKILLs the shard's detached process group and an aggregate that distinguishes passed / failed / timed-out / never-started — partial execution can no longer read as a pass. Bun's native --shard/--isolate covers none of this: no process-group kill (hung claude/codex PTY grandchildren survive in-process isolation), no never-started taxonomy, no per-shard env. Each shard child gets GSTACK_EVAL_DIR=<evalDir>/shards/<slug>/ (slug = test filename sans extension, stable across runs) so shard baselines compare against their own prior runs. Output classification lives in scripts/test-strict-output.ts (strict exit-code derivation, incremental fail-line classifier, child signal forwarding) so the runner and any future strict bun-test wrapper share one implementation. Enumeration derives from the shared paid-test-set module; tier exclusion fires only on an explicit whole-file EVALS_TIER === '<other>' guard. package.json gains test:gate:sharded / test:periodic:sharded, and eval:bg:gate / eval:bg:periodic now run the sharded scripts with detach timeouts sized to the worst case (gate: 49 shards x 30min / 4 jobs ~ 6.2h -> 25200s; periodic: 59 -> 28800s). test/paid-shards.test.ts pins enumeration, tier classification, and the kill-and-continue property with a real busy-loop shard. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> (cherry picked from commit 5e76bd5931836257f896cedfe4e93912cb759c70)
This commit is contained in:
Garry Tan
parent
9ff2b9b162
commit
4e0725479f
|
|
@ -27,6 +27,8 @@
|
|||
"test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts",
|
||||
"test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts",
|
||||
"test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts",
|
||||
"test:gate:sharded": "bun run scripts/test-paid-shards.ts --tier gate",
|
||||
"test:periodic:sharded": "bun run scripts/test-paid-shards.ts --tier periodic",
|
||||
"test:codex": "EVALS=1 bun test test/codex-e2e.test.ts",
|
||||
"test:codex:all": "EVALS=1 EVALS_ALL=1 bun test test/codex-e2e.test.ts",
|
||||
"test:gemini": "EVALS=1 bun test test/gemini-e2e.test.ts",
|
||||
|
|
@ -36,8 +38,8 @@
|
|||
"start": "bun run browse/src/server.ts",
|
||||
"eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals",
|
||||
"eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all",
|
||||
"eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 3600 -- bun run test:gate",
|
||||
"eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 5400 -- bun run test:periodic",
|
||||
"eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded",
|
||||
"eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 28800 -- bun run test:periodic:sharded",
|
||||
"eval:list": "bun run scripts/eval-list.ts",
|
||||
"eval:compare": "bun run scripts/eval-compare.ts",
|
||||
"eval:summary": "bun run scripts/eval-summary.ts",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,459 @@
|
|||
#!/usr/bin/env bun
|
||||
/**
|
||||
* test-paid-shards — enumerate, shard, and run the paid (gate/periodic) tier.
|
||||
*
|
||||
* The single-process `test:gate` fan-out has never completed a run: one wedged
|
||||
* or spinning file takes the whole tier down, and an in-process `--timeout`
|
||||
* cannot save it because a spinning main thread never fires a timer. This
|
||||
* runner applies the free tier's proven fix — one Bun process per shard — plus
|
||||
* the two things the paid tier additionally needs:
|
||||
*
|
||||
* - an EXTERNAL wall-clock timeout that kills the shard's process GROUP, and
|
||||
* - an aggregate that distinguishes failed from timed-out from never-started,
|
||||
* so 26% execution can never again look like a pass.
|
||||
*
|
||||
* Why not Bun 1.3.13's native `--shard` / isolated runs? Three gaps, each one
|
||||
* fatal for this tier:
|
||||
* 1. No detached-process-group SIGKILL. Paid tests spawn `claude` / `codex`
|
||||
* PTY grandchildren; when a shard hangs, in-process isolation kills the
|
||||
* Bun worker but the grandchildren survive and burn cores for hours.
|
||||
* 2. No never-started taxonomy. A run that aborts partway reports only what
|
||||
* executed — the shards that never ran are invisible, which is exactly
|
||||
* the 26%-execution-looks-like-a-pass bug.
|
||||
* 3. No per-shard env / eval dir. Each shard needs its own GSTACK_EVAL_DIR
|
||||
* so eval baselines are per-test-file instead of last-flush-wins.
|
||||
*
|
||||
* Worst-case wall clock (all shards hit the 30min timeout, 4 parallel jobs):
|
||||
* gate tier is 49 shards × 30min / 4 jobs ≈ 6.2h; periodic is 59 shards ≈ 7.4h.
|
||||
* The eval:bg:* detach timeouts (25200s / 28800s) are sized against these.
|
||||
*
|
||||
* Enumeration matches package.json's `test:gate` globs (via the shared
|
||||
* test/helpers/paid-test-set.ts) and honors EVALS_TIER against the E2E_TIERS
|
||||
* map in test/helpers/touchfiles.ts. Output classification reuses
|
||||
* scripts/test-strict-output.ts rather than reimplementing it.
|
||||
*
|
||||
* Parallelism now lives ACROSS shards (--jobs), not inside one Bun process, so
|
||||
* each shard runs its own file sequentially and can be killed independently.
|
||||
*
|
||||
* Usage:
|
||||
* bun run scripts/test-paid-shards.ts --list # shard plan only
|
||||
* bun run scripts/test-paid-shards.ts --tier gate # run gate tier
|
||||
* bun run scripts/test-paid-shards.ts --timeout 600 --jobs 2
|
||||
*/
|
||||
|
||||
import { spawn, type ChildProcess } from 'node:child_process';
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { normalizeRelativePath } from './test-free-shards';
|
||||
import {
|
||||
BunTestOutputClassifier,
|
||||
exactTestFileSelectors,
|
||||
forwardAndClassify,
|
||||
installChildSignalForwarding,
|
||||
strictTestExitCode,
|
||||
} from './test-strict-output';
|
||||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
|
||||
export type PaidTier = 'gate' | 'periodic';
|
||||
|
||||
export const DEFAULT_TIER: PaidTier = 'gate';
|
||||
export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000;
|
||||
export const DEFAULT_MAX_FILES_PER_SHARD = 1;
|
||||
export const DEFAULT_JOBS = 4;
|
||||
|
||||
export function collectPaidTestFiles(rootDir = ROOT): string[] {
|
||||
const testDir = path.join(rootDir, 'test');
|
||||
if (!fs.existsSync(testDir)) return [];
|
||||
return fs.readdirSync(testDir)
|
||||
.map((name) => `test/${name}`)
|
||||
.filter(isPaidTestFile)
|
||||
.sort();
|
||||
}
|
||||
|
||||
export interface TierClassification {
|
||||
included: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a paid test file has anything to run in `tier`.
|
||||
*
|
||||
* Per-TEST tier filtering already happens at runtime: test/helpers/e2e-helpers.ts
|
||||
* intersects the selected tests with E2E_TIERS whenever EVALS_TIER is set, and
|
||||
* this runner passes EVALS_TIER down to every shard. So this file-level pass is
|
||||
* only an optimization — skipping a file merely saves one near-instant shard.
|
||||
*
|
||||
* Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly
|
||||
* the invisible-non-execution bug this runner exists to kill), so the only
|
||||
* exclusion evidence accepted is an explicit whole-file `EVALS_TIER === '<other>'`
|
||||
* guard. Inferring a file's tier from which E2E_TIERS names appear in its source
|
||||
* is guesswork that silently drops real work: short keys like 'retro' match
|
||||
* unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and
|
||||
* carry no E2E_TIERS name at all. Everything without an explicit other-tier
|
||||
* guard runs and self-skips.
|
||||
*/
|
||||
export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification {
|
||||
const other: PaidTier = tier === 'gate' ? 'periodic' : 'gate';
|
||||
const declares = (candidate: PaidTier) =>
|
||||
new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source);
|
||||
|
||||
if (declares(tier)) return { included: true, reason: `declares EVALS_TIER === '${tier}'` };
|
||||
if (declares(other)) return { included: false, reason: `declares EVALS_TIER === '${other}' only` };
|
||||
return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' };
|
||||
}
|
||||
|
||||
export interface TierSelection {
|
||||
selected: string[];
|
||||
excluded: Array<{ file: string; reason: string }>;
|
||||
}
|
||||
|
||||
export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection {
|
||||
const selected: string[] = [];
|
||||
const excluded: Array<{ file: string; reason: string }> = [];
|
||||
for (const file of files) {
|
||||
const source = fs.readFileSync(path.join(rootDir, file), 'utf8');
|
||||
const classification = classifyPaidTestFile(source, tier);
|
||||
if (classification.included) selected.push(file);
|
||||
else excluded.push({ file, reason: classification.reason });
|
||||
}
|
||||
return { selected, excluded };
|
||||
}
|
||||
|
||||
export function planPaidShards(
|
||||
files: string[],
|
||||
options: { maxFilesPerShard?: number } = {},
|
||||
): string[][] {
|
||||
const size = Math.max(1, options.maxFilesPerShard ?? DEFAULT_MAX_FILES_PER_SHARD);
|
||||
const unique = [...new Set(files.map(normalizeRelativePath))].sort();
|
||||
const shards: string[][] = [];
|
||||
for (let index = 0; index < unique.length; index += size) shards.push(unique.slice(index, index + size));
|
||||
return shards;
|
||||
}
|
||||
|
||||
export function buildPaidShardArgs(files: string[], timeoutMs: number): string[] {
|
||||
return ['test', ...files, '--retry', '2', `--timeout=${timeoutMs}`];
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable per-shard eval-dir slug: test filename sans extension, sanitized.
|
||||
* Stable across runs so each shard baselines against its own prior run.
|
||||
*/
|
||||
export function shardSlug(files: string[]): string {
|
||||
return files
|
||||
.map((file) => path.basename(normalizeRelativePath(file)).replace(/\.test\.(?:[cm]?[jt]s|tsx|jsx)$/, ''))
|
||||
.join('+')
|
||||
.replace(/[^a-zA-Z0-9._+-]/g, '-');
|
||||
}
|
||||
|
||||
export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started';
|
||||
|
||||
export interface ShardOutcome {
|
||||
shard: number;
|
||||
files: string[];
|
||||
status: ShardStatus;
|
||||
exitCode: number | null;
|
||||
elapsedMs: number;
|
||||
groupPid: number | null;
|
||||
}
|
||||
|
||||
export interface ShardCommand {
|
||||
command: string;
|
||||
args: string[];
|
||||
}
|
||||
|
||||
export interface RunShardsOptions {
|
||||
tier?: PaidTier;
|
||||
timeoutMs?: number;
|
||||
jobs?: number;
|
||||
rootDir?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
/** When set, each shard child gets GSTACK_EVAL_DIR=<evalDirBase>/shards/<slug>/. */
|
||||
evalDirBase?: string;
|
||||
/** Override the spawned command. Tests inject fake slow/spinning commands. */
|
||||
commandFor?: (files: string[]) => ShardCommand;
|
||||
log?: (line: string) => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* SIGKILL the shard's whole process group. Orphaned grandchildren (browsers,
|
||||
* claude sessions) are how a stalled run once burned a core for 15.7 hours.
|
||||
*/
|
||||
function killProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void {
|
||||
if (process.platform === 'win32' || typeof child.pid !== 'number') {
|
||||
child.kill(signal);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
process.kill(-child.pid, signal);
|
||||
} catch (err) {
|
||||
const code = (err as NodeJS.ErrnoException).code;
|
||||
if (code === 'ESRCH') return; // group already gone
|
||||
if (code !== 'EPERM') throw err;
|
||||
// Observed on macOS after a SIGKILLed group is reaped: signalling the
|
||||
// now-empty group id returns EPERM, not ESRCH. Throwing here loses the
|
||||
// shard's real outcome (a timeout gets recorded as a failure) and, from
|
||||
// the timeout timer, leaves the shard promise unsettled — a hang, which
|
||||
// is the exact failure class this runner exists to kill. Fall back to the
|
||||
// direct pid so a genuinely-live child is still signalled.
|
||||
try {
|
||||
child.kill(signal);
|
||||
} catch {
|
||||
// Best-effort reap: nothing actionable is left if this fails too.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function runPaidShard(
|
||||
files: string[],
|
||||
shardNumber: number,
|
||||
totalShards: number,
|
||||
options: RunShardsOptions = {},
|
||||
): Promise<ShardOutcome> {
|
||||
if (files.length === 0) throw new Error('Cannot run an empty paid-test shard.');
|
||||
const rootDir = options.rootDir ?? ROOT;
|
||||
const timeoutMs = options.timeoutMs ?? DEFAULT_SHARD_TIMEOUT_MS;
|
||||
const streamLive = (options.jobs ?? DEFAULT_JOBS) === 1;
|
||||
const log = options.log ?? ((line: string) => console.log(line));
|
||||
const label = `[test:paid] shard ${shardNumber}/${totalShards}`;
|
||||
|
||||
const { command, args } = options.commandFor
|
||||
? options.commandFor(files)
|
||||
: {
|
||||
command: process.execPath,
|
||||
args: buildPaidShardArgs(exactTestFileSelectors(files, rootDir), timeoutMs),
|
||||
};
|
||||
|
||||
const env = { ...(options.env ?? process.env) };
|
||||
if (options.evalDirBase) {
|
||||
env.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files));
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
log(`${label} START ${files.join(' ')} (timeout ${Math.round(timeoutMs / 1000)}s)`);
|
||||
|
||||
const child = spawn(command, args, {
|
||||
cwd: rootDir,
|
||||
env,
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
detached: process.platform !== 'win32',
|
||||
windowsHide: true,
|
||||
});
|
||||
const groupPid = child.pid ?? null;
|
||||
// Group-kill on parent SIGINT/SIGTERM too, not just on timeout.
|
||||
const forwarding = installChildSignalForwarding({
|
||||
kill: (signal?: NodeJS.Signals | number) => {
|
||||
killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM');
|
||||
return true;
|
||||
},
|
||||
});
|
||||
|
||||
const classifier = new BunTestOutputClassifier();
|
||||
const buffered: Buffer[] = [];
|
||||
const sink = (destination: NodeJS.WriteStream): NodeJS.WriteStream => (streamLive
|
||||
? destination
|
||||
: ({ write: (chunk: Buffer | string) => buffered.push(Buffer.from(chunk)) } as unknown as NodeJS.WriteStream));
|
||||
|
||||
let timedOut = false;
|
||||
const killTimer = setTimeout(() => {
|
||||
timedOut = true;
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
}, timeoutMs);
|
||||
|
||||
let exitCode: number | null = null;
|
||||
try {
|
||||
const streams: Array<Promise<void>> = [];
|
||||
if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier));
|
||||
if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier));
|
||||
exitCode = await new Promise<number | null>((resolve, reject) => {
|
||||
child.once('error', reject);
|
||||
child.once('close', (code) => resolve(code));
|
||||
});
|
||||
await Promise.all(streams);
|
||||
} finally {
|
||||
clearTimeout(killTimer);
|
||||
forwarding.dispose();
|
||||
// Reap survivors of this shard even on the clean path.
|
||||
killProcessGroup(child, 'SIGKILL');
|
||||
}
|
||||
|
||||
const summary = classifier.end();
|
||||
if (!streamLive && buffered.length > 0) process.stdout.write(Buffer.concat(buffered));
|
||||
|
||||
const status: ShardStatus = timedOut
|
||||
? 'timed-out'
|
||||
: strictTestExitCode(exitCode ?? 1, summary) === 0 ? 'passed' : 'failed';
|
||||
const elapsedMs = Date.now() - startedAt;
|
||||
log(`${label} ${status.toUpperCase()} in ${Math.round(elapsedMs / 1000)}s (exit ${exitCode ?? 'signal'})`);
|
||||
|
||||
return { shard: shardNumber, files, status, exitCode, elapsedMs, groupPid };
|
||||
}
|
||||
|
||||
export interface RunSummary {
|
||||
total: number;
|
||||
executed: number;
|
||||
passed: number;
|
||||
failed: number;
|
||||
timedOut: number;
|
||||
neverStarted: number;
|
||||
outcomes: ShardOutcome[];
|
||||
}
|
||||
|
||||
export function summarize(outcomes: ShardOutcome[]): RunSummary {
|
||||
const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length;
|
||||
return {
|
||||
total: outcomes.length,
|
||||
executed: outcomes.length - count('never-started'),
|
||||
passed: count('passed'),
|
||||
failed: count('failed'),
|
||||
timedOut: count('timed-out'),
|
||||
neverStarted: count('never-started'),
|
||||
outcomes,
|
||||
};
|
||||
}
|
||||
|
||||
/** Run every shard in its own process. A timeout or failure never aborts the run. */
|
||||
export async function runPaidShards(
|
||||
shards: string[][],
|
||||
options: RunShardsOptions = {},
|
||||
): Promise<RunSummary> {
|
||||
const jobs = Math.max(1, options.jobs ?? DEFAULT_JOBS);
|
||||
const outcomes: ShardOutcome[] = shards.map((files, index) => ({
|
||||
shard: index + 1,
|
||||
files,
|
||||
status: 'never-started',
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
}));
|
||||
|
||||
let next = 0;
|
||||
const worker = async (): Promise<void> => {
|
||||
while (true) {
|
||||
const index = next;
|
||||
next += 1;
|
||||
if (index >= shards.length) return;
|
||||
try {
|
||||
outcomes[index] = await runPaidShard(shards[index], index + 1, shards.length, { ...options, jobs });
|
||||
} catch (error) {
|
||||
outcomes[index] = {
|
||||
shard: index + 1,
|
||||
files: shards[index],
|
||||
status: 'failed',
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
};
|
||||
console.error(`[test:paid] shard ${index + 1} could not run: ${error instanceof Error ? error.message : String(error)}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await Promise.all(Array.from({ length: Math.min(jobs, shards.length) }, worker));
|
||||
return summarize(outcomes);
|
||||
}
|
||||
|
||||
export function formatSummary(summary: RunSummary): string[] {
|
||||
const lines = [
|
||||
'',
|
||||
`[test:paid] ${summary.executed}/${summary.total} shards executed — `
|
||||
+ `${summary.passed} passed, ${summary.failed} failed, `
|
||||
+ `${summary.timedOut} timed out, ${summary.neverStarted} never started`,
|
||||
];
|
||||
for (const outcome of summary.outcomes) {
|
||||
lines.push(
|
||||
` ${outcome.status.padEnd(13)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s `
|
||||
+ outcome.files.join(' '),
|
||||
);
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
type CliOptions = {
|
||||
tier: PaidTier;
|
||||
listOnly: boolean;
|
||||
timeoutMs: number;
|
||||
jobs: number;
|
||||
maxFilesPerShard: number;
|
||||
};
|
||||
|
||||
function parsePositiveInt(value: string | undefined, flag: string): number {
|
||||
const parsed = Number.parseInt(value ?? '', 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} needs a positive integer. Received: ${value}`);
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process.env): CliOptions {
|
||||
const options: CliOptions = {
|
||||
tier: (env.EVALS_TIER as PaidTier) || DEFAULT_TIER,
|
||||
listOnly: false,
|
||||
timeoutMs: env.EVALS_SHARD_TIMEOUT_MS
|
||||
? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS')
|
||||
: DEFAULT_SHARD_TIMEOUT_MS,
|
||||
jobs: env.EVALS_CONCURRENCY ? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY') : DEFAULT_JOBS,
|
||||
maxFilesPerShard: DEFAULT_MAX_FILES_PER_SHARD,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--list') { options.listOnly = true; continue; }
|
||||
if (arg === '--tier') {
|
||||
const value = argv[index += 1];
|
||||
if (value !== 'gate' && value !== 'periodic') throw new Error(`--tier must be gate or periodic. Received: ${value}`);
|
||||
options.tier = value;
|
||||
continue;
|
||||
}
|
||||
if (arg === '--timeout') { options.timeoutMs = parsePositiveInt(argv[index += 1], '--timeout') * 1000; continue; }
|
||||
if (arg === '--jobs') { options.jobs = parsePositiveInt(argv[index += 1], '--jobs'); continue; }
|
||||
if (arg === '--files-per-shard') { options.maxFilesPerShard = parsePositiveInt(argv[index += 1], '--files-per-shard'); continue; }
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const options = parseCliOptions(process.argv.slice(2));
|
||||
const discovered = collectPaidTestFiles();
|
||||
if (discovered.length === 0) throw new Error('No paid test files were discovered.');
|
||||
|
||||
const { selected, excluded } = selectPaidTestFiles(discovered, options.tier);
|
||||
const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard });
|
||||
console.log(
|
||||
`[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, `
|
||||
+ `${shards.length} shards, jobs=${options.jobs}, timeout=${Math.round(options.timeoutMs / 1000)}s`,
|
||||
);
|
||||
|
||||
if (options.listOnly) {
|
||||
for (let index = 0; index < shards.length; index += 1) {
|
||||
console.log(` shard ${index + 1}/${shards.length}: ${shards[index].join(' ')}`);
|
||||
}
|
||||
if (excluded.length > 0) {
|
||||
console.log(`\nExcluded (${excluded.length}):`);
|
||||
for (const { file, reason } of excluded) console.log(` - ${file} [${reason}]`);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
const summary = await runPaidShards(shards, {
|
||||
tier: options.tier,
|
||||
timeoutMs: options.timeoutMs,
|
||||
jobs: options.jobs,
|
||||
env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier },
|
||||
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
|
||||
});
|
||||
for (const line of formatSummary(summary)) console.log(line);
|
||||
return summary.passed === summary.total ? 0 : 1;
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
try {
|
||||
process.exitCode = await main();
|
||||
} catch (error) {
|
||||
console.error(`[test:paid] ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,200 @@
|
|||
/**
|
||||
* Strict Bun-test output classification + child lifecycle helpers.
|
||||
*
|
||||
* Works around a Bun test runner bug where failures can be printed even though
|
||||
* the child exits successfully: output is forwarded byte-for-byte as it
|
||||
* arrives, and only complete Bun result lines and terminal summaries are
|
||||
* classified. `strictTestExitCode` then refuses to trust a zero exit when the
|
||||
* output shows failures (or when fewer files ran than expected).
|
||||
*
|
||||
* Shared by the sharded paid-tier runner (scripts/test-paid-shards.ts) and any
|
||||
* future strict wrapper around `bun test`.
|
||||
*/
|
||||
|
||||
import { type ChildProcess } from 'node:child_process';
|
||||
import { StringDecoder } from 'node:string_decoder';
|
||||
import * as path from 'node:path';
|
||||
|
||||
const ROOT = path.resolve(import.meta.dir, '..');
|
||||
const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g;
|
||||
const BUN_FAIL_RESULT = /^\(fail\) .+ \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests';
|
||||
const BUN_TERMINAL_SUMMARY = /^Ran \d+ tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/;
|
||||
|
||||
export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests';
|
||||
|
||||
export interface BunTestOutputSummary {
|
||||
failedTests: number;
|
||||
unhandledBetweenTests: number;
|
||||
terminalFileCounts: number[];
|
||||
}
|
||||
|
||||
export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM';
|
||||
|
||||
export interface TerminationSignalSource {
|
||||
on(event: string, listener: () => void): unknown;
|
||||
off(event: string, listener: () => void): unknown;
|
||||
}
|
||||
|
||||
export interface TerminationTimerApi {
|
||||
schedule(callback: () => void, delayMs: number): unknown;
|
||||
cancel(handle: unknown): void;
|
||||
}
|
||||
|
||||
export interface ChildSignalForwarding {
|
||||
readonly receivedSignal: ForwardedTerminationSignal | null;
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
const DEFAULT_TERMINATION_TIMER: TerminationTimerApi = {
|
||||
schedule: (callback, delayMs) => setTimeout(callback, delayMs),
|
||||
cancel: (handle) => clearTimeout(handle as ReturnType<typeof setTimeout>),
|
||||
};
|
||||
|
||||
/**
|
||||
* Bind one active child to the parent's termination lifecycle. SIGINT and
|
||||
* SIGTERM get a grace period so Bun can clean up; a repeated signal, timeout,
|
||||
* or synchronous parent exit uses SIGKILL so the child cannot be orphaned.
|
||||
*/
|
||||
export function installChildSignalForwarding(
|
||||
child: Pick<ChildProcess, 'kill'>,
|
||||
source: TerminationSignalSource = process,
|
||||
timer: TerminationTimerApi = DEFAULT_TERMINATION_TIMER,
|
||||
graceMs = 5_000,
|
||||
): ChildSignalForwarding {
|
||||
let receivedSignal: ForwardedTerminationSignal | null = null;
|
||||
let forceTimer: unknown = null;
|
||||
let disposed = false;
|
||||
|
||||
const forward = (signal: ForwardedTerminationSignal): void => {
|
||||
if (disposed) return;
|
||||
if (receivedSignal !== null) {
|
||||
child.kill('SIGKILL');
|
||||
return;
|
||||
}
|
||||
receivedSignal = signal;
|
||||
child.kill(signal);
|
||||
forceTimer = timer.schedule(() => {
|
||||
forceTimer = null;
|
||||
child.kill('SIGKILL');
|
||||
}, graceMs);
|
||||
};
|
||||
const onSigint = () => forward('SIGINT');
|
||||
const onSigterm = () => forward('SIGTERM');
|
||||
const onExit = () => { child.kill('SIGKILL'); };
|
||||
|
||||
source.on('SIGINT', onSigint);
|
||||
source.on('SIGTERM', onSigterm);
|
||||
source.on('exit', onExit);
|
||||
|
||||
return {
|
||||
get receivedSignal() {
|
||||
return receivedSignal;
|
||||
},
|
||||
dispose() {
|
||||
if (disposed) return;
|
||||
disposed = true;
|
||||
source.off('SIGINT', onSigint);
|
||||
source.off('SIGTERM', onSigterm);
|
||||
source.off('exit', onExit);
|
||||
if (forceTimer !== null) timer.cancel(forceTimer);
|
||||
forceTimer = null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding | null {
|
||||
const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
|
||||
if (BUN_FAIL_RESULT.test(line)) return 'failed-test';
|
||||
if (line === BUN_BETWEEN_TESTS_ERROR) return 'unhandled-between-tests';
|
||||
return null;
|
||||
}
|
||||
|
||||
export function parseBunTerminalSummaryLine(rawLine: string): number | null {
|
||||
const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, '');
|
||||
const match = BUN_TERMINAL_SUMMARY.exec(line);
|
||||
return match ? Number.parseInt(match[1], 10) : null;
|
||||
}
|
||||
|
||||
/** Incrementally classifies output without assuming process chunks align to lines. */
|
||||
export class BunTestOutputClassifier {
|
||||
private readonly decoder = new StringDecoder('utf8');
|
||||
private pending = '';
|
||||
private failedTests = 0;
|
||||
private unhandledBetweenTests = 0;
|
||||
private terminalFileCounts: number[] = [];
|
||||
|
||||
write(chunk: Uint8Array | string): void {
|
||||
this.pending += typeof chunk === 'string'
|
||||
? chunk
|
||||
: this.decoder.write(Buffer.from(chunk));
|
||||
this.consumeCompleteLines();
|
||||
}
|
||||
|
||||
end(): BunTestOutputSummary {
|
||||
this.pending += this.decoder.end();
|
||||
if (this.pending.length > 0) this.classify(this.pending);
|
||||
this.pending = '';
|
||||
return this.summary();
|
||||
}
|
||||
|
||||
summary(): BunTestOutputSummary {
|
||||
return {
|
||||
failedTests: this.failedTests,
|
||||
unhandledBetweenTests: this.unhandledBetweenTests,
|
||||
terminalFileCounts: [...this.terminalFileCounts],
|
||||
};
|
||||
}
|
||||
|
||||
private consumeCompleteLines(): void {
|
||||
let newline = this.pending.indexOf('\n');
|
||||
while (newline !== -1) {
|
||||
this.classify(this.pending.slice(0, newline));
|
||||
this.pending = this.pending.slice(newline + 1);
|
||||
newline = this.pending.indexOf('\n');
|
||||
}
|
||||
}
|
||||
|
||||
private classify(line: string): void {
|
||||
const finding = classifyBunTestOutputLine(line);
|
||||
if (finding === 'failed-test') this.failedTests += 1;
|
||||
if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1;
|
||||
const terminalFileCount = parseBunTerminalSummaryLine(line);
|
||||
if (terminalFileCount !== null) this.terminalFileCounts.push(terminalFileCount);
|
||||
}
|
||||
}
|
||||
|
||||
export function strictTestExitCode(
|
||||
childExitCode: number,
|
||||
summary: BunTestOutputSummary,
|
||||
expectedFiles?: number,
|
||||
): number {
|
||||
if (childExitCode !== 0) return childExitCode;
|
||||
if (summary.failedTests > 0 || summary.unhandledBetweenTests > 0) return 1;
|
||||
if (expectedFiles !== undefined && !summary.terminalFileCounts.includes(expectedFiles)) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bun treats positional test paths as substring filters. Resolve every
|
||||
* canonical relative path before spawning so `test/foo.test.ts` cannot also
|
||||
* select `browse/test/foo.test.ts`.
|
||||
*/
|
||||
export function exactTestFileSelectors(files: string[], rootDir = ROOT): string[] {
|
||||
return files.map((file) => path.isAbsolute(file) ? path.normalize(file) : path.resolve(rootDir, file));
|
||||
}
|
||||
|
||||
export function forwardAndClassify(
|
||||
stream: NodeJS.ReadableStream,
|
||||
destination: NodeJS.WriteStream,
|
||||
classifier: BunTestOutputClassifier,
|
||||
): Promise<void> {
|
||||
return new Promise((resolve, reject) => {
|
||||
stream.on('data', (chunk: Buffer | string) => {
|
||||
classifier.write(chunk);
|
||||
destination.write(chunk);
|
||||
});
|
||||
stream.on('end', resolve);
|
||||
stream.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,117 @@
|
|||
/**
|
||||
* Pins the paid-tier sharded runner (scripts/test-paid-shards.ts).
|
||||
*
|
||||
* Two properties matter, and both are why `test:gate` has never finished a run:
|
||||
* 1. Enumeration + sharding — every file `test:gate`'s globs expand to gets
|
||||
* its own process, and tier exclusion only ever fires on explicit evidence.
|
||||
* 2. A spinning shard is killed externally and the run CONTINUES. The fake
|
||||
* command here is a real busy loop, so an in-process timer could not save
|
||||
* it — exactly the failure mode `sample` caught on the wedged run.
|
||||
*/
|
||||
|
||||
import { describe, test, expect } from 'bun:test';
|
||||
import {
|
||||
PAID_TEST_GLOBS,
|
||||
classifyPaidTestFile,
|
||||
collectPaidTestFiles,
|
||||
isPaidTestFile,
|
||||
planPaidShards,
|
||||
runPaidShards,
|
||||
summarize,
|
||||
type ShardOutcome,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
describe('paid test enumeration', () => {
|
||||
test('matches the globs package.json test:gate expands', () => {
|
||||
expect(isPaidTestFile('test/skill-e2e-qa-workflow.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-llm-eval.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true);
|
||||
expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true);
|
||||
// Outside the globs: no dash, extra suffix, or a free test.
|
||||
expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false);
|
||||
expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false);
|
||||
});
|
||||
|
||||
test('discovers files and gives each one its own shard', () => {
|
||||
const files = collectPaidTestFiles();
|
||||
expect(files.length).toBeGreaterThan(0);
|
||||
expect(files.every(isPaidTestFile)).toBe(true);
|
||||
expect(PAID_TEST_GLOBS.length).toBe(5);
|
||||
|
||||
const shards = planPaidShards(files);
|
||||
expect(shards.flat().sort()).toEqual([...files].sort());
|
||||
expect(shards.every((shard) => shard.length === 1)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('tier classification', () => {
|
||||
test('excludes only on an explicit other-tier guard', () => {
|
||||
const gateGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';";
|
||||
const periodicGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';";
|
||||
|
||||
expect(classifyPaidTestFile(gateGuard, 'gate').included).toBe(true);
|
||||
expect(classifyPaidTestFile(periodicGuard, 'gate').included).toBe(false);
|
||||
expect(classifyPaidTestFile(gateGuard, 'periodic').included).toBe(false);
|
||||
expect(classifyPaidTestFile(periodicGuard, 'periodic').included).toBe(true);
|
||||
});
|
||||
|
||||
test('keeps files whose tier is decided per-test at runtime', () => {
|
||||
// Naming an E2E_TIERS key is not evidence — 'retro' appears in the
|
||||
// LLM-judge file, which test:gate does run.
|
||||
const noGuard = "runSkillTest('retro', async () => {});";
|
||||
expect(classifyPaidTestFile(noGuard, 'gate').included).toBe(true);
|
||||
expect(classifyPaidTestFile(noGuard, 'periodic').included).toBe(true);
|
||||
expect(classifyPaidTestFile('', 'gate').included).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shard execution', () => {
|
||||
const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}';
|
||||
|
||||
const commandFor = (files: string[]) => {
|
||||
if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] };
|
||||
if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] };
|
||||
return { command: process.execPath, args: ['-e', 'console.log("ok")'] };
|
||||
};
|
||||
|
||||
test('a spinning shard times out, is killed, and the run continues', async () => {
|
||||
const lines: string[] = [];
|
||||
const summary = await runPaidShards([['spin'], ['fail'], ['pass']], {
|
||||
timeoutMs: 1_200,
|
||||
jobs: 1,
|
||||
commandFor,
|
||||
log: (line) => lines.push(line),
|
||||
});
|
||||
|
||||
const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome;
|
||||
expect(byName('spin').status).toBe('timed-out');
|
||||
expect(byName('fail').status).toBe('failed');
|
||||
expect(byName('pass').status).toBe('passed');
|
||||
|
||||
// The run never aborted: every shard reports, none is 'never-started'.
|
||||
expect(summary).toMatchObject({
|
||||
total: 3, executed: 3, passed: 1, failed: 1, timedOut: 1, neverStarted: 0,
|
||||
});
|
||||
|
||||
// The spinner was killed at the deadline, not left to burn a core.
|
||||
expect(byName('spin').elapsedMs).toBeLessThan(30_000);
|
||||
expect(byName('spin').groupPid).toBeGreaterThan(0);
|
||||
if (process.platform !== 'win32') {
|
||||
expect(() => process.kill(byName('spin').groupPid as number, 0)).toThrow();
|
||||
}
|
||||
|
||||
// Heartbeat: a START and a terminal line per shard, with elapsed seconds.
|
||||
expect(lines.filter((l) => l.includes(' START ')).length).toBe(3);
|
||||
expect(lines.some((l) => /TIMED-OUT in \d+s/.test(l))).toBe(true);
|
||||
expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true);
|
||||
}, 30_000);
|
||||
|
||||
test('summarize reports shards that never ran', () => {
|
||||
const summary = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 });
|
||||
});
|
||||
});
|
||||
Loading…
Reference in New Issue