diff --git a/package.json b/package.json index db5d2a5d0..cc84d3d0b 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,8 @@ "test:e2e:all": "EVALS=1 EVALS_ALL=1 bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts", "test:gate": "EVALS=1 EVALS_TIER=gate bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-llm-eval.test.ts test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts", "test:periodic": "EVALS=1 EVALS_TIER=periodic EVALS_ALL=1 bun test --retry 2 --concurrent --max-concurrency ${EVALS_CONCURRENCY:-15} test/skill-e2e-*.test.ts test/skill-routing-e2e.test.ts test/codex-e2e.test.ts test/gemini-e2e.test.ts", + "test:gate:sharded": "bun run scripts/test-paid-shards.ts --tier gate", + "test:periodic:sharded": "bun run scripts/test-paid-shards.ts --tier periodic", "test:codex": "EVALS=1 bun test test/codex-e2e.test.ts", "test:codex:all": "EVALS=1 EVALS_ALL=1 bun test test/codex-e2e.test.ts", "test:gemini": "EVALS=1 bun test test/gemini-e2e.test.ts", @@ -36,8 +38,8 @@ "start": "bun run browse/src/server.ts", "eval:bg": "bin/gstack-detach --label evals --lock gstack-evals --timeout 5400 -- bun run test:evals", "eval:bg:all": "bin/gstack-detach --label evals-all --lock gstack-evals --timeout 7200 -- bun run test:evals:all", - "eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 3600 -- bun run test:gate", - "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 5400 -- bun run test:periodic", + "eval:bg:gate": "bin/gstack-detach --label evals-gate --lock gstack-evals --timeout 25200 -- bun run test:gate:sharded", + "eval:bg:periodic": "bin/gstack-detach --label evals-periodic --lock gstack-evals --timeout 28800 -- bun run test:periodic:sharded", "eval:list": "bun run scripts/eval-list.ts", "eval:compare": "bun run scripts/eval-compare.ts", "eval:summary": "bun run scripts/eval-summary.ts", diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts new file mode 100644 index 000000000..975ec5267 --- /dev/null +++ b/scripts/test-paid-shards.ts @@ -0,0 +1,459 @@ +#!/usr/bin/env bun +/** + * test-paid-shards — enumerate, shard, and run the paid (gate/periodic) tier. + * + * The single-process `test:gate` fan-out has never completed a run: one wedged + * or spinning file takes the whole tier down, and an in-process `--timeout` + * cannot save it because a spinning main thread never fires a timer. This + * runner applies the free tier's proven fix — one Bun process per shard — plus + * the two things the paid tier additionally needs: + * + * - an EXTERNAL wall-clock timeout that kills the shard's process GROUP, and + * - an aggregate that distinguishes failed from timed-out from never-started, + * so 26% execution can never again look like a pass. + * + * Why not Bun 1.3.13's native `--shard` / isolated runs? Three gaps, each one + * fatal for this tier: + * 1. No detached-process-group SIGKILL. Paid tests spawn `claude` / `codex` + * PTY grandchildren; when a shard hangs, in-process isolation kills the + * Bun worker but the grandchildren survive and burn cores for hours. + * 2. No never-started taxonomy. A run that aborts partway reports only what + * executed — the shards that never ran are invisible, which is exactly + * the 26%-execution-looks-like-a-pass bug. + * 3. No per-shard env / eval dir. Each shard needs its own GSTACK_EVAL_DIR + * so eval baselines are per-test-file instead of last-flush-wins. + * + * Worst-case wall clock (all shards hit the 30min timeout, 4 parallel jobs): + * gate tier is 49 shards × 30min / 4 jobs ≈ 6.2h; periodic is 59 shards ≈ 7.4h. + * The eval:bg:* detach timeouts (25200s / 28800s) are sized against these. + * + * Enumeration matches package.json's `test:gate` globs (via the shared + * test/helpers/paid-test-set.ts) and honors EVALS_TIER against the E2E_TIERS + * map in test/helpers/touchfiles.ts. Output classification reuses + * scripts/test-strict-output.ts rather than reimplementing it. + * + * Parallelism now lives ACROSS shards (--jobs), not inside one Bun process, so + * each shard runs its own file sequentially and can be killed independently. + * + * Usage: + * bun run scripts/test-paid-shards.ts --list # shard plan only + * bun run scripts/test-paid-shards.ts --tier gate # run gate tier + * bun run scripts/test-paid-shards.ts --timeout 600 --jobs 2 + */ + +import { spawn, type ChildProcess } from 'node:child_process'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { normalizeRelativePath } from './test-free-shards'; +import { + BunTestOutputClassifier, + exactTestFileSelectors, + forwardAndClassify, + installChildSignalForwarding, + strictTestExitCode, +} from './test-strict-output'; +import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; +import { getProjectEvalDir } from '../test/helpers/eval-store'; + +export { PAID_TEST_GLOBS, isPaidTestFile }; + +const ROOT = path.resolve(import.meta.dir, '..'); + +export type PaidTier = 'gate' | 'periodic'; + +export const DEFAULT_TIER: PaidTier = 'gate'; +export const DEFAULT_SHARD_TIMEOUT_MS = 30 * 60_000; +export const DEFAULT_MAX_FILES_PER_SHARD = 1; +export const DEFAULT_JOBS = 4; + +export function collectPaidTestFiles(rootDir = ROOT): string[] { + const testDir = path.join(rootDir, 'test'); + if (!fs.existsSync(testDir)) return []; + return fs.readdirSync(testDir) + .map((name) => `test/${name}`) + .filter(isPaidTestFile) + .sort(); +} + +export interface TierClassification { + included: boolean; + reason: string; +} + +/** + * Decide whether a paid test file has anything to run in `tier`. + * + * Per-TEST tier filtering already happens at runtime: test/helpers/e2e-helpers.ts + * intersects the selected tests with E2E_TIERS whenever EVALS_TIER is set, and + * this runner passes EVALS_TIER down to every shard. So this file-level pass is + * only an optimization — skipping a file merely saves one near-instant shard. + * + * Exclusion is the dangerous direction (a wrongly-skipped gate test is exactly + * the invisible-non-execution bug this runner exists to kill), so the only + * exclusion evidence accepted is an explicit whole-file `EVALS_TIER === ''` + * guard. Inferring a file's tier from which E2E_TIERS names appear in its source + * is guesswork that silently drops real work: short keys like 'retro' match + * unrelated strings, and LLM-judge tests are keyed off LLM_JUDGE_TOUCHFILES and + * carry no E2E_TIERS name at all. Everything without an explicit other-tier + * guard runs and self-skips. + */ +export function classifyPaidTestFile(source: string, tier: PaidTier): TierClassification { + const other: PaidTier = tier === 'gate' ? 'periodic' : 'gate'; + const declares = (candidate: PaidTier) => + new RegExp(`EVALS_TIER\\s*===\\s*['"\`]${candidate}['"\`]`).test(source); + + if (declares(tier)) return { included: true, reason: `declares EVALS_TIER === '${tier}'` }; + if (declares(other)) return { included: false, reason: `declares EVALS_TIER === '${other}' only` }; + return { included: true, reason: 'no whole-file tier guard — runtime E2E_TIERS filter decides' }; +} + +export interface TierSelection { + selected: string[]; + excluded: Array<{ file: string; reason: string }>; +} + +export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = ROOT): TierSelection { + const selected: string[] = []; + const excluded: Array<{ file: string; reason: string }> = []; + for (const file of files) { + const source = fs.readFileSync(path.join(rootDir, file), 'utf8'); + const classification = classifyPaidTestFile(source, tier); + if (classification.included) selected.push(file); + else excluded.push({ file, reason: classification.reason }); + } + return { selected, excluded }; +} + +export function planPaidShards( + files: string[], + options: { maxFilesPerShard?: number } = {}, +): string[][] { + const size = Math.max(1, options.maxFilesPerShard ?? DEFAULT_MAX_FILES_PER_SHARD); + const unique = [...new Set(files.map(normalizeRelativePath))].sort(); + const shards: string[][] = []; + for (let index = 0; index < unique.length; index += size) shards.push(unique.slice(index, index + size)); + return shards; +} + +export function buildPaidShardArgs(files: string[], timeoutMs: number): string[] { + return ['test', ...files, '--retry', '2', `--timeout=${timeoutMs}`]; +} + +/** + * Stable per-shard eval-dir slug: test filename sans extension, sanitized. + * Stable across runs so each shard baselines against its own prior run. + */ +export function shardSlug(files: string[]): string { + return files + .map((file) => path.basename(normalizeRelativePath(file)).replace(/\.test\.(?:[cm]?[jt]s|tsx|jsx)$/, '')) + .join('+') + .replace(/[^a-zA-Z0-9._+-]/g, '-'); +} + +export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started'; + +export interface ShardOutcome { + shard: number; + files: string[]; + status: ShardStatus; + exitCode: number | null; + elapsedMs: number; + groupPid: number | null; +} + +export interface ShardCommand { + command: string; + args: string[]; +} + +export interface RunShardsOptions { + tier?: PaidTier; + timeoutMs?: number; + jobs?: number; + rootDir?: string; + env?: NodeJS.ProcessEnv; + /** When set, each shard child gets GSTACK_EVAL_DIR=/shards//. */ + evalDirBase?: string; + /** Override the spawned command. Tests inject fake slow/spinning commands. */ + commandFor?: (files: string[]) => ShardCommand; + log?: (line: string) => void; +} + +/** + * SIGKILL the shard's whole process group. Orphaned grandchildren (browsers, + * claude sessions) are how a stalled run once burned a core for 15.7 hours. + */ +function killProcessGroup(child: ChildProcess, signal: NodeJS.Signals): void { + if (process.platform === 'win32' || typeof child.pid !== 'number') { + child.kill(signal); + return; + } + try { + process.kill(-child.pid, signal); + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === 'ESRCH') return; // group already gone + if (code !== 'EPERM') throw err; + // Observed on macOS after a SIGKILLed group is reaped: signalling the + // now-empty group id returns EPERM, not ESRCH. Throwing here loses the + // shard's real outcome (a timeout gets recorded as a failure) and, from + // the timeout timer, leaves the shard promise unsettled — a hang, which + // is the exact failure class this runner exists to kill. Fall back to the + // direct pid so a genuinely-live child is still signalled. + try { + child.kill(signal); + } catch { + // Best-effort reap: nothing actionable is left if this fails too. + } + } +} + +export async function runPaidShard( + files: string[], + shardNumber: number, + totalShards: number, + options: RunShardsOptions = {}, +): Promise { + if (files.length === 0) throw new Error('Cannot run an empty paid-test shard.'); + const rootDir = options.rootDir ?? ROOT; + const timeoutMs = options.timeoutMs ?? DEFAULT_SHARD_TIMEOUT_MS; + const streamLive = (options.jobs ?? DEFAULT_JOBS) === 1; + const log = options.log ?? ((line: string) => console.log(line)); + const label = `[test:paid] shard ${shardNumber}/${totalShards}`; + + const { command, args } = options.commandFor + ? options.commandFor(files) + : { + command: process.execPath, + args: buildPaidShardArgs(exactTestFileSelectors(files, rootDir), timeoutMs), + }; + + const env = { ...(options.env ?? process.env) }; + if (options.evalDirBase) { + env.GSTACK_EVAL_DIR = path.join(options.evalDirBase, 'shards', shardSlug(files)); + } + + const startedAt = Date.now(); + log(`${label} START ${files.join(' ')} (timeout ${Math.round(timeoutMs / 1000)}s)`); + + const child = spawn(command, args, { + cwd: rootDir, + env, + stdio: ['ignore', 'pipe', 'pipe'], + detached: process.platform !== 'win32', + windowsHide: true, + }); + const groupPid = child.pid ?? null; + // Group-kill on parent SIGINT/SIGTERM too, not just on timeout. + const forwarding = installChildSignalForwarding({ + kill: (signal?: NodeJS.Signals | number) => { + killProcessGroup(child, (signal as NodeJS.Signals) ?? 'SIGTERM'); + return true; + }, + }); + + const classifier = new BunTestOutputClassifier(); + const buffered: Buffer[] = []; + const sink = (destination: NodeJS.WriteStream): NodeJS.WriteStream => (streamLive + ? destination + : ({ write: (chunk: Buffer | string) => buffered.push(Buffer.from(chunk)) } as unknown as NodeJS.WriteStream)); + + let timedOut = false; + const killTimer = setTimeout(() => { + timedOut = true; + killProcessGroup(child, 'SIGKILL'); + }, timeoutMs); + + let exitCode: number | null = null; + try { + const streams: Array> = []; + if (child.stdout) streams.push(forwardAndClassify(child.stdout, sink(process.stdout), classifier)); + if (child.stderr) streams.push(forwardAndClassify(child.stderr, sink(process.stderr), classifier)); + exitCode = await new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', (code) => resolve(code)); + }); + await Promise.all(streams); + } finally { + clearTimeout(killTimer); + forwarding.dispose(); + // Reap survivors of this shard even on the clean path. + killProcessGroup(child, 'SIGKILL'); + } + + const summary = classifier.end(); + if (!streamLive && buffered.length > 0) process.stdout.write(Buffer.concat(buffered)); + + const status: ShardStatus = timedOut + ? 'timed-out' + : strictTestExitCode(exitCode ?? 1, summary) === 0 ? 'passed' : 'failed'; + const elapsedMs = Date.now() - startedAt; + log(`${label} ${status.toUpperCase()} in ${Math.round(elapsedMs / 1000)}s (exit ${exitCode ?? 'signal'})`); + + return { shard: shardNumber, files, status, exitCode, elapsedMs, groupPid }; +} + +export interface RunSummary { + total: number; + executed: number; + passed: number; + failed: number; + timedOut: number; + neverStarted: number; + outcomes: ShardOutcome[]; +} + +export function summarize(outcomes: ShardOutcome[]): RunSummary { + const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length; + return { + total: outcomes.length, + executed: outcomes.length - count('never-started'), + passed: count('passed'), + failed: count('failed'), + timedOut: count('timed-out'), + neverStarted: count('never-started'), + outcomes, + }; +} + +/** Run every shard in its own process. A timeout or failure never aborts the run. */ +export async function runPaidShards( + shards: string[][], + options: RunShardsOptions = {}, +): Promise { + const jobs = Math.max(1, options.jobs ?? DEFAULT_JOBS); + const outcomes: ShardOutcome[] = shards.map((files, index) => ({ + shard: index + 1, + files, + status: 'never-started', + exitCode: null, + elapsedMs: 0, + groupPid: null, + })); + + let next = 0; + const worker = async (): Promise => { + while (true) { + const index = next; + next += 1; + if (index >= shards.length) return; + try { + outcomes[index] = await runPaidShard(shards[index], index + 1, shards.length, { ...options, jobs }); + } catch (error) { + outcomes[index] = { + shard: index + 1, + files: shards[index], + status: 'failed', + exitCode: null, + elapsedMs: 0, + groupPid: null, + }; + console.error(`[test:paid] shard ${index + 1} could not run: ${error instanceof Error ? error.message : String(error)}`); + } + } + }; + + await Promise.all(Array.from({ length: Math.min(jobs, shards.length) }, worker)); + return summarize(outcomes); +} + +export function formatSummary(summary: RunSummary): string[] { + const lines = [ + '', + `[test:paid] ${summary.executed}/${summary.total} shards executed — ` + + `${summary.passed} passed, ${summary.failed} failed, ` + + `${summary.timedOut} timed out, ${summary.neverStarted} never started`, + ]; + for (const outcome of summary.outcomes) { + lines.push( + ` ${outcome.status.padEnd(13)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ` + + outcome.files.join(' '), + ); + } + return lines; +} + +type CliOptions = { + tier: PaidTier; + listOnly: boolean; + timeoutMs: number; + jobs: number; + maxFilesPerShard: number; +}; + +function parsePositiveInt(value: string | undefined, flag: string): number { + const parsed = Number.parseInt(value ?? '', 10); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} needs a positive integer. Received: ${value}`); + return parsed; +} + +export function parseCliOptions(argv: string[], env: NodeJS.ProcessEnv = process.env): CliOptions { + const options: CliOptions = { + tier: (env.EVALS_TIER as PaidTier) || DEFAULT_TIER, + listOnly: false, + timeoutMs: env.EVALS_SHARD_TIMEOUT_MS + ? parsePositiveInt(env.EVALS_SHARD_TIMEOUT_MS, 'EVALS_SHARD_TIMEOUT_MS') + : DEFAULT_SHARD_TIMEOUT_MS, + jobs: env.EVALS_CONCURRENCY ? parsePositiveInt(env.EVALS_CONCURRENCY, 'EVALS_CONCURRENCY') : DEFAULT_JOBS, + maxFilesPerShard: DEFAULT_MAX_FILES_PER_SHARD, + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--list') { options.listOnly = true; continue; } + if (arg === '--tier') { + const value = argv[index += 1]; + if (value !== 'gate' && value !== 'periodic') throw new Error(`--tier must be gate or periodic. Received: ${value}`); + options.tier = value; + continue; + } + if (arg === '--timeout') { options.timeoutMs = parsePositiveInt(argv[index += 1], '--timeout') * 1000; continue; } + if (arg === '--jobs') { options.jobs = parsePositiveInt(argv[index += 1], '--jobs'); continue; } + if (arg === '--files-per-shard') { options.maxFilesPerShard = parsePositiveInt(argv[index += 1], '--files-per-shard'); continue; } + throw new Error(`Unknown argument: ${arg}`); + } + return options; +} + +async function main(): Promise { + const options = parseCliOptions(process.argv.slice(2)); + const discovered = collectPaidTestFiles(); + if (discovered.length === 0) throw new Error('No paid test files were discovered.'); + + const { selected, excluded } = selectPaidTestFiles(discovered, options.tier); + const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard }); + console.log( + `[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, ` + + `${shards.length} shards, jobs=${options.jobs}, timeout=${Math.round(options.timeoutMs / 1000)}s`, + ); + + if (options.listOnly) { + for (let index = 0; index < shards.length; index += 1) { + console.log(` shard ${index + 1}/${shards.length}: ${shards[index].join(' ')}`); + } + if (excluded.length > 0) { + console.log(`\nExcluded (${excluded.length}):`); + for (const { file, reason } of excluded) console.log(` - ${file} [${reason}]`); + } + return 0; + } + + const summary = await runPaidShards(shards, { + tier: options.tier, + timeoutMs: options.timeoutMs, + jobs: options.jobs, + env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier }, + evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), + }); + for (const line of formatSummary(summary)) console.log(line); + return summary.passed === summary.total ? 0 : 1; +} + +if (import.meta.main) { + try { + process.exitCode = await main(); + } catch (error) { + console.error(`[test:paid] ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 1; + } +} diff --git a/scripts/test-strict-output.ts b/scripts/test-strict-output.ts new file mode 100644 index 000000000..16c073c0a --- /dev/null +++ b/scripts/test-strict-output.ts @@ -0,0 +1,200 @@ +/** + * Strict Bun-test output classification + child lifecycle helpers. + * + * Works around a Bun test runner bug where failures can be printed even though + * the child exits successfully: output is forwarded byte-for-byte as it + * arrives, and only complete Bun result lines and terminal summaries are + * classified. `strictTestExitCode` then refuses to trust a zero exit when the + * output shows failures (or when fewer files ran than expected). + * + * Shared by the sharded paid-tier runner (scripts/test-paid-shards.ts) and any + * future strict wrapper around `bun test`. + */ + +import { type ChildProcess } from 'node:child_process'; +import { StringDecoder } from 'node:string_decoder'; +import * as path from 'node:path'; + +const ROOT = path.resolve(import.meta.dir, '..'); +const ANSI_ESCAPE = /\u001B\[[0-?]*[ -/]*[@-~]/g; +const BUN_FAIL_RESULT = /^\(fail\) .+ \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/; +const BUN_BETWEEN_TESTS_ERROR = '# Unhandled error between tests'; +const BUN_TERMINAL_SUMMARY = /^Ran \d+ tests? across (\d+) files?\. \[(?:\d+(?:\.\d+)?)(?:ns|us|µs|ms|s)\]$/; + +export type BunTestOutputFinding = 'failed-test' | 'unhandled-between-tests'; + +export interface BunTestOutputSummary { + failedTests: number; + unhandledBetweenTests: number; + terminalFileCounts: number[]; +} + +export type ForwardedTerminationSignal = 'SIGINT' | 'SIGTERM'; + +export interface TerminationSignalSource { + on(event: string, listener: () => void): unknown; + off(event: string, listener: () => void): unknown; +} + +export interface TerminationTimerApi { + schedule(callback: () => void, delayMs: number): unknown; + cancel(handle: unknown): void; +} + +export interface ChildSignalForwarding { + readonly receivedSignal: ForwardedTerminationSignal | null; + dispose(): void; +} + +const DEFAULT_TERMINATION_TIMER: TerminationTimerApi = { + schedule: (callback, delayMs) => setTimeout(callback, delayMs), + cancel: (handle) => clearTimeout(handle as ReturnType), +}; + +/** + * Bind one active child to the parent's termination lifecycle. SIGINT and + * SIGTERM get a grace period so Bun can clean up; a repeated signal, timeout, + * or synchronous parent exit uses SIGKILL so the child cannot be orphaned. + */ +export function installChildSignalForwarding( + child: Pick, + source: TerminationSignalSource = process, + timer: TerminationTimerApi = DEFAULT_TERMINATION_TIMER, + graceMs = 5_000, +): ChildSignalForwarding { + let receivedSignal: ForwardedTerminationSignal | null = null; + let forceTimer: unknown = null; + let disposed = false; + + const forward = (signal: ForwardedTerminationSignal): void => { + if (disposed) return; + if (receivedSignal !== null) { + child.kill('SIGKILL'); + return; + } + receivedSignal = signal; + child.kill(signal); + forceTimer = timer.schedule(() => { + forceTimer = null; + child.kill('SIGKILL'); + }, graceMs); + }; + const onSigint = () => forward('SIGINT'); + const onSigterm = () => forward('SIGTERM'); + const onExit = () => { child.kill('SIGKILL'); }; + + source.on('SIGINT', onSigint); + source.on('SIGTERM', onSigterm); + source.on('exit', onExit); + + return { + get receivedSignal() { + return receivedSignal; + }, + dispose() { + if (disposed) return; + disposed = true; + source.off('SIGINT', onSigint); + source.off('SIGTERM', onSigterm); + source.off('exit', onExit); + if (forceTimer !== null) timer.cancel(forceTimer); + forceTimer = null; + }, + }; +} + +export function classifyBunTestOutputLine(rawLine: string): BunTestOutputFinding | null { + const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, ''); + if (BUN_FAIL_RESULT.test(line)) return 'failed-test'; + if (line === BUN_BETWEEN_TESTS_ERROR) return 'unhandled-between-tests'; + return null; +} + +export function parseBunTerminalSummaryLine(rawLine: string): number | null { + const line = rawLine.replace(ANSI_ESCAPE, '').replace(/\r$/, ''); + const match = BUN_TERMINAL_SUMMARY.exec(line); + return match ? Number.parseInt(match[1], 10) : null; +} + +/** Incrementally classifies output without assuming process chunks align to lines. */ +export class BunTestOutputClassifier { + private readonly decoder = new StringDecoder('utf8'); + private pending = ''; + private failedTests = 0; + private unhandledBetweenTests = 0; + private terminalFileCounts: number[] = []; + + write(chunk: Uint8Array | string): void { + this.pending += typeof chunk === 'string' + ? chunk + : this.decoder.write(Buffer.from(chunk)); + this.consumeCompleteLines(); + } + + end(): BunTestOutputSummary { + this.pending += this.decoder.end(); + if (this.pending.length > 0) this.classify(this.pending); + this.pending = ''; + return this.summary(); + } + + summary(): BunTestOutputSummary { + return { + failedTests: this.failedTests, + unhandledBetweenTests: this.unhandledBetweenTests, + terminalFileCounts: [...this.terminalFileCounts], + }; + } + + private consumeCompleteLines(): void { + let newline = this.pending.indexOf('\n'); + while (newline !== -1) { + this.classify(this.pending.slice(0, newline)); + this.pending = this.pending.slice(newline + 1); + newline = this.pending.indexOf('\n'); + } + } + + private classify(line: string): void { + const finding = classifyBunTestOutputLine(line); + if (finding === 'failed-test') this.failedTests += 1; + if (finding === 'unhandled-between-tests') this.unhandledBetweenTests += 1; + const terminalFileCount = parseBunTerminalSummaryLine(line); + if (terminalFileCount !== null) this.terminalFileCounts.push(terminalFileCount); + } +} + +export function strictTestExitCode( + childExitCode: number, + summary: BunTestOutputSummary, + expectedFiles?: number, +): number { + if (childExitCode !== 0) return childExitCode; + if (summary.failedTests > 0 || summary.unhandledBetweenTests > 0) return 1; + if (expectedFiles !== undefined && !summary.terminalFileCounts.includes(expectedFiles)) return 1; + return 0; +} + +/** + * Bun treats positional test paths as substring filters. Resolve every + * canonical relative path before spawning so `test/foo.test.ts` cannot also + * select `browse/test/foo.test.ts`. + */ +export function exactTestFileSelectors(files: string[], rootDir = ROOT): string[] { + return files.map((file) => path.isAbsolute(file) ? path.normalize(file) : path.resolve(rootDir, file)); +} + +export function forwardAndClassify( + stream: NodeJS.ReadableStream, + destination: NodeJS.WriteStream, + classifier: BunTestOutputClassifier, +): Promise { + return new Promise((resolve, reject) => { + stream.on('data', (chunk: Buffer | string) => { + classifier.write(chunk); + destination.write(chunk); + }); + stream.on('end', resolve); + stream.on('error', reject); + }); +} diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts new file mode 100644 index 000000000..2f58bef6b --- /dev/null +++ b/test/paid-shards.test.ts @@ -0,0 +1,117 @@ +/** + * Pins the paid-tier sharded runner (scripts/test-paid-shards.ts). + * + * Two properties matter, and both are why `test:gate` has never finished a run: + * 1. Enumeration + sharding — every file `test:gate`'s globs expand to gets + * its own process, and tier exclusion only ever fires on explicit evidence. + * 2. A spinning shard is killed externally and the run CONTINUES. The fake + * command here is a real busy loop, so an in-process timer could not save + * it — exactly the failure mode `sample` caught on the wedged run. + */ + +import { describe, test, expect } from 'bun:test'; +import { + PAID_TEST_GLOBS, + classifyPaidTestFile, + collectPaidTestFiles, + isPaidTestFile, + planPaidShards, + runPaidShards, + summarize, + type ShardOutcome, +} from '../scripts/test-paid-shards'; + +describe('paid test enumeration', () => { + test('matches the globs package.json test:gate expands', () => { + expect(isPaidTestFile('test/skill-e2e-qa-workflow.test.ts')).toBe(true); + expect(isPaidTestFile('test/skill-llm-eval.test.ts')).toBe(true); + expect(isPaidTestFile('test/codex-e2e.test.ts')).toBe(true); + expect(isPaidTestFile('test/skill-e2e-triage-audit.test.ts')).toBe(true); + // Outside the globs: no dash, extra suffix, or a free test. + expect(isPaidTestFile('test/skill-e2e.test.ts')).toBe(false); + expect(isPaidTestFile('test/codex-e2e-recommendation-substance.test.ts')).toBe(false); + expect(isPaidTestFile('test/paid-shards.test.ts')).toBe(false); + }); + + test('discovers files and gives each one its own shard', () => { + const files = collectPaidTestFiles(); + expect(files.length).toBeGreaterThan(0); + expect(files.every(isPaidTestFile)).toBe(true); + expect(PAID_TEST_GLOBS.length).toBe(5); + + const shards = planPaidShards(files); + expect(shards.flat().sort()).toEqual([...files].sort()); + expect(shards.every((shard) => shard.length === 1)).toBe(true); + }); +}); + +describe('tier classification', () => { + test('excludes only on an explicit other-tier guard', () => { + const gateGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'gate';"; + const periodicGuard = "const shouldRun = !!process.env.EVALS && process.env.EVALS_TIER === 'periodic';"; + + expect(classifyPaidTestFile(gateGuard, 'gate').included).toBe(true); + expect(classifyPaidTestFile(periodicGuard, 'gate').included).toBe(false); + expect(classifyPaidTestFile(gateGuard, 'periodic').included).toBe(false); + expect(classifyPaidTestFile(periodicGuard, 'periodic').included).toBe(true); + }); + + test('keeps files whose tier is decided per-test at runtime', () => { + // Naming an E2E_TIERS key is not evidence — 'retro' appears in the + // LLM-judge file, which test:gate does run. + const noGuard = "runSkillTest('retro', async () => {});"; + expect(classifyPaidTestFile(noGuard, 'gate').included).toBe(true); + expect(classifyPaidTestFile(noGuard, 'periodic').included).toBe(true); + expect(classifyPaidTestFile('', 'gate').included).toBe(true); + }); +}); + +describe('shard execution', () => { + const BUSY_LOOP = 'const end = Date.now() + 600000; while (Date.now() < end) {}'; + + const commandFor = (files: string[]) => { + if (files[0] === 'spin') return { command: process.execPath, args: ['-e', BUSY_LOOP] }; + if (files[0] === 'fail') return { command: process.execPath, args: ['-e', 'process.exit(3)'] }; + return { command: process.execPath, args: ['-e', 'console.log("ok")'] }; + }; + + test('a spinning shard times out, is killed, and the run continues', async () => { + const lines: string[] = []; + const summary = await runPaidShards([['spin'], ['fail'], ['pass']], { + timeoutMs: 1_200, + jobs: 1, + commandFor, + log: (line) => lines.push(line), + }); + + const byName = (name: string) => summary.outcomes.find((o) => o.files[0] === name) as ShardOutcome; + expect(byName('spin').status).toBe('timed-out'); + expect(byName('fail').status).toBe('failed'); + expect(byName('pass').status).toBe('passed'); + + // The run never aborted: every shard reports, none is 'never-started'. + expect(summary).toMatchObject({ + total: 3, executed: 3, passed: 1, failed: 1, timedOut: 1, neverStarted: 0, + }); + + // The spinner was killed at the deadline, not left to burn a core. + expect(byName('spin').elapsedMs).toBeLessThan(30_000); + expect(byName('spin').groupPid).toBeGreaterThan(0); + if (process.platform !== 'win32') { + expect(() => process.kill(byName('spin').groupPid as number, 0)).toThrow(); + } + + // Heartbeat: a START and a terminal line per shard, with elapsed seconds. + expect(lines.filter((l) => l.includes(' START ')).length).toBe(3); + expect(lines.some((l) => /TIMED-OUT in \d+s/.test(l))).toBe(true); + expect(lines.some((l) => /PASSED in \d+s/.test(l))).toBe(true); + }, 30_000); + + test('summarize reports shards that never ran', () => { + const summary = summarize([ + { shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 }, + { shard: 2, files: ['b'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null }, + ]); + expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 }); + }); +});