From 9b9623e30d09129a1eafacad14c23f1da463de41 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 08:51:20 -0700 Subject: [PATCH] =?UTF-8?q?evals:=20parent-side=20shard=20skipping=20?= =?UTF-8?q?=E2=80=94=20a=20one-test=20diff=20runs=203=20of=2044=20shards?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sharded runner spawned every shard regardless of diff; only the child self-skipped, so a typical single-skill change still paid 44 Bun boots + container-equivalent setup for shards with zero selected tests. The parent now computes selection once (mirroring e2e-helpers exactly: EVALS_ALL -> run-all, empty union -> run-all, git errors propagate the fail-closed throw) and drops shards where no selected test name maps in. Mapping = quoted E2E map keys in the file's source UNION keys whose dep list registers the file (constructed-name families need the second direction). FAIL-OPEN everywhere it matters: run-all, non-skill-e2e files, unreadable source, zero mapped names all keep the shard — the child filter stays authoritative, so a parent bug can only run extra. New taxonomy status skipped-by-diff (never conflated with never-started); selection banner prints once; --list is selection-aware. C6 lands in the same commit: a HARD tier-alignment test — every paid skill-e2e file must be parent-mappable or provably fail-open-safe. Note: this change-set's 14 dep-list registrations in touchfiles-data.ts rode along in f945c841 (concurrent-agent staging); they belong to this change logically. Demo: selection of one test -> 'running 3 of 44 shards, 41 skipped-by-diff'. 13 new $0 tests via injected seams. Co-Authored-By: Claude Fable 5 --- scripts/test-paid-shards.ts | 216 ++++++++++++++++++++++++++++++-- test/e2e-tier-alignment.test.ts | 45 +++++++ test/paid-shards.test.ts | 143 +++++++++++++++++++++ 3 files changed, 396 insertions(+), 8 deletions(-) diff --git a/scripts/test-paid-shards.ts b/scripts/test-paid-shards.ts index dc3a89516..6a7f1b814 100644 --- a/scripts/test-paid-shards.ts +++ b/scripts/test-paid-shards.ts @@ -65,6 +65,14 @@ import { import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set'; import { getProjectEvalDir } from '../test/helpers/eval-store'; import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight'; +import { + detectBaseBranch, + getChangedFiles, + selectTests, + E2E_TOUCHFILES, + E2E_TIERS, + GLOBAL_TOUCHFILES, +} from '../test/helpers/touchfiles'; export { PAID_TEST_GLOBS, isPaidTestFile }; @@ -139,6 +147,159 @@ export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = R return { selected, excluded }; } +// --- Parent-side diff selection (shard skipping) --- + +/** + * The test names the parent mapper recognizes: every E2E map key. LLM-judge + * keys are deliberately excluded — skill-llm-eval.test.ts is not a + * skill-e2e-* file, so it is always kept (child self-skip authoritative). + */ +export const PARENT_MAPPER_TEST_NAMES: string[] = [ + ...new Set([...Object.keys(E2E_TOUCHFILES), ...Object.keys(E2E_TIERS)]), +]; + +/** + * Which of `names` appear in `source` as a quoted string ('x', "x", or `x`). + * Same class of detection test/e2e-tier-alignment.test.ts uses: exact + * quote-delimited match, raw source (comments count — a false hit can only + * KEEP a shard, and the registration union below covers constructed names). + */ +export function knownTestNamesInSource(source: string, names: Iterable): string[] { + const hits: string[] = []; + for (const name of names) { + if ( + source.includes(`'${name}'`) + || source.includes(`"${name}"`) + || source.includes(`\`${name}\``) + ) hits.push(name); + } + return hits; +} + +export interface PaidDiffSelection { + /** null = run everything (EVALS_ALL, or no changes vs base). */ + selectedNames: Set | null; + reason: string; + totalTests: number; +} + +/** + * Compute diff selection in the PARENT, mirroring the module-scope selection + * block in test/helpers/e2e-helpers.ts exactly: EVALS_ALL → run all; + * base = EVALS_BASE || detectBaseBranch || 'main'; empty changed-file union → + * run all. (e2e-helpers additionally gates on EVALS=1, which this runner sets + * for every child unconditionally, so the parent mirror omits it.) + * + * getChangedFiles THROWS on git errors (fail-closed) — the children would hit + * the same throw at module load, so the parent surfaces it before any shard + * spawns. + */ +export function computePaidDiffSelection( + env: NodeJS.ProcessEnv = process.env, + rootDir = ROOT, +): PaidDiffSelection { + const totalTests = Object.keys(E2E_TOUCHFILES).length; + if (env.EVALS_ALL) { + return { selectedNames: null, reason: 'run-all (EVALS_ALL=1)', totalTests }; + } + const baseBranch = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main'; + const changedFiles = getChangedFiles(baseBranch, rootDir); + if (changedFiles.length === 0) { + return { selectedNames: null, reason: `run-all (no changes vs ${baseBranch})`, totalTests }; + } + const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, { + baseRef: baseBranch, cwd: rootDir, + }); + return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests }; +} + +export interface ShardSkipDecision { + file: string; + kept: boolean; + reason: string; +} + +export interface DiffSkipOptions { + rootDir?: string; + /** Injectable for tests. Throwing reads fail OPEN (shard kept). */ + readSource?: (file: string) => string; + /** Injectable name census (default: PARENT_MAPPER_TEST_NAMES). */ + allNames?: string[]; + /** Injectable registration map (default: E2E_TOUCHFILES). */ + e2eTouchfiles?: Record; +} + +/** + * Decide whether a paid test file can be skipped under the current diff + * selection. A file's MAPPED names are the union of: + * - E2E map keys quoted in its source, and + * - E2E map keys whose dep list registers the file (the tier-alignment + * mapping) — this covers files whose testNames are constructed rather + * than literal. + * + * FAIL-OPEN by construction: run-all selection, non-skill-e2e paid files + * (llm-judge / codex-e2e / gemini-e2e / routing, keyed off other maps), + * unreadable sources, and files with zero mapped names all KEEP their shard — + * the child's self-skip stays authoritative. A parent bug may only run + * extra work, never drop it. + */ +export function diffSkipDecisionForFile( + file: string, + selectedNames: Set | null, + options: DiffSkipOptions = {}, +): ShardSkipDecision { + if (selectedNames === null) return { file, kept: true, reason: 'run-all selection' }; + const rel = normalizeRelativePath(file); + if (!/^test\/skill-e2e-.*\.test\.ts$/.test(rel)) { + return { file, kept: true, reason: 'non-skill-e2e paid file — child self-skip authoritative' }; + } + let source: string; + try { + const read = options.readSource + ?? ((f: string) => fs.readFileSync(path.join(options.rootDir ?? ROOT, f), 'utf8')); + source = read(file); + } catch { + return { file, kept: true, reason: 'source unreadable — fail-open' }; + } + const allNames = options.allNames ?? PARENT_MAPPER_TEST_NAMES; + const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES; + const quoted = knownTestNamesInSource(source, allNames); + const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel)); + const mapped = [...new Set([...quoted, ...registered])]; + if (mapped.length === 0) { + return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' }; + } + const selectedHere = mapped.filter((n) => selectedNames.has(n)); + if (selectedHere.length > 0) { + const shown = selectedHere.slice(0, 3).join(', ') + (selectedHere.length > 3 ? ', …' : ''); + return { file, kept: true, reason: `selected: ${shown}` }; + } + return { file, kept: false, reason: `none of its ${mapped.length} mapped test(s) selected` }; +} + +/** + * Partition planned shards into runnable vs skipped-by-diff. A shard is + * skipped only when EVERY file in it is skippable. + */ +export function partitionShardsByDiffSelection( + shards: string[][], + selectedNames: Set | null, + options: DiffSkipOptions = {}, +): { runnable: string[][]; skipped: Array<{ files: string[]; reason: string }> } { + if (selectedNames === null) return { runnable: shards, skipped: [] }; + const runnable: string[][] = []; + const skipped: Array<{ files: string[]; reason: string }> = []; + for (const shard of shards) { + const decisions = shard.map((file) => diffSkipDecisionForFile(file, selectedNames, options)); + if (decisions.every((d) => !d.kept)) { + skipped.push({ files: shard, reason: [...new Set(decisions.map((d) => d.reason))].join('; ') }); + } else { + runnable.push(shard); + } + } + return { runnable, skipped }; +} + export function planPaidShards( files: string[], options: { maxFilesPerShard?: number } = {}, @@ -172,7 +333,7 @@ export function shardSlug(files: string[]): string { .replace(/[^a-zA-Z0-9._+-]/g, '-'); } -export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started'; +export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started' | 'skipped-by-diff'; export interface ShardOutcome { shard: number; @@ -306,6 +467,8 @@ export interface RunSummary { failed: number; timedOut: number; neverStarted: number; + /** Shards the parent skipped via diff selection — successes, never conflated with never-started. */ + skippedByDiff: number; outcomes: ShardOutcome[]; } @@ -313,15 +476,25 @@ export function summarize(outcomes: ShardOutcome[]): RunSummary { const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length; return { total: outcomes.length, - executed: outcomes.length - count('never-started'), + executed: outcomes.length - count('never-started') - count('skipped-by-diff'), passed: count('passed'), failed: count('failed'), timedOut: count('timed-out'), neverStarted: count('never-started'), + skippedByDiff: count('skipped-by-diff'), outcomes, }; } +/** + * Exit code for a finished run: skipped-by-diff shards are successes (the + * parent proved none of their tests were selected); everything else must + * have passed. + */ +export function summaryExitCode(summary: RunSummary): number { + return summary.passed + summary.skippedByDiff === summary.total ? 0 : 1; +} + /** Run every shard in its own process. A timeout or failure never aborts the run. */ export async function runPaidShards( shards: string[][], @@ -368,11 +541,12 @@ export function formatSummary(summary: RunSummary): string[] { '', `[test:paid] ${summary.executed}/${summary.total} shards executed — ` + `${summary.passed} passed, ${summary.failed} failed, ` - + `${summary.timedOut} timed out, ${summary.neverStarted} never started`, + + `${summary.timedOut} timed out, ${summary.neverStarted} never started, ` + + `${summary.skippedByDiff} skipped by diff`, ]; for (const outcome of summary.outcomes) { lines.push( - ` ${outcome.status.padEnd(13)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ` + ` ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s ` + outcome.files.join(' '), ); } @@ -448,14 +622,30 @@ async function main(): Promise { const { selected, excluded } = selectPaidTestFiles(discovered, options.tier); const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard }); + + // Parent-side diff selection (D9): skip whole shards whose mapped tests are + // all unselected. Fail-open everywhere — the child's self-skip stays + // authoritative for anything the mapper can't attribute. + const diffSelection = computePaidDiffSelection(process.env); + const { runnable, skipped } = partitionShardsByDiffSelection(shards, diffSelection.selectedNames); + const selectedCount = diffSelection.selectedNames + ? diffSelection.selectedNames.size + : diffSelection.totalTests; + console.log( + `[test:paid] selection: selected ${selectedCount} of ${diffSelection.totalTests} tests -> ` + + `running ${runnable.length} of ${shards.length} shards, reason: ${diffSelection.reason}`, + ); console.log( `[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, ` + `${shards.length} shards, jobs=${options.jobs}, timeout=${Math.round(options.timeoutMs / 1000)}s`, ); if (options.listOnly) { + const skipReasons = new Map(skipped.map((s) => [s.files.join(' '), s.reason])); for (let index = 0; index < shards.length; index += 1) { - console.log(` shard ${index + 1}/${shards.length}: ${shards[index].join(' ')}`); + const key = shards[index].join(' '); + const note = skipReasons.has(key) ? ` [would skip: ${skipReasons.get(key)}]` : ''; + console.log(` shard ${index + 1}/${shards.length}: ${key}${note}`); } if (excluded.length > 0) { console.log(`\nExcluded (${excluded.length}):`); @@ -468,9 +658,10 @@ async function main(): Promise { // Before this, every shard's e2e-helpers module load re-pinged the API — // ~30 paid claude -p calls (30s timeout each) per full run for one bit of // information. A dead API now fails here, before any shard spawns. - preflightAnthropicApi(process.env); + // Nothing runnable → nothing to ping. + if (runnable.length > 0) preflightAnthropicApi(process.env); - const summary = await runPaidShards(shards, { + const runSummary = await runPaidShards(runnable, { // Tier reaches the children only via EVALS_TIER below; the runtime // E2E_TIERS filter inside each child is the real selection mechanism. timeoutMs: options.timeoutMs, @@ -479,8 +670,17 @@ async function main(): Promise { env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' }, evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(), }); + const skippedOutcomes: ShardOutcome[] = skipped.map((s, index) => ({ + shard: runnable.length + index + 1, + files: s.files, + status: 'skipped-by-diff', + exitCode: null, + elapsedMs: 0, + groupPid: null, + })); + const summary = summarize([...runSummary.outcomes, ...skippedOutcomes]); for (const line of formatSummary(summary)) console.log(line); - return summary.passed === summary.total ? 0 : 1; + return summaryExitCode(summary); } if (import.meta.main) { diff --git a/test/e2e-tier-alignment.test.ts b/test/e2e-tier-alignment.test.ts index 34451a0d4..84e452c20 100644 --- a/test/e2e-tier-alignment.test.ts +++ b/test/e2e-tier-alignment.test.ts @@ -20,6 +20,8 @@ import { describe, test, expect } from 'bun:test'; import { readdirSync, readFileSync } from 'fs'; import * as path from 'path'; import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles'; +import { isPaidTestFile } from './helpers/paid-test-set'; +import { knownTestNamesInSource, PARENT_MAPPER_TEST_NAMES } from '../scripts/test-paid-shards'; const TEST_DIR = import.meta.dir; // Both quote styles — a mechanical refactor to double quotes must not @@ -88,4 +90,47 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () => expect(misaligned).toEqual([]); }); + + // HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when + // none of the file's MAPPED test names (E2E map keys quoted in its source, + // union E2E map keys whose dep list registers the file) are diff-selected. + // A skill-e2e file the mapper cannot see at all is only safe if it provably + // opts out of name-based selection: it must not touch the e2e-helpers + // selection surface (describeIfSelected / runSkillTest / selectedTests) AND + // it must carry an explicit whole-file EVALS_TIER self-gate (the child-side + // gate that makes the parent's fail-open keep semantically correct). + // + // Anything else is an invisible-test-names hole: the parent could drop a + // shard whose child would have run real work. Fix by either quoting the + // test's E2E map key as a string literal in the file, or adding the file's + // path to its key's dep list in test/helpers/touchfiles-data.ts. + test('every paid skill-e2e file is visible to the parent diff mapper (or provably fail-open-safe)', () => { + const invisible: string[] = []; + + for (const file of testFiles) { + const repoPath = `test/${file}`; + if (!isPaidTestFile(repoPath)) continue; + const content = readFileSync(path.join(TEST_DIR, file), 'utf-8'); + + const quoted = knownTestNamesInSource(content, PARENT_MAPPER_TEST_NAMES); + const registered = Object.keys(E2E_TOUCHFILES).filter((k) => E2E_TOUCHFILES[k].includes(repoPath)); + if (quoted.length + registered.length > 0) continue; // parent-mappable + + const usesNameSelection = /\b(describeIfSelected|runSkillTest|selectedTests)\b/.test(content); + const selfGated = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/.test(content); + if (!usesNameSelection && selfGated) continue; // fail-open-safe standalone + + invisible.push( + `${repoPath}: invisible to the parent diff mapper — no E2E map key quoted in the file, ` + + 'not registered in any E2E_TOUCHFILES dep list, and it ' + + (usesNameSelection + ? 'uses name-based selection (describeIfSelected/runSkillTest/selectedTests)' + : 'has no whole-file EVALS_TIER self-gate') + + '. Quote the test\'s E2E map key as a string literal, or add this file path to its ' + + 'key\'s dep list in test/helpers/touchfiles-data.ts.', + ); + } + + expect(invisible).toEqual([]); + }); }); diff --git a/test/paid-shards.test.ts b/test/paid-shards.test.ts index 2f58bef6b..df3cd921f 100644 --- a/test/paid-shards.test.ts +++ b/test/paid-shards.test.ts @@ -14,10 +14,16 @@ import { PAID_TEST_GLOBS, classifyPaidTestFile, collectPaidTestFiles, + computePaidDiffSelection, + diffSkipDecisionForFile, + formatSummary, isPaidTestFile, + knownTestNamesInSource, + partitionShardsByDiffSelection, planPaidShards, runPaidShards, summarize, + summaryExitCode, type ShardOutcome, } from '../scripts/test-paid-shards'; @@ -115,3 +121,140 @@ describe('shard execution', () => { expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 }); }); }); + +describe('parent-side diff shard skipping', () => { + const ALL_NAMES = ['alpha-test', 'beta-test', 'gamma-registered']; + const TOUCHFILES: Record = { + 'alpha-test': ['a/**'], + 'beta-test': ['b/**'], + 'gamma-registered': ['g/**', 'test/skill-e2e-gamma.test.ts'], + }; + const SOURCES: Record = { + 'test/skill-e2e-alpha.test.ts': "runSkillTest('alpha-test', async () => {});", + 'test/skill-e2e-beta.test.ts': 'describeIfSelected("beta", ["beta-test"], () => {});', + // Constructed testName — invisible by quotes, mapped only via registration. + 'test/skill-e2e-gamma.test.ts': 'const name = buildName(); test(name, async () => {});', + // No recognizable names, no registration — the fail-open class. + 'test/skill-e2e-opaque.test.ts': "const shouldRun = process.env.EVALS_TIER === 'periodic';", + 'test/codex-e2e.test.ts': 'codex tests keyed off CODEX_E2E_TOUCHFILES', + }; + const opts = { + readSource: (file: string) => { + if (!(file in SOURCES)) throw new Error(`unreadable: ${file}`); + return SOURCES[file]; + }, + allNames: ALL_NAMES, + e2eTouchfiles: TOUCHFILES, + }; + + test('knownTestNamesInSource matches only exact quoted strings', () => { + expect(knownTestNamesInSource("x 'alpha-test' y", ['alpha-test', 'beta-test'])).toEqual(['alpha-test']); + expect(knownTestNamesInSource('x "beta-test" y', ['alpha-test', 'beta-test'])).toEqual(['beta-test']); + expect(knownTestNamesInSource('`alpha-test`', ['alpha-test'])).toEqual(['alpha-test']); + // Substring inside a longer quoted string is not a hit. + expect(knownTestNamesInSource("'alpha-test-extended'", ['alpha-test'])).toEqual([]); + }); + + test('selected name in file → shard kept', () => { + const d = diffSkipDecisionForFile('test/skill-e2e-alpha.test.ts', new Set(['alpha-test']), opts); + expect(d.kept).toBe(true); + expect(d.reason).toContain('alpha-test'); + }); + + test('no selected names in file → skipped-by-diff', () => { + const d = diffSkipDecisionForFile('test/skill-e2e-beta.test.ts', new Set(['alpha-test']), opts); + expect(d.kept).toBe(false); + expect(d.reason).toContain('mapped test(s)'); + }); + + test('dep-list registration maps files with constructed test names', () => { + const selected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['gamma-registered']), opts); + expect(selected.kept).toBe(true); + const unselected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['alpha-test']), opts); + expect(unselected.kept).toBe(false); + }); + + test('FAIL-OPEN: unmapped file kept, child self-skip authoritative', () => { + const d = diffSkipDecisionForFile('test/skill-e2e-opaque.test.ts', new Set(['alpha-test']), opts); + expect(d.kept).toBe(true); + expect(d.reason).toContain('fail-open'); + }); + + test('FAIL-OPEN: unreadable source kept', () => { + const d = diffSkipDecisionForFile('test/skill-e2e-missing.test.ts', new Set(['alpha-test']), opts); + expect(d.kept).toBe(true); + expect(d.reason).toContain('fail-open'); + }); + + test('FAIL-OPEN: non-skill-e2e paid files always kept', () => { + const d = diffSkipDecisionForFile('test/codex-e2e.test.ts', new Set(['alpha-test']), opts); + expect(d.kept).toBe(true); + expect(d.reason).toContain('non-skill-e2e'); + }); + + test('run-all selection (null) bypasses skipping entirely', () => { + const shards = [['test/skill-e2e-alpha.test.ts'], ['test/skill-e2e-beta.test.ts']]; + const { runnable, skipped } = partitionShardsByDiffSelection(shards, null, opts); + expect(runnable).toEqual(shards); + expect(skipped).toEqual([]); + }); + + test('EVALS_ALL=1 yields run-all selection (no git consulted)', () => { + const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv); + expect(selection.selectedNames).toBeNull(); + expect(selection.reason).toContain('EVALS_ALL=1'); + expect(selection.totalTests).toBeGreaterThan(0); + }); + + test('partition drops only all-skippable shards', () => { + const shards = [ + ['test/skill-e2e-alpha.test.ts'], + ['test/skill-e2e-beta.test.ts'], + ['test/skill-e2e-opaque.test.ts'], + ['test/codex-e2e.test.ts'], + ]; + const { runnable, skipped } = partitionShardsByDiffSelection(shards, new Set(['alpha-test']), opts); + expect(runnable).toEqual([ + ['test/skill-e2e-alpha.test.ts'], + ['test/skill-e2e-opaque.test.ts'], + ['test/codex-e2e.test.ts'], + ]); + expect(skipped.length).toBe(1); + expect(skipped[0].files).toEqual(['test/skill-e2e-beta.test.ts']); + }); + + test('taxonomy: skipped-by-diff counted separately, never conflated with never-started', () => { + const summary = summarize([ + { shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 }, + { shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null }, + { shard: 3, files: ['c'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null }, + ]); + expect(summary).toMatchObject({ + total: 3, executed: 1, passed: 1, skippedByDiff: 1, neverStarted: 1, + }); + const lines = formatSummary(summary); + expect(lines[1]).toContain('1 skipped by diff'); + expect(lines[1]).toContain('1 never started'); + expect(lines.some((l) => l.includes('skipped-by-diff') && l.includes('b'))).toBe(true); + }); + + test('exit code ignores skipped-by-diff shards (they are successes)', () => { + const allGood = summarize([ + { shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 }, + { shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null }, + ]); + expect(summaryExitCode(allGood)).toBe(0); + + const withFailure = summarize([ + { shard: 1, files: ['a'], status: 'failed', exitCode: 1, elapsedMs: 1, groupPid: 1 }, + { shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null }, + ]); + expect(summaryExitCode(withFailure)).toBe(1); + + const withNeverStarted = summarize([ + { shard: 1, files: ['a'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null }, + { shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null }, + ]); + expect(summaryExitCode(withNeverStarted)).toBe(1); + }); +});