mirror of https://github.com/garrytan/gstack.git
evals: parent-side shard skipping — a one-test diff runs 3 of 44 shards
The sharded runner spawned every shard regardless of diff; only the
child self-skipped, so a typical single-skill change still paid 44 Bun
boots + container-equivalent setup for shards with zero selected tests.
The parent now computes selection once (mirroring e2e-helpers exactly:
EVALS_ALL -> run-all, empty union -> run-all, git errors propagate the
fail-closed throw) and drops shards where no selected test name maps in.
Mapping = quoted E2E map keys in the file's source UNION keys whose dep
list registers the file (constructed-name families need the second
direction). FAIL-OPEN everywhere it matters: run-all, non-skill-e2e
files, unreadable source, zero mapped names all keep the shard — the
child filter stays authoritative, so a parent bug can only run extra.
New taxonomy status skipped-by-diff (never conflated with
never-started); selection banner prints once; --list is selection-aware.
C6 lands in the same commit: a HARD tier-alignment test — every paid
skill-e2e file must be parent-mappable or provably fail-open-safe.
Note: this change-set's 14 dep-list registrations in touchfiles-data.ts
rode along in f945c841 (concurrent-agent staging); they belong to this
change logically.
Demo: selection of one test -> 'running 3 of 44 shards, 41
skipped-by-diff'. 13 new $0 tests via injected seams.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan
parent
0337aaf05d
commit
9b9623e30d
|
|
@ -65,6 +65,14 @@ import {
|
|||
import { PAID_TEST_GLOBS, isPaidTestFile } from '../test/helpers/paid-test-set';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import { preflightAnthropicApi } from '../test/helpers/anthropic-preflight';
|
||||
import {
|
||||
detectBaseBranch,
|
||||
getChangedFiles,
|
||||
selectTests,
|
||||
E2E_TOUCHFILES,
|
||||
E2E_TIERS,
|
||||
GLOBAL_TOUCHFILES,
|
||||
} from '../test/helpers/touchfiles';
|
||||
|
||||
export { PAID_TEST_GLOBS, isPaidTestFile };
|
||||
|
||||
|
|
@ -139,6 +147,159 @@ export function selectPaidTestFiles(files: string[], tier: PaidTier, rootDir = R
|
|||
return { selected, excluded };
|
||||
}
|
||||
|
||||
// --- Parent-side diff selection (shard skipping) ---
|
||||
|
||||
/**
|
||||
* The test names the parent mapper recognizes: every E2E map key. LLM-judge
|
||||
* keys are deliberately excluded — skill-llm-eval.test.ts is not a
|
||||
* skill-e2e-* file, so it is always kept (child self-skip authoritative).
|
||||
*/
|
||||
export const PARENT_MAPPER_TEST_NAMES: string[] = [
|
||||
...new Set([...Object.keys(E2E_TOUCHFILES), ...Object.keys(E2E_TIERS)]),
|
||||
];
|
||||
|
||||
/**
|
||||
* Which of `names` appear in `source` as a quoted string ('x', "x", or `x`).
|
||||
* Same class of detection test/e2e-tier-alignment.test.ts uses: exact
|
||||
* quote-delimited match, raw source (comments count — a false hit can only
|
||||
* KEEP a shard, and the registration union below covers constructed names).
|
||||
*/
|
||||
export function knownTestNamesInSource(source: string, names: Iterable<string>): string[] {
|
||||
const hits: string[] = [];
|
||||
for (const name of names) {
|
||||
if (
|
||||
source.includes(`'${name}'`)
|
||||
|| source.includes(`"${name}"`)
|
||||
|| source.includes(`\`${name}\``)
|
||||
) hits.push(name);
|
||||
}
|
||||
return hits;
|
||||
}
|
||||
|
||||
export interface PaidDiffSelection {
|
||||
/** null = run everything (EVALS_ALL, or no changes vs base). */
|
||||
selectedNames: Set<string> | null;
|
||||
reason: string;
|
||||
totalTests: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute diff selection in the PARENT, mirroring the module-scope selection
|
||||
* block in test/helpers/e2e-helpers.ts exactly: EVALS_ALL → run all;
|
||||
* base = EVALS_BASE || detectBaseBranch || 'main'; empty changed-file union →
|
||||
* run all. (e2e-helpers additionally gates on EVALS=1, which this runner sets
|
||||
* for every child unconditionally, so the parent mirror omits it.)
|
||||
*
|
||||
* getChangedFiles THROWS on git errors (fail-closed) — the children would hit
|
||||
* the same throw at module load, so the parent surfaces it before any shard
|
||||
* spawns.
|
||||
*/
|
||||
export function computePaidDiffSelection(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
rootDir = ROOT,
|
||||
): PaidDiffSelection {
|
||||
const totalTests = Object.keys(E2E_TOUCHFILES).length;
|
||||
if (env.EVALS_ALL) {
|
||||
return { selectedNames: null, reason: 'run-all (EVALS_ALL=1)', totalTests };
|
||||
}
|
||||
const baseBranch = env.EVALS_BASE || detectBaseBranch(rootDir) || 'main';
|
||||
const changedFiles = getChangedFiles(baseBranch, rootDir);
|
||||
if (changedFiles.length === 0) {
|
||||
return { selectedNames: null, reason: `run-all (no changes vs ${baseBranch})`, totalTests };
|
||||
}
|
||||
const selection = selectTests(changedFiles, E2E_TOUCHFILES, GLOBAL_TOUCHFILES, {
|
||||
baseRef: baseBranch, cwd: rootDir,
|
||||
});
|
||||
return { selectedNames: new Set(selection.selected), reason: selection.reason, totalTests };
|
||||
}
|
||||
|
||||
export interface ShardSkipDecision {
|
||||
file: string;
|
||||
kept: boolean;
|
||||
reason: string;
|
||||
}
|
||||
|
||||
export interface DiffSkipOptions {
|
||||
rootDir?: string;
|
||||
/** Injectable for tests. Throwing reads fail OPEN (shard kept). */
|
||||
readSource?: (file: string) => string;
|
||||
/** Injectable name census (default: PARENT_MAPPER_TEST_NAMES). */
|
||||
allNames?: string[];
|
||||
/** Injectable registration map (default: E2E_TOUCHFILES). */
|
||||
e2eTouchfiles?: Record<string, string[]>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a paid test file can be skipped under the current diff
|
||||
* selection. A file's MAPPED names are the union of:
|
||||
* - E2E map keys quoted in its source, and
|
||||
* - E2E map keys whose dep list registers the file (the tier-alignment
|
||||
* mapping) — this covers files whose testNames are constructed rather
|
||||
* than literal.
|
||||
*
|
||||
* FAIL-OPEN by construction: run-all selection, non-skill-e2e paid files
|
||||
* (llm-judge / codex-e2e / gemini-e2e / routing, keyed off other maps),
|
||||
* unreadable sources, and files with zero mapped names all KEEP their shard —
|
||||
* the child's self-skip stays authoritative. A parent bug may only run
|
||||
* extra work, never drop it.
|
||||
*/
|
||||
export function diffSkipDecisionForFile(
|
||||
file: string,
|
||||
selectedNames: Set<string> | null,
|
||||
options: DiffSkipOptions = {},
|
||||
): ShardSkipDecision {
|
||||
if (selectedNames === null) return { file, kept: true, reason: 'run-all selection' };
|
||||
const rel = normalizeRelativePath(file);
|
||||
if (!/^test\/skill-e2e-.*\.test\.ts$/.test(rel)) {
|
||||
return { file, kept: true, reason: 'non-skill-e2e paid file — child self-skip authoritative' };
|
||||
}
|
||||
let source: string;
|
||||
try {
|
||||
const read = options.readSource
|
||||
?? ((f: string) => fs.readFileSync(path.join(options.rootDir ?? ROOT, f), 'utf8'));
|
||||
source = read(file);
|
||||
} catch {
|
||||
return { file, kept: true, reason: 'source unreadable — fail-open' };
|
||||
}
|
||||
const allNames = options.allNames ?? PARENT_MAPPER_TEST_NAMES;
|
||||
const touchfiles = options.e2eTouchfiles ?? E2E_TOUCHFILES;
|
||||
const quoted = knownTestNamesInSource(source, allNames);
|
||||
const registered = Object.keys(touchfiles).filter((k) => touchfiles[k].includes(rel));
|
||||
const mapped = [...new Set([...quoted, ...registered])];
|
||||
if (mapped.length === 0) {
|
||||
return { file, kept: true, reason: 'no mappable test names — fail-open, child self-skip authoritative' };
|
||||
}
|
||||
const selectedHere = mapped.filter((n) => selectedNames.has(n));
|
||||
if (selectedHere.length > 0) {
|
||||
const shown = selectedHere.slice(0, 3).join(', ') + (selectedHere.length > 3 ? ', …' : '');
|
||||
return { file, kept: true, reason: `selected: ${shown}` };
|
||||
}
|
||||
return { file, kept: false, reason: `none of its ${mapped.length} mapped test(s) selected` };
|
||||
}
|
||||
|
||||
/**
|
||||
* Partition planned shards into runnable vs skipped-by-diff. A shard is
|
||||
* skipped only when EVERY file in it is skippable.
|
||||
*/
|
||||
export function partitionShardsByDiffSelection(
|
||||
shards: string[][],
|
||||
selectedNames: Set<string> | null,
|
||||
options: DiffSkipOptions = {},
|
||||
): { runnable: string[][]; skipped: Array<{ files: string[]; reason: string }> } {
|
||||
if (selectedNames === null) return { runnable: shards, skipped: [] };
|
||||
const runnable: string[][] = [];
|
||||
const skipped: Array<{ files: string[]; reason: string }> = [];
|
||||
for (const shard of shards) {
|
||||
const decisions = shard.map((file) => diffSkipDecisionForFile(file, selectedNames, options));
|
||||
if (decisions.every((d) => !d.kept)) {
|
||||
skipped.push({ files: shard, reason: [...new Set(decisions.map((d) => d.reason))].join('; ') });
|
||||
} else {
|
||||
runnable.push(shard);
|
||||
}
|
||||
}
|
||||
return { runnable, skipped };
|
||||
}
|
||||
|
||||
export function planPaidShards(
|
||||
files: string[],
|
||||
options: { maxFilesPerShard?: number } = {},
|
||||
|
|
@ -172,7 +333,7 @@ export function shardSlug(files: string[]): string {
|
|||
.replace(/[^a-zA-Z0-9._+-]/g, '-');
|
||||
}
|
||||
|
||||
export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started';
|
||||
export type ShardStatus = 'passed' | 'failed' | 'timed-out' | 'never-started' | 'skipped-by-diff';
|
||||
|
||||
export interface ShardOutcome {
|
||||
shard: number;
|
||||
|
|
@ -306,6 +467,8 @@ export interface RunSummary {
|
|||
failed: number;
|
||||
timedOut: number;
|
||||
neverStarted: number;
|
||||
/** Shards the parent skipped via diff selection — successes, never conflated with never-started. */
|
||||
skippedByDiff: number;
|
||||
outcomes: ShardOutcome[];
|
||||
}
|
||||
|
||||
|
|
@ -313,15 +476,25 @@ export function summarize(outcomes: ShardOutcome[]): RunSummary {
|
|||
const count = (status: ShardStatus) => outcomes.filter((o) => o.status === status).length;
|
||||
return {
|
||||
total: outcomes.length,
|
||||
executed: outcomes.length - count('never-started'),
|
||||
executed: outcomes.length - count('never-started') - count('skipped-by-diff'),
|
||||
passed: count('passed'),
|
||||
failed: count('failed'),
|
||||
timedOut: count('timed-out'),
|
||||
neverStarted: count('never-started'),
|
||||
skippedByDiff: count('skipped-by-diff'),
|
||||
outcomes,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Exit code for a finished run: skipped-by-diff shards are successes (the
|
||||
* parent proved none of their tests were selected); everything else must
|
||||
* have passed.
|
||||
*/
|
||||
export function summaryExitCode(summary: RunSummary): number {
|
||||
return summary.passed + summary.skippedByDiff === summary.total ? 0 : 1;
|
||||
}
|
||||
|
||||
/** Run every shard in its own process. A timeout or failure never aborts the run. */
|
||||
export async function runPaidShards(
|
||||
shards: string[][],
|
||||
|
|
@ -368,11 +541,12 @@ export function formatSummary(summary: RunSummary): string[] {
|
|||
'',
|
||||
`[test:paid] ${summary.executed}/${summary.total} shards executed — `
|
||||
+ `${summary.passed} passed, ${summary.failed} failed, `
|
||||
+ `${summary.timedOut} timed out, ${summary.neverStarted} never started`,
|
||||
+ `${summary.timedOut} timed out, ${summary.neverStarted} never started, `
|
||||
+ `${summary.skippedByDiff} skipped by diff`,
|
||||
];
|
||||
for (const outcome of summary.outcomes) {
|
||||
lines.push(
|
||||
` ${outcome.status.padEnd(13)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s `
|
||||
` ${outcome.status.padEnd(15)} ${String(Math.round(outcome.elapsedMs / 1000)).padStart(5)}s `
|
||||
+ outcome.files.join(' '),
|
||||
);
|
||||
}
|
||||
|
|
@ -448,14 +622,30 @@ async function main(): Promise<number> {
|
|||
|
||||
const { selected, excluded } = selectPaidTestFiles(discovered, options.tier);
|
||||
const shards = planPaidShards(selected, { maxFilesPerShard: options.maxFilesPerShard });
|
||||
|
||||
// Parent-side diff selection (D9): skip whole shards whose mapped tests are
|
||||
// all unselected. Fail-open everywhere — the child's self-skip stays
|
||||
// authoritative for anything the mapper can't attribute.
|
||||
const diffSelection = computePaidDiffSelection(process.env);
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, diffSelection.selectedNames);
|
||||
const selectedCount = diffSelection.selectedNames
|
||||
? diffSelection.selectedNames.size
|
||||
: diffSelection.totalTests;
|
||||
console.log(
|
||||
`[test:paid] selection: selected ${selectedCount} of ${diffSelection.totalTests} tests -> `
|
||||
+ `running ${runnable.length} of ${shards.length} shards, reason: ${diffSelection.reason}`,
|
||||
);
|
||||
console.log(
|
||||
`[test:paid] tier=${options.tier}: ${selected.length}/${discovered.length} files, `
|
||||
+ `${shards.length} shards, jobs=${options.jobs}, timeout=${Math.round(options.timeoutMs / 1000)}s`,
|
||||
);
|
||||
|
||||
if (options.listOnly) {
|
||||
const skipReasons = new Map(skipped.map((s) => [s.files.join(' '), s.reason]));
|
||||
for (let index = 0; index < shards.length; index += 1) {
|
||||
console.log(` shard ${index + 1}/${shards.length}: ${shards[index].join(' ')}`);
|
||||
const key = shards[index].join(' ');
|
||||
const note = skipReasons.has(key) ? ` [would skip: ${skipReasons.get(key)}]` : '';
|
||||
console.log(` shard ${index + 1}/${shards.length}: ${key}${note}`);
|
||||
}
|
||||
if (excluded.length > 0) {
|
||||
console.log(`\nExcluded (${excluded.length}):`);
|
||||
|
|
@ -468,9 +658,10 @@ async function main(): Promise<number> {
|
|||
// Before this, every shard's e2e-helpers module load re-pinged the API —
|
||||
// ~30 paid claude -p calls (30s timeout each) per full run for one bit of
|
||||
// information. A dead API now fails here, before any shard spawns.
|
||||
preflightAnthropicApi(process.env);
|
||||
// Nothing runnable → nothing to ping.
|
||||
if (runnable.length > 0) preflightAnthropicApi(process.env);
|
||||
|
||||
const summary = await runPaidShards(shards, {
|
||||
const runSummary = await runPaidShards(runnable, {
|
||||
// Tier reaches the children only via EVALS_TIER below; the runtime
|
||||
// E2E_TIERS filter inside each child is the real selection mechanism.
|
||||
timeoutMs: options.timeoutMs,
|
||||
|
|
@ -479,8 +670,17 @@ async function main(): Promise<number> {
|
|||
env: { ...process.env, EVALS: '1', EVALS_TIER: options.tier, EVALS_PREFLIGHT_OK: '1' },
|
||||
evalDirBase: process.env.GSTACK_EVAL_DIR || getProjectEvalDir(),
|
||||
});
|
||||
const skippedOutcomes: ShardOutcome[] = skipped.map((s, index) => ({
|
||||
shard: runnable.length + index + 1,
|
||||
files: s.files,
|
||||
status: 'skipped-by-diff',
|
||||
exitCode: null,
|
||||
elapsedMs: 0,
|
||||
groupPid: null,
|
||||
}));
|
||||
const summary = summarize([...runSummary.outcomes, ...skippedOutcomes]);
|
||||
for (const line of formatSummary(summary)) console.log(line);
|
||||
return summary.passed === summary.total ? 0 : 1;
|
||||
return summaryExitCode(summary);
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import { describe, test, expect } from 'bun:test';
|
|||
import { readdirSync, readFileSync } from 'fs';
|
||||
import * as path from 'path';
|
||||
import { E2E_TOUCHFILES, E2E_TIERS, LLM_JUDGE_TOUCHFILES } from './helpers/touchfiles';
|
||||
import { isPaidTestFile } from './helpers/paid-test-set';
|
||||
import { knownTestNamesInSource, PARENT_MAPPER_TEST_NAMES } from '../scripts/test-paid-shards';
|
||||
|
||||
const TEST_DIR = import.meta.dir;
|
||||
// Both quote styles — a mechanical refactor to double quotes must not
|
||||
|
|
@ -88,4 +90,47 @@ describe('E2E tier alignment (touchfiles declaration vs test self-gate)', () =>
|
|||
|
||||
expect(misaligned).toEqual([]);
|
||||
});
|
||||
|
||||
// HARD invariant (C6): the paid sharded runner skips a skill-e2e shard when
|
||||
// none of the file's MAPPED test names (E2E map keys quoted in its source,
|
||||
// union E2E map keys whose dep list registers the file) are diff-selected.
|
||||
// A skill-e2e file the mapper cannot see at all is only safe if it provably
|
||||
// opts out of name-based selection: it must not touch the e2e-helpers
|
||||
// selection surface (describeIfSelected / runSkillTest / selectedTests) AND
|
||||
// it must carry an explicit whole-file EVALS_TIER self-gate (the child-side
|
||||
// gate that makes the parent's fail-open keep semantically correct).
|
||||
//
|
||||
// Anything else is an invisible-test-names hole: the parent could drop a
|
||||
// shard whose child would have run real work. Fix by either quoting the
|
||||
// test's E2E map key as a string literal in the file, or adding the file's
|
||||
// path to its key's dep list in test/helpers/touchfiles-data.ts.
|
||||
test('every paid skill-e2e file is visible to the parent diff mapper (or provably fail-open-safe)', () => {
|
||||
const invisible: string[] = [];
|
||||
|
||||
for (const file of testFiles) {
|
||||
const repoPath = `test/${file}`;
|
||||
if (!isPaidTestFile(repoPath)) continue;
|
||||
const content = readFileSync(path.join(TEST_DIR, file), 'utf-8');
|
||||
|
||||
const quoted = knownTestNamesInSource(content, PARENT_MAPPER_TEST_NAMES);
|
||||
const registered = Object.keys(E2E_TOUCHFILES).filter((k) => E2E_TOUCHFILES[k].includes(repoPath));
|
||||
if (quoted.length + registered.length > 0) continue; // parent-mappable
|
||||
|
||||
const usesNameSelection = /\b(describeIfSelected|runSkillTest|selectedTests)\b/.test(content);
|
||||
const selfGated = /EVALS_TIER\s*===\s*['"](gate|periodic)['"]/.test(content);
|
||||
if (!usesNameSelection && selfGated) continue; // fail-open-safe standalone
|
||||
|
||||
invisible.push(
|
||||
`${repoPath}: invisible to the parent diff mapper — no E2E map key quoted in the file, `
|
||||
+ 'not registered in any E2E_TOUCHFILES dep list, and it '
|
||||
+ (usesNameSelection
|
||||
? 'uses name-based selection (describeIfSelected/runSkillTest/selectedTests)'
|
||||
: 'has no whole-file EVALS_TIER self-gate')
|
||||
+ '. Quote the test\'s E2E map key as a string literal, or add this file path to its '
|
||||
+ 'key\'s dep list in test/helpers/touchfiles-data.ts.',
|
||||
);
|
||||
}
|
||||
|
||||
expect(invisible).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -14,10 +14,16 @@ import {
|
|||
PAID_TEST_GLOBS,
|
||||
classifyPaidTestFile,
|
||||
collectPaidTestFiles,
|
||||
computePaidDiffSelection,
|
||||
diffSkipDecisionForFile,
|
||||
formatSummary,
|
||||
isPaidTestFile,
|
||||
knownTestNamesInSource,
|
||||
partitionShardsByDiffSelection,
|
||||
planPaidShards,
|
||||
runPaidShards,
|
||||
summarize,
|
||||
summaryExitCode,
|
||||
type ShardOutcome,
|
||||
} from '../scripts/test-paid-shards';
|
||||
|
||||
|
|
@ -115,3 +121,140 @@ describe('shard execution', () => {
|
|||
expect(summary).toMatchObject({ total: 2, executed: 1, passed: 1, neverStarted: 1 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('parent-side diff shard skipping', () => {
|
||||
const ALL_NAMES = ['alpha-test', 'beta-test', 'gamma-registered'];
|
||||
const TOUCHFILES: Record<string, string[]> = {
|
||||
'alpha-test': ['a/**'],
|
||||
'beta-test': ['b/**'],
|
||||
'gamma-registered': ['g/**', 'test/skill-e2e-gamma.test.ts'],
|
||||
};
|
||||
const SOURCES: Record<string, string> = {
|
||||
'test/skill-e2e-alpha.test.ts': "runSkillTest('alpha-test', async () => {});",
|
||||
'test/skill-e2e-beta.test.ts': 'describeIfSelected("beta", ["beta-test"], () => {});',
|
||||
// Constructed testName — invisible by quotes, mapped only via registration.
|
||||
'test/skill-e2e-gamma.test.ts': 'const name = buildName(); test(name, async () => {});',
|
||||
// No recognizable names, no registration — the fail-open class.
|
||||
'test/skill-e2e-opaque.test.ts': "const shouldRun = process.env.EVALS_TIER === 'periodic';",
|
||||
'test/codex-e2e.test.ts': 'codex tests keyed off CODEX_E2E_TOUCHFILES',
|
||||
};
|
||||
const opts = {
|
||||
readSource: (file: string) => {
|
||||
if (!(file in SOURCES)) throw new Error(`unreadable: ${file}`);
|
||||
return SOURCES[file];
|
||||
},
|
||||
allNames: ALL_NAMES,
|
||||
e2eTouchfiles: TOUCHFILES,
|
||||
};
|
||||
|
||||
test('knownTestNamesInSource matches only exact quoted strings', () => {
|
||||
expect(knownTestNamesInSource("x 'alpha-test' y", ['alpha-test', 'beta-test'])).toEqual(['alpha-test']);
|
||||
expect(knownTestNamesInSource('x "beta-test" y', ['alpha-test', 'beta-test'])).toEqual(['beta-test']);
|
||||
expect(knownTestNamesInSource('`alpha-test`', ['alpha-test'])).toEqual(['alpha-test']);
|
||||
// Substring inside a longer quoted string is not a hit.
|
||||
expect(knownTestNamesInSource("'alpha-test-extended'", ['alpha-test'])).toEqual([]);
|
||||
});
|
||||
|
||||
test('selected name in file → shard kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-alpha.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('alpha-test');
|
||||
});
|
||||
|
||||
test('no selected names in file → skipped-by-diff', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-beta.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(false);
|
||||
expect(d.reason).toContain('mapped test(s)');
|
||||
});
|
||||
|
||||
test('dep-list registration maps files with constructed test names', () => {
|
||||
const selected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['gamma-registered']), opts);
|
||||
expect(selected.kept).toBe(true);
|
||||
const unselected = diffSkipDecisionForFile('test/skill-e2e-gamma.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(unselected.kept).toBe(false);
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: unmapped file kept, child self-skip authoritative', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-opaque.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('fail-open');
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: unreadable source kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/skill-e2e-missing.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('fail-open');
|
||||
});
|
||||
|
||||
test('FAIL-OPEN: non-skill-e2e paid files always kept', () => {
|
||||
const d = diffSkipDecisionForFile('test/codex-e2e.test.ts', new Set(['alpha-test']), opts);
|
||||
expect(d.kept).toBe(true);
|
||||
expect(d.reason).toContain('non-skill-e2e');
|
||||
});
|
||||
|
||||
test('run-all selection (null) bypasses skipping entirely', () => {
|
||||
const shards = [['test/skill-e2e-alpha.test.ts'], ['test/skill-e2e-beta.test.ts']];
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, null, opts);
|
||||
expect(runnable).toEqual(shards);
|
||||
expect(skipped).toEqual([]);
|
||||
});
|
||||
|
||||
test('EVALS_ALL=1 yields run-all selection (no git consulted)', () => {
|
||||
const selection = computePaidDiffSelection({ EVALS_ALL: '1' } as NodeJS.ProcessEnv);
|
||||
expect(selection.selectedNames).toBeNull();
|
||||
expect(selection.reason).toContain('EVALS_ALL=1');
|
||||
expect(selection.totalTests).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('partition drops only all-skippable shards', () => {
|
||||
const shards = [
|
||||
['test/skill-e2e-alpha.test.ts'],
|
||||
['test/skill-e2e-beta.test.ts'],
|
||||
['test/skill-e2e-opaque.test.ts'],
|
||||
['test/codex-e2e.test.ts'],
|
||||
];
|
||||
const { runnable, skipped } = partitionShardsByDiffSelection(shards, new Set(['alpha-test']), opts);
|
||||
expect(runnable).toEqual([
|
||||
['test/skill-e2e-alpha.test.ts'],
|
||||
['test/skill-e2e-opaque.test.ts'],
|
||||
['test/codex-e2e.test.ts'],
|
||||
]);
|
||||
expect(skipped.length).toBe(1);
|
||||
expect(skipped[0].files).toEqual(['test/skill-e2e-beta.test.ts']);
|
||||
});
|
||||
|
||||
test('taxonomy: skipped-by-diff counted separately, never conflated with never-started', () => {
|
||||
const summary = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
{ shard: 3, files: ['c'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summary).toMatchObject({
|
||||
total: 3, executed: 1, passed: 1, skippedByDiff: 1, neverStarted: 1,
|
||||
});
|
||||
const lines = formatSummary(summary);
|
||||
expect(lines[1]).toContain('1 skipped by diff');
|
||||
expect(lines[1]).toContain('1 never started');
|
||||
expect(lines.some((l) => l.includes('skipped-by-diff') && l.includes('b'))).toBe(true);
|
||||
});
|
||||
|
||||
test('exit code ignores skipped-by-diff shards (they are successes)', () => {
|
||||
const allGood = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'passed', exitCode: 0, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(allGood)).toBe(0);
|
||||
|
||||
const withFailure = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'failed', exitCode: 1, elapsedMs: 1, groupPid: 1 },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(withFailure)).toBe(1);
|
||||
|
||||
const withNeverStarted = summarize([
|
||||
{ shard: 1, files: ['a'], status: 'never-started', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
{ shard: 2, files: ['b'], status: 'skipped-by-diff', exitCode: null, elapsedMs: 0, groupPid: null },
|
||||
]);
|
||||
expect(summaryExitCode(withNeverStarted)).toBe(1);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue