From 9ff2b9b162c00b821631e61e6d9dd5d10382b1ec Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Wed, 12 Aug 2026 11:14:02 -0700 Subject: [PATCH] feat(evals): env-driven lazy eval dir + shard-aware store and tooling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing eval-store no longer spawns the gstack-slug subprocess: the module-level DEFAULT_EVAL_DIR constant is now a memoized defaultEvalDir() resolved at collector construction. Resolution order: explicit constructor arg, then GSTACK_EVAL_DIR, then slug detection — so the sharded paid runner can point each shard child at its own /shards// dir with plain env, no --preload. Runs collected under a shards/ subdir record their slug in the eval JSON (EvalResult.shard). findPreviousRun scans one shards// level and prefers same-slug priors, so each shard baselines against its own history instead of whichever shard flushed last. eval:list, eval:summary, and eval:compare enumerate the same one level of shard subdirs; eval:compare's no-arg mode also stops picking an in-progress accumulator as the after-run. eval-watch stays flat (documented follow-up): it tails a single dir for live progress and gains nothing from per-shard baselines until the runner emits a merged stream. Co-Authored-By: Claude Fable 5 (cherry picked from commit e1f53f7d9c7fe6b65877d843f2e25bd2e2d12ffd) --- scripts/eval-compare.ts | 34 +++++++---- scripts/eval-list.ts | 16 ++--- scripts/eval-summary.ts | 15 ++--- test/helpers/eval-store.test.ts | 104 ++++++++++++++++++++++++++++++++ test/helpers/eval-store.ts | 72 +++++++++++++++------- 5 files changed, 185 insertions(+), 56 deletions(-) diff --git a/scripts/eval-compare.ts b/scripts/eval-compare.ts index 3cb30d5fb..a7b0dba03 100644 --- a/scripts/eval-compare.ts +++ b/scripts/eval-compare.ts @@ -10,12 +10,13 @@ import * as fs from 'fs'; import * as path from 'path'; -import * as os from 'os'; import { findPreviousRun, compareEvalResults, formatComparison, getProjectEvalDir, + isPartialEval, + listEvalJsonFiles, } from '../test/helpers/eval-store'; import type { EvalResult } from '../test/helpers/eval-store'; @@ -52,25 +53,34 @@ if (args.length === 2) { } beforeFile = prev; } else { - // No args — find two most recent of the same tier - let files: string[]; - try { - files = fs.readdirSync(EVAL_DIR) - .filter(f => f.endsWith('.json')) - .sort() - .reverse(); - } catch { + // No args — find two most recent of the same tier. Scans the flat dir plus + // one level of shards//; in-progress accumulators are never the + // "after" run (comparing against a half-finished run says nothing). + const files = listEvalJsonFiles(EVAL_DIR) + .sort((a, b) => path.basename(b).localeCompare(path.basename(a))); + + if (files.length === 0) { console.log('No eval runs yet. Run: EVALS=1 bun run test:evals'); process.exit(0); } - if (files.length < 2) { console.log('Need at least 2 eval runs to compare. Run evals again.'); process.exit(0); } - // Most recent file - afterFile = path.join(EVAL_DIR, files[0]); + // Most recent finalized file + const latest = files.find(f => { + try { + return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f); + } catch { + return false; + } + }); + if (!latest) { + console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals'); + process.exit(0); + } + afterFile = latest; const afterResult = loadResult(afterFile); const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile); if (!prev) { diff --git a/scripts/eval-list.ts b/scripts/eval-list.ts index 67d3f71a2..bb4ee9a83 100644 --- a/scripts/eval-list.ts +++ b/scripts/eval-list.ts @@ -6,9 +6,7 @@ */ import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; -import { getProjectEvalDir } from '../test/helpers/eval-store'; +import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store'; const EVAL_DIR = getProjectEvalDir(); @@ -37,14 +35,8 @@ for (let i = 0; i < args.length; i++) { else if (args[i] === '--limit') { limit = parseLimit(args[++i]); } } -// Read eval files -let files: string[]; -try { - files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json')); -} catch { - console.log('No eval runs yet. Run: EVALS=1 bun run test:evals'); - process.exit(0); -} +// Read eval files (flat dir plus one level of shards//) +const files = listEvalJsonFiles(EVAL_DIR); if (files.length === 0) { console.log('No eval runs yet. Run: EVALS=1 bun run test:evals'); @@ -68,7 +60,7 @@ interface RunSummary { const runs: RunSummary[] = []; for (const file of files) { try { - const data = JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8')); + const data = JSON.parse(fs.readFileSync(file, 'utf-8')); if (filterBranch && data.branch !== filterBranch) continue; if (filterTier && data.tier !== filterTier) continue; const totalTurns = (data.tests || []).reduce((s: number, t: any) => s + (t.turns_used || 0), 0); diff --git a/scripts/eval-summary.ts b/scripts/eval-summary.ts index fba682c21..a3b8d8bc6 100644 --- a/scripts/eval-summary.ts +++ b/scripts/eval-summary.ts @@ -6,20 +6,13 @@ */ import * as fs from 'fs'; -import * as path from 'path'; -import * as os from 'os'; import type { EvalResult } from '../test/helpers/eval-store'; -import { getProjectEvalDir } from '../test/helpers/eval-store'; +import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store'; const EVAL_DIR = getProjectEvalDir(); -let files: string[]; -try { - files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json')); -} catch { - console.log('No eval runs yet. Run: EVALS=1 bun run test:evals'); - process.exit(0); -} +// Flat dir plus one level of shards// +const files = listEvalJsonFiles(EVAL_DIR); if (files.length === 0) { console.log('No eval runs yet. Run: EVALS=1 bun run test:evals'); @@ -30,7 +23,7 @@ if (files.length === 0) { const results: EvalResult[] = []; for (const file of files) { try { - results.push(JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8'))); + results.push(JSON.parse(fs.readFileSync(file, 'utf-8'))); } catch { continue; } } diff --git a/test/helpers/eval-store.test.ts b/test/helpers/eval-store.test.ts index 5e8cc6633..540143aad 100644 --- a/test/helpers/eval-store.test.ts +++ b/test/helpers/eval-store.test.ts @@ -182,6 +182,59 @@ describe('EvalCollector', () => { }); }); +// --- GSTACK_EVAL_DIR + shard slug tests --- + +describe('EvalCollector eval-dir resolution', () => { + const savedEnv = process.env.GSTACK_EVAL_DIR; + + afterEach(() => { + if (savedEnv === undefined) delete process.env.GSTACK_EVAL_DIR; + else process.env.GSTACK_EVAL_DIR = savedEnv; + }); + + test('honors GSTACK_EVAL_DIR set after import — no --preload needed', async () => { + // The default eval dir must resolve lazily at construction, not at module + // load: the sharded runner sets GSTACK_EVAL_DIR in each shard child's env + // and shard tests import this module long before any collector exists. + const envDir = path.join(tmpDir, 'env-dir'); + process.env.GSTACK_EVAL_DIR = envDir; + const collector = new EvalCollector('e2e'); + collector.addTest(makeEntry()); + await captureStderr(async () => { await collector.finalize(); }); + expect(fs.readdirSync(envDir).filter(f => !f.startsWith('_partial'))).toHaveLength(1); + }); + + test('explicit constructor arg beats GSTACK_EVAL_DIR', async () => { + process.env.GSTACK_EVAL_DIR = path.join(tmpDir, 'env-dir'); + const explicit = path.join(tmpDir, 'explicit'); + const collector = new EvalCollector('e2e', explicit); + collector.addTest(makeEntry()); + await captureStderr(async () => { await collector.finalize(); }); + expect(fs.existsSync(path.join(tmpDir, 'env-dir'))).toBe(false); + expect(fs.readdirSync(explicit).length).toBeGreaterThan(0); + }); + + test('writes the shard slug when the eval dir is a shards/ subdir', async () => { + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + const collector = new EvalCollector('e2e', shardDir); + collector.addTest(makeEntry()); + await captureStderr(async () => { await collector.finalize(); }); + + const partial = JSON.parse(fs.readFileSync(path.join(shardDir, '_partial-e2e.json'), 'utf-8')); + expect(partial.shard).toBe('skill-e2e-qa'); + const final = fs.readdirSync(shardDir).find(f => !f.startsWith('_partial'))!; + expect(JSON.parse(fs.readFileSync(path.join(shardDir, final), 'utf-8')).shard).toBe('skill-e2e-qa'); + }); + + test('writes no shard slug for a flat eval dir', async () => { + const collector = new EvalCollector('e2e', tmpDir); + collector.addTest(makeEntry()); + await captureStderr(async () => { await collector.finalize(); }); + const final = fs.readdirSync(tmpDir).find(f => f.endsWith('.json') && !f.startsWith('_partial'))!; + expect(JSON.parse(fs.readFileSync(path.join(tmpDir, final), 'utf-8')).shard).toBeUndefined(); + }); +}); + // --- judgePassed tests --- describe('judgePassed', () => { @@ -347,6 +400,57 @@ describe('findPreviousRun', () => { const result = findPreviousRun(tmpDir, 'e2e', 'main', 'current.json'); expect(result).toBeNull(); // only llm-judge file, looking for e2e }); + + test('a shard run prefers its own shard history over newer other-shard or flat priors', () => { + const mine = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + const other = path.join(tmpDir, 'shards', 'codex-e2e'); + fs.mkdirSync(mine, { recursive: true }); + fs.mkdirSync(other, { recursive: true }); + // Same-shard prior — oldest of the three, must still win. + fs.writeFileSync( + path.join(mine, '0.3.4-main-e2e-20260311-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-11T10:00:00Z' })), + ); + fs.writeFileSync( + path.join(other, '0.3.5-main-e2e-20260312-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })), + ); + fs.writeFileSync( + path.join(tmpDir, '0.3.6-main-e2e-20260313-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-13T10:00:00Z' })), + ); + + const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(mine, 'current.json')); + expect(result).toContain(path.join('shards', 'skill-e2e-qa')); + }); + + test('a flat run prefers flat history over a newer shard prior', () => { + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync( + path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })), + ); + fs.writeFileSync( + path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })), + ); + + const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json')); + expect(result).toContain('0.3.5-main-e2e'); + }); + + test('falls back to a shard prior when the flat dir has no candidate', () => { + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync( + path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })), + ); + + const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json')); + expect(result).toContain(path.join('shards', 'skill-e2e-qa')); + }); }); // --- isPartialEval tests --- diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index fde939594..3f1c8e24c 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -39,7 +39,16 @@ export function getProjectEvalDir(): string { return LEGACY_EVAL_DIR; } -const DEFAULT_EVAL_DIR = getProjectEvalDir(); +/** + * Lazy + memoized so importing this module never spawns the gstack-slug + * subprocess. Callers that pass an explicit dir or set GSTACK_EVAL_DIR + * (the sharded paid runner does, per shard) never pay for slug detection. + */ +let memoizedDefaultEvalDir: string | null = null; +function defaultEvalDir(): string { + if (memoizedDefaultEvalDir === null) memoizedDefaultEvalDir = getProjectEvalDir(); + return memoizedDefaultEvalDir; +} // --- Interfaces --- @@ -104,6 +113,8 @@ export interface EvalResult { total_duration_ms: number; wall_clock_ms?: number; // wall-clock from collector creation to finalization (shows parallelism) tests: EvalTestEntry[]; + /** Shard slug when the run was collected under /shards//. */ + shard?: string; _partial?: boolean; // true for incremental saves, absent in final } @@ -184,6 +195,16 @@ export function listEvalJsonFiles(evalDir: string): string[] { return files; } +/** + * Shard slug for an eval dir: when the dir is directly under a `shards/` + * directory (the sharded paid runner's per-shard GSTACK_EVAL_DIR layout), + * the dir name is the slug; otherwise null. + */ +export function shardSlugOfEvalDir(evalDir: string): string | null { + const normalized = path.resolve(evalDir); + return path.basename(path.dirname(normalized)) === 'shards' ? path.basename(normalized) : null; +} + /** * Find the most recent finalized (non-partial) eval file for a tier, scanning * `evalDir` and one level of `shards//` subdirs. Shared by the budget @@ -246,7 +267,9 @@ export function extractToolSummary(transcript: any[]): Record { /** * Find the most recent prior COMPLETED eval file for comparison. - * Prefers same branch, falls back to any branch. + * Scans the eval dir plus one level of `shards//` subdirs. Prefers + * same shard slug (a shard's own history over another shard's or the flat + * dir's), then same branch, then falls back to anything. * * In-progress accumulators (`_partial: true`, written by savePartial after every * test) are never candidates: the current run's own partial carries the current @@ -260,25 +283,22 @@ export function findPreviousRun( branch: string, excludeFile: string, ): string | null { - let files: string[]; - try { - files = fs.readdirSync(evalDir).filter(f => f.endsWith('.json')); - } catch { - return null; // dir doesn't exist - } - // Parse top-level fields from each file (cheap — no full tests array needed) - const entries: Array<{ file: string; branch: string; timestamp: string }> = []; - for (const file of files) { - if (file === path.basename(excludeFile)) continue; - const fullPath = path.join(evalDir, file); + const entries: Array<{ file: string; branch: string; timestamp: string; shard: string | null }> = []; + for (const fullPath of listEvalJsonFiles(evalDir)) { + if (path.resolve(fullPath) === path.resolve(excludeFile)) continue; try { const raw = fs.readFileSync(fullPath, 'utf-8'); // Quick parse — only grab the fields we need const data = JSON.parse(raw); - if (isPartialEval(data, file)) continue; // in-progress run, not a baseline + if (isPartialEval(data, fullPath)) continue; // in-progress run, not a baseline if (data.tier !== tier) continue; - entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' }); + entries.push({ + file: fullPath, + branch: data.branch || '', + timestamp: data.timestamp || '', + shard: data.shard || shardSlugOfEvalDir(path.dirname(fullPath)), + }); } catch { continue; } } @@ -287,11 +307,17 @@ export function findPreviousRun( // Sort by timestamp descending entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp)); - // Prefer same branch - const sameBranch = entries.find(e => e.branch === branch); - if (sameBranch) return sameBranch.file; - - // Fallback: any branch + // Prefer same shard slug (null = the flat dir), then same branch, then any. + const targetShard = shardSlugOfEvalDir(path.dirname(excludeFile)); + const preferences: Array<(e: typeof entries[number]) => boolean> = [ + e => e.shard === targetShard && e.branch === branch, + e => e.shard === targetShard, + e => e.branch === branch, + ]; + for (const matches of preferences) { + const hit = entries.find(matches); + if (hit) return hit.file; + } return entries[0].file; } @@ -743,11 +769,13 @@ export class EvalCollector { private tests: EvalTestEntry[] = []; private finalized = false; private evalDir: string; + private shard: string | null; private createdAt = Date.now(); constructor(tier: 'e2e' | 'llm-judge', evalDir?: string) { this.tier = tier; - this.evalDir = evalDir || DEFAULT_EVAL_DIR; + this.evalDir = evalDir || process.env.GSTACK_EVAL_DIR || defaultEvalDir(); + this.shard = shardSlugOfEvalDir(this.evalDir); } addTest(entry: EvalTestEntry): void { @@ -778,6 +806,7 @@ export class EvalCollector { total_cost_usd: Math.round(totalCost * 100) / 100, total_duration_ms: totalDuration, tests: this.tests, + ...(this.shard ? { shard: this.shard } : {}), _partial: true, }; @@ -815,6 +844,7 @@ export class EvalCollector { total_duration_ms: totalDuration, wall_clock_ms: Date.now() - this.createdAt, tests: this.tests, + ...(this.shard ? { shard: this.shard } : {}), }; // Write eval file