mirror of https://github.com/garrytan/gstack.git
feat(evals): env-driven lazy eval dir + shard-aware store and tooling
Importing eval-store no longer spawns the gstack-slug subprocess: the module-level DEFAULT_EVAL_DIR constant is now a memoized defaultEvalDir() resolved at collector construction. Resolution order: explicit constructor arg, then GSTACK_EVAL_DIR, then slug detection — so the sharded paid runner can point each shard child at its own <evalDir>/shards/<slug>/ dir with plain env, no --preload. Runs collected under a shards/ subdir record their slug in the eval JSON (EvalResult.shard). findPreviousRun scans one shards/<slug>/ level and prefers same-slug priors, so each shard baselines against its own history instead of whichever shard flushed last. eval:list, eval:summary, and eval:compare enumerate the same one level of shard subdirs; eval:compare's no-arg mode also stops picking an in-progress accumulator as the after-run. eval-watch stays flat (documented follow-up): it tails a single dir for live progress and gains nothing from per-shard baselines until the runner emits a merged stream. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> (cherry picked from commit e1f53f7d9c7fe6b65877d843f2e25bd2e2d12ffd)
This commit is contained in:
Garry Tan
parent
ea1c2851e5
commit
9ff2b9b162
|
|
@ -10,12 +10,13 @@
|
|||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import {
|
||||
findPreviousRun,
|
||||
compareEvalResults,
|
||||
formatComparison,
|
||||
getProjectEvalDir,
|
||||
isPartialEval,
|
||||
listEvalJsonFiles,
|
||||
} from '../test/helpers/eval-store';
|
||||
import type { EvalResult } from '../test/helpers/eval-store';
|
||||
|
||||
|
|
@ -52,25 +53,34 @@ if (args.length === 2) {
|
|||
}
|
||||
beforeFile = prev;
|
||||
} else {
|
||||
// No args — find two most recent of the same tier
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(EVAL_DIR)
|
||||
.filter(f => f.endsWith('.json'))
|
||||
.sort()
|
||||
.reverse();
|
||||
} catch {
|
||||
// No args — find two most recent of the same tier. Scans the flat dir plus
|
||||
// one level of shards/<slug>/; in-progress accumulators are never the
|
||||
// "after" run (comparing against a half-finished run says nothing).
|
||||
const files = listEvalJsonFiles(EVAL_DIR)
|
||||
.sort((a, b) => path.basename(b).localeCompare(path.basename(a)));
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
if (files.length < 2) {
|
||||
console.log('Need at least 2 eval runs to compare. Run evals again.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Most recent file
|
||||
afterFile = path.join(EVAL_DIR, files[0]);
|
||||
// Most recent finalized file
|
||||
const latest = files.find(f => {
|
||||
try {
|
||||
return !isPartialEval(JSON.parse(fs.readFileSync(f, 'utf-8')), f);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
if (!latest) {
|
||||
console.log('No completed eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
process.exit(0);
|
||||
}
|
||||
afterFile = latest;
|
||||
const afterResult = loadResult(afterFile);
|
||||
const prev = findPreviousRun(EVAL_DIR, afterResult.tier, afterResult.branch, afterFile);
|
||||
if (!prev) {
|
||||
|
|
|
|||
|
|
@ -6,9 +6,7 @@
|
|||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
|
||||
|
||||
const EVAL_DIR = getProjectEvalDir();
|
||||
|
||||
|
|
@ -37,14 +35,8 @@ for (let i = 0; i < args.length; i++) {
|
|||
else if (args[i] === '--limit') { limit = parseLimit(args[++i]); }
|
||||
}
|
||||
|
||||
// Read eval files
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
|
||||
} catch {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
process.exit(0);
|
||||
}
|
||||
// Read eval files (flat dir plus one level of shards/<slug>/)
|
||||
const files = listEvalJsonFiles(EVAL_DIR);
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
|
|
@ -68,7 +60,7 @@ interface RunSummary {
|
|||
const runs: RunSummary[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
const data = JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8'));
|
||||
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
|
||||
if (filterBranch && data.branch !== filterBranch) continue;
|
||||
if (filterTier && data.tier !== filterTier) continue;
|
||||
const totalTurns = (data.tests || []).reduce((s: number, t: any) => s + (t.turns_used || 0), 0);
|
||||
|
|
|
|||
|
|
@ -6,20 +6,13 @@
|
|||
*/
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as os from 'os';
|
||||
import type { EvalResult } from '../test/helpers/eval-store';
|
||||
import { getProjectEvalDir } from '../test/helpers/eval-store';
|
||||
import { getProjectEvalDir, listEvalJsonFiles } from '../test/helpers/eval-store';
|
||||
|
||||
const EVAL_DIR = getProjectEvalDir();
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(EVAL_DIR).filter(f => f.endsWith('.json'));
|
||||
} catch {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
process.exit(0);
|
||||
}
|
||||
// Flat dir plus one level of shards/<slug>/
|
||||
const files = listEvalJsonFiles(EVAL_DIR);
|
||||
|
||||
if (files.length === 0) {
|
||||
console.log('No eval runs yet. Run: EVALS=1 bun run test:evals');
|
||||
|
|
@ -30,7 +23,7 @@ if (files.length === 0) {
|
|||
const results: EvalResult[] = [];
|
||||
for (const file of files) {
|
||||
try {
|
||||
results.push(JSON.parse(fs.readFileSync(path.join(EVAL_DIR, file), 'utf-8')));
|
||||
results.push(JSON.parse(fs.readFileSync(file, 'utf-8')));
|
||||
} catch { continue; }
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -182,6 +182,59 @@ describe('EvalCollector', () => {
|
|||
});
|
||||
});
|
||||
|
||||
// --- GSTACK_EVAL_DIR + shard slug tests ---
|
||||
|
||||
describe('EvalCollector eval-dir resolution', () => {
|
||||
const savedEnv = process.env.GSTACK_EVAL_DIR;
|
||||
|
||||
afterEach(() => {
|
||||
if (savedEnv === undefined) delete process.env.GSTACK_EVAL_DIR;
|
||||
else process.env.GSTACK_EVAL_DIR = savedEnv;
|
||||
});
|
||||
|
||||
test('honors GSTACK_EVAL_DIR set after import — no --preload needed', async () => {
|
||||
// The default eval dir must resolve lazily at construction, not at module
|
||||
// load: the sharded runner sets GSTACK_EVAL_DIR in each shard child's env
|
||||
// and shard tests import this module long before any collector exists.
|
||||
const envDir = path.join(tmpDir, 'env-dir');
|
||||
process.env.GSTACK_EVAL_DIR = envDir;
|
||||
const collector = new EvalCollector('e2e');
|
||||
collector.addTest(makeEntry());
|
||||
await captureStderr(async () => { await collector.finalize(); });
|
||||
expect(fs.readdirSync(envDir).filter(f => !f.startsWith('_partial'))).toHaveLength(1);
|
||||
});
|
||||
|
||||
test('explicit constructor arg beats GSTACK_EVAL_DIR', async () => {
|
||||
process.env.GSTACK_EVAL_DIR = path.join(tmpDir, 'env-dir');
|
||||
const explicit = path.join(tmpDir, 'explicit');
|
||||
const collector = new EvalCollector('e2e', explicit);
|
||||
collector.addTest(makeEntry());
|
||||
await captureStderr(async () => { await collector.finalize(); });
|
||||
expect(fs.existsSync(path.join(tmpDir, 'env-dir'))).toBe(false);
|
||||
expect(fs.readdirSync(explicit).length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
test('writes the shard slug when the eval dir is a shards/ subdir', async () => {
|
||||
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
|
||||
const collector = new EvalCollector('e2e', shardDir);
|
||||
collector.addTest(makeEntry());
|
||||
await captureStderr(async () => { await collector.finalize(); });
|
||||
|
||||
const partial = JSON.parse(fs.readFileSync(path.join(shardDir, '_partial-e2e.json'), 'utf-8'));
|
||||
expect(partial.shard).toBe('skill-e2e-qa');
|
||||
const final = fs.readdirSync(shardDir).find(f => !f.startsWith('_partial'))!;
|
||||
expect(JSON.parse(fs.readFileSync(path.join(shardDir, final), 'utf-8')).shard).toBe('skill-e2e-qa');
|
||||
});
|
||||
|
||||
test('writes no shard slug for a flat eval dir', async () => {
|
||||
const collector = new EvalCollector('e2e', tmpDir);
|
||||
collector.addTest(makeEntry());
|
||||
await captureStderr(async () => { await collector.finalize(); });
|
||||
const final = fs.readdirSync(tmpDir).find(f => f.endsWith('.json') && !f.startsWith('_partial'))!;
|
||||
expect(JSON.parse(fs.readFileSync(path.join(tmpDir, final), 'utf-8')).shard).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
// --- judgePassed tests ---
|
||||
|
||||
describe('judgePassed', () => {
|
||||
|
|
@ -347,6 +400,57 @@ describe('findPreviousRun', () => {
|
|||
const result = findPreviousRun(tmpDir, 'e2e', 'main', 'current.json');
|
||||
expect(result).toBeNull(); // only llm-judge file, looking for e2e
|
||||
});
|
||||
|
||||
test('a shard run prefers its own shard history over newer other-shard or flat priors', () => {
|
||||
const mine = path.join(tmpDir, 'shards', 'skill-e2e-qa');
|
||||
const other = path.join(tmpDir, 'shards', 'codex-e2e');
|
||||
fs.mkdirSync(mine, { recursive: true });
|
||||
fs.mkdirSync(other, { recursive: true });
|
||||
// Same-shard prior — oldest of the three, must still win.
|
||||
fs.writeFileSync(
|
||||
path.join(mine, '0.3.4-main-e2e-20260311-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-11T10:00:00Z' })),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(other, '0.3.5-main-e2e-20260312-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, '0.3.6-main-e2e-20260313-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-13T10:00:00Z' })),
|
||||
);
|
||||
|
||||
const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(mine, 'current.json'));
|
||||
expect(result).toContain(path.join('shards', 'skill-e2e-qa'));
|
||||
});
|
||||
|
||||
test('a flat run prefers flat history over a newer shard prior', () => {
|
||||
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
|
||||
fs.mkdirSync(shardDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
|
||||
);
|
||||
fs.writeFileSync(
|
||||
path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
|
||||
);
|
||||
|
||||
const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json'));
|
||||
expect(result).toContain('0.3.5-main-e2e');
|
||||
});
|
||||
|
||||
test('falls back to a shard prior when the flat dir has no candidate', () => {
|
||||
const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
|
||||
fs.mkdirSync(shardDir, { recursive: true });
|
||||
fs.writeFileSync(
|
||||
path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
|
||||
JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
|
||||
);
|
||||
|
||||
const result = findPreviousRun(tmpDir, 'e2e', 'main', path.join(tmpDir, 'current.json'));
|
||||
expect(result).toContain(path.join('shards', 'skill-e2e-qa'));
|
||||
});
|
||||
});
|
||||
|
||||
// --- isPartialEval tests ---
|
||||
|
|
|
|||
|
|
@ -39,7 +39,16 @@ export function getProjectEvalDir(): string {
|
|||
return LEGACY_EVAL_DIR;
|
||||
}
|
||||
|
||||
const DEFAULT_EVAL_DIR = getProjectEvalDir();
|
||||
/**
|
||||
* Lazy + memoized so importing this module never spawns the gstack-slug
|
||||
* subprocess. Callers that pass an explicit dir or set GSTACK_EVAL_DIR
|
||||
* (the sharded paid runner does, per shard) never pay for slug detection.
|
||||
*/
|
||||
let memoizedDefaultEvalDir: string | null = null;
|
||||
function defaultEvalDir(): string {
|
||||
if (memoizedDefaultEvalDir === null) memoizedDefaultEvalDir = getProjectEvalDir();
|
||||
return memoizedDefaultEvalDir;
|
||||
}
|
||||
|
||||
// --- Interfaces ---
|
||||
|
||||
|
|
@ -104,6 +113,8 @@ export interface EvalResult {
|
|||
total_duration_ms: number;
|
||||
wall_clock_ms?: number; // wall-clock from collector creation to finalization (shows parallelism)
|
||||
tests: EvalTestEntry[];
|
||||
/** Shard slug when the run was collected under <evalDir>/shards/<slug>/. */
|
||||
shard?: string;
|
||||
_partial?: boolean; // true for incremental saves, absent in final
|
||||
}
|
||||
|
||||
|
|
@ -184,6 +195,16 @@ export function listEvalJsonFiles(evalDir: string): string[] {
|
|||
return files;
|
||||
}
|
||||
|
||||
/**
|
||||
* Shard slug for an eval dir: when the dir is directly under a `shards/`
|
||||
* directory (the sharded paid runner's per-shard GSTACK_EVAL_DIR layout),
|
||||
* the dir name is the slug; otherwise null.
|
||||
*/
|
||||
export function shardSlugOfEvalDir(evalDir: string): string | null {
|
||||
const normalized = path.resolve(evalDir);
|
||||
return path.basename(path.dirname(normalized)) === 'shards' ? path.basename(normalized) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Find the most recent finalized (non-partial) eval file for a tier, scanning
|
||||
* `evalDir` and one level of `shards/<slug>/` subdirs. Shared by the budget
|
||||
|
|
@ -246,7 +267,9 @@ export function extractToolSummary(transcript: any[]): Record<string, number> {
|
|||
|
||||
/**
|
||||
* Find the most recent prior COMPLETED eval file for comparison.
|
||||
* Prefers same branch, falls back to any branch.
|
||||
* Scans the eval dir plus one level of `shards/<slug>/` subdirs. Prefers
|
||||
* same shard slug (a shard's own history over another shard's or the flat
|
||||
* dir's), then same branch, then falls back to anything.
|
||||
*
|
||||
* In-progress accumulators (`_partial: true`, written by savePartial after every
|
||||
* test) are never candidates: the current run's own partial carries the current
|
||||
|
|
@ -260,25 +283,22 @@ export function findPreviousRun(
|
|||
branch: string,
|
||||
excludeFile: string,
|
||||
): string | null {
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(evalDir).filter(f => f.endsWith('.json'));
|
||||
} catch {
|
||||
return null; // dir doesn't exist
|
||||
}
|
||||
|
||||
// Parse top-level fields from each file (cheap — no full tests array needed)
|
||||
const entries: Array<{ file: string; branch: string; timestamp: string }> = [];
|
||||
for (const file of files) {
|
||||
if (file === path.basename(excludeFile)) continue;
|
||||
const fullPath = path.join(evalDir, file);
|
||||
const entries: Array<{ file: string; branch: string; timestamp: string; shard: string | null }> = [];
|
||||
for (const fullPath of listEvalJsonFiles(evalDir)) {
|
||||
if (path.resolve(fullPath) === path.resolve(excludeFile)) continue;
|
||||
try {
|
||||
const raw = fs.readFileSync(fullPath, 'utf-8');
|
||||
// Quick parse — only grab the fields we need
|
||||
const data = JSON.parse(raw);
|
||||
if (isPartialEval(data, file)) continue; // in-progress run, not a baseline
|
||||
if (isPartialEval(data, fullPath)) continue; // in-progress run, not a baseline
|
||||
if (data.tier !== tier) continue;
|
||||
entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' });
|
||||
entries.push({
|
||||
file: fullPath,
|
||||
branch: data.branch || '',
|
||||
timestamp: data.timestamp || '',
|
||||
shard: data.shard || shardSlugOfEvalDir(path.dirname(fullPath)),
|
||||
});
|
||||
} catch { continue; }
|
||||
}
|
||||
|
||||
|
|
@ -287,11 +307,17 @@ export function findPreviousRun(
|
|||
// Sort by timestamp descending
|
||||
entries.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
|
||||
|
||||
// Prefer same branch
|
||||
const sameBranch = entries.find(e => e.branch === branch);
|
||||
if (sameBranch) return sameBranch.file;
|
||||
|
||||
// Fallback: any branch
|
||||
// Prefer same shard slug (null = the flat dir), then same branch, then any.
|
||||
const targetShard = shardSlugOfEvalDir(path.dirname(excludeFile));
|
||||
const preferences: Array<(e: typeof entries[number]) => boolean> = [
|
||||
e => e.shard === targetShard && e.branch === branch,
|
||||
e => e.shard === targetShard,
|
||||
e => e.branch === branch,
|
||||
];
|
||||
for (const matches of preferences) {
|
||||
const hit = entries.find(matches);
|
||||
if (hit) return hit.file;
|
||||
}
|
||||
return entries[0].file;
|
||||
}
|
||||
|
||||
|
|
@ -743,11 +769,13 @@ export class EvalCollector {
|
|||
private tests: EvalTestEntry[] = [];
|
||||
private finalized = false;
|
||||
private evalDir: string;
|
||||
private shard: string | null;
|
||||
private createdAt = Date.now();
|
||||
|
||||
constructor(tier: 'e2e' | 'llm-judge', evalDir?: string) {
|
||||
this.tier = tier;
|
||||
this.evalDir = evalDir || DEFAULT_EVAL_DIR;
|
||||
this.evalDir = evalDir || process.env.GSTACK_EVAL_DIR || defaultEvalDir();
|
||||
this.shard = shardSlugOfEvalDir(this.evalDir);
|
||||
}
|
||||
|
||||
addTest(entry: EvalTestEntry): void {
|
||||
|
|
@ -778,6 +806,7 @@ export class EvalCollector {
|
|||
total_cost_usd: Math.round(totalCost * 100) / 100,
|
||||
total_duration_ms: totalDuration,
|
||||
tests: this.tests,
|
||||
...(this.shard ? { shard: this.shard } : {}),
|
||||
_partial: true,
|
||||
};
|
||||
|
||||
|
|
@ -815,6 +844,7 @@ export class EvalCollector {
|
|||
total_duration_ms: totalDuration,
|
||||
wall_clock_ms: Date.now() - this.createdAt,
|
||||
tests: this.tests,
|
||||
...(this.shard ? { shard: this.shard } : {}),
|
||||
};
|
||||
|
||||
// Write eval file
|
||||
|
|
|
|||
Loading…
Reference in New Issue