diff --git a/test/helpers/eval-store.test.ts b/test/helpers/eval-store.test.ts index 067a56281..5e8cc6633 100644 --- a/test/helpers/eval-store.test.ts +++ b/test/helpers/eval-store.test.ts @@ -6,6 +6,9 @@ import { EvalCollector, extractToolSummary, findPreviousRun, + findLatestFinalizedRun, + isPartialEval, + listEvalJsonFiles, compareEvalResults, formatComparison, generateCommentary, @@ -346,6 +349,92 @@ describe('findPreviousRun', () => { }); }); +// --- isPartialEval tests --- + +describe('isPartialEval', () => { + test('flag set but file renamed — still partial', () => { + expect(isPartialEval({ _partial: true }, 'renamed-to-look-final.json')).toBe(true); + }); + + test('name matches but no flag — still partial', () => { + expect(isPartialEval({}, '_partial-e2e.json')).toBe(true); + expect(isPartialEval(null, path.join('/some/dir', '_partial-e2e.json'))).toBe(true); + }); + + test('finalized run is not partial', () => { + expect(isPartialEval(makeResult(), '0.3.6-main-e2e-20260314-100000.json')).toBe(false); + }); +}); + +// --- listEvalJsonFiles / findLatestFinalizedRun tests --- + +describe('findLatestFinalizedRun', () => { + test('listEvalJsonFiles recurses exactly one shards/*/ level', () => { + fs.writeFileSync(path.join(tmpDir, 'flat.json'), '{}'); + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync(path.join(shardDir, 'sharded.json'), '{}'); + // Nested one level deeper than the contract — must NOT be picked up. + const tooDeep = path.join(shardDir, 'shards', 'nested'); + fs.mkdirSync(tooDeep, { recursive: true }); + fs.writeFileSync(path.join(tooDeep, 'too-deep.json'), '{}'); + fs.writeFileSync(path.join(tmpDir, 'not-json.txt'), ''); + + const files = listEvalJsonFiles(tmpDir).map(f => path.basename(f)).sort(); + expect(files).toEqual(['flat.json', 'sharded.json']); + }); + + test('finds the newest finalized run across flat dir and shard subdirs', () => { + fs.writeFileSync( + path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })), + ); + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync( + path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })), + ); + + const latest = findLatestFinalizedRun(tmpDir, 'e2e'); + expect(latest?.filepath).toContain('skill-e2e-qa'); + expect(latest?.result.timestamp).toBe('2026-03-14T10:00:00Z'); + }); + + test('skips partials by flag and by name, filters by tier', () => { + // Newest by timestamp, but partial by flag under a shard dir. + const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa'); + fs.mkdirSync(shardDir, { recursive: true }); + fs.writeFileSync( + path.join(shardDir, 'flagged.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-16T10:00:00Z', _partial: true })), + ); + // Partial by name only. + fs.writeFileSync( + path.join(tmpDir, '_partial-e2e.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-15T10:00:00Z' })), + ); + // Wrong tier. + fs.writeFileSync( + path.join(tmpDir, '0.3.6-main-llm-judge-20260317-100000.json'), + JSON.stringify(makeResult({ tier: 'llm-judge', timestamp: '2026-03-17T10:00:00Z' })), + ); + // The genuine baseline. + fs.writeFileSync( + path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'), + JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })), + ); + + const latest = findLatestFinalizedRun(tmpDir, 'e2e'); + expect(latest?.result.timestamp).toBe('2026-03-12T10:00:00Z'); + }); + + test('returns null for missing dir or no finalized runs', () => { + expect(findLatestFinalizedRun('/nonexistent/path', 'e2e')).toBeNull(); + expect(findLatestFinalizedRun(tmpDir, 'e2e')).toBeNull(); + }); +}); + // --- compareEvalResults tests --- describe('compareEvalResults', () => { diff --git a/test/helpers/eval-store.ts b/test/helpers/eval-store.ts index 7a42a9bde..fde939594 100644 --- a/test/helpers/eval-store.ts +++ b/test/helpers/eval-store.ts @@ -138,6 +138,77 @@ export interface ComparisonResult { // --- Shared helpers --- +/** + * Is this eval file an in-progress accumulator rather than a finalized run? + * + * True on either signal: the `_partial` flag inside the JSON (the authoritative + * role marker) OR a filename starting with `_partial` (catches accumulators + * whose body predates the flag, and flagged files that were renamed keep being + * caught by the flag). Every baseline lookup must exclude these — an + * accumulator carries the current run's tier, branch, and freshest timestamp, + * so treating it as a baseline makes the run compare against itself. + */ +export function isPartialEval(data: unknown, filename: string): boolean { + if (path.basename(filename).startsWith('_partial')) return true; + return Boolean((data as { _partial?: unknown } | null)?._partial); +} + +/** + * List eval JSON files in `evalDir` plus one level of `/shards//` + * subdirectories (where the sharded paid runner points each shard's collector). + * Returns absolute paths. Missing dirs yield []. + */ +export function listEvalJsonFiles(evalDir: string): string[] { + const jsonIn = (dir: string): string[] => { + let names: string[]; + try { + names = fs.readdirSync(dir); + } catch { + return []; + } + return names.filter(f => f.endsWith('.json')).map(f => path.join(dir, f)); + }; + + const files = jsonIn(evalDir); + const shardsRoot = path.join(evalDir, 'shards'); + let shardDirs: fs.Dirent[]; + try { + shardDirs = fs.readdirSync(shardsRoot, { withFileTypes: true }); + } catch { + return files; + } + for (const entry of shardDirs) { + if (!entry.isDirectory()) continue; + files.push(...jsonIn(path.join(shardsRoot, entry.name))); + } + return files; +} + +/** + * Find the most recent finalized (non-partial) eval file for a tier, scanning + * `evalDir` and one level of `shards//` subdirs. Shared by the budget + * regression gate and any tooling that needs "the latest real run". + */ +export function findLatestFinalizedRun( + evalDir: string, + tier: 'e2e' | 'llm-judge', +): { filepath: string; result: EvalResult } | null { + let latest: { filepath: string; result: EvalResult; timestamp: string } | null = null; + for (const filepath of listEvalJsonFiles(evalDir)) { + let data: EvalResult; + try { + data = JSON.parse(fs.readFileSync(filepath, 'utf-8')) as EvalResult; + } catch { continue; } + if (isPartialEval(data, filepath)) continue; + if (data.tier !== tier) continue; + const timestamp = data.timestamp ?? ''; + if (!latest || timestamp.localeCompare(latest.timestamp) > 0) { + latest = { filepath, result: data, timestamp }; + } + } + return latest ? { filepath: latest.filepath, result: latest.result } : null; +} + /** * Determine if a planted-bug eval passed based on judge results vs ground truth thresholds. * Centralizes the pass/fail logic so all planted-bug tests use the same criteria. @@ -205,7 +276,7 @@ export function findPreviousRun( const raw = fs.readFileSync(fullPath, 'utf-8'); // Quick parse — only grab the fields we need const data = JSON.parse(raw); - if (data._partial) continue; // in-progress run, not a baseline + if (isPartialEval(data, file)) continue; // in-progress run, not a baseline if (data.tier !== tier) continue; entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' }); } catch { continue; } diff --git a/test/skill-budget-regression.test.ts b/test/skill-budget-regression.test.ts index 85391bfc2..318875cd9 100644 --- a/test/skill-budget-regression.test.ts +++ b/test/skill-budget-regression.test.ts @@ -27,10 +27,10 @@ import { describe, test } from 'bun:test'; import { spawnSync } from 'child_process'; import * as fs from 'fs'; -import * as path from 'path'; import { getProjectEvalDir, findPreviousRun, + findLatestFinalizedRun, compareEvalResults, assertNoBudgetRegression, type EvalResult, @@ -72,42 +72,9 @@ function currentGitBranch(): string { } } -interface LatestRun { - filepath: string; - result: EvalResult; -} - -/** Find the most recent finalized (non-_partial) eval file for a tier. */ -function findLatestRun(evalDir: string, tier: 'e2e' | 'llm-judge'): LatestRun | null { - let entries: string[]; - try { - entries = fs.readdirSync(evalDir); - } catch { - return null; - } - const candidates: Array<{ filepath: string; timestamp: string }> = []; - for (const f of entries) { - if (!f.endsWith('.json')) continue; - if (f.startsWith('_partial')) continue; - const fullPath = path.join(evalDir, f); - try { - const data = JSON.parse(fs.readFileSync(fullPath, 'utf-8')) as EvalResult; - if (data.tier !== tier) continue; - candidates.push({ filepath: fullPath, timestamp: data.timestamp ?? '' }); - } catch { /* ignore corrupt */ } - } - if (candidates.length === 0) return null; - candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp)); - const top = candidates[0]!; - return { - filepath: top.filepath, - result: JSON.parse(fs.readFileSync(top.filepath, 'utf-8')) as EvalResult, - }; -} - function checkTier(tier: 'e2e' | 'llm-judge'): void { const evalDir = getProjectEvalDir(); - const latest = findLatestRun(evalDir, tier); + const latest = findLatestFinalizedRun(evalDir, tier); if (!latest) { // eslint-disable-next-line no-console console.log(`[budget-regression:${tier}] no current run in ${evalDir} — skipping`); @@ -165,7 +132,7 @@ function checkTier(tier: 'e2e' | 'llm-judge'): void { /** Enforce a hard dollar cap on per-run eval cost. */ function checkHardCap(tier: 'e2e' | 'llm-judge'): void { const evalDir = getProjectEvalDir(); - const latest = findLatestRun(evalDir, tier); + const latest = findLatestFinalizedRun(evalDir, tier); if (!latest) return; const cap = TIER_CAPS[tier]; const cost = latest.result.total_cost_usd;