refactor(evals): shared partial-run predicate + finalized-run lookup

isPartialEval(data, filename) is the one place that decides what counts
as an in-progress accumulator (the _partial flag OR a _partial-prefixed
filename), and findLatestFinalizedRun(evalDir, tier) is the one place
that finds the newest real run — scanning the eval dir plus one level of
shards/<slug>/ subdirs, where the sharded paid runner points each
shard's collector. skill-budget-regression.test.ts's hand-rolled
findLatestRun (flag-blind: a flagged-but-renamed accumulator passed its
name check) is replaced by the shared helper.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
(cherry picked from commit b55fcf6966366fd21a8cdc46de61aab6e1b1d100)

test/helpers/eval-store.test.ts

@@ -6,6 +6,9 @@ import {
   EvalCollector,
   extractToolSummary,
   findPreviousRun,
+  findLatestFinalizedRun,
+  isPartialEval,
+  listEvalJsonFiles,
   compareEvalResults,
   formatComparison,
   generateCommentary,
@@ -346,6 +349,92 @@ describe('findPreviousRun', () => {
   });
 });
 
+// --- isPartialEval tests ---
+
+describe('isPartialEval', () => {
+  test('flag set but file renamed — still partial', () => {
+    expect(isPartialEval({ _partial: true }, 'renamed-to-look-final.json')).toBe(true);
+  });
+
+  test('name matches but no flag — still partial', () => {
+    expect(isPartialEval({}, '_partial-e2e.json')).toBe(true);
+    expect(isPartialEval(null, path.join('/some/dir', '_partial-e2e.json'))).toBe(true);
+  });
+
+  test('finalized run is not partial', () => {
+    expect(isPartialEval(makeResult(), '0.3.6-main-e2e-20260314-100000.json')).toBe(false);
+  });
+});
+
+// --- listEvalJsonFiles / findLatestFinalizedRun tests ---
+
+describe('findLatestFinalizedRun', () => {
+  test('listEvalJsonFiles recurses exactly one shards/*/ level', () => {
+    fs.writeFileSync(path.join(tmpDir, 'flat.json'), '{}');
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(path.join(shardDir, 'sharded.json'), '{}');
+    // Nested one level deeper than the contract — must NOT be picked up.
+    const tooDeep = path.join(shardDir, 'shards', 'nested');
+    fs.mkdirSync(tooDeep, { recursive: true });
+    fs.writeFileSync(path.join(tooDeep, 'too-deep.json'), '{}');
+    fs.writeFileSync(path.join(tmpDir, 'not-json.txt'), '');
+
+    const files = listEvalJsonFiles(tmpDir).map(f => path.basename(f)).sort();
+    expect(files).toEqual(['flat.json', 'sharded.json']);
+  });
+
+  test('finds the newest finalized run across flat dir and shard subdirs', () => {
+    fs.writeFileSync(
+      path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
+    );
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(shardDir, '0.3.6-main-e2e-20260314-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-14T10:00:00Z' })),
+    );
+
+    const latest = findLatestFinalizedRun(tmpDir, 'e2e');
+    expect(latest?.filepath).toContain('skill-e2e-qa');
+    expect(latest?.result.timestamp).toBe('2026-03-14T10:00:00Z');
+  });
+
+  test('skips partials by flag and by name, filters by tier', () => {
+    // Newest by timestamp, but partial by flag under a shard dir.
+    const shardDir = path.join(tmpDir, 'shards', 'skill-e2e-qa');
+    fs.mkdirSync(shardDir, { recursive: true });
+    fs.writeFileSync(
+      path.join(shardDir, 'flagged.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-16T10:00:00Z', _partial: true })),
+    );
+    // Partial by name only.
+    fs.writeFileSync(
+      path.join(tmpDir, '_partial-e2e.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-15T10:00:00Z' })),
+    );
+    // Wrong tier.
+    fs.writeFileSync(
+      path.join(tmpDir, '0.3.6-main-llm-judge-20260317-100000.json'),
+      JSON.stringify(makeResult({ tier: 'llm-judge', timestamp: '2026-03-17T10:00:00Z' })),
+    );
+    // The genuine baseline.
+    fs.writeFileSync(
+      path.join(tmpDir, '0.3.5-main-e2e-20260312-100000.json'),
+      JSON.stringify(makeResult({ timestamp: '2026-03-12T10:00:00Z' })),
+    );
+
+    const latest = findLatestFinalizedRun(tmpDir, 'e2e');
+    expect(latest?.result.timestamp).toBe('2026-03-12T10:00:00Z');
+  });
+
+  test('returns null for missing dir or no finalized runs', () => {
+    expect(findLatestFinalizedRun('/nonexistent/path', 'e2e')).toBeNull();
+    expect(findLatestFinalizedRun(tmpDir, 'e2e')).toBeNull();
+  });
+});
+
 // --- compareEvalResults tests ---
 
 describe('compareEvalResults', () => {

test/helpers/eval-store.ts

@@ -138,6 +138,77 @@ export interface ComparisonResult {
 
 // --- Shared helpers ---
 
+/**
+ * Is this eval file an in-progress accumulator rather than a finalized run?
+ *
+ * True on either signal: the `_partial` flag inside the JSON (the authoritative
+ * role marker) OR a filename starting with `_partial` (catches accumulators
+ * whose body predates the flag, and flagged files that were renamed keep being
+ * caught by the flag). Every baseline lookup must exclude these — an
+ * accumulator carries the current run's tier, branch, and freshest timestamp,
+ * so treating it as a baseline makes the run compare against itself.
+ */
+export function isPartialEval(data: unknown, filename: string): boolean {
+  if (path.basename(filename).startsWith('_partial')) return true;
+  return Boolean((data as { _partial?: unknown } | null)?._partial);
+}
+
+/**
+ * List eval JSON files in `evalDir` plus one level of `<evalDir>/shards/<slug>/`
+ * subdirectories (where the sharded paid runner points each shard's collector).
+ * Returns absolute paths. Missing dirs yield [].
+ */
+export function listEvalJsonFiles(evalDir: string): string[] {
+  const jsonIn = (dir: string): string[] => {
+    let names: string[];
+    try {
+      names = fs.readdirSync(dir);
+    } catch {
+      return [];
+    }
+    return names.filter(f => f.endsWith('.json')).map(f => path.join(dir, f));
+  };
+
+  const files = jsonIn(evalDir);
+  const shardsRoot = path.join(evalDir, 'shards');
+  let shardDirs: fs.Dirent[];
+  try {
+    shardDirs = fs.readdirSync(shardsRoot, { withFileTypes: true });
+  } catch {
+    return files;
+  }
+  for (const entry of shardDirs) {
+    if (!entry.isDirectory()) continue;
+    files.push(...jsonIn(path.join(shardsRoot, entry.name)));
+  }
+  return files;
+}
+
+/**
+ * Find the most recent finalized (non-partial) eval file for a tier, scanning
+ * `evalDir` and one level of `shards/<slug>/` subdirs. Shared by the budget
+ * regression gate and any tooling that needs "the latest real run".
+ */
+export function findLatestFinalizedRun(
+  evalDir: string,
+  tier: 'e2e' | 'llm-judge',
+): { filepath: string; result: EvalResult } | null {
+  let latest: { filepath: string; result: EvalResult; timestamp: string } | null = null;
+  for (const filepath of listEvalJsonFiles(evalDir)) {
+    let data: EvalResult;
+    try {
+      data = JSON.parse(fs.readFileSync(filepath, 'utf-8')) as EvalResult;
+    } catch { continue; }
+    if (isPartialEval(data, filepath)) continue;
+    if (data.tier !== tier) continue;
+    const timestamp = data.timestamp ?? '';
+    if (!latest || timestamp.localeCompare(latest.timestamp) > 0) {
+      latest = { filepath, result: data, timestamp };
+    }
+  }
+  return latest ? { filepath: latest.filepath, result: latest.result } : null;
+}
+
 /**
  * Determine if a planted-bug eval passed based on judge results vs ground truth thresholds.
  * Centralizes the pass/fail logic so all planted-bug tests use the same criteria.
@@ -205,7 +276,7 @@ export function findPreviousRun(
       const raw = fs.readFileSync(fullPath, 'utf-8');
       // Quick parse — only grab the fields we need
       const data = JSON.parse(raw);
-      if (data._partial) continue; // in-progress run, not a baseline
+      if (isPartialEval(data, file)) continue; // in-progress run, not a baseline
       if (data.tier !== tier) continue;
       entries.push({ file: fullPath, branch: data.branch || '', timestamp: data.timestamp || '' });
     } catch { continue; }

test/skill-budget-regression.test.ts

@@ -27,10 +27,10 @@
 import { describe, test } from 'bun:test';
 import { spawnSync } from 'child_process';
 import * as fs from 'fs';
-import * as path from 'path';
 import {
   getProjectEvalDir,
   findPreviousRun,
+  findLatestFinalizedRun,
   compareEvalResults,
   assertNoBudgetRegression,
   type EvalResult,
@@ -72,42 +72,9 @@ function currentGitBranch(): string {
   }
 }
 
-interface LatestRun {
-  filepath: string;
-  result: EvalResult;
-}
-
-/** Find the most recent finalized (non-_partial) eval file for a tier. */
-function findLatestRun(evalDir: string, tier: 'e2e' | 'llm-judge'): LatestRun | null {
-  let entries: string[];
-  try {
-    entries = fs.readdirSync(evalDir);
-  } catch {
-    return null;
-  }
-  const candidates: Array<{ filepath: string; timestamp: string }> = [];
-  for (const f of entries) {
-    if (!f.endsWith('.json')) continue;
-    if (f.startsWith('_partial')) continue;
-    const fullPath = path.join(evalDir, f);
-    try {
-      const data = JSON.parse(fs.readFileSync(fullPath, 'utf-8')) as EvalResult;
-      if (data.tier !== tier) continue;
-      candidates.push({ filepath: fullPath, timestamp: data.timestamp ?? '' });
-    } catch { /* ignore corrupt */ }
-  }
-  if (candidates.length === 0) return null;
-  candidates.sort((a, b) => b.timestamp.localeCompare(a.timestamp));
-  const top = candidates[0]!;
-  return {
-    filepath: top.filepath,
-    result: JSON.parse(fs.readFileSync(top.filepath, 'utf-8')) as EvalResult,
-  };
-}
-
 function checkTier(tier: 'e2e' | 'llm-judge'): void {
   const evalDir = getProjectEvalDir();
-  const latest = findLatestRun(evalDir, tier);
+  const latest = findLatestFinalizedRun(evalDir, tier);
   if (!latest) {
     // eslint-disable-next-line no-console
     console.log(`[budget-regression:${tier}] no current run in ${evalDir} — skipping`);
@@ -165,7 +132,7 @@ function checkTier(tier: 'e2e' | 'llm-judge'): void {
 /** Enforce a hard dollar cap on per-run eval cost. */
 function checkHardCap(tier: 'e2e' | 'llm-judge'): void {
   const evalDir = getProjectEvalDir();
-  const latest = findLatestRun(evalDir, tier);
+  const latest = findLatestFinalizedRun(evalDir, tier);
   if (!latest) return;
   const cap = TIER_CAPS[tier];
   const cost = latest.result.total_cost_usd;