From 576112e7cff6c55c3e6da98da2b63702f45aa358 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 09:38:22 -0700 Subject: [PATCH] test: catalog ratchet reads committed content; SDK unit pins follow D1a default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups from the verification runs: 1. skill-size-budget's catalog estimate still flaked under --parallel (8356, then 8041, vs 4177 solo) even after filtering to tracked skills: sibling workers REGENERATE real SKILL.md files mid-run, so any live-tree read is a moving target. The ratchet now reads each tracked skill's frontmatter from git show HEAD: — the catalog that ships — which no concurrent worker can perturb. 2. agent-sdk-runner unit pins asserted the old Opus default through the default-flow fixtures; flipped to the Sonnet default (the explicit- override pass-through pins keep Opus — that path is unchanged). Co-Authored-By: Claude Fable 5 --- test/agent-sdk-runner.test.ts | 10 +++--- test/helpers/capture-parity-baseline.ts | 2 +- test/skill-size-budget.test.ts | 43 +++++++++++++------------ 3 files changed, 29 insertions(+), 26 deletions(-) diff --git a/test/agent-sdk-runner.test.ts b/test/agent-sdk-runner.test.ts index b760d6833..5677a06c5 100644 --- a/test/agent-sdk-runner.test.ts +++ b/test/agent-sdk-runner.test.ts @@ -45,7 +45,7 @@ function uuid(): string { return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`; } -function systemInit(model = 'claude-opus-4-7', version = '2.1.117'): SDKMessage { +function systemInit(model = 'claude-sonnet-4-6', version = '2.1.117'): SDKMessage { return { type: 'system', subtype: 'init', @@ -77,7 +77,7 @@ function assistantTurn( id: 'msg_' + uuid(), type: 'message', role: 'assistant', - model: 'claude-opus-4-7', + model: 'claude-sonnet-4-6', content: blocks.map((b) => ({ ...b })), stop_reason: 'end_turn', stop_sequence: null, @@ -259,7 +259,7 @@ describe('runAgentSdkTest — happy path', () => { expect(result.turnsUsed).toBe(2); expect(result.costUsd).toBe(0.05); expect(result.sdkClaudeCodeVersion).toBe('2.1.117'); - expect(result.model).toBe('claude-opus-4-7'); + expect(result.model).toBe('claude-sonnet-4-6'); expect(result.firstResponseMs).toBeGreaterThanOrEqual(0); }); @@ -699,7 +699,7 @@ describe('toSkillTestResult', () => { expect(s.output).toBe('hi'); expect(s.costEstimate.estimatedCost).toBe(0.02); expect(s.costEstimate.turnsUsed).toBe(1); - expect(s.model).toBe('claude-opus-4-7'); + expect(s.model).toBe('claude-sonnet-4-6'); expect(s.firstResponseMs).toBeNumber(); expect(s.maxInterTurnMs).toBeNumber(); expect(s.transcript).toBeArray(); @@ -715,7 +715,7 @@ describe('validateFixtures', () => { return { id: 'test-fixture', overlayPath: 'model-overlays/opus-4-7.md', - model: 'claude-opus-4-7', + model: 'claude-sonnet-4-6', trials: 10, setupWorkspace: () => {}, userPrompt: 'go', diff --git a/test/helpers/capture-parity-baseline.ts b/test/helpers/capture-parity-baseline.ts index ca54d9cf0..ce94fc48f 100644 --- a/test/helpers/capture-parity-baseline.ts +++ b/test/helpers/capture-parity-baseline.ts @@ -57,7 +57,7 @@ export interface CaptureOptions { } /** Extract the frontmatter description from a SKILL.md file. Empty string if none. */ -function extractDescription(content: string): string { +export function extractDescription(content: string): string { if (!content.startsWith('---\n')) return ''; const fmEnd = content.indexOf('\n---', 4); if (fmEnd === -1) return ''; diff --git a/test/skill-size-budget.test.ts b/test/skill-size-budget.test.ts index 7703bb838..31251379b 100644 --- a/test/skill-size-budget.test.ts +++ b/test/skill-size-budget.test.ts @@ -32,7 +32,7 @@ import { describe, test, expect } from 'bun:test'; import * as fs from 'fs'; import * as path from 'path'; import { execSync } from 'child_process'; -import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline'; +import { captureBaseline, extractDescription, type ParityBaseline } from './helpers/capture-parity-baseline'; import { logBudgetOverride } from './helpers/budget-override'; import { CARVED_SKILLS } from './helpers/carve-guards'; @@ -212,28 +212,31 @@ describe('SKILL.md size budget regression (gate, free)', () => { }); test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => { - const current = captureBaseline({ repoRoot: REPO_ROOT }); - // Count only git-TRACKED skills. Under the parallel free-suite runner, - // concurrent test files can leave transient skill-shaped dirs in the live - // repo mid-run (observed: this estimate exactly DOUBLED, 8356 vs 4177, - // while a solo run passed). A repo-budget ratchet should measure the - // catalog that ships, not another worker's scratch state. - const tracked = new Set( - execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' }) - .split('\n') - .filter(Boolean) - .filter((p) => p.split('/').length === 2) - .map((p) => p.split('/')[0]), - ); - const catalogTokens = Math.round( - Object.values(current.skills) - .filter((s) => tracked.has(s.skill)) - .reduce((sum, s) => sum + s.descriptionLen, 0) / 4, - ); + // Measure COMMITTED content (git show HEAD:), not the live tree. Under + // the parallel free-suite runner, sibling workers regenerate real + // SKILL.md files mid-run (gen-skill-docs regen tests), so the live-tree + // estimate was a moving target: 4177 solo, 8356 and 8041 in two parallel + // runs. A repo-budget ratchet measures the catalog that ships; CI always + // checks the PR's committed tree anyway. + const trackedPaths = execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' }) + .split('\n') + .filter(Boolean) + .filter((p) => p.split('/').length === 2); + let descriptionBytes = 0; + for (const rel of trackedPaths) { + const committed = execSync(`git show HEAD:${JSON.stringify(rel)}`, { + cwd: REPO_ROOT, + encoding: 'utf-8', + maxBuffer: 8 * 1024 * 1024, + }); + descriptionBytes += Buffer.byteLength(extractDescription(committed), 'utf-8'); + } + const catalogTokens = Math.round(descriptionBytes / 4); + const trackedCount = trackedPaths.length; const v145Target = 7000; if (catalogTokens <= v145Target) { // eslint-disable-next-line no-console - console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${tracked.size} tracked skills)`); + console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${trackedCount} tracked skills)`); return; } const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();