test: catalog ratchet reads committed content; SDK unit pins follow D1a default

Two follow-ups from the verification runs:

1. skill-size-budget's catalog estimate still flaked under --parallel
   (8356, then 8041, vs 4177 solo) even after filtering to tracked
   skills: sibling workers REGENERATE real SKILL.md files mid-run, so
   any live-tree read is a moving target. The ratchet now reads each
   tracked skill's frontmatter from git show HEAD: — the catalog that
   ships — which no concurrent worker can perturb.

2. agent-sdk-runner unit pins asserted the old Opus default through the
   default-flow fixtures; flipped to the Sonnet default (the explicit-
   override pass-through pins keep Opus — that path is unchanged).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan 2026-08-15 09:38:22 -07:00
parent c51bb2452c
commit 576112e7cf
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
3 changed files with 29 additions and 26 deletions

View File

@ -45,7 +45,7 @@ function uuid(): string {
return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
}
function systemInit(model = 'claude-opus-4-7', version = '2.1.117'): SDKMessage {
function systemInit(model = 'claude-sonnet-4-6', version = '2.1.117'): SDKMessage {
return {
type: 'system',
subtype: 'init',
@ -77,7 +77,7 @@ function assistantTurn(
id: 'msg_' + uuid(),
type: 'message',
role: 'assistant',
model: 'claude-opus-4-7',
model: 'claude-sonnet-4-6',
content: blocks.map((b) => ({ ...b })),
stop_reason: 'end_turn',
stop_sequence: null,
@ -259,7 +259,7 @@ describe('runAgentSdkTest — happy path', () => {
expect(result.turnsUsed).toBe(2);
expect(result.costUsd).toBe(0.05);
expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
expect(result.model).toBe('claude-opus-4-7');
expect(result.model).toBe('claude-sonnet-4-6');
expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
});
@ -699,7 +699,7 @@ describe('toSkillTestResult', () => {
expect(s.output).toBe('hi');
expect(s.costEstimate.estimatedCost).toBe(0.02);
expect(s.costEstimate.turnsUsed).toBe(1);
expect(s.model).toBe('claude-opus-4-7');
expect(s.model).toBe('claude-sonnet-4-6');
expect(s.firstResponseMs).toBeNumber();
expect(s.maxInterTurnMs).toBeNumber();
expect(s.transcript).toBeArray();
@ -715,7 +715,7 @@ describe('validateFixtures', () => {
return {
id: 'test-fixture',
overlayPath: 'model-overlays/opus-4-7.md',
model: 'claude-opus-4-7',
model: 'claude-sonnet-4-6',
trials: 10,
setupWorkspace: () => {},
userPrompt: 'go',

View File

@ -57,7 +57,7 @@ export interface CaptureOptions {
}
/** Extract the frontmatter description from a SKILL.md file. Empty string if none. */
function extractDescription(content: string): string {
export function extractDescription(content: string): string {
if (!content.startsWith('---\n')) return '';
const fmEnd = content.indexOf('\n---', 4);
if (fmEnd === -1) return '';

View File

@ -32,7 +32,7 @@ import { describe, test, expect } from 'bun:test';
import * as fs from 'fs';
import * as path from 'path';
import { execSync } from 'child_process';
import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline';
import { captureBaseline, extractDescription, type ParityBaseline } from './helpers/capture-parity-baseline';
import { logBudgetOverride } from './helpers/budget-override';
import { CARVED_SKILLS } from './helpers/carve-guards';
@ -212,28 +212,31 @@ describe('SKILL.md size budget regression (gate, free)', () => {
});
test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => {
const current = captureBaseline({ repoRoot: REPO_ROOT });
// Count only git-TRACKED skills. Under the parallel free-suite runner,
// concurrent test files can leave transient skill-shaped dirs in the live
// repo mid-run (observed: this estimate exactly DOUBLED, 8356 vs 4177,
// while a solo run passed). A repo-budget ratchet should measure the
// catalog that ships, not another worker's scratch state.
const tracked = new Set(
execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
.split('\n')
.filter(Boolean)
.filter((p) => p.split('/').length === 2)
.map((p) => p.split('/')[0]),
);
const catalogTokens = Math.round(
Object.values(current.skills)
.filter((s) => tracked.has(s.skill))
.reduce((sum, s) => sum + s.descriptionLen, 0) / 4,
);
// Measure COMMITTED content (git show HEAD:), not the live tree. Under
// the parallel free-suite runner, sibling workers regenerate real
// SKILL.md files mid-run (gen-skill-docs regen tests), so the live-tree
// estimate was a moving target: 4177 solo, 8356 and 8041 in two parallel
// runs. A repo-budget ratchet measures the catalog that ships; CI always
// checks the PR's committed tree anyway.
const trackedPaths = execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
.split('\n')
.filter(Boolean)
.filter((p) => p.split('/').length === 2);
let descriptionBytes = 0;
for (const rel of trackedPaths) {
const committed = execSync(`git show HEAD:${JSON.stringify(rel)}`, {
cwd: REPO_ROOT,
encoding: 'utf-8',
maxBuffer: 8 * 1024 * 1024,
});
descriptionBytes += Buffer.byteLength(extractDescription(committed), 'utf-8');
}
const catalogTokens = Math.round(descriptionBytes / 4);
const trackedCount = trackedPaths.length;
const v145Target = 7000;
if (catalogTokens <= v145Target) {
// eslint-disable-next-line no-console
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${tracked.size} tracked skills)`);
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${trackedCount} tracked skills)`);
return;
}
const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();