mirror of https://github.com/garrytan/gstack.git
test: catalog ratchet reads committed content; SDK unit pins follow D1a default
Two follow-ups from the verification runs: 1. skill-size-budget's catalog estimate still flaked under --parallel (8356, then 8041, vs 4177 solo) even after filtering to tracked skills: sibling workers REGENERATE real SKILL.md files mid-run, so any live-tree read is a moving target. The ratchet now reads each tracked skill's frontmatter from git show HEAD: — the catalog that ships — which no concurrent worker can perturb. 2. agent-sdk-runner unit pins asserted the old Opus default through the default-flow fixtures; flipped to the Sonnet default (the explicit- override pass-through pins keep Opus — that path is unchanged). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
c51bb2452c
commit
576112e7cf
|
|
@ -45,7 +45,7 @@ function uuid(): string {
|
|||
return `00000000-0000-0000-0000-${String(++uuidCounter).padStart(12, '0')}`;
|
||||
}
|
||||
|
||||
function systemInit(model = 'claude-opus-4-7', version = '2.1.117'): SDKMessage {
|
||||
function systemInit(model = 'claude-sonnet-4-6', version = '2.1.117'): SDKMessage {
|
||||
return {
|
||||
type: 'system',
|
||||
subtype: 'init',
|
||||
|
|
@ -77,7 +77,7 @@ function assistantTurn(
|
|||
id: 'msg_' + uuid(),
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
model: 'claude-opus-4-7',
|
||||
model: 'claude-sonnet-4-6',
|
||||
content: blocks.map((b) => ({ ...b })),
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
|
|
@ -259,7 +259,7 @@ describe('runAgentSdkTest — happy path', () => {
|
|||
expect(result.turnsUsed).toBe(2);
|
||||
expect(result.costUsd).toBe(0.05);
|
||||
expect(result.sdkClaudeCodeVersion).toBe('2.1.117');
|
||||
expect(result.model).toBe('claude-opus-4-7');
|
||||
expect(result.model).toBe('claude-sonnet-4-6');
|
||||
expect(result.firstResponseMs).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
|
|
@ -699,7 +699,7 @@ describe('toSkillTestResult', () => {
|
|||
expect(s.output).toBe('hi');
|
||||
expect(s.costEstimate.estimatedCost).toBe(0.02);
|
||||
expect(s.costEstimate.turnsUsed).toBe(1);
|
||||
expect(s.model).toBe('claude-opus-4-7');
|
||||
expect(s.model).toBe('claude-sonnet-4-6');
|
||||
expect(s.firstResponseMs).toBeNumber();
|
||||
expect(s.maxInterTurnMs).toBeNumber();
|
||||
expect(s.transcript).toBeArray();
|
||||
|
|
@ -715,7 +715,7 @@ describe('validateFixtures', () => {
|
|||
return {
|
||||
id: 'test-fixture',
|
||||
overlayPath: 'model-overlays/opus-4-7.md',
|
||||
model: 'claude-opus-4-7',
|
||||
model: 'claude-sonnet-4-6',
|
||||
trials: 10,
|
||||
setupWorkspace: () => {},
|
||||
userPrompt: 'go',
|
||||
|
|
|
|||
|
|
@ -57,7 +57,7 @@ export interface CaptureOptions {
|
|||
}
|
||||
|
||||
/** Extract the frontmatter description from a SKILL.md file. Empty string if none. */
|
||||
function extractDescription(content: string): string {
|
||||
export function extractDescription(content: string): string {
|
||||
if (!content.startsWith('---\n')) return '';
|
||||
const fmEnd = content.indexOf('\n---', 4);
|
||||
if (fmEnd === -1) return '';
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ import { describe, test, expect } from 'bun:test';
|
|||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import { execSync } from 'child_process';
|
||||
import { captureBaseline, type ParityBaseline } from './helpers/capture-parity-baseline';
|
||||
import { captureBaseline, extractDescription, type ParityBaseline } from './helpers/capture-parity-baseline';
|
||||
import { logBudgetOverride } from './helpers/budget-override';
|
||||
import { CARVED_SKILLS } from './helpers/carve-guards';
|
||||
|
||||
|
|
@ -212,28 +212,31 @@ describe('SKILL.md size budget regression (gate, free)', () => {
|
|||
});
|
||||
|
||||
test('catalog token estimate stays compressed (v1.45 target ≤ 7000)', () => {
|
||||
const current = captureBaseline({ repoRoot: REPO_ROOT });
|
||||
// Count only git-TRACKED skills. Under the parallel free-suite runner,
|
||||
// concurrent test files can leave transient skill-shaped dirs in the live
|
||||
// repo mid-run (observed: this estimate exactly DOUBLED, 8356 vs 4177,
|
||||
// while a solo run passed). A repo-budget ratchet should measure the
|
||||
// catalog that ships, not another worker's scratch state.
|
||||
const tracked = new Set(
|
||||
execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.filter((p) => p.split('/').length === 2)
|
||||
.map((p) => p.split('/')[0]),
|
||||
);
|
||||
const catalogTokens = Math.round(
|
||||
Object.values(current.skills)
|
||||
.filter((s) => tracked.has(s.skill))
|
||||
.reduce((sum, s) => sum + s.descriptionLen, 0) / 4,
|
||||
);
|
||||
// Measure COMMITTED content (git show HEAD:), not the live tree. Under
|
||||
// the parallel free-suite runner, sibling workers regenerate real
|
||||
// SKILL.md files mid-run (gen-skill-docs regen tests), so the live-tree
|
||||
// estimate was a moving target: 4177 solo, 8356 and 8041 in two parallel
|
||||
// runs. A repo-budget ratchet measures the catalog that ships; CI always
|
||||
// checks the PR's committed tree anyway.
|
||||
const trackedPaths = execSync('git ls-files -- "*/SKILL.md"', { cwd: REPO_ROOT, encoding: 'utf-8' })
|
||||
.split('\n')
|
||||
.filter(Boolean)
|
||||
.filter((p) => p.split('/').length === 2);
|
||||
let descriptionBytes = 0;
|
||||
for (const rel of trackedPaths) {
|
||||
const committed = execSync(`git show HEAD:${JSON.stringify(rel)}`, {
|
||||
cwd: REPO_ROOT,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
});
|
||||
descriptionBytes += Buffer.byteLength(extractDescription(committed), 'utf-8');
|
||||
}
|
||||
const catalogTokens = Math.round(descriptionBytes / 4);
|
||||
const trackedCount = trackedPaths.length;
|
||||
const v145Target = 7000;
|
||||
if (catalogTokens <= v145Target) {
|
||||
// eslint-disable-next-line no-console
|
||||
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${tracked.size} tracked skills)`);
|
||||
console.log(`[skill-size-budget] catalog OK: ~${catalogTokens} tokens (target ≤${v145Target}, ${trackedCount} tracked skills)`);
|
||||
return;
|
||||
}
|
||||
const overrideReason = process.env.GSTACK_SIZE_BUDGET_OVERRIDE_REASON?.trim();
|
||||
|
|
|
|||
Loading…
Reference in New Issue