mirror of https://github.com/garrytan/gstack.git
evals: SDK runner default Opus -> Sonnet (D1a)
agent-sdk-runner defaulted to Opus 4.7 while session-runner (the claude -p path) defaulted to Sonnet — an inconsistency between the two runners, not a decision anyone made. Unpinned tests were implicitly asserting the expensive model. The 30+ tests that genuinely need Opus already pin it via opts.model. Re-baselined by the periodic run in this branch's final verification; regressors get explicit Opus pins. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
d464412010
commit
07f7d30e80
|
|
@ -299,7 +299,10 @@ export async function runAgentSdkTest(
|
|||
const sem = getApiSemaphore();
|
||||
const maxRetries = opts.maxRetries ?? 3;
|
||||
const queryImpl: QueryProvider = opts.queryProvider ?? query;
|
||||
const model = opts.model ?? 'claude-opus-4-7';
|
||||
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
|
||||
// default was an inconsistency between the two runners, not a choice —
|
||||
// tests that need Opus pin it via opts.model (30+ already do).
|
||||
const model = opts.model ?? 'claude-sonnet-4-6';
|
||||
|
||||
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
|
||||
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ This is a capture test, not an interactive session. Skip any system-audit / envi
|
|||
timeout: 240_000,
|
||||
testName: opts.testName,
|
||||
runId: opts.runId,
|
||||
model: opts.model ?? 'claude-opus-4-7',
|
||||
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
@ -253,7 +253,7 @@ Rules for this run:
|
|||
timeout: opts.timeout ?? 300_000,
|
||||
testName: opts.testName,
|
||||
runId: opts.runId,
|
||||
model: opts.model ?? 'claude-opus-4-7',
|
||||
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||
});
|
||||
|
||||
const readSections = new Set<string>();
|
||||
|
|
@ -334,7 +334,7 @@ Write the verbatim text of that AskUserQuestion (the full decision brief: title,
|
|||
timeout: 240_000,
|
||||
testName: opts.testName,
|
||||
runId: opts.runId,
|
||||
model: opts.model ?? 'claude-opus-4-7',
|
||||
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
|
||||
});
|
||||
|
||||
try {
|
||||
|
|
|
|||
Loading…
Reference in New Issue