evals: SDK runner default Opus -> Sonnet (D1a)

agent-sdk-runner defaulted to Opus 4.7 while session-runner (the claude
-p path) defaulted to Sonnet — an inconsistency between the two runners,
not a decision anyone made. Unpinned tests were implicitly asserting the
expensive model. The 30+ tests that genuinely need Opus already pin it
via opts.model. Re-baselined by the periodic run in this branch's final
verification; regressors get explicit Opus pins.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Garry Tan 2026-08-15 09:14:36 -07:00
parent d464412010
commit 07f7d30e80
No known key found for this signature in database
GPG Key ID: C1F69E85C74EFE1D
2 changed files with 7 additions and 4 deletions

View File

@ -299,7 +299,10 @@ export async function runAgentSdkTest(
const sem = getApiSemaphore();
const maxRetries = opts.maxRetries ?? 3;
const queryImpl: QueryProvider = opts.queryProvider ?? query;
const model = opts.model ?? 'claude-opus-4-7';
// Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus
// default was an inconsistency between the two runners, not a choice —
// tests that need Opus pin it via opts.model (30+ already do).
const model = opts.model ?? 'claude-sonnet-4-6';
// NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist
// scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with

View File

@ -191,7 +191,7 @@ This is a capture test, not an interactive session. Skip any system-audit / envi
timeout: 240_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-opus-4-7',
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
});
try {
@ -253,7 +253,7 @@ Rules for this run:
timeout: opts.timeout ?? 300_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-opus-4-7',
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
});
const readSections = new Set<string>();
@ -334,7 +334,7 @@ Write the verbatim text of that AskUserQuestion (the full decision brief: title,
timeout: 240_000,
testName: opts.testName,
runId: opts.runId,
model: opts.model ?? 'claude-opus-4-7',
model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in
});
try {