From 07f7d30e80cbbe723e56affb1b0258c721bccea7 Mon Sep 17 00:00:00 2001 From: Garry Tan Date: Sat, 15 Aug 2026 09:14:36 -0700 Subject: [PATCH] evals: SDK runner default Opus -> Sonnet (D1a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit agent-sdk-runner defaulted to Opus 4.7 while session-runner (the claude -p path) defaulted to Sonnet — an inconsistency between the two runners, not a decision anyone made. Unpinned tests were implicitly asserting the expensive model. The 30+ tests that genuinely need Opus already pin it via opts.model. Re-baselined by the periodic run in this branch's final verification; regressors get explicit Opus pins. Co-Authored-By: Claude Fable 5 --- test/helpers/agent-sdk-runner.ts | 5 ++++- test/helpers/auq-sdk-capture.ts | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/test/helpers/agent-sdk-runner.ts b/test/helpers/agent-sdk-runner.ts index 9bd5eb8f8..f7f0b79ac 100644 --- a/test/helpers/agent-sdk-runner.ts +++ b/test/helpers/agent-sdk-runner.ts @@ -299,7 +299,10 @@ export async function runAgentSdkTest( const sem = getApiSemaphore(); const maxRetries = opts.maxRetries ?? 3; const queryImpl: QueryProvider = opts.queryProvider ?? query; - const model = opts.model ?? 'claude-opus-4-7'; + // Default matches session-runner's Sonnet (D1a, 2026-08): the old Opus + // default was an inconsistency between the two runners, not a choice — + // tests that need Opus pin it via opts.model (30+ already do). + const model = opts.model ?? 'claude-sonnet-4-6'; // NOTE on env: the SDK child gets the COMPLETE hermetic env (allowlist // scrub + ANTHROPIC_API_KEY + hermetic CLAUDE_CONFIG_DIR/GSTACK_HOME), with diff --git a/test/helpers/auq-sdk-capture.ts b/test/helpers/auq-sdk-capture.ts index a95a4b05b..c94985123 100644 --- a/test/helpers/auq-sdk-capture.ts +++ b/test/helpers/auq-sdk-capture.ts @@ -191,7 +191,7 @@ This is a capture test, not an interactive session. Skip any system-audit / envi timeout: 240_000, testName: opts.testName, runId: opts.runId, - model: opts.model ?? 'claude-opus-4-7', + model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in }); try { @@ -253,7 +253,7 @@ Rules for this run: timeout: opts.timeout ?? 300_000, testName: opts.testName, runId: opts.runId, - model: opts.model ?? 'claude-opus-4-7', + model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in }); const readSections = new Set(); @@ -334,7 +334,7 @@ Write the verbatim text of that AskUserQuestion (the full decision brief: title, timeout: 240_000, testName: opts.testName, runId: opts.runId, - model: opts.model ?? 'claude-opus-4-7', + model: opts.model ?? 'claude-sonnet-4-6', // D1a: align with session-runner default; Opus is opt-in }); try {