From cd5433add19d189bad72d149791a3d7cc4fcf586 Mon Sep 17 00:00:00 2001
From: Garry Tan <garrytan@gmail.com>
Date: Fri, 14 Aug 2026 12:02:28 -0700
Subject: [PATCH] fix(model-benchmark): recognize macOS Keychain auth in the
 claude adapter (#1890)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The default macOS Claude Code install stores OAuth under the Keychain
generic-password service "Claude Code-credentials" and never writes
~/.claude/.credentials.json, so available()'s file-or-env sniff reported
"No Claude auth found" while claude -p worked fine. On darwin the sniff
now also probes the Keychain entry — metadata only (no -w, the secret is
never read), 5s timeout, any security(1) failure degrades to not-found.

Verified live on this machine (subscription install, no creds file,
Keychain entry present).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 test/helpers/providers/claude.ts | 25 ++++++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts
index ce77767c2..858a30e0d 100644
--- a/test/helpers/providers/claude.ts
+++ b/test/helpers/providers/claude.ts
@@ -1,6 +1,6 @@
 import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
 import { estimateCostUsd } from '../pricing';
-import { execFileSync } from 'child_process';
+import { execFileSync, spawnSync } from 'child_process';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
@@ -25,11 +25,30 @@ export class ClaudeAdapter implements ProviderAdapter {
     if (!resolved) {
       return { ok: false, reason: 'claude CLI not found on PATH. Install from https://claude.ai/download or npm i -g @anthropic-ai/claude-code (or set GSTACK_CLAUDE_BIN)' };
     }
-    // Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY
+    // Auth sniff: ~/.claude/.credentials.json OR ANTHROPIC_API_KEY OR (macOS)
+    // the Keychain entry subscription installs use instead of the creds file.
+    // #1890: the default macOS install stores OAuth under the generic-password
+    // service "Claude Code-credentials" and never writes .credentials.json,
+    // so the file-or-env sniff reported "No Claude auth found" while
+    // `claude -p` worked fine. Metadata probe only (no -w — never reads the
+    // secret), and any failure of `security` itself falls through to the
+    // not-found reason rather than throwing.
     const credsPath = path.join(os.homedir(), '.claude', '.credentials.json');
     const hasCreds = fs.existsSync(credsPath);
     const hasKey = !!process.env.ANTHROPIC_API_KEY;
-    if (!hasCreds && !hasKey) {
+    let hasKeychain = false;
+    if (!hasCreds && !hasKey && process.platform === 'darwin') {
+      try {
+        const probe = spawnSync('security', ['find-generic-password', '-s', 'Claude Code-credentials'], {
+          stdio: 'ignore',
+          timeout: 5000,
+        });
+        hasKeychain = probe.status === 0;
+      } catch {
+        hasKeychain = false;
+      }
+    }
+    if (!hasCreds && !hasKey && !hasKeychain) {
       return { ok: false, reason: 'No Claude auth found. Log in via `claude` interactive session, or export ANTHROPIC_API_KEY.' };
     }
     return { ok: true };