From 9c2414016c3cea5299ab43c951acc0d5b8e2f7a3 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Thu, 16 Jul 2026 09:06:58 +0200 Subject: [PATCH] fix benchmark adapters and stdin handling --- test/benchmark-cli.test.ts | 48 ++++++++++++++++++++++ test/benchmark-runner.test.ts | 11 ++++- test/helpers/benchmark-runner.ts | 20 +++++---- test/helpers/providers/gemini.ts | 14 +++++-- test/helpers/providers/gpt.ts | 18 ++++++-- test/helpers/secret-sink-harness.ts | 10 ++--- test/skill-e2e-benchmark-providers.test.ts | 16 ++++++++ 7 files changed, 117 insertions(+), 20 deletions(-) diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index 1912f3d8f..aa86a9c42 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -142,6 +142,54 @@ describe('gstack-model-benchmark --dry-run', () => { } }); + test('Gemini OAuth-only auth is reported as NOT READY with Antigravity remediation', () => { + const oauthHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-oauth-home-')); + try { + const oauthDir = path.join(oauthHome, '.gemini'); + fs.mkdirSync(oauthDir, { recursive: true }); + fs.writeFileSync(path.join(oauthDir, 'oauth_creds.json'), '{}'); + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], { + cwd: ROOT, + env: { + PATH: process.env.PATH ?? '', + TERM: process.env.TERM ?? 'xterm', + HOME: oauthHome, + }, + encoding: 'utf-8', + timeout: 15000, + }); + expect(result.status).toBe(0); + const out = result.stdout?.toString() ?? ''; + expect(out).toMatch(/gemini:\s+NOT READY/); + expect(out).toContain('UNSUPPORTED_CLIENT'); + expect(out).toContain('`agy` adapter'); + expect(out).toContain('GOOGLE_API_KEY'); + } finally { + fs.rmSync(oauthHome, { recursive: true, force: true }); + } + }); + + test('Gemini API-key auth remains locally ready', () => { + const emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-key-home-')); + try { + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], { + cwd: ROOT, + env: { + PATH: process.env.PATH ?? '', + TERM: process.env.TERM ?? 'xterm', + HOME: emptyHome, + GOOGLE_API_KEY: 'offline-readiness-test-key', + }, + encoding: 'utf-8', + timeout: 15000, + }); + expect(result.status).toBe(0); + expect(result.stdout?.toString() ?? '').toMatch(/gemini:\s+OK/); + } finally { + fs.rmSync(emptyHome, { recursive: true, force: true }); + } + }); + test('long prompt is truncated in dry-run display', () => { const longPrompt = 'x'.repeat(200); const r = run(['--prompt', longPrompt, '--dry-run']); diff --git a/test/benchmark-runner.test.ts b/test/benchmark-runner.test.ts index ecd503ea8..cb6c0c197 100644 --- a/test/benchmark-runner.test.ts +++ b/test/benchmark-runner.test.ts @@ -37,6 +37,13 @@ test('estimateCostUsd applies cached input discount alongside uncached input', ( expect(cost2).toBeCloseTo(8.25, 2); }); +test('estimateCostUsd prices normalized Codex cache reads at the discount', () => { + // Raw Codex usage was 21,155 total input with 9,984 cached. The adapter + // normalizes that to 11,171 uncached + 9,984 cached before pricing. + const cost = estimateCostUsd({ input: 11_171, cached: 9_984, output: 5 }, 'gpt-5.4'); + expect(cost).toBeCloseTo(0.030474, 6); +}); + test('PRICING table covers the key model families', () => { expect(PRICING['claude-opus-4-7']).toBeDefined(); expect(PRICING['claude-sonnet-4-6']).toBeDefined(); @@ -72,7 +79,7 @@ test('formatTable handles a report with mixed success/error/unavailable entries' available: true, result: { output: 'ok', - tokens: { input: 100, output: 200 }, + tokens: { input: 100, cached: 50, output: 200 }, durationMs: 800, toolCalls: 3, modelUsed: 'claude-opus-4-7', @@ -107,6 +114,8 @@ test('formatTable handles a report with mixed success/error/unavailable entries' expect(table).toContain('ERROR auth'); expect(table).toContain('unavailable'); expect(table).toContain('9.2/10'); + expect(table).toContain('100+50→200'); + expect(formatMarkdown(report)).toContain('100+50→200'); }); test('formatJson produces parseable JSON', () => { diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index 01ca00b73..d00ac7ab0 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -101,21 +101,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise { diff --git a/test/helpers/providers/gpt.ts b/test/helpers/providers/gpt.ts index 07757dc2f..8a3a66e9f 100644 --- a/test/helpers/providers/gpt.ts +++ b/test/helpers/providers/gpt.ts @@ -46,6 +46,11 @@ export class GptAdapter implements ProviderAdapter { timeout: opts.timeoutMs, encoding: 'utf-8', maxBuffer: 32 * 1024 * 1024, + // Benchmarks can themselves run under an interactive agent process. + // Never let nested Codex inherit that process's stdin: it may wait for + // additional input instead of executing the prompt passed in `args`. + // Keep stdout/stderr piped so JSONL parsing and error routing still work. + stdio: ['ignore', 'pipe', 'pipe'], }); const parsed = this.parseJsonl(out); return { @@ -84,9 +89,10 @@ export class GptAdapter implements ProviderAdapter { * - turn.completed → usage.input_tokens, usage.output_tokens * - thread.started → session id (not used here) */ - private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } { + private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } { let output = ''; let input = 0; + let cached = 0; let out = 0; let toolCalls = 0; let modelUsed: string | undefined; @@ -103,7 +109,13 @@ export class GptAdapter implements ProviderAdapter { } } else if (obj.type === 'turn.completed') { const u = obj.usage ?? {}; - input += u.input_tokens ?? 0; + // Codex reports cached_input_tokens as a subset of input_tokens. + // Normalize to the benchmark contract, where input is uncached-only + // and cached is a separate bucket billed at the cache-read rate. + const turnInput = u.input_tokens ?? 0; + const turnCached = u.cached_input_tokens ?? 0; + input += Math.max(0, turnInput - turnCached); + cached += turnCached; out += u.output_tokens ?? 0; if (obj.model) modelUsed = obj.model; } @@ -111,7 +123,7 @@ export class GptAdapter implements ProviderAdapter { // skip malformed lines — codex stderr can leak in } } - return { output, tokens: { input, output: out }, toolCalls, modelUsed }; + return { output, tokens: { input, output: out, cached }, toolCalls, modelUsed }; } private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { diff --git a/test/helpers/secret-sink-harness.ts b/test/helpers/secret-sink-harness.ts index d97ffd91c..7c69a48b9 100644 --- a/test/helpers/secret-sink-harness.ts +++ b/test/helpers/secret-sink-harness.ts @@ -88,12 +88,12 @@ export async function runWithSecretSink(opts: SecretSinkOptions): Promise { diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index bd60c71cf..cef28e363 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -56,6 +56,22 @@ test('agy adapter enforces sandboxed plan mode and rejects permission bypass', a expect(result.error?.reason).toContain('not allowed'); }); +test('gpt adapter disables inherited stdin for nested codex', () => { + const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/gpt.ts'), 'utf8'); + expect(source).toContain("stdio: ['ignore', 'pipe', 'pipe']"); +}); + +test('gpt adapter normalizes cached input out of Codex total input', () => { + const parseJsonl = (gpt as unknown as { + parseJsonl(raw: string): { tokens: { input: number; output: number; cached?: number } }; + }).parseJsonl.bind(gpt); + const parsed = parseJsonl([ + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } }), + JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } }), + ].join('\n')); + expect(parsed.tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 }); +}); + // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. let workdir: string;