fix(benchmark): parse current gemini stream-json content/stats shape

GeminiAdapter was reading message.text and result.usage, so current CLI
content/stats events produced empty $0 success rows. Accept content with
an assistant role guard, stats token fallbacks, init model, and treat
empty exit-0 output as an error (#2159).

Co-authored-by: Cursor <cursoragent@cursor.com>

test/helpers/providers/gemini.test.ts

@@ -0,0 +1,95 @@
+import { describe, test, expect } from 'bun:test';
+import { parseGeminiStreamJson } from './gemini';
+
+// Current CLI shape (gemini ≥ ~1.58): content + role, tokens under result.stats
+const CURRENT_CLI_FIXTURE = [
+  '{"type":"init","timestamp":"2026-03-20T15:14:46.455Z","session_id":"test-session-123","model":"gemini-2.5-pro"}',
+  '{"type":"message","role":"user","content":"Reply with exactly the word: PONG"}',
+  '{"type":"message","role":"assistant","content":"PONG","delta":true}',
+  '{"type":"result","status":"success","stats":{"input_tokens":24946,"output_tokens":32,"total_tokens":24978}}',
+].join('\n');
+
+// Legacy shape: text field, tokens under result.usage
+const LEGACY_FIXTURE = [
+  '{"type":"message","text":"hello"}',
+  '{"type":"tool_use","name":"run_shell_command"}',
+  '{"type":"result","usage":{"input_token_count":100,"output_token_count":5},"model":"gemini-2.5-flash"}',
+].join('\n');
+
+describe('parseGeminiStreamJson', () => {
+  test('current CLI: reads assistant content, ignores user echo, stats tokens, init model', () => {
+    const parsed = parseGeminiStreamJson(CURRENT_CLI_FIXTURE);
+    expect(parsed.output).toBe('PONG');
+    expect(parsed.tokens.input).toBe(24946);
+    expect(parsed.tokens.output).toBe(32);
+    expect(parsed.modelUsed).toBe('gemini-2.5-pro');
+    expect(parsed.toolCalls).toBe(0);
+  });
+
+  test('current CLI: user role content must not concat into output', () => {
+    const raw = [
+      '{"type":"message","role":"user","content":"PROMPT ECHO"}',
+      '{"type":"message","role":"assistant","content":"ok","delta":true}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('ok');
+    expect(parsed.output).not.toContain('PROMPT ECHO');
+  });
+
+  test('legacy: still accepts text + usage.input_token_count', () => {
+    const parsed = parseGeminiStreamJson(LEGACY_FIXTURE);
+    expect(parsed.output).toBe('hello');
+    expect(parsed.tokens.input).toBe(100);
+    expect(parsed.tokens.output).toBe(5);
+    expect(parsed.toolCalls).toBe(1);
+    expect(parsed.modelUsed).toBe('gemini-2.5-flash');
+  });
+
+  test('legacy text with role:user is ignored', () => {
+    const raw = [
+      '{"type":"message","role":"user","text":"echo"}',
+      '{"type":"message","role":"assistant","text":"kept"}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('kept');
+  });
+
+  test('concatenates multiple assistant content deltas', () => {
+    const raw = [
+      '{"type":"message","role":"assistant","content":"A","delta":true}',
+      '{"type":"message","role":"assistant","content":"B","delta":true}',
+    ].join('\n');
+    expect(parseGeminiStreamJson(raw).output).toBe('AB');
+  });
+
+  test('skips malformed lines without throwing', () => {
+    const raw = [
+      '{"type":"init","model":"m1"}',
+      'not json',
+      '{"type":"message","role":"assistant","content":"x","delta":true}',
+      '{incomplete',
+      '{"type":"result","stats":{"input_tokens":1,"output_tokens":2}}',
+    ].join('\n');
+    const parsed = parseGeminiStreamJson(raw);
+    expect(parsed.output).toBe('x');
+    expect(parsed.tokens).toEqual({ input: 1, output: 2 });
+    expect(parsed.modelUsed).toBe('m1');
+  });
+
+  test('empty / whitespace-only input yields empty parse (no throw)', () => {
+    const parsed = parseGeminiStreamJson('');
+    expect(parsed.output).toBe('');
+    expect(parsed.tokens).toEqual({ input: 0, output: 0 });
+    expect(parsed.toolCalls).toBe(0);
+    expect(parsed.modelUsed).toBeUndefined();
+  });
+
+  test('result.model overrides init.model when both present', () => {
+    const raw = [
+      '{"type":"init","model":"from-init"}',
+      '{"type":"message","role":"assistant","content":"hi"}',
+      '{"type":"result","model":"from-result","stats":{"input_tokens":1,"output_tokens":1}}',
+    ].join('\n');
+    expect(parseGeminiStreamJson(raw).modelUsed).toBe('from-result');
+  });
+});

test/helpers/providers/gemini.ts

@@ -5,6 +5,63 @@ import * as fs from 'fs';
 import * as path from 'path';
 import * as os from 'os';
 
+export type GeminiStreamParse = {
+  output: string;
+  tokens: { input: number; output: number };
+  toolCalls: number;
+  modelUsed?: string;
+};
+
+/**
+ * Parse gemini NDJSON stream events (exported for unit tests).
+ *
+ * Current CLI (`--output-format stream-json`) emits:
+ *   init  → model
+ *   message { role, content, delta? } → concat assistant content
+ *   tool_use → increment toolCalls
+ *   result { stats: { input_tokens, output_tokens } } → tokens
+ *
+ * Legacy shape (still accepted):
+ *   message { text } → concat text
+ *   result { usage: { input_token_count, output_token_count } } → tokens
+ */
+export function parseGeminiStreamJson(raw: string): GeminiStreamParse {
+  let output = '';
+  let input = 0;
+  let out = 0;
+  let toolCalls = 0;
+  let modelUsed: string | undefined;
+  for (const line of raw.split('\n')) {
+    const s = line.trim();
+    if (!s) continue;
+    try {
+      const obj = JSON.parse(s);
+      if (obj.type === 'init') {
+        if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
+      } else if (obj.type === 'message') {
+        // Current CLI: content + role. Role guard is required — the CLI echoes
+        // the user prompt as role:'user', which must not land in output.
+        if (obj.role === 'assistant' && typeof obj.content === 'string') {
+          output += obj.content;
+        } else if (typeof obj.text === 'string' && obj.role !== 'user') {
+          // Legacy text field (no role, or assistant).
+          output += obj.text;
+        }
+      } else if (obj.type === 'tool_use') {
+        toolCalls += 1;
+      } else if (obj.type === 'result') {
+        const u = obj.usage ?? obj.stats ?? {};
+        input += u.input_token_count ?? u.input_tokens ?? u.prompt_tokens ?? 0;
+        out += u.output_token_count ?? u.output_tokens ?? u.completion_tokens ?? 0;
+        if (typeof obj.model === 'string' && obj.model) modelUsed = obj.model;
+      }
+    } catch {
+      // skip malformed lines
+    }
+  }
+  return { output, tokens: { input, output: out }, toolCalls, modelUsed };
+}
+
 /**
  * Gemini adapter — wraps the `gemini` CLI.
  *
@@ -48,13 +105,23 @@ export class GeminiAdapter implements ProviderAdapter {
         encoding: 'utf-8',
         maxBuffer: 32 * 1024 * 1024,
       });
-      const parsed = this.parseStreamJson(out);
+      const parsed = parseGeminiStreamJson(out);
+      const modelUsed = parsed.modelUsed || opts.model || 'gemini-2.5-pro';
+      // Empty-success is indistinguishable from a healthy cheap run in the
+      // comparison table — never report it as a $0 success row (#2159).
+      if (!parsed.output.trim()) {
+        return this.emptyResult(
+          Date.now() - start,
+          { code: 'unknown', reason: 'empty output from gemini CLI (exit 0)' },
+          modelUsed,
+        );
+      }
       return {
         output: parsed.output,
         tokens: parsed.tokens,
         durationMs: Date.now() - start,
         toolCalls: parsed.toolCalls,
-        modelUsed: parsed.modelUsed || opts.model || 'gemini-2.5-pro',
+        modelUsed,
       };
     } catch (err: unknown) {
       const durationMs = Date.now() - start;
@@ -77,41 +144,6 @@ export class GeminiAdapter implements ProviderAdapter {
     return estimateCostUsd(tokens, model ?? 'gemini-2.5-pro');
   }
 
-  /**
-   * Parse gemini NDJSON stream events:
-   *   init  → session id (discarded here)
-   *   message { delta: true, text } → concat to output
-   *   tool_use { name } → increment toolCalls
-   *   result { usage: { input_token_count, output_token_count } } → tokens
-   */
-  private parseStreamJson(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
-    let output = '';
-    let input = 0;
-    let out = 0;
-    let toolCalls = 0;
-    let modelUsed: string | undefined;
-    for (const line of raw.split('\n')) {
-      const s = line.trim();
-      if (!s) continue;
-      try {
-        const obj = JSON.parse(s);
-        if (obj.type === 'message' && typeof obj.text === 'string') {
-          output += obj.text;
-        } else if (obj.type === 'tool_use') {
-          toolCalls += 1;
-        } else if (obj.type === 'result') {
-          const u = obj.usage ?? {};
-          input += u.input_token_count ?? u.prompt_tokens ?? 0;
-          out += u.output_token_count ?? u.completion_tokens ?? 0;
-          if (obj.model) modelUsed = obj.model;
-        }
-      } catch {
-        // skip malformed lines
-      }
-    }
-    return { output, tokens: { input, output: out }, toolCalls, modelUsed };
-  }
-
   private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
     return {
       output: '',

test/skill-e2e-benchmark-providers.test.ts

@@ -129,19 +129,15 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     if (result.error) {
       throw new Error(`gemini errored: ${result.error.code} — ${result.error.reason}`);
     }
-    // Gemini CLI occasionally returns empty output even on successful runs
+    // Adapter must never report empty-success (#2159). After content/stats
-    // (model returned content the CLI parser missed, intermittent stream issues).
+    // parsing, a healthy run has non-empty assistant text + token counts.
-    // We assert the adapter ran end-to-end without erroring and reports a non-
+    expect(result.output.trim().length).toBeGreaterThan(0);
-    // empty token count instead of grepping the literal "ok" — that string
+    expect(result.output.toLowerCase()).toContain('ok');
-    // assertion was too brittle for a smoke that's really about "did the
+    expect(result.tokens.input).toBeGreaterThan(0);
-    // adapter wire up and the run terminate successfully?"
+    expect(result.tokens.output).toBeGreaterThan(0);
-    expect(typeof result.output).toBe('string');
-    // Gemini CLI sometimes returns 0 tokens in the result event (older responses);
-    // assert non-negative instead of strictly positive.
-    expect(result.tokens.input).toBeGreaterThanOrEqual(0);
-    expect(result.tokens.output).toBeGreaterThanOrEqual(0);
     expect(result.durationMs).toBeGreaterThan(0);
     expect(typeof result.modelUsed).toBe('string');
+    expect(result.modelUsed.length).toBeGreaterThan(0);
   }, 150_000);
 
   test('timeout error surfaces as error.code=timeout (no exception)', async () => {