feat(benchmark): add Agy model benchmark provider and runner adapter

bin/gstack-model-benchmark

@@ -31,11 +31,13 @@ import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkIn
 import { ClaudeAdapter } from '../test/helpers/providers/claude';
 import { GptAdapter } from '../test/helpers/providers/gpt';
 import { GeminiAdapter } from '../test/helpers/providers/gemini';
+import { AgyAdapter } from '../test/helpers/providers/agy';
 
 const ADAPTER_FACTORIES = {
   claude: () => new ClaudeAdapter(),
   gpt: () => new GptAdapter(),
   gemini: () => new GeminiAdapter(),
+  agy: () => new AgyAdapter(),
 };
 
 type OutputFormat = 'table' | 'json' | 'markdown';
@@ -76,13 +78,13 @@ function positionalArgs(args: string[]): string[] {
   return positional;
 }
 
-function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> {
+function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini' | 'agy'> {
   if (!s) return ['claude'];
-  const seen = new Set<'claude' | 'gpt' | 'gemini'>();
+  const seen = new Set<'claude' | 'gpt' | 'gemini' | 'agy'>();
   for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) {
-    if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p);
+    if (p === 'claude' || p === 'gpt' || p === 'gemini' || p === 'agy') seen.add(p);
     else {
-      console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`);
+      console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini, agy.`);
     }
   }
   return seen.size ? Array.from(seen) : ['claude'];
@@ -149,7 +151,7 @@ async function main(): Promise<void> {
 
 async function dryRunReport(opts: {
   prompt: string;
-  providers: Array<'claude' | 'gpt' | 'gemini'>;
+  providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>;
   workdir: string;
   timeoutMs: number;
   output: OutputFormat;

test/benchmark-cli.test.ts

@@ -36,12 +36,13 @@ function run(args: string[], opts: { env?: Record<string, string> } = {}): { sta
 
 describe('gstack-model-benchmark --dry-run', () => {
   test('prints provider availability report and exits 0', () => {
-    const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']);
+    const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']);
     expect(r.status).toBe(0);
     expect(r.stdout).toContain('gstack-model-benchmark --dry-run');
     expect(r.stdout).toContain('claude');
     expect(r.stdout).toContain('gpt');
     expect(r.stdout).toContain('gemini');
+    expect(r.stdout).toContain('agy');
     expect(r.stdout).toContain('no prompts sent');
   });
 
@@ -86,12 +87,12 @@ describe('gstack-model-benchmark --dry-run', () => {
   });
 
   test('each adapter reports either OK or NOT READY, never crashes', () => {
-    const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']);
+    const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']);
     expect(r.status).toBe(0);
     // Each provider line must end in OK or NOT READY
     const lines = r.stdout.split('\n');
-    const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini):/.test(l));
+    const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini|agy):/.test(l));
-    expect(adapterLines.length).toBe(3);
+    expect(adapterLines.length).toBe(4);
     for (const line of adapterLines) {
       expect(line).toMatch(/(OK|NOT READY)/);
     }
@@ -116,7 +117,7 @@ describe('gstack-model-benchmark --dry-run', () => {
         TERM: process.env.TERM ?? 'xterm',
         HOME: emptyHome,
       };
-      const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run'], {
+      const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run'], {
         cwd: ROOT,
         env: minimalEnv,
         encoding: 'utf-8',
@@ -124,14 +125,15 @@ describe('gstack-model-benchmark --dry-run', () => {
       });
       expect(result.status).toBe(0);
       const out = result.stdout?.toString() ?? '';
-      // gpt + gemini must report NOT READY in this clean env (their auth check
+      // gpt + gemini + agy must report NOT READY in this clean env (their auth check
       // reads paths under the overridden HOME).
       expect(out).toMatch(/gpt:\s+NOT READY/);
       expect(out).toMatch(/gemini:\s+NOT READY/);
+      expect(out).toMatch(/agy:\s+NOT READY/);
       // Every NOT READY line must include a concrete remediation hint so users
       // can resolve the missing auth. This is the regression we care about.
       const notReadyLines = out.split('\n').filter(l => l.includes('NOT READY'));
-      expect(notReadyLines.length).toBeGreaterThanOrEqual(2);
+      expect(notReadyLines.length).toBeGreaterThanOrEqual(3);
       for (const line of notReadyLines) {
         expect(line).toMatch(/(install|Install|login|export|Run|Log in)/);
       }

test/helpers/benchmark-runner.ts

@@ -11,15 +11,16 @@ import type { ProviderAdapter, RunOpts, RunResult } from './providers/types';
 import { ClaudeAdapter } from './providers/claude';
 import { GptAdapter } from './providers/gpt';
 import { GeminiAdapter } from './providers/gemini';
+import { AgyAdapter } from './providers/agy';
 
 export interface BenchmarkInput {
   prompt: string;
   workdir: string;
   timeoutMs?: number;
-  /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */
+  /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini', 'agy']). */
-  providers: Array<'claude' | 'gpt' | 'gemini'>;
+  providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>;
   /** Optional per-provider model overrides. */
-  models?: Partial<Record<'claude' | 'gpt' | 'gemini', string>>;
+  models?: Partial<Record<'claude' | 'gpt' | 'gemini' | 'agy', string>>;
   /** If true, skip providers whose available() returns !ok. If false, include them with error. */
   skipUnavailable?: boolean;
 }
@@ -44,10 +45,11 @@ export interface BenchmarkReport {
   entries: BenchmarkEntry[];
 }
 
-const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = {
+const ADAPTERS: Record<'claude' | 'gpt' | 'gemini' | 'agy', () => ProviderAdapter> = {
   claude: () => new ClaudeAdapter(),
   gpt: () => new GptAdapter(),
   gemini: () => new GeminiAdapter(),
+  agy: () => new AgyAdapter(),
 };
 
 export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkReport> {

test/helpers/providers/agy.ts

@@ -0,0 +1,87 @@
+import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types';
+import { estimateCostUsd } from '../pricing';
+import { execFileSync, spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+/**
+ * Agy adapter — wraps the `agy` CLI using `agy --print`.
+ *
+ * Auth comes from ~/.gemini/oauth_creds.json.
+ */
+export class AgyAdapter implements ProviderAdapter {
+  readonly name = 'agy';
+  readonly family = 'gemini' as const;
+
+  async available(): Promise<AvailabilityCheck> {
+    const res = spawnSync('sh', ['-c', 'command -v agy'], { timeout: 2000 });
+    if (res.status !== 0) {
+      return { ok: false, reason: 'agy CLI not found on PATH. Install per Google Antigravity instructions.' };
+    }
+    const newCfgDir = path.join(os.homedir(), '.gemini');
+    const newOauth = path.join(newCfgDir, 'oauth_creds.json');
+    const hasCfg = fs.existsSync(newOauth);
+    if (!hasCfg) {
+      return { ok: false, reason: 'No Agy auth found. Log in via `agy` or authenticate.' };
+    }
+    return { ok: true };
+  }
+
+  async run(opts: RunOpts): Promise<RunResult> {
+    const start = Date.now();
+    const args = ['--print', opts.prompt, '--dangerously-skip-permissions'];
+    if (opts.model) args.push('--model', opts.model);
+    if (opts.extraArgs) args.push(...opts.extraArgs);
+
+    try {
+      const out = execFileSync('agy', args, {
+        cwd: opts.workdir,
+        timeout: opts.timeoutMs,
+        encoding: 'utf-8',
+        maxBuffer: 32 * 1024 * 1024,
+      });
+
+      // Estimate tokens as agy does not output NDJSON/JSON tokens yet in print mode
+      const inputTokens = Math.ceil(opts.prompt.length / 4);
+      const outputTokens = Math.ceil(out.length / 4);
+
+      return {
+        output: out.trim(),
+        tokens: { input: inputTokens, output: outputTokens },
+        durationMs: Date.now() - start,
+        toolCalls: 0,
+        modelUsed: opts.model || 'gemini-2.5-flash',
+      };
+    } catch (err: unknown) {
+      const durationMs = Date.now() - start;
+      const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
+      const stderr = e.stderr?.toString() ?? '';
+      if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
+        return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
+      }
+      if (/unauthorized|auth|login|api key/i.test(stderr)) {
+        return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
+      }
+      if (/rate[- ]?limit|429|quota/i.test(stderr)) {
+        return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
+      }
+      return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
+    }
+  }
+
+  estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number {
+    return estimateCostUsd(tokens, model ?? 'gemini-2.5-flash');
+  }
+
+  private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
+    return {
+      output: '',
+      tokens: { input: 0, output: 0 },
+      durationMs,
+      toolCalls: 0,
+      modelUsed: model ?? 'gemini-2.5-flash',
+      error,
+    };
+  }
+}

test/skill-e2e-benchmark-providers.test.ts

@@ -22,6 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test';
 import { ClaudeAdapter } from './helpers/providers/claude';
 import { GptAdapter } from './helpers/providers/gpt';
 import { GeminiAdapter } from './helpers/providers/gemini';
+import { AgyAdapter } from './helpers/providers/agy';
 import { runBenchmark } from './helpers/benchmark-runner';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -39,6 +40,7 @@ const PROMPT = 'Reply with exactly this text and nothing else: ok';
 const claude = new ClaudeAdapter();
 const gpt = new GptAdapter();
 const gemini = new GeminiAdapter();
+const agy = new AgyAdapter();
 
 // Use a temp working directory so provider CLIs can't accidentally touch the repo.
 // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak.
@@ -80,6 +82,14 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     }
   });
 
+  test('agy: available() returns structured ok/reason', async () => {
+    const check = await agy.available();
+    expect(check).toHaveProperty('ok');
+    if (!check.ok) {
+      expect(typeof check.reason).toBe('string');
+    }
+  });
+
   test('claude: trivial prompt produces parseable output', async () => {
     const check = await claude.available();
     if (!check.ok) {
@@ -144,11 +154,31 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
     expect(typeof result.modelUsed).toBe('string');
   }, 150_000);
 
+  test('agy: trivial prompt produces parseable output', async () => {
+    const check = await agy.available();
+    if (!check.ok) {
+      process.stderr.write(`\nagy live smoke: SKIPPED — ${check.reason}\n`);
+      return;
+    }
+    const result = await agy.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 });
+    if (result.error) {
+      throw new Error(`agy errored: ${result.error.code} — ${result.error.reason}`);
+    }
+    expect(typeof result.output).toBe('string');
+    expect(result.tokens.input).toBeGreaterThan(0);
+    expect(result.tokens.output).toBeGreaterThan(0);
+    expect(result.durationMs).toBeGreaterThan(0);
+    expect(typeof result.modelUsed).toBe('string');
+    const cost = agy.estimateCost(result.tokens, result.modelUsed);
+    expect(cost).toBeGreaterThan(0);
+  }, 150_000);
+
   test('timeout error surfaces as error.code=timeout (no exception)', async () => {
-    // Use whatever adapter is available first — all three should share timeout semantics.
+    // Use whatever adapter is available first — all should share timeout semantics.
     const adapter = (await claude.available()).ok ? claude
       : (await gpt.available()).ok ? gpt
       : (await gemini.available()).ok ? gemini
+      : (await agy.available()).ok ? agy
       : null;
     if (!adapter) {
       process.stderr.write('\ntimeout smoke: SKIPPED — no provider available\n');
@@ -164,16 +194,16 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => {
   }, 30_000);
 
   test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => {
-    // Use the full runner with all three providers — whichever are unauthed should
+    // Use the full runner with all providers — whichever are unauthed should
     // return entries with available=false and not crash the batch.
     const report = await runBenchmark({
       prompt: PROMPT,
       workdir,
-      providers: ['claude', 'gpt', 'gemini'],
+      providers: ['claude', 'gpt', 'gemini', 'agy'],
       timeoutMs: 120_000,
       skipUnavailable: false,
     });
-    expect(report.entries).toHaveLength(3);
+    expect(report.entries).toHaveLength(4);
     for (const e of report.entries) {
       expect(['claude', 'gpt', 'gemini']).toContain(e.family);
       if (e.available) {