diff --git a/bin/gstack-model-benchmark b/bin/gstack-model-benchmark index c5f5cb5b6..a102f2dcd 100755 --- a/bin/gstack-model-benchmark +++ b/bin/gstack-model-benchmark @@ -31,11 +31,13 @@ import { runBenchmark, formatTable, formatJson, formatMarkdown, type BenchmarkIn import { ClaudeAdapter } from '../test/helpers/providers/claude'; import { GptAdapter } from '../test/helpers/providers/gpt'; import { GeminiAdapter } from '../test/helpers/providers/gemini'; +import { AgyAdapter } from '../test/helpers/providers/agy'; const ADAPTER_FACTORIES = { claude: () => new ClaudeAdapter(), gpt: () => new GptAdapter(), gemini: () => new GeminiAdapter(), + agy: () => new AgyAdapter(), }; type OutputFormat = 'table' | 'json' | 'markdown'; @@ -76,13 +78,13 @@ function positionalArgs(args: string[]): string[] { return positional; } -function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini'> { +function parseProviders(s: string | undefined): Array<'claude' | 'gpt' | 'gemini' | 'agy'> { if (!s) return ['claude']; - const seen = new Set<'claude' | 'gpt' | 'gemini'>(); + const seen = new Set<'claude' | 'gpt' | 'gemini' | 'agy'>(); for (const p of s.split(',').map(x => x.trim()).filter(Boolean)) { - if (p === 'claude' || p === 'gpt' || p === 'gemini') seen.add(p); + if (p === 'claude' || p === 'gpt' || p === 'gemini' || p === 'agy') seen.add(p); else { - console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini.`); + console.error(`WARN: unknown provider '${p}' — skipping. Valid: claude, gpt, gemini, agy.`); } } return seen.size ? Array.from(seen) : ['claude']; @@ -149,7 +151,7 @@ async function main(): Promise { async function dryRunReport(opts: { prompt: string; - providers: Array<'claude' | 'gpt' | 'gemini'>; + providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>; workdir: string; timeoutMs: number; output: OutputFormat; diff --git a/test/benchmark-cli.test.ts b/test/benchmark-cli.test.ts index 8edea3b24..1912f3d8f 100644 --- a/test/benchmark-cli.test.ts +++ b/test/benchmark-cli.test.ts @@ -36,12 +36,13 @@ function run(args: string[], opts: { env?: Record } = {}): { sta describe('gstack-model-benchmark --dry-run', () => { test('prints provider availability report and exits 0', () => { - const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']); + const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']); expect(r.status).toBe(0); expect(r.stdout).toContain('gstack-model-benchmark --dry-run'); expect(r.stdout).toContain('claude'); expect(r.stdout).toContain('gpt'); expect(r.stdout).toContain('gemini'); + expect(r.stdout).toContain('agy'); expect(r.stdout).toContain('no prompts sent'); }); @@ -86,12 +87,12 @@ describe('gstack-model-benchmark --dry-run', () => { }); test('each adapter reports either OK or NOT READY, never crashes', () => { - const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run']); + const r = run(['--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run']); expect(r.status).toBe(0); // Each provider line must end in OK or NOT READY const lines = r.stdout.split('\n'); - const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini):/.test(l)); - expect(adapterLines.length).toBe(3); + const adapterLines = lines.filter(l => /^\s+(claude|gpt|gemini|agy):/.test(l)); + expect(adapterLines.length).toBe(4); for (const line of adapterLines) { expect(line).toMatch(/(OK|NOT READY)/); } @@ -116,7 +117,7 @@ describe('gstack-model-benchmark --dry-run', () => { TERM: process.env.TERM ?? 'xterm', HOME: emptyHome, }; - const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini', '--dry-run'], { + const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'claude,gpt,gemini,agy', '--dry-run'], { cwd: ROOT, env: minimalEnv, encoding: 'utf-8', @@ -124,14 +125,15 @@ describe('gstack-model-benchmark --dry-run', () => { }); expect(result.status).toBe(0); const out = result.stdout?.toString() ?? ''; - // gpt + gemini must report NOT READY in this clean env (their auth check + // gpt + gemini + agy must report NOT READY in this clean env (their auth check // reads paths under the overridden HOME). expect(out).toMatch(/gpt:\s+NOT READY/); expect(out).toMatch(/gemini:\s+NOT READY/); + expect(out).toMatch(/agy:\s+NOT READY/); // Every NOT READY line must include a concrete remediation hint so users // can resolve the missing auth. This is the regression we care about. const notReadyLines = out.split('\n').filter(l => l.includes('NOT READY')); - expect(notReadyLines.length).toBeGreaterThanOrEqual(2); + expect(notReadyLines.length).toBeGreaterThanOrEqual(3); for (const line of notReadyLines) { expect(line).toMatch(/(install|Install|login|export|Run|Log in)/); } diff --git a/test/helpers/benchmark-runner.ts b/test/helpers/benchmark-runner.ts index cbef4107b..01ca00b73 100644 --- a/test/helpers/benchmark-runner.ts +++ b/test/helpers/benchmark-runner.ts @@ -11,15 +11,16 @@ import type { ProviderAdapter, RunOpts, RunResult } from './providers/types'; import { ClaudeAdapter } from './providers/claude'; import { GptAdapter } from './providers/gpt'; import { GeminiAdapter } from './providers/gemini'; +import { AgyAdapter } from './providers/agy'; export interface BenchmarkInput { prompt: string; workdir: string; timeoutMs?: number; - /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini']). */ - providers: Array<'claude' | 'gpt' | 'gemini'>; + /** Adapter names to run (e.g., ['claude', 'gpt', 'gemini', 'agy']). */ + providers: Array<'claude' | 'gpt' | 'gemini' | 'agy'>; /** Optional per-provider model overrides. */ - models?: Partial>; + models?: Partial>; /** If true, skip providers whose available() returns !ok. If false, include them with error. */ skipUnavailable?: boolean; } @@ -44,10 +45,11 @@ export interface BenchmarkReport { entries: BenchmarkEntry[]; } -const ADAPTERS: Record<'claude' | 'gpt' | 'gemini', () => ProviderAdapter> = { +const ADAPTERS: Record<'claude' | 'gpt' | 'gemini' | 'agy', () => ProviderAdapter> = { claude: () => new ClaudeAdapter(), gpt: () => new GptAdapter(), gemini: () => new GeminiAdapter(), + agy: () => new AgyAdapter(), }; export async function runBenchmark(input: BenchmarkInput): Promise { diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts new file mode 100644 index 000000000..3a50efe6d --- /dev/null +++ b/test/helpers/providers/agy.ts @@ -0,0 +1,87 @@ +import type { ProviderAdapter, RunOpts, RunResult, AvailabilityCheck } from './types'; +import { estimateCostUsd } from '../pricing'; +import { execFileSync, spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; + +/** + * Agy adapter — wraps the `agy` CLI using `agy --print`. + * + * Auth comes from ~/.gemini/oauth_creds.json. + */ +export class AgyAdapter implements ProviderAdapter { + readonly name = 'agy'; + readonly family = 'gemini' as const; + + async available(): Promise { + const res = spawnSync('sh', ['-c', 'command -v agy'], { timeout: 2000 }); + if (res.status !== 0) { + return { ok: false, reason: 'agy CLI not found on PATH. Install per Google Antigravity instructions.' }; + } + const newCfgDir = path.join(os.homedir(), '.gemini'); + const newOauth = path.join(newCfgDir, 'oauth_creds.json'); + const hasCfg = fs.existsSync(newOauth); + if (!hasCfg) { + return { ok: false, reason: 'No Agy auth found. Log in via `agy` or authenticate.' }; + } + return { ok: true }; + } + + async run(opts: RunOpts): Promise { + const start = Date.now(); + const args = ['--print', opts.prompt, '--dangerously-skip-permissions']; + if (opts.model) args.push('--model', opts.model); + if (opts.extraArgs) args.push(...opts.extraArgs); + + try { + const out = execFileSync('agy', args, { + cwd: opts.workdir, + timeout: opts.timeoutMs, + encoding: 'utf-8', + maxBuffer: 32 * 1024 * 1024, + }); + + // Estimate tokens as agy does not output NDJSON/JSON tokens yet in print mode + const inputTokens = Math.ceil(opts.prompt.length / 4); + const outputTokens = Math.ceil(out.length / 4); + + return { + output: out.trim(), + tokens: { input: inputTokens, output: outputTokens }, + durationMs: Date.now() - start, + toolCalls: 0, + modelUsed: opts.model || 'gemini-2.5-flash', + }; + } catch (err: unknown) { + const durationMs = Date.now() - start; + const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; + const stderr = e.stderr?.toString() ?? ''; + if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { + return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); + } + if (/unauthorized|auth|login|api key/i.test(stderr)) { + return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + } + if (/rate[- ]?limit|429|quota/i.test(stderr)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + } + } + + estimateCost(tokens: { input: number; output: number; cached?: number }, model?: string): number { + return estimateCostUsd(tokens, model ?? 'gemini-2.5-flash'); + } + + private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult { + return { + output: '', + tokens: { input: 0, output: 0 }, + durationMs, + toolCalls: 0, + modelUsed: model ?? 'gemini-2.5-flash', + error, + }; + } +} diff --git a/test/skill-e2e-benchmark-providers.test.ts b/test/skill-e2e-benchmark-providers.test.ts index 12456ec23..446762d11 100644 --- a/test/skill-e2e-benchmark-providers.test.ts +++ b/test/skill-e2e-benchmark-providers.test.ts @@ -22,6 +22,7 @@ import { describe, test, expect, beforeAll, afterAll } from 'bun:test'; import { ClaudeAdapter } from './helpers/providers/claude'; import { GptAdapter } from './helpers/providers/gpt'; import { GeminiAdapter } from './helpers/providers/gemini'; +import { AgyAdapter } from './helpers/providers/agy'; import { runBenchmark } from './helpers/benchmark-runner'; import * as fs from 'fs'; import * as path from 'path'; @@ -39,6 +40,7 @@ const PROMPT = 'Reply with exactly this text and nothing else: ok'; const claude = new ClaudeAdapter(); const gpt = new GptAdapter(); const gemini = new GeminiAdapter(); +const agy = new AgyAdapter(); // Use a temp working directory so provider CLIs can't accidentally touch the repo. // Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak. @@ -80,6 +82,14 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { } }); + test('agy: available() returns structured ok/reason', async () => { + const check = await agy.available(); + expect(check).toHaveProperty('ok'); + if (!check.ok) { + expect(typeof check.reason).toBe('string'); + } + }); + test('claude: trivial prompt produces parseable output', async () => { const check = await claude.available(); if (!check.ok) { @@ -144,11 +154,31 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { expect(typeof result.modelUsed).toBe('string'); }, 150_000); + test('agy: trivial prompt produces parseable output', async () => { + const check = await agy.available(); + if (!check.ok) { + process.stderr.write(`\nagy live smoke: SKIPPED — ${check.reason}\n`); + return; + } + const result = await agy.run({ prompt: PROMPT, workdir, timeoutMs: 120_000 }); + if (result.error) { + throw new Error(`agy errored: ${result.error.code} — ${result.error.reason}`); + } + expect(typeof result.output).toBe('string'); + expect(result.tokens.input).toBeGreaterThan(0); + expect(result.tokens.output).toBeGreaterThan(0); + expect(result.durationMs).toBeGreaterThan(0); + expect(typeof result.modelUsed).toBe('string'); + const cost = agy.estimateCost(result.tokens, result.modelUsed); + expect(cost).toBeGreaterThan(0); + }, 150_000); + test('timeout error surfaces as error.code=timeout (no exception)', async () => { - // Use whatever adapter is available first — all three should share timeout semantics. + // Use whatever adapter is available first — all should share timeout semantics. const adapter = (await claude.available()).ok ? claude : (await gpt.available()).ok ? gpt : (await gemini.available()).ok ? gemini + : (await agy.available()).ok ? agy : null; if (!adapter) { process.stderr.write('\ntimeout smoke: SKIPPED — no provider available\n'); @@ -164,16 +194,16 @@ describeIfEvals('multi-provider benchmark adapters (live)', () => { }, 30_000); test('runBenchmark: Promise.allSettled means one unavailable provider does not block others', async () => { - // Use the full runner with all three providers — whichever are unauthed should + // Use the full runner with all providers — whichever are unauthed should // return entries with available=false and not crash the batch. const report = await runBenchmark({ prompt: PROMPT, workdir, - providers: ['claude', 'gpt', 'gemini'], + providers: ['claude', 'gpt', 'gemini', 'agy'], timeoutMs: 120_000, skipUnavailable: false, }); - expect(report.entries).toHaveLength(3); + expect(report.entries).toHaveLength(4); for (const e of report.entries) { expect(['claude', 'gpt', 'gemini']).toContain(e.family); if (e.available) {