mirror of https://github.com/garrytan/gstack.git
fix benchmark adapters and stdin handling
This commit is contained in:
Mattias Petersson
parent
49964ee712
commit
9c2414016c
|
|
@ -142,6 +142,54 @@ describe('gstack-model-benchmark --dry-run', () => {
|
|||
}
|
||||
});
|
||||
|
||||
test('Gemini OAuth-only auth is reported as NOT READY with Antigravity remediation', () => {
|
||||
const oauthHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-oauth-home-'));
|
||||
try {
|
||||
const oauthDir = path.join(oauthHome, '.gemini');
|
||||
fs.mkdirSync(oauthDir, { recursive: true });
|
||||
fs.writeFileSync(path.join(oauthDir, 'oauth_creds.json'), '{}');
|
||||
const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], {
|
||||
cwd: ROOT,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? '',
|
||||
TERM: process.env.TERM ?? 'xterm',
|
||||
HOME: oauthHome,
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 15000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
const out = result.stdout?.toString() ?? '';
|
||||
expect(out).toMatch(/gemini:\s+NOT READY/);
|
||||
expect(out).toContain('UNSUPPORTED_CLIENT');
|
||||
expect(out).toContain('`agy` adapter');
|
||||
expect(out).toContain('GOOGLE_API_KEY');
|
||||
} finally {
|
||||
fs.rmSync(oauthHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('Gemini API-key auth remains locally ready', () => {
|
||||
const emptyHome = fs.mkdtempSync(path.join(os.tmpdir(), 'bench-gemini-key-home-'));
|
||||
try {
|
||||
const result = spawnSync('bun', ['run', BIN, '--prompt', 'hi', '--models', 'gemini', '--dry-run'], {
|
||||
cwd: ROOT,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? '',
|
||||
TERM: process.env.TERM ?? 'xterm',
|
||||
HOME: emptyHome,
|
||||
GOOGLE_API_KEY: 'offline-readiness-test-key',
|
||||
},
|
||||
encoding: 'utf-8',
|
||||
timeout: 15000,
|
||||
});
|
||||
expect(result.status).toBe(0);
|
||||
expect(result.stdout?.toString() ?? '').toMatch(/gemini:\s+OK/);
|
||||
} finally {
|
||||
fs.rmSync(emptyHome, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('long prompt is truncated in dry-run display', () => {
|
||||
const longPrompt = 'x'.repeat(200);
|
||||
const r = run(['--prompt', longPrompt, '--dry-run']);
|
||||
|
|
|
|||
|
|
@ -37,6 +37,13 @@ test('estimateCostUsd applies cached input discount alongside uncached input', (
|
|||
expect(cost2).toBeCloseTo(8.25, 2);
|
||||
});
|
||||
|
||||
test('estimateCostUsd prices normalized Codex cache reads at the discount', () => {
|
||||
// Raw Codex usage was 21,155 total input with 9,984 cached. The adapter
|
||||
// normalizes that to 11,171 uncached + 9,984 cached before pricing.
|
||||
const cost = estimateCostUsd({ input: 11_171, cached: 9_984, output: 5 }, 'gpt-5.4');
|
||||
expect(cost).toBeCloseTo(0.030474, 6);
|
||||
});
|
||||
|
||||
test('PRICING table covers the key model families', () => {
|
||||
expect(PRICING['claude-opus-4-7']).toBeDefined();
|
||||
expect(PRICING['claude-sonnet-4-6']).toBeDefined();
|
||||
|
|
@ -72,7 +79,7 @@ test('formatTable handles a report with mixed success/error/unavailable entries'
|
|||
available: true,
|
||||
result: {
|
||||
output: 'ok',
|
||||
tokens: { input: 100, output: 200 },
|
||||
tokens: { input: 100, cached: 50, output: 200 },
|
||||
durationMs: 800,
|
||||
toolCalls: 3,
|
||||
modelUsed: 'claude-opus-4-7',
|
||||
|
|
@ -107,6 +114,8 @@ test('formatTable handles a report with mixed success/error/unavailable entries'
|
|||
expect(table).toContain('ERROR auth');
|
||||
expect(table).toContain('unavailable');
|
||||
expect(table).toContain('9.2/10');
|
||||
expect(table).toContain('100+50→200');
|
||||
expect(formatMarkdown(report)).toContain('100+50→200');
|
||||
});
|
||||
|
||||
test('formatJson produces parseable JSON', () => {
|
||||
|
|
|
|||
|
|
@ -101,21 +101,21 @@ export async function runBenchmark(input: BenchmarkInput): Promise<BenchmarkRepo
|
|||
}
|
||||
|
||||
export function formatTable(report: BenchmarkReport): string {
|
||||
const header = `Model Latency In→Out Tokens Cost Quality Tool Calls Notes`;
|
||||
const header = `Model Latency Input (new+cached)→Out Cost Quality Tool Calls Notes`;
|
||||
const sep = '-'.repeat(header.length);
|
||||
const rows: string[] = [header, sep];
|
||||
for (const e of report.entries) {
|
||||
if (!e.available) {
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 20)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
rows.push(`${pad(e.provider, 20)} ${pad('-', 9)} ${pad('-', 24)} ${pad('-', 10)} ${pad('-', 9)} ${pad('-', 12)} unavailable: ${e.unavailable_reason ?? 'unknown'}`);
|
||||
continue;
|
||||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad('-', 9)} ${pad(String(r.toolCalls), 12)} ERROR ${r.error.code}: ${r.error.reason.slice(0, 40)}`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(`${r.tokens.input}→${r.tokens.output}`, 20)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
rows.push(`${pad(r.modelUsed, 20)} ${pad(msToStr(r.durationMs), 9)} ${pad(fmtTokens(r.tokens), 24)} ${pad(fmtCost(e.costUsd), 10)} ${pad(quality, 9)} ${pad(String(r.toolCalls), 12)}`);
|
||||
}
|
||||
return rows.join('\n');
|
||||
}
|
||||
|
|
@ -132,7 +132,7 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
|||
`**Workdir:** \`${report.workdir}\``,
|
||||
`**Total duration:** ${msToStr(report.durationMs)}`,
|
||||
'',
|
||||
'| Model | Latency | Tokens (in→out) | Cost | Quality | Tools | Notes |',
|
||||
'| Model | Latency | Tokens (new+cached→out) | Cost | Quality | Tools | Notes |',
|
||||
'|-------|---------|-----------------|------|---------|-------|-------|',
|
||||
];
|
||||
for (const e of report.entries) {
|
||||
|
|
@ -142,11 +142,11 @@ export function formatMarkdown(report: BenchmarkReport): string {
|
|||
}
|
||||
const r = e.result!;
|
||||
if (r.error) {
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | - | ${r.toolCalls} | ERROR ${r.error.code}: ${r.error.reason.slice(0, 80)} |`);
|
||||
continue;
|
||||
}
|
||||
const quality = e.qualityScore !== undefined ? `${e.qualityScore.toFixed(1)}/10` : '-';
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${r.tokens.input}→${r.tokens.output} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
lines.push(`| ${r.modelUsed} | ${msToStr(r.durationMs)} | ${fmtTokens(r.tokens)} | ${fmtCost(e.costUsd)} | ${quality} | ${r.toolCalls} | |`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
|
@ -160,6 +160,12 @@ function msToStr(ms: number): string {
|
|||
return `${(ms / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
function fmtTokens(tokens: { input: number; output: number; cached?: number }): string {
|
||||
return tokens.cached === undefined
|
||||
? `${tokens.input}→${tokens.output}`
|
||||
: `${tokens.input}+${tokens.cached}→${tokens.output}`;
|
||||
}
|
||||
|
||||
function fmtCost(usd?: number): string {
|
||||
if (usd === undefined) return '-';
|
||||
if (usd < 0.01) return `$${usd.toFixed(4)}`;
|
||||
|
|
|
|||
|
|
@ -25,12 +25,18 @@ export class GeminiAdapter implements ProviderAdapter {
|
|||
const legacyCfgDir = path.join(os.homedir(), '.config', 'gemini');
|
||||
const newCfgDir = path.join(os.homedir(), '.gemini');
|
||||
const newOauth = path.join(newCfgDir, 'oauth_creds.json');
|
||||
const hasCfg = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
|
||||
const hasOauth = fs.existsSync(legacyCfgDir) || fs.existsSync(newOauth);
|
||||
const hasKey = !!process.env.GOOGLE_API_KEY;
|
||||
if (!hasCfg && !hasKey) {
|
||||
return { ok: false, reason: 'No Gemini auth found. Log in via `gemini login` or export GOOGLE_API_KEY.' };
|
||||
if (hasKey) {
|
||||
return { ok: true };
|
||||
}
|
||||
return { ok: true };
|
||||
if (hasOauth) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: 'Stored Gemini OAuth is not a reliable readiness signal and may fail with UNSUPPORTED_CLIENT. Use the `agy` adapter for Google subscription auth, or export GOOGLE_API_KEY.',
|
||||
};
|
||||
}
|
||||
return { ok: false, reason: 'No Gemini auth found. Use the `agy` adapter for Google subscription auth, or export GOOGLE_API_KEY.' };
|
||||
}
|
||||
|
||||
async run(opts: RunOpts): Promise<RunResult> {
|
||||
|
|
|
|||
|
|
@ -46,6 +46,11 @@ export class GptAdapter implements ProviderAdapter {
|
|||
timeout: opts.timeoutMs,
|
||||
encoding: 'utf-8',
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
// Benchmarks can themselves run under an interactive agent process.
|
||||
// Never let nested Codex inherit that process's stdin: it may wait for
|
||||
// additional input instead of executing the prompt passed in `args`.
|
||||
// Keep stdout/stderr piped so JSONL parsing and error routing still work.
|
||||
stdio: ['ignore', 'pipe', 'pipe'],
|
||||
});
|
||||
const parsed = this.parseJsonl(out);
|
||||
return {
|
||||
|
|
@ -84,9 +89,10 @@ export class GptAdapter implements ProviderAdapter {
|
|||
* - turn.completed → usage.input_tokens, usage.output_tokens
|
||||
* - thread.started → session id (not used here)
|
||||
*/
|
||||
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number }; toolCalls: number; modelUsed?: string } {
|
||||
private parseJsonl(raw: string): { output: string; tokens: { input: number; output: number; cached?: number }; toolCalls: number; modelUsed?: string } {
|
||||
let output = '';
|
||||
let input = 0;
|
||||
let cached = 0;
|
||||
let out = 0;
|
||||
let toolCalls = 0;
|
||||
let modelUsed: string | undefined;
|
||||
|
|
@ -103,7 +109,13 @@ export class GptAdapter implements ProviderAdapter {
|
|||
}
|
||||
} else if (obj.type === 'turn.completed') {
|
||||
const u = obj.usage ?? {};
|
||||
input += u.input_tokens ?? 0;
|
||||
// Codex reports cached_input_tokens as a subset of input_tokens.
|
||||
// Normalize to the benchmark contract, where input is uncached-only
|
||||
// and cached is a separate bucket billed at the cache-read rate.
|
||||
const turnInput = u.input_tokens ?? 0;
|
||||
const turnCached = u.cached_input_tokens ?? 0;
|
||||
input += Math.max(0, turnInput - turnCached);
|
||||
cached += turnCached;
|
||||
out += u.output_tokens ?? 0;
|
||||
if (obj.model) modelUsed = obj.model;
|
||||
}
|
||||
|
|
@ -111,7 +123,7 @@ export class GptAdapter implements ProviderAdapter {
|
|||
// skip malformed lines — codex stderr can leak in
|
||||
}
|
||||
}
|
||||
return { output, tokens: { input, output: out }, toolCalls, modelUsed };
|
||||
return { output, tokens: { input, output: out, cached }, toolCalls, modelUsed };
|
||||
}
|
||||
|
||||
private emptyResult(durationMs: number, error: RunResult['error'], model?: string): RunResult {
|
||||
|
|
|
|||
|
|
@ -88,12 +88,12 @@ export async function runWithSecretSink(opts: SecretSinkOptions): Promise<SinkRe
|
|||
env,
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
stdin: opts.stdin ? 'pipe' : 'ignore',
|
||||
// Supplying the complete buffer at spawn time is reliable across Bun
|
||||
// versions and restricted runners. A writable `stdin: 'pipe'` can close
|
||||
// before buffered bytes reach the child (Bun 1.3.14), producing empty
|
||||
// input even when write() and end() are called in order.
|
||||
stdin: opts.stdin === undefined ? 'ignore' : Buffer.from(opts.stdin),
|
||||
});
|
||||
if (opts.stdin) {
|
||||
proc.stdin!.write(opts.stdin);
|
||||
proc.stdin!.end();
|
||||
}
|
||||
|
||||
const timeoutMs = opts.timeoutMs ?? 10_000;
|
||||
const timeoutHandle = setTimeout(() => {
|
||||
|
|
|
|||
|
|
@ -56,6 +56,22 @@ test('agy adapter enforces sandboxed plan mode and rejects permission bypass', a
|
|||
expect(result.error?.reason).toContain('not allowed');
|
||||
});
|
||||
|
||||
test('gpt adapter disables inherited stdin for nested codex', () => {
|
||||
const source = fs.readFileSync(path.join(import.meta.dir, 'helpers/providers/gpt.ts'), 'utf8');
|
||||
expect(source).toContain("stdio: ['ignore', 'pipe', 'pipe']");
|
||||
});
|
||||
|
||||
test('gpt adapter normalizes cached input out of Codex total input', () => {
|
||||
const parseJsonl = (gpt as unknown as {
|
||||
parseJsonl(raw: string): { tokens: { input: number; output: number; cached?: number } };
|
||||
}).parseJsonl.bind(gpt);
|
||||
const parsed = parseJsonl([
|
||||
JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 21_155, cached_input_tokens: 9_984, output_tokens: 5 } }),
|
||||
JSON.stringify({ type: 'turn.completed', usage: { input_tokens: 200, cached_input_tokens: 150, output_tokens: 10 } }),
|
||||
].join('\n'));
|
||||
expect(parsed.tokens).toEqual({ input: 11_221, cached: 10_134, output: 15 });
|
||||
});
|
||||
|
||||
// Use a temp working directory so provider CLIs can't accidentally touch the repo.
|
||||
// Created in beforeAll / cleaned in afterAll so concurrent CI runs don't leak.
|
||||
let workdir: string;
|
||||
|
|
|
|||
Loading…
Reference in New Issue