From e9fb8f311bcbe69a831fa022cd1b85548d4c7678 Mon Sep 17 00:00:00 2001 From: Mattias Petersson Date: Thu, 16 Jul 2026 09:18:36 +0200 Subject: [PATCH] classify provider capacity errors --- test/benchmark-provider-errors.test.ts | 27 ++++++++++++++++++++++++++ test/gen-skill-docs.test.ts | 2 +- test/helpers/providers/agy.ts | 16 +++++++++------ test/helpers/providers/claude.ts | 16 +++++++++------ test/helpers/providers/errors.ts | 20 +++++++++++++++++++ test/helpers/providers/gemini.ts | 16 +++++++++------ test/helpers/providers/gpt.ts | 16 +++++++++------ test/helpers/providers/types.ts | 1 + 8 files changed, 89 insertions(+), 25 deletions(-) create mode 100644 test/benchmark-provider-errors.test.ts create mode 100644 test/helpers/providers/errors.ts diff --git a/test/benchmark-provider-errors.test.ts b/test/benchmark-provider-errors.test.ts new file mode 100644 index 000000000..ff2b02f04 --- /dev/null +++ b/test/benchmark-provider-errors.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, test } from 'bun:test'; +import { isCapacityError, providerErrorDetail } from './helpers/providers/errors'; + +describe('benchmark provider error classification', () => { + test('recognizes the Antigravity model-capacity message', () => { + expect(isCapacityError('Selected model is at capacity. Please try a different model')).toBe(true); + }); + + test('recognizes overloaded and high-load variants', () => { + expect(isCapacityError('The provider is overloaded right now')).toBe(true); + expect(isCapacityError('Model temporarily unavailable due to high demand')).toBe(true); + }); + + test('does not misclassify auth, quota, or generic failures as capacity', () => { + expect(isCapacityError('Error 401: login required')).toBe(false); + expect(isCapacityError('429 quota exceeded')).toBe(false); + expect(isCapacityError('connection reset by peer')).toBe(false); + }); + + test('combines stderr and message without duplicating subprocess output', () => { + const stderr = 'Selected model is at capacity.'; + expect(providerErrorDetail({ stderr: Buffer.from(stderr), message: `Command failed\n${stderr}` })) + .toBe(`Command failed\n${stderr}`); + expect(providerErrorDetail({ stderr: Buffer.from('stderr only'), message: 'message only' })) + .toBe('stderr only\nmessage only'); + }); +}); diff --git a/test/gen-skill-docs.test.ts b/test/gen-skill-docs.test.ts index c06b69f8e..0548b686a 100644 --- a/test/gen-skill-docs.test.ts +++ b/test/gen-skill-docs.test.ts @@ -2452,7 +2452,7 @@ describe('setup script validation', () => { expect(runtimeFn).toContain('agy_version="$(tr -d \'[:space:]\' < "$gstack_dir/VERSION")"'); expect(runtimeFn).toContain('"$agy_version" > "$agy_gstack/plugin.json"'); - expect(runtimeFn).not.toContain('"version": "1.58.5.0"'); + expect(runtimeFn).not.toMatch(/"version": "\d+\.\d+\.\d+\.\d+"/); expect(linkFn).toContain('-name SKILL.md'); expect(linkFn).toContain('bun run gen:skill-docs --host agy'); }); diff --git a/test/helpers/providers/agy.ts b/test/helpers/providers/agy.ts index c37129f08..e30a34bfa 100644 --- a/test/helpers/providers/agy.ts +++ b/test/helpers/providers/agy.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Agy adapter — wraps the `agy` CLI using `agy --print`. @@ -64,17 +65,20 @@ export class AgyAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login|api key/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429|quota/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429|quota/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/claude.ts b/test/helpers/providers/claude.ts index ce77767c2..07eb7e6ca 100644 --- a/test/helpers/providers/claude.ts +++ b/test/helpers/providers/claude.ts @@ -5,6 +5,7 @@ import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; import { resolveClaudeCommand } from '../../../browse/src/claude-bin'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Claude adapter — wraps the `claude` CLI via claude -p. @@ -67,17 +68,20 @@ export class ClaudeAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/errors.ts b/test/helpers/providers/errors.ts new file mode 100644 index 000000000..29b6a9d23 --- /dev/null +++ b/test/helpers/providers/errors.ts @@ -0,0 +1,20 @@ +export interface ProviderProcessError { + stderr?: Buffer | string; + message?: string; +} + +/** Combine subprocess diagnostics without duplicating identical text. */ +export function providerErrorDetail(error: ProviderProcessError): string { + const stderr = typeof error.stderr === 'string' + ? error.stderr + : error.stderr?.toString() ?? ''; + const message = error.message ?? ''; + return stderr && message.includes(stderr) + ? message + : [stderr, message].filter(Boolean).join('\n'); +} + +/** Provider/model saturation is transient and distinct from auth or quota. */ +export function isCapacityError(detail: string): boolean { + return /(?:selected\s+)?model\s+is\s+at\s+capacity|\bat\s+capacity\b|\bmodel\s+capacity\b|\boverloaded\b|temporarily unavailable due to (?:high )?(?:load|demand)/i.test(detail); +} diff --git a/test/helpers/providers/gemini.ts b/test/helpers/providers/gemini.ts index 46f8dc9a0..93d866c7d 100644 --- a/test/helpers/providers/gemini.ts +++ b/test/helpers/providers/gemini.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * Gemini adapter — wraps the `gemini` CLI. @@ -65,17 +66,20 @@ export class GeminiAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login|api key/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login|api key/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429|quota/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429|quota/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/gpt.ts b/test/helpers/providers/gpt.ts index 8a3a66e9f..7dc19faa8 100644 --- a/test/helpers/providers/gpt.ts +++ b/test/helpers/providers/gpt.ts @@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process'; import * as fs from 'fs'; import * as path from 'path'; import * as os from 'os'; +import { isCapacityError, providerErrorDetail } from './errors'; /** * GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output). @@ -63,17 +64,20 @@ export class GptAdapter implements ProviderAdapter { } catch (err: unknown) { const durationMs = Date.now() - start; const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string }; - const stderr = e.stderr?.toString() ?? ''; + const detail = providerErrorDetail(e); if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') { return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model); } - if (/unauthorized|auth|login/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model); + if (/unauthorized|auth|login/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model); } - if (/rate[- ]?limit|429/i.test(stderr)) { - return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model); + if (/rate[- ]?limit|429/i.test(detail)) { + return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model); } - return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model); + if (isCapacityError(detail)) { + return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model); + } + return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model); } } diff --git a/test/helpers/providers/types.ts b/test/helpers/providers/types.ts index 1680d0ceb..ddaa01f09 100644 --- a/test/helpers/providers/types.ts +++ b/test/helpers/providers/types.ts @@ -31,6 +31,7 @@ export type RunError = | 'auth' // Credentials missing or invalid. | 'timeout' // Exceeded timeoutMs. | 'rate_limit' // Provider rate-limited us; backoff exceeded. + | 'capacity' // Selected model/provider is temporarily saturated. | 'binary_missing' // CLI not found on PATH. | 'unknown'; // Catch-all with reason populated.