RomanNum3ral
/
gstack
mirror of https://github.com/garrytan/gstack.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

classify provider capacity errors

This commit is contained in:
Mattias Petersson 2026-07-16 09:18:36 +02:00
parent 9c2414016c
commit e9fb8f311b
8 changed files with 89 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
test/benchmark-provider-errors.test.ts Normal file
View File

@ -0,0 +1,27 @@
import { describe, expect, test } from 'bun:test';
import { isCapacityError, providerErrorDetail } from './helpers/providers/errors';
describe('benchmark provider error classification', () => {
test('recognizes the Antigravity model-capacity message', () => {
expect(isCapacityError('Selected model is at capacity. Please try a different model')).toBe(true);
});
test('recognizes overloaded and high-load variants', () => {
expect(isCapacityError('The provider is overloaded right now')).toBe(true);
expect(isCapacityError('Model temporarily unavailable due to high demand')).toBe(true);
});
test('does not misclassify auth, quota, or generic failures as capacity', () => {
expect(isCapacityError('Error 401: login required')).toBe(false);
expect(isCapacityError('429 quota exceeded')).toBe(false);
expect(isCapacityError('connection reset by peer')).toBe(false);
});
test('combines stderr and message without duplicating subprocess output', () => {
const stderr = 'Selected model is at capacity.';
expect(providerErrorDetail({ stderr: Buffer.from(stderr), message: `Command failed\n${stderr}` }))
.toBe(`Command failed\n${stderr}`);
expect(providerErrorDetail({ stderr: Buffer.from('stderr only'), message: 'message only' }))
.toBe('stderr only\nmessage only');
});
});

2
test/gen-skill-docs.test.ts
View File

@ -2452,7 +2452,7 @@ describe('setup script validation', () => {
expect(runtimeFn).toContain('agy_version="$(tr -d \'[:space:]\' < "$gstack_dir/VERSION")"');
expect(runtimeFn).toContain('"$agy_version" > "$agy_gstack/plugin.json"');
expect(runtimeFn).not.toContain('"version": "1.58.5.0"');
expect(runtimeFn).not.toMatch(/"version": "\d+\.\d+\.\d+\.\d+"/);
expect(linkFn).toContain('-name SKILL.md');
expect(linkFn).toContain('bun run gen:skill-docs --host agy');
});

16
test/helpers/providers/agy.ts
View File

@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { isCapacityError, providerErrorDetail } from './errors';
/**
* Agy adapter — wraps the `agy` CLI using `agy --print`.
@ -64,17 +65,20 @@ export class AgyAdapter implements ProviderAdapter {
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
const detail = providerErrorDetail(e);
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
}
if (/unauthorized|auth|login|api key/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
if (/unauthorized|auth|login|api key/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model);
}
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
if (/rate[- ]?limit|429|quota/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
if (isCapacityError(detail)) {
return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model);
}
}

16
test/helpers/providers/claude.ts
View File

@ -5,6 +5,7 @@ import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { resolveClaudeCommand } from '../../../browse/src/claude-bin';
import { isCapacityError, providerErrorDetail } from './errors';
/**
* Claude adapter — wraps the `claude` CLI via claude -p.
@ -67,17 +68,20 @@ export class ClaudeAdapter implements ProviderAdapter {
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
const detail = providerErrorDetail(e);
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
}
if (/unauthorized|auth|login/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
if (/unauthorized|auth|login/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model);
}
if (/rate[- ]?limit|429/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
if (/rate[- ]?limit|429/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
if (isCapacityError(detail)) {
return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model);
}
}

20
test/helpers/providers/errors.ts Normal file
View File

@ -0,0 +1,20 @@
export interface ProviderProcessError {
stderr?: Buffer | string;
message?: string;
}
/** Combine subprocess diagnostics without duplicating identical text. */
export function providerErrorDetail(error: ProviderProcessError): string {
const stderr = typeof error.stderr === 'string'
? error.stderr
: error.stderr?.toString() ?? '';
const message = error.message ?? '';
return stderr && message.includes(stderr)
? message
: [stderr, message].filter(Boolean).join('\n');
}
/** Provider/model saturation is transient and distinct from auth or quota. */
export function isCapacityError(detail: string): boolean {
return /(?:selected\s+)?model\s+is\s+at\s+capacity|\bat\s+capacity\b|\bmodel\s+capacity\b|\boverloaded\b|temporarily unavailable due to (?:high )?(?:load|demand)/i.test(detail);
}

16
test/helpers/providers/gemini.ts
View File

@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { isCapacityError, providerErrorDetail } from './errors';
/**
* Gemini adapter — wraps the `gemini` CLI.
@ -65,17 +66,20 @@ export class GeminiAdapter implements ProviderAdapter {
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
const detail = providerErrorDetail(e);
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
}
if (/unauthorized|auth|login|api key/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
if (/unauthorized|auth|login|api key/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model);
}
if (/rate[- ]?limit|429|quota/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
if (/rate[- ]?limit|429|quota/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
if (isCapacityError(detail)) {
return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model);
}
}

16
test/helpers/providers/gpt.ts
View File

@ -4,6 +4,7 @@ import { execFileSync, spawnSync } from 'child_process';
import * as fs from 'fs';
import * as path from 'path';
import * as os from 'os';
import { isCapacityError, providerErrorDetail } from './errors';
/**
* GPT adapter — wraps the OpenAI `codex` CLI (codex exec with --json output).
@ -63,17 +64,20 @@ export class GptAdapter implements ProviderAdapter {
} catch (err: unknown) {
const durationMs = Date.now() - start;
const e = err as { code?: string; stderr?: Buffer; signal?: string; message?: string };
const stderr = e.stderr?.toString() ?? '';
const detail = providerErrorDetail(e);
if (e.signal === 'SIGTERM' || e.code === 'ETIMEDOUT') {
return this.emptyResult(durationMs, { code: 'timeout', reason: `exceeded ${opts.timeoutMs}ms` }, opts.model);
}
if (/unauthorized|auth|login/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'auth', reason: stderr.slice(0, 400) }, opts.model);
if (/unauthorized|auth|login/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'auth', reason: detail.slice(0, 400) }, opts.model);
}
if (/rate[- ]?limit|429/i.test(stderr)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: stderr.slice(0, 400) }, opts.model);
if (/rate[- ]?limit|429/i.test(detail)) {
return this.emptyResult(durationMs, { code: 'rate_limit', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (e.message ?? stderr ?? 'unknown').slice(0, 400) }, opts.model);
if (isCapacityError(detail)) {
return this.emptyResult(durationMs, { code: 'capacity', reason: detail.slice(0, 400) }, opts.model);
}
return this.emptyResult(durationMs, { code: 'unknown', reason: (detail || 'unknown').slice(0, 400) }, opts.model);
}
}

1
test/helpers/providers/types.ts
View File

@ -31,6 +31,7 @@ export type RunError =
| 'auth' // Credentials missing or invalid.
| 'timeout' // Exceeded timeoutMs.
| 'rate_limit' // Provider rate-limited us; backoff exceeded.
| 'capacity' // Selected model/provider is temporarily saturated.
| 'binary_missing' // CLI not found on PATH.
| 'unknown'; // Catch-all with reason populated.