fix(adapter-hermes-gateway): honor provider-quota backoff on terminal 429 events
When the Hermes gateway completes a run with status=failed carrying an upstream provider quota-exhaustion message (e.g. Codex "quota exhausted (429); retry after 4825s", "HTTP 429", "usage limit reached"), the adapter emits errorCode=hermes_gateway_run_failed with no errorFamily or retryNotBefore. The heartbeat scheduler treats that as a generic failure and immediately fires the next run, which hits the same 429, producing a storm of failed runs during the entire cooldown window. This change: 1. Adds detectProviderQuotaExhaustion() that recognises the three known forms of the quota message. When matched, the result is upgraded to errorCode=hermes_gateway_rate_limited + errorFamily=provider_quota and a real retryNotBefore is derived from the seconds hint in the message (fallback: 60 s cool-down). 2. Adds parseHermesRetryAfterHeader() so the HTTP 429 path (which already reads Retry-After) normalises delta-seconds and HTTP-date values into an absolute ISO datetime. Passing the raw value "120" was making the downstream new Date(value) parse it as year 120 CE, so the scheduler saw a past timestamp and skipped the backoff. 3. Applies the same quota upgrade in errorResult() when a real HTTP 429 is received without a Retry-After header, so the scheduler still gets a family + retryNotBefore. Adds unit tests for mapFinalResultForTest, parseHermesRetryAfterHeader, and detectProviderQuotaExhaustion covering both explicit-retry-after and fallback code paths.
This commit is contained in:
Builder (NUBLO)
parent
1a74719309
commit
411f9535ed
|
|
@ -1,6 +1,12 @@
|
|||
import { describe, expect, it, vi, afterEach } from "vitest";
|
||||
import type { AdapterExecutionContext } from "@paperclipai/adapter-utils";
|
||||
import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js";
|
||||
import {
|
||||
__providerQuotaInternals,
|
||||
execute,
|
||||
mapFinalResultForTest,
|
||||
parseSseFramesForTest,
|
||||
resolveSessionKey,
|
||||
} from "./execute.js";
|
||||
import { testEnvironment } from "./test.js";
|
||||
|
||||
function makeCtx(config: Record<string, unknown>): AdapterExecutionContext {
|
||||
|
|
@ -730,4 +736,106 @@ describe("mapFinalResultForTest", () => {
|
|||
expect(result.errorCode).toBe("hermes_gateway_run_failed");
|
||||
expect(result.errorMessage).toBe("boom");
|
||||
});
|
||||
|
||||
it("promotes provider-quota exhaustion in terminal failed events to a backoff-friendly result", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-quota",
|
||||
status: "failed",
|
||||
payload: {
|
||||
status: "failed",
|
||||
error: "Codex provider quota exhausted (429); retry after 4825s. Credentials still valid.",
|
||||
},
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 4825 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("falls back to a 60s cool-down when the quota message omits an explicit retry-after", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-quota-no-hint",
|
||||
status: "failed",
|
||||
payload: { status: "failed", error: "HTTP 429: The usage limit has been reached" },
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("leaves unrelated terminal failures untouched", () => {
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-other",
|
||||
status: "failed",
|
||||
payload: { status: "failed", error: "internal boom" },
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_run_failed");
|
||||
expect(result.errorFamily).toBeUndefined();
|
||||
expect(result.retryNotBefore).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseHermesRetryAfterHeader", () => {
|
||||
const { parseHermesRetryAfterHeader } = __providerQuotaInternals;
|
||||
|
||||
it("interprets delta-seconds values as an absolute future ISO datetime", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
expect(parseHermesRetryAfterHeader("120", now)).toBe(new Date(now + 120_000).toISOString());
|
||||
});
|
||||
|
||||
it("parses HTTP-date values into an ISO datetime", () => {
|
||||
expect(parseHermesRetryAfterHeader("Fri, 04 Sep 2026 00:02:00 GMT")).toBe(
|
||||
new Date("2026-09-04T00:02:00Z").toISOString(),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns null for empty, missing, or malformed values", () => {
|
||||
expect(parseHermesRetryAfterHeader(null)).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader(undefined)).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader("")).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader(" ")).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectProviderQuotaExhaustion", () => {
|
||||
const { detectProviderQuotaExhaustion } = __providerQuotaInternals;
|
||||
|
||||
it("returns null for messages that do not match a known quota signature", () => {
|
||||
expect(detectProviderQuotaExhaustion(null)).toBeNull();
|
||||
expect(detectProviderQuotaExhaustion("")).toBeNull();
|
||||
expect(detectProviderQuotaExhaustion("random failure")).toBeNull();
|
||||
});
|
||||
|
||||
it("extracts an explicit retry-after from the Codex quota message", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
const result = detectProviderQuotaExhaustion(
|
||||
"Codex provider quota exhausted (429); retry after 1653s. Credentials still valid.",
|
||||
now,
|
||||
);
|
||||
expect(result).toEqual({
|
||||
errorCode: "hermes_gateway_rate_limited",
|
||||
errorFamily: "provider_quota",
|
||||
retryNotBefore: new Date(now + 1653 * 1000).toISOString(),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -353,6 +353,73 @@ function classifyHttpError(status: number): { code: string; family: AdapterExecu
|
|||
return { code: "hermes_gateway_protocol_error", family: null };
|
||||
}
|
||||
|
||||
// The Hermes gateway surfaces upstream provider rate-limit exhaustion by
|
||||
// completing the run with status=failed and an error message like:
|
||||
// "Codex provider quota exhausted (429); retry after 4825s. Credentials..."
|
||||
// "HTTP 429: The usage limit has been reached"
|
||||
// Without special handling the adapter emits errorCode=hermes_gateway_run_failed
|
||||
// with no errorFamily / retryNotBefore, so the downstream heartbeat scheduler
|
||||
// immediately fires another run — which hits the same 429 — producing a storm
|
||||
// of failed runs during the entire cooldown window (~1h). We tag those
|
||||
// terminal-event failures with errorFamily=provider_quota and a real
|
||||
// retryNotBefore timestamp so the scheduler honours the backoff.
|
||||
const CODEX_QUOTA_MESSAGE_RE =
|
||||
/(?:codex\s+provider\s+)?quota\s+exhausted\s*\(?\s*429\s*\)?(?:[^0-9]*?retry\s*after\s+(\d+)\s*s)?/i;
|
||||
const HTTP_429_MESSAGE_RE = /\bhttp\s*429\b|\b429\s*:/i;
|
||||
const USAGE_LIMIT_MESSAGE_RE = /\busage[-_\s]?limit(?:\s+has\s+been)?\s+reached\b/i;
|
||||
const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i;
|
||||
|
||||
// Fallback cool-down (seconds) when the gateway did not include an explicit
|
||||
// retry-after. Kept modest so a real reset-time reported by a later attempt
|
||||
// can shorten it, but long enough to break the immediate hot-loop.
|
||||
const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60;
|
||||
|
||||
function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null {
|
||||
if (raw === null || raw === undefined) return null;
|
||||
const value = String(raw).trim();
|
||||
if (!value) return null;
|
||||
// delta-seconds form (e.g. "120")
|
||||
if (/^\d+$/.test(value)) {
|
||||
const seconds = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(seconds) || seconds < 0) return null;
|
||||
return new Date(now + seconds * 1000).toISOString();
|
||||
}
|
||||
// HTTP-date form
|
||||
const parsed = Date.parse(value);
|
||||
if (!Number.isFinite(parsed)) return null;
|
||||
return new Date(parsed).toISOString();
|
||||
}
|
||||
|
||||
function detectProviderQuotaExhaustion(
|
||||
message: string | null | undefined,
|
||||
now = Date.now(),
|
||||
): { errorCode: string; errorFamily: AdapterExecutionResult["errorFamily"]; retryNotBefore: string | null } | null {
|
||||
if (!message) return null;
|
||||
const trimmed = String(message).trim();
|
||||
if (!trimmed) return null;
|
||||
const isQuotaExhausted =
|
||||
CODEX_QUOTA_MESSAGE_RE.test(trimmed) ||
|
||||
HTTP_429_MESSAGE_RE.test(trimmed) ||
|
||||
USAGE_LIMIT_MESSAGE_RE.test(trimmed);
|
||||
if (!isQuotaExhausted) return null;
|
||||
const explicit = CODEX_QUOTA_MESSAGE_RE.exec(trimmed);
|
||||
const hint = RETRY_AFTER_HINT_RE.exec(trimmed);
|
||||
const rawSeconds = (explicit && explicit[1]) ?? (hint && hint[1]) ?? null;
|
||||
let seconds: number | null = null;
|
||||
if (rawSeconds !== null) {
|
||||
const parsed = Number.parseInt(rawSeconds, 10);
|
||||
if (Number.isFinite(parsed) && parsed > 0) seconds = parsed;
|
||||
}
|
||||
if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC;
|
||||
const retryNotBefore = new Date(now + seconds * 1000).toISOString();
|
||||
return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore };
|
||||
}
|
||||
|
||||
export const __providerQuotaInternals = {
|
||||
parseHermesRetryAfterHeader,
|
||||
detectProviderQuotaExhaustion,
|
||||
};
|
||||
|
||||
function fetchFailureMessage(err: unknown): string {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : null;
|
||||
|
|
@ -380,7 +447,13 @@ async function fetchJson(input: RequestInfo | URL, init: RequestInit): Promise<u
|
|||
const err = new Error(`Hermes gateway HTTP ${response.status}`) as HermesHttpError;
|
||||
err.status = response.status;
|
||||
err.code = classified.code;
|
||||
err.retryNotBefore = response.headers.get("retry-after");
|
||||
// The `retry-after` header value is either delta-seconds (e.g. "120") or an
|
||||
// HTTP-date. The downstream consumer (heartbeat.readTransientRetryNotBefore)
|
||||
// parses via `new Date(value)`, which accepts "1653" as year 1653 → the
|
||||
// date lands in the past and the scheduler ignores the backoff. Normalise
|
||||
// to an absolute ISO datetime up front so the scheduler sees a real
|
||||
// future instant.
|
||||
err.retryNotBefore = parseHermesRetryAfterHeader(response.headers.get("retry-after"));
|
||||
err.body = body;
|
||||
throw err;
|
||||
}
|
||||
|
|
@ -671,16 +744,26 @@ export function mapFinalResultForTest(input: {
|
|||
const mapped = terminalResultCode(input.terminal.status);
|
||||
const usage = parseUsage(payload);
|
||||
const costUsd = parseCostUsd(payload);
|
||||
const errorMessage = mapped.errorCode
|
||||
? redactText(extractErrorMessage(payload) ?? `Hermes run ${input.terminal.status}`)
|
||||
const rawErrorMessage = mapped.errorCode
|
||||
? extractErrorMessage(payload) ?? `Hermes run ${input.terminal.status}`
|
||||
: null;
|
||||
const errorMessage = rawErrorMessage ? redactText(rawErrorMessage) : null;
|
||||
// Detect provider-quota exhaustion (e.g. Codex 429) inside a terminal
|
||||
// "failed" event and tag the result so the scheduler honours the backoff.
|
||||
const quotaOverride =
|
||||
mapped.errorCode && FAILURE_STATUSES.has(input.terminal.status)
|
||||
? detectProviderQuotaExhaustion(rawErrorMessage)
|
||||
: null;
|
||||
const finalErrorCode = quotaOverride?.errorCode ?? mapped.errorCode;
|
||||
return {
|
||||
exitCode: mapped.exitCode,
|
||||
signal: mapped.signal,
|
||||
timedOut: false,
|
||||
provider: "hermes_gateway",
|
||||
model: extractModel(payload),
|
||||
...(mapped.errorCode ? { errorCode: mapped.errorCode } : {}),
|
||||
...(finalErrorCode ? { errorCode: finalErrorCode } : {}),
|
||||
...(quotaOverride?.errorFamily ? { errorFamily: quotaOverride.errorFamily } : {}),
|
||||
...(quotaOverride?.retryNotBefore ? { retryNotBefore: quotaOverride.retryNotBefore } : {}),
|
||||
...(errorMessage ? { errorMessage } : {}),
|
||||
...(usage ? { usage } : {}),
|
||||
...(costUsd !== null ? { costUsd } : {}),
|
||||
|
|
@ -757,16 +840,27 @@ function errorResult(err: unknown, redactText: TextRedactor = sanitizeSensitiveT
|
|||
const hermesError = err as HermesHttpError;
|
||||
const code = hermesError.code ?? "hermes_gateway_protocol_error";
|
||||
const classified = hermesError.status ? classifyHttpError(hermesError.status) : null;
|
||||
const rawMessage = err instanceof Error ? err.message : String(err);
|
||||
const errorMessage = code === "hermes_gateway_auth_failed"
|
||||
? `${redactErrorMessage(err, redactText)}. Check adapterConfig.apiKey matches the Hermes API_SERVER_KEY for the running gateway.`
|
||||
: redactErrorMessage(err, redactText);
|
||||
// On real HTTP 429s, upgrade the family to provider_quota (more specific than
|
||||
// transient_upstream) when the message or body signals it, and synthesise
|
||||
// retryNotBefore from the message if the header was missing.
|
||||
const quotaOverride =
|
||||
hermesError.status === 429 && !hermesError.retryNotBefore
|
||||
? detectProviderQuotaExhaustion(rawMessage)
|
||||
: null;
|
||||
return {
|
||||
exitCode: 1,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorCode: code,
|
||||
errorFamily: classified?.family ?? (code === "hermes_gateway_connect_failed" ? "transient_upstream" : null),
|
||||
retryNotBefore: hermesError.retryNotBefore ?? null,
|
||||
errorFamily:
|
||||
quotaOverride?.errorFamily ??
|
||||
classified?.family ??
|
||||
(code === "hermes_gateway_connect_failed" ? "transient_upstream" : null),
|
||||
retryNotBefore: hermesError.retryNotBefore ?? quotaOverride?.retryNotBefore ?? null,
|
||||
errorMessage,
|
||||
errorMeta: {
|
||||
...(hermesError.status ? { status: hermesError.status } : {}),
|
||||
|
|
|
|||
Loading…
Reference in New Issue