Merge 43f729bcdf into c9e3bb7ca4
This commit is contained in:
Sergio-LPA
GitHub
commit
47ace78182
|
|
@ -1,6 +1,12 @@
|
|||
import { describe, expect, it, vi, afterEach } from "vitest";
|
||||
import type { AdapterExecutionContext } from "@paperclipai/adapter-utils";
|
||||
import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js";
|
||||
import {
|
||||
__providerQuotaInternals,
|
||||
execute,
|
||||
mapFinalResultForTest,
|
||||
parseSseFramesForTest,
|
||||
resolveSessionKey,
|
||||
} from "./execute.js";
|
||||
import { testEnvironment } from "./test.js";
|
||||
|
||||
function makeCtx(config: Record<string, unknown>): AdapterExecutionContext {
|
||||
|
|
@ -764,4 +770,223 @@ describe("mapFinalResultForTest", () => {
|
|||
expect(result.errorCode).toBe("hermes_gateway_run_failed");
|
||||
expect(result.errorMessage).toBe("boom");
|
||||
});
|
||||
|
||||
it("promotes provider-quota exhaustion in terminal failed events to a backoff-friendly result", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-quota",
|
||||
status: "failed",
|
||||
payload: {
|
||||
status: "failed",
|
||||
error: "Codex provider quota exhausted (429); retry after 4825s. Credentials still valid.",
|
||||
},
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 4825 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("falls back to a 60s cool-down when the quota message omits an explicit retry-after", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-quota-no-hint",
|
||||
status: "failed",
|
||||
payload: { status: "failed", error: "HTTP 429: The usage limit has been reached" },
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("degrades an out-of-range retry hint in a terminal failed event to the fallback cool-down instead of throwing", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-quota-absurd",
|
||||
status: "failed",
|
||||
payload: {
|
||||
status: "failed",
|
||||
error: "Codex provider quota exhausted (429); retry after 9999999999999999s.",
|
||||
},
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("leaves unrelated terminal failures untouched", () => {
|
||||
const result = mapFinalResultForTest({
|
||||
terminal: {
|
||||
runId: "run-other",
|
||||
status: "failed",
|
||||
payload: { status: "failed", error: "internal boom" },
|
||||
},
|
||||
outputChunks: [],
|
||||
sessionKey: "session-key",
|
||||
strategy: "issue",
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_run_failed");
|
||||
expect(result.errorFamily).toBeUndefined();
|
||||
expect(result.retryNotBefore).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseHermesRetryAfterHeader", () => {
|
||||
const { parseHermesRetryAfterHeader } = __providerQuotaInternals;
|
||||
|
||||
it("interprets delta-seconds values as an absolute future ISO datetime", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
expect(parseHermesRetryAfterHeader("120", now)).toBe(new Date(now + 120_000).toISOString());
|
||||
});
|
||||
|
||||
it("parses HTTP-date values into an ISO datetime", () => {
|
||||
expect(parseHermesRetryAfterHeader("Fri, 04 Sep 2026 00:02:00 GMT")).toBe(
|
||||
new Date("2026-09-04T00:02:00Z").toISOString(),
|
||||
);
|
||||
});
|
||||
|
||||
it("returns null for empty, missing, or malformed values", () => {
|
||||
expect(parseHermesRetryAfterHeader(null)).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader(undefined)).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader("")).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader(" ")).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull();
|
||||
});
|
||||
|
||||
it("returns null instead of throwing for finite values beyond the Date range", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
expect(() => parseHermesRetryAfterHeader("9999999999999999", now)).not.toThrow();
|
||||
expect(parseHermesRetryAfterHeader("9999999999999999", now)).toBeNull();
|
||||
expect(parseHermesRetryAfterHeader("99999999999999999999999", now)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("extractQuotaSignalFromBody", () => {
|
||||
const { extractQuotaSignalFromBody } = __providerQuotaInternals;
|
||||
|
||||
it("reads the upstream message from the shapes the gateway uses", () => {
|
||||
expect(extractQuotaSignalFromBody("HTTP 429: The usage limit has been reached")).toBe(
|
||||
"HTTP 429: The usage limit has been reached",
|
||||
);
|
||||
expect(extractQuotaSignalFromBody({ text: "quota exhausted (429)" })).toBe("quota exhausted (429)");
|
||||
expect(extractQuotaSignalFromBody({ error: "quota exhausted (429)" })).toBe("quota exhausted (429)");
|
||||
expect(extractQuotaSignalFromBody({ error: { message: "quota exhausted (429)" } })).toBe(
|
||||
"quota exhausted (429)",
|
||||
);
|
||||
expect(extractQuotaSignalFromBody({ message: "quota exhausted (429)" })).toBe("quota exhausted (429)");
|
||||
expect(extractQuotaSignalFromBody({ detail: "quota exhausted (429)" })).toBe("quota exhausted (429)");
|
||||
});
|
||||
|
||||
it("returns null for empty or unrelated bodies", () => {
|
||||
expect(extractQuotaSignalFromBody(null)).toBeNull();
|
||||
expect(extractQuotaSignalFromBody(undefined)).toBeNull();
|
||||
expect(extractQuotaSignalFromBody("")).toBeNull();
|
||||
expect(extractQuotaSignalFromBody({})).toBeNull();
|
||||
expect(extractQuotaSignalFromBody([])).toBeNull();
|
||||
expect(extractQuotaSignalFromBody({ error: 42 })).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectProviderQuotaExhaustion", () => {
|
||||
const { detectProviderQuotaExhaustion } = __providerQuotaInternals;
|
||||
|
||||
it("returns null for messages that do not match a known quota signature", () => {
|
||||
expect(detectProviderQuotaExhaustion(null)).toBeNull();
|
||||
expect(detectProviderQuotaExhaustion("")).toBeNull();
|
||||
expect(detectProviderQuotaExhaustion("random failure")).toBeNull();
|
||||
});
|
||||
|
||||
it("degrades an out-of-range retry hint to the fallback cool-down instead of throwing", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
const message = "Codex provider quota exhausted (429); retry after 9999999999999999s.";
|
||||
expect(() => detectProviderQuotaExhaustion(message, now)).not.toThrow();
|
||||
expect(detectProviderQuotaExhaustion(message, now)).toEqual({
|
||||
errorCode: "hermes_gateway_rate_limited",
|
||||
errorFamily: "provider_quota",
|
||||
retryNotBefore: new Date(now + 60 * 1000).toISOString(),
|
||||
});
|
||||
});
|
||||
|
||||
it("extracts an explicit retry-after from the Codex quota message", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
const result = detectProviderQuotaExhaustion(
|
||||
"Codex provider quota exhausted (429); retry after 1653s. Credentials still valid.",
|
||||
now,
|
||||
);
|
||||
expect(result).toEqual({
|
||||
errorCode: "hermes_gateway_rate_limited",
|
||||
errorFamily: "provider_quota",
|
||||
retryNotBefore: new Date(now + 1653 * 1000).toISOString(),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("execute: direct HTTP 429 classification", () => {
|
||||
const now = new Date("2026-09-04T00:00:00Z").getTime();
|
||||
|
||||
function run429(init: { body?: string; headers?: Record<string, string> }) {
|
||||
vi.spyOn(Date, "now").mockReturnValue(now);
|
||||
const fetchMock = vi.fn(async () =>
|
||||
new Response(init.body ?? "", {
|
||||
status: 429,
|
||||
headers: { "content-type": "application/json", ...(init.headers ?? {}) },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetchMock);
|
||||
return execute(makeCtx({ apiBaseUrl: "http://127.0.0.1:8642", apiKey: "secret-key", timeoutSec: 5 }));
|
||||
}
|
||||
|
||||
it("keeps a headerless 429 with no upstream quota signal as transient gateway throttling", async () => {
|
||||
const result = await run429({});
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps a headerless 429 whose body only describes gateway throttling as transient_upstream", async () => {
|
||||
const result = await run429({ body: JSON.stringify({ error: "Too many requests to the gateway, slow down" }) });
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBeNull();
|
||||
});
|
||||
|
||||
it("promotes a headerless 429 to provider_quota when the body carries the upstream quota signature", async () => {
|
||||
const result = await run429({
|
||||
body: JSON.stringify({ error: "Codex provider quota exhausted (429); retry after 120s. Credentials still valid." }),
|
||||
});
|
||||
expect(result.errorCode).toBe("hermes_gateway_rate_limited");
|
||||
expect(result.errorFamily).toBe("provider_quota");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 120 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("honours the retry-after header on a 429 and leaves the family as transient_upstream", async () => {
|
||||
const result = await run429({ headers: { "retry-after": "30" } });
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBe(new Date(now + 30 * 1000).toISOString());
|
||||
});
|
||||
|
||||
it("does not reject execute() when the retry-after header is beyond the Date range", async () => {
|
||||
const result = await run429({ headers: { "retry-after": "9999999999999999" } });
|
||||
expect(result.exitCode).toBe(1);
|
||||
expect(result.errorFamily).toBe("transient_upstream");
|
||||
expect(result.retryNotBefore).toBeNull();
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -357,6 +357,108 @@ function classifyHttpError(status: number): { code: string; family: AdapterExecu
|
|||
return { code: "hermes_gateway_protocol_error", family: null };
|
||||
}
|
||||
|
||||
// The Hermes gateway surfaces upstream provider rate-limit exhaustion by
|
||||
// completing the run with status=failed and an error message like:
|
||||
// "Codex provider quota exhausted (429); retry after 4825s. Credentials..."
|
||||
// "HTTP 429: The usage limit has been reached"
|
||||
// Without special handling the adapter emits errorCode=hermes_gateway_run_failed
|
||||
// with no errorFamily / retryNotBefore, so the downstream heartbeat scheduler
|
||||
// immediately fires another run — which hits the same 429 — producing a storm
|
||||
// of failed runs during the entire cooldown window (~1h). We tag those
|
||||
// terminal-event failures with errorFamily=provider_quota and a real
|
||||
// retryNotBefore timestamp so the scheduler honours the backoff.
|
||||
const CODEX_QUOTA_MESSAGE_RE =
|
||||
/(?:codex\s+provider\s+)?quota\s+exhausted\s*\(?\s*429\s*\)?(?:[^0-9]*?retry\s*after\s+(\d+)\s*s)?/i;
|
||||
const HTTP_429_MESSAGE_RE = /\bhttp\s*429\b|\b429\s*:/i;
|
||||
const USAGE_LIMIT_MESSAGE_RE = /\busage[-_\s]?limit(?:\s+has\s+been)?\s+reached\b/i;
|
||||
const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i;
|
||||
|
||||
// Fallback cool-down (seconds) when the gateway did not include an explicit
|
||||
// retry-after. Kept modest so a real reset-time reported by a later attempt
|
||||
// can shorten it, but long enough to break the immediate hot-loop.
|
||||
const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60;
|
||||
|
||||
// ECMAScript Dates only cover ±8.64e15 ms around the epoch. A finite but
|
||||
// out-of-range value (e.g. "Retry-After: 9999999999999999") builds an Invalid
|
||||
// Date whose toISOString() throws, and on the terminal-event path that
|
||||
// exception would escape result mapping and reject execute() instead of
|
||||
// yielding a handled failure. Serialise through this guard instead.
|
||||
const MAX_DATE_MS = 8_640_000_000_000_000;
|
||||
|
||||
function toSafeIsoTimestamp(ms: number): string | null {
|
||||
if (!Number.isFinite(ms) || Math.abs(ms) > MAX_DATE_MS) return null;
|
||||
return new Date(ms).toISOString();
|
||||
}
|
||||
|
||||
function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null {
|
||||
if (raw === null || raw === undefined) return null;
|
||||
const value = String(raw).trim();
|
||||
if (!value) return null;
|
||||
// delta-seconds form (e.g. "120")
|
||||
if (/^\d+$/.test(value)) {
|
||||
const seconds = Number.parseInt(value, 10);
|
||||
if (!Number.isFinite(seconds) || seconds < 0) return null;
|
||||
return toSafeIsoTimestamp(now + seconds * 1000);
|
||||
}
|
||||
// HTTP-date form
|
||||
const parsed = Date.parse(value);
|
||||
if (!Number.isFinite(parsed)) return null;
|
||||
return toSafeIsoTimestamp(parsed);
|
||||
}
|
||||
|
||||
// The direct HTTP 429 path only sees the synthetic "Hermes gateway HTTP 429"
|
||||
// message, which says nothing about *who* is throttling. The upstream quota
|
||||
// signal, when present, lives in the response body — either as a plain-text
|
||||
// body (wrapped as { text }) or as an { error | message | detail } record.
|
||||
function extractQuotaSignalFromBody(body: unknown): string | null {
|
||||
if (typeof body === "string") return nonEmpty(body);
|
||||
const record = asRecord(body);
|
||||
if (!record) return null;
|
||||
const nestedError = asRecord(record.error);
|
||||
return (
|
||||
nonEmpty(record.error) ??
|
||||
nonEmpty(nestedError?.message) ??
|
||||
nonEmpty(record.message) ??
|
||||
nonEmpty(record.detail) ??
|
||||
nonEmpty(record.text)
|
||||
);
|
||||
}
|
||||
|
||||
function detectProviderQuotaExhaustion(
|
||||
message: string | null | undefined,
|
||||
now = Date.now(),
|
||||
): { errorCode: string; errorFamily: AdapterExecutionResult["errorFamily"]; retryNotBefore: string | null } | null {
|
||||
if (!message) return null;
|
||||
const trimmed = String(message).trim();
|
||||
if (!trimmed) return null;
|
||||
const isQuotaExhausted =
|
||||
CODEX_QUOTA_MESSAGE_RE.test(trimmed) ||
|
||||
HTTP_429_MESSAGE_RE.test(trimmed) ||
|
||||
USAGE_LIMIT_MESSAGE_RE.test(trimmed);
|
||||
if (!isQuotaExhausted) return null;
|
||||
const explicit = CODEX_QUOTA_MESSAGE_RE.exec(trimmed);
|
||||
const hint = RETRY_AFTER_HINT_RE.exec(trimmed);
|
||||
const rawSeconds = (explicit && explicit[1]) ?? (hint && hint[1]) ?? null;
|
||||
let seconds: number | null = null;
|
||||
if (rawSeconds !== null) {
|
||||
const parsed = Number.parseInt(rawSeconds, 10);
|
||||
if (Number.isFinite(parsed) && parsed > 0) seconds = parsed;
|
||||
}
|
||||
if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC;
|
||||
// An absurd retry hint must not turn into an exception (or a null backoff
|
||||
// that re-enables the hot-loop): degrade to the fallback cool-down.
|
||||
const retryNotBefore =
|
||||
toSafeIsoTimestamp(now + seconds * 1000) ??
|
||||
toSafeIsoTimestamp(now + HERMES_GATEWAY_QUOTA_FALLBACK_SEC * 1000);
|
||||
return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore };
|
||||
}
|
||||
|
||||
export const __providerQuotaInternals = {
|
||||
parseHermesRetryAfterHeader,
|
||||
detectProviderQuotaExhaustion,
|
||||
extractQuotaSignalFromBody,
|
||||
};
|
||||
|
||||
function fetchFailureMessage(err: unknown): string {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
const cause = err instanceof Error ? (err as { cause?: unknown }).cause : null;
|
||||
|
|
@ -384,7 +486,13 @@ async function fetchJson(input: RequestInfo | URL, init: RequestInit): Promise<u
|
|||
const err = new Error(`Hermes gateway HTTP ${response.status}`) as HermesHttpError;
|
||||
err.status = response.status;
|
||||
err.code = classified.code;
|
||||
err.retryNotBefore = response.headers.get("retry-after");
|
||||
// The `retry-after` header value is either delta-seconds (e.g. "120") or an
|
||||
// HTTP-date. The downstream consumer (heartbeat.readTransientRetryNotBefore)
|
||||
// parses via `new Date(value)`, which accepts "1653" as year 1653 → the
|
||||
// date lands in the past and the scheduler ignores the backoff. Normalise
|
||||
// to an absolute ISO datetime up front so the scheduler sees a real
|
||||
// future instant.
|
||||
err.retryNotBefore = parseHermesRetryAfterHeader(response.headers.get("retry-after"));
|
||||
err.body = body;
|
||||
throw err;
|
||||
}
|
||||
|
|
@ -675,16 +783,26 @@ export function mapFinalResultForTest(input: {
|
|||
const mapped = terminalResultCode(input.terminal.status);
|
||||
const usage = parseUsage(payload);
|
||||
const costUsd = parseCostUsd(payload);
|
||||
const errorMessage = mapped.errorCode
|
||||
? redactText(extractErrorMessage(payload) ?? `Hermes run ${input.terminal.status}`)
|
||||
const rawErrorMessage = mapped.errorCode
|
||||
? extractErrorMessage(payload) ?? `Hermes run ${input.terminal.status}`
|
||||
: null;
|
||||
const errorMessage = rawErrorMessage ? redactText(rawErrorMessage) : null;
|
||||
// Detect provider-quota exhaustion (e.g. Codex 429) inside a terminal
|
||||
// "failed" event and tag the result so the scheduler honours the backoff.
|
||||
const quotaOverride =
|
||||
mapped.errorCode && FAILURE_STATUSES.has(input.terminal.status)
|
||||
? detectProviderQuotaExhaustion(rawErrorMessage)
|
||||
: null;
|
||||
const finalErrorCode = quotaOverride?.errorCode ?? mapped.errorCode;
|
||||
return {
|
||||
exitCode: mapped.exitCode,
|
||||
signal: mapped.signal,
|
||||
timedOut: false,
|
||||
provider: "hermes_gateway",
|
||||
model: extractModel(payload),
|
||||
...(mapped.errorCode ? { errorCode: mapped.errorCode } : {}),
|
||||
...(finalErrorCode ? { errorCode: finalErrorCode } : {}),
|
||||
...(quotaOverride?.errorFamily ? { errorFamily: quotaOverride.errorFamily } : {}),
|
||||
...(quotaOverride?.retryNotBefore ? { retryNotBefore: quotaOverride.retryNotBefore } : {}),
|
||||
...(errorMessage ? { errorMessage } : {}),
|
||||
...(usage ? { usage } : {}),
|
||||
...(costUsd !== null ? { costUsd } : {}),
|
||||
|
|
@ -764,13 +882,27 @@ function errorResult(err: unknown, redactText: TextRedactor = sanitizeSensitiveT
|
|||
const errorMessage = code === "hermes_gateway_auth_failed"
|
||||
? `${redactErrorMessage(err, redactText)}. Check adapterConfig.apiKey matches the Hermes API_SERVER_KEY for the running gateway.`
|
||||
: redactErrorMessage(err, redactText);
|
||||
// On real HTTP 429s without a retry-after header, upgrade the family to
|
||||
// provider_quota (more specific than transient_upstream) only when the
|
||||
// response *body* carries an upstream quota signature, and synthesise
|
||||
// retryNotBefore from it. The synthetic "Hermes gateway HTTP 429" message
|
||||
// must not be consulted: it matches the broad 429 matcher for every
|
||||
// headerless 429, including the gateway merely throttling requests, which
|
||||
// has to stay transient_upstream.
|
||||
const quotaOverride =
|
||||
hermesError.status === 429 && !hermesError.retryNotBefore
|
||||
? detectProviderQuotaExhaustion(extractQuotaSignalFromBody(hermesError.body))
|
||||
: null;
|
||||
return {
|
||||
exitCode: 1,
|
||||
signal: null,
|
||||
timedOut: false,
|
||||
errorCode: code,
|
||||
errorFamily: classified?.family ?? (code === "hermes_gateway_connect_failed" ? "transient_upstream" : null),
|
||||
retryNotBefore: hermesError.retryNotBefore ?? null,
|
||||
errorFamily:
|
||||
quotaOverride?.errorFamily ??
|
||||
classified?.family ??
|
||||
(code === "hermes_gateway_connect_failed" ? "transient_upstream" : null),
|
||||
retryNotBefore: hermesError.retryNotBefore ?? quotaOverride?.retryNotBefore ?? null,
|
||||
errorMessage,
|
||||
errorMeta: {
|
||||
...(hermesError.status ? { status: hermesError.status } : {}),
|
||||
|
|
|
|||
Loading…
Reference in New Issue