diff --git a/packages/adapters/hermes/src/gateway/server/execute.test.ts b/packages/adapters/hermes/src/gateway/server/execute.test.ts index 7efdac23bd..08b2424001 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.test.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi, afterEach } from "vitest"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; -import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js"; +import { + __providerQuotaInternals, + execute, + mapFinalResultForTest, + parseSseFramesForTest, + resolveSessionKey, +} from "./execute.js"; import { testEnvironment } from "./test.js"; function makeCtx(config: Record): AdapterExecutionContext { @@ -764,4 +770,223 @@ describe("mapFinalResultForTest", () => { expect(result.errorCode).toBe("hermes_gateway_run_failed"); expect(result.errorMessage).toBe("boom"); }); + + it("promotes provider-quota exhaustion in terminal failed events to a backoff-friendly result", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota", + status: "failed", + payload: { + status: "failed", + error: "Codex provider quota exhausted (429); retry after 4825s. Credentials still valid.", + }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.exitCode).toBe(1); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 4825 * 1000).toISOString()); + }); + + it("falls back to a 60s cool-down when the quota message omits an explicit retry-after", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota-no-hint", + status: "failed", + payload: { status: "failed", error: "HTTP 429: The usage limit has been reached" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); + }); + + it("degrades an out-of-range retry hint in a terminal failed event to the fallback cool-down instead of throwing", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota-absurd", + status: "failed", + payload: { + status: "failed", + error: "Codex provider quota exhausted (429); retry after 9999999999999999s.", + }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); + }); + + it("leaves unrelated terminal failures untouched", () => { + const result = mapFinalResultForTest({ + terminal: { + runId: "run-other", + status: "failed", + payload: { status: "failed", error: "internal boom" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_run_failed"); + expect(result.errorFamily).toBeUndefined(); + expect(result.retryNotBefore).toBeUndefined(); + }); +}); + +describe("parseHermesRetryAfterHeader", () => { + const { parseHermesRetryAfterHeader } = __providerQuotaInternals; + + it("interprets delta-seconds values as an absolute future ISO datetime", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + expect(parseHermesRetryAfterHeader("120", now)).toBe(new Date(now + 120_000).toISOString()); + }); + + it("parses HTTP-date values into an ISO datetime", () => { + expect(parseHermesRetryAfterHeader("Fri, 04 Sep 2026 00:02:00 GMT")).toBe( + new Date("2026-09-04T00:02:00Z").toISOString(), + ); + }); + + it("returns null for empty, missing, or malformed values", () => { + expect(parseHermesRetryAfterHeader(null)).toBeNull(); + expect(parseHermesRetryAfterHeader(undefined)).toBeNull(); + expect(parseHermesRetryAfterHeader("")).toBeNull(); + expect(parseHermesRetryAfterHeader(" ")).toBeNull(); + expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull(); + }); + + it("returns null instead of throwing for finite values beyond the Date range", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + expect(() => parseHermesRetryAfterHeader("9999999999999999", now)).not.toThrow(); + expect(parseHermesRetryAfterHeader("9999999999999999", now)).toBeNull(); + expect(parseHermesRetryAfterHeader("99999999999999999999999", now)).toBeNull(); + }); +}); + +describe("extractQuotaSignalFromBody", () => { + const { extractQuotaSignalFromBody } = __providerQuotaInternals; + + it("reads the upstream message from the shapes the gateway uses", () => { + expect(extractQuotaSignalFromBody("HTTP 429: The usage limit has been reached")).toBe( + "HTTP 429: The usage limit has been reached", + ); + expect(extractQuotaSignalFromBody({ text: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ error: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ error: { message: "quota exhausted (429)" } })).toBe( + "quota exhausted (429)", + ); + expect(extractQuotaSignalFromBody({ message: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ detail: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + }); + + it("returns null for empty or unrelated bodies", () => { + expect(extractQuotaSignalFromBody(null)).toBeNull(); + expect(extractQuotaSignalFromBody(undefined)).toBeNull(); + expect(extractQuotaSignalFromBody("")).toBeNull(); + expect(extractQuotaSignalFromBody({})).toBeNull(); + expect(extractQuotaSignalFromBody([])).toBeNull(); + expect(extractQuotaSignalFromBody({ error: 42 })).toBeNull(); + }); +}); + +describe("detectProviderQuotaExhaustion", () => { + const { detectProviderQuotaExhaustion } = __providerQuotaInternals; + + it("returns null for messages that do not match a known quota signature", () => { + expect(detectProviderQuotaExhaustion(null)).toBeNull(); + expect(detectProviderQuotaExhaustion("")).toBeNull(); + expect(detectProviderQuotaExhaustion("random failure")).toBeNull(); + }); + + it("degrades an out-of-range retry hint to the fallback cool-down instead of throwing", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + const message = "Codex provider quota exhausted (429); retry after 9999999999999999s."; + expect(() => detectProviderQuotaExhaustion(message, now)).not.toThrow(); + expect(detectProviderQuotaExhaustion(message, now)).toEqual({ + errorCode: "hermes_gateway_rate_limited", + errorFamily: "provider_quota", + retryNotBefore: new Date(now + 60 * 1000).toISOString(), + }); + }); + + it("extracts an explicit retry-after from the Codex quota message", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + const result = detectProviderQuotaExhaustion( + "Codex provider quota exhausted (429); retry after 1653s. Credentials still valid.", + now, + ); + expect(result).toEqual({ + errorCode: "hermes_gateway_rate_limited", + errorFamily: "provider_quota", + retryNotBefore: new Date(now + 1653 * 1000).toISOString(), + }); + }); +}); + +describe("execute: direct HTTP 429 classification", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + + function run429(init: { body?: string; headers?: Record }) { + vi.spyOn(Date, "now").mockReturnValue(now); + const fetchMock = vi.fn(async () => + new Response(init.body ?? "", { + status: 429, + headers: { "content-type": "application/json", ...(init.headers ?? {}) }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + return execute(makeCtx({ apiBaseUrl: "http://127.0.0.1:8642", apiKey: "secret-key", timeoutSec: 5 })); + } + + it("keeps a headerless 429 with no upstream quota signal as transient gateway throttling", async () => { + const result = await run429({}); + expect(result.exitCode).toBe(1); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); + + it("keeps a headerless 429 whose body only describes gateway throttling as transient_upstream", async () => { + const result = await run429({ body: JSON.stringify({ error: "Too many requests to the gateway, slow down" }) }); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); + + it("promotes a headerless 429 to provider_quota when the body carries the upstream quota signature", async () => { + const result = await run429({ + body: JSON.stringify({ error: "Codex provider quota exhausted (429); retry after 120s. Credentials still valid." }), + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 120 * 1000).toISOString()); + }); + + it("honours the retry-after header on a 429 and leaves the family as transient_upstream", async () => { + const result = await run429({ headers: { "retry-after": "30" } }); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBe(new Date(now + 30 * 1000).toISOString()); + }); + + it("does not reject execute() when the retry-after header is beyond the Date range", async () => { + const result = await run429({ headers: { "retry-after": "9999999999999999" } }); + expect(result.exitCode).toBe(1); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); }); diff --git a/packages/adapters/hermes/src/gateway/server/execute.ts b/packages/adapters/hermes/src/gateway/server/execute.ts index 0a30e7235c..2be5b4ac71 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.ts @@ -357,6 +357,108 @@ function classifyHttpError(status: number): { code: string; family: AdapterExecu return { code: "hermes_gateway_protocol_error", family: null }; } +// The Hermes gateway surfaces upstream provider rate-limit exhaustion by +// completing the run with status=failed and an error message like: +// "Codex provider quota exhausted (429); retry after 4825s. Credentials..." +// "HTTP 429: The usage limit has been reached" +// Without special handling the adapter emits errorCode=hermes_gateway_run_failed +// with no errorFamily / retryNotBefore, so the downstream heartbeat scheduler +// immediately fires another run — which hits the same 429 — producing a storm +// of failed runs during the entire cooldown window (~1h). We tag those +// terminal-event failures with errorFamily=provider_quota and a real +// retryNotBefore timestamp so the scheduler honours the backoff. +const CODEX_QUOTA_MESSAGE_RE = + /(?:codex\s+provider\s+)?quota\s+exhausted\s*\(?\s*429\s*\)?(?:[^0-9]*?retry\s*after\s+(\d+)\s*s)?/i; +const HTTP_429_MESSAGE_RE = /\bhttp\s*429\b|\b429\s*:/i; +const USAGE_LIMIT_MESSAGE_RE = /\busage[-_\s]?limit(?:\s+has\s+been)?\s+reached\b/i; +const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i; + +// Fallback cool-down (seconds) when the gateway did not include an explicit +// retry-after. Kept modest so a real reset-time reported by a later attempt +// can shorten it, but long enough to break the immediate hot-loop. +const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60; + +// ECMAScript Dates only cover ±8.64e15 ms around the epoch. A finite but +// out-of-range value (e.g. "Retry-After: 9999999999999999") builds an Invalid +// Date whose toISOString() throws, and on the terminal-event path that +// exception would escape result mapping and reject execute() instead of +// yielding a handled failure. Serialise through this guard instead. +const MAX_DATE_MS = 8_640_000_000_000_000; + +function toSafeIsoTimestamp(ms: number): string | null { + if (!Number.isFinite(ms) || Math.abs(ms) > MAX_DATE_MS) return null; + return new Date(ms).toISOString(); +} + +function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null { + if (raw === null || raw === undefined) return null; + const value = String(raw).trim(); + if (!value) return null; + // delta-seconds form (e.g. "120") + if (/^\d+$/.test(value)) { + const seconds = Number.parseInt(value, 10); + if (!Number.isFinite(seconds) || seconds < 0) return null; + return toSafeIsoTimestamp(now + seconds * 1000); + } + // HTTP-date form + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) return null; + return toSafeIsoTimestamp(parsed); +} + +// The direct HTTP 429 path only sees the synthetic "Hermes gateway HTTP 429" +// message, which says nothing about *who* is throttling. The upstream quota +// signal, when present, lives in the response body — either as a plain-text +// body (wrapped as { text }) or as an { error | message | detail } record. +function extractQuotaSignalFromBody(body: unknown): string | null { + if (typeof body === "string") return nonEmpty(body); + const record = asRecord(body); + if (!record) return null; + const nestedError = asRecord(record.error); + return ( + nonEmpty(record.error) ?? + nonEmpty(nestedError?.message) ?? + nonEmpty(record.message) ?? + nonEmpty(record.detail) ?? + nonEmpty(record.text) + ); +} + +function detectProviderQuotaExhaustion( + message: string | null | undefined, + now = Date.now(), +): { errorCode: string; errorFamily: AdapterExecutionResult["errorFamily"]; retryNotBefore: string | null } | null { + if (!message) return null; + const trimmed = String(message).trim(); + if (!trimmed) return null; + const isQuotaExhausted = + CODEX_QUOTA_MESSAGE_RE.test(trimmed) || + HTTP_429_MESSAGE_RE.test(trimmed) || + USAGE_LIMIT_MESSAGE_RE.test(trimmed); + if (!isQuotaExhausted) return null; + const explicit = CODEX_QUOTA_MESSAGE_RE.exec(trimmed); + const hint = RETRY_AFTER_HINT_RE.exec(trimmed); + const rawSeconds = (explicit && explicit[1]) ?? (hint && hint[1]) ?? null; + let seconds: number | null = null; + if (rawSeconds !== null) { + const parsed = Number.parseInt(rawSeconds, 10); + if (Number.isFinite(parsed) && parsed > 0) seconds = parsed; + } + if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC; + // An absurd retry hint must not turn into an exception (or a null backoff + // that re-enables the hot-loop): degrade to the fallback cool-down. + const retryNotBefore = + toSafeIsoTimestamp(now + seconds * 1000) ?? + toSafeIsoTimestamp(now + HERMES_GATEWAY_QUOTA_FALLBACK_SEC * 1000); + return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore }; +} + +export const __providerQuotaInternals = { + parseHermesRetryAfterHeader, + detectProviderQuotaExhaustion, + extractQuotaSignalFromBody, +}; + function fetchFailureMessage(err: unknown): string { const message = err instanceof Error ? err.message : String(err); const cause = err instanceof Error ? (err as { cause?: unknown }).cause : null; @@ -384,7 +486,13 @@ async function fetchJson(input: RequestInfo | URL, init: RequestInit): Promise