From 411f9535edcc7468c4ac706ef05456da9ef49265 Mon Sep 17 00:00:00 2001 From: "Builder (NUBLO)" Date: Fri, 4 Sep 2026 05:59:17 +0000 Subject: [PATCH 1/2] fix(adapter-hermes-gateway): honor provider-quota backoff on terminal 429 events When the Hermes gateway completes a run with status=failed carrying an upstream provider quota-exhaustion message (e.g. Codex "quota exhausted (429); retry after 4825s", "HTTP 429", "usage limit reached"), the adapter emits errorCode=hermes_gateway_run_failed with no errorFamily or retryNotBefore. The heartbeat scheduler treats that as a generic failure and immediately fires the next run, which hits the same 429, producing a storm of failed runs during the entire cooldown window. This change: 1. Adds detectProviderQuotaExhaustion() that recognises the three known forms of the quota message. When matched, the result is upgraded to errorCode=hermes_gateway_rate_limited + errorFamily=provider_quota and a real retryNotBefore is derived from the seconds hint in the message (fallback: 60 s cool-down). 2. Adds parseHermesRetryAfterHeader() so the HTTP 429 path (which already reads Retry-After) normalises delta-seconds and HTTP-date values into an absolute ISO datetime. Passing the raw value "120" was making the downstream new Date(value) parse it as year 120 CE, so the scheduler saw a past timestamp and skipped the backoff. 3. Applies the same quota upgrade in errorResult() when a real HTTP 429 is received without a Retry-After header, so the scheduler still gets a family + retryNotBefore. Adds unit tests for mapFinalResultForTest, parseHermesRetryAfterHeader, and detectProviderQuotaExhaustion covering both explicit-retry-after and fallback code paths. --- .../hermes/src/gateway/server/execute.test.ts | 110 +++++++++++++++++- .../hermes/src/gateway/server/execute.ts | 106 ++++++++++++++++- 2 files changed, 209 insertions(+), 7 deletions(-) diff --git a/packages/adapters/hermes/src/gateway/server/execute.test.ts b/packages/adapters/hermes/src/gateway/server/execute.test.ts index e824cf2620..a22bb5aabb 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.test.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi, afterEach } from "vitest"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; -import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js"; +import { + __providerQuotaInternals, + execute, + mapFinalResultForTest, + parseSseFramesForTest, + resolveSessionKey, +} from "./execute.js"; import { testEnvironment } from "./test.js"; function makeCtx(config: Record): AdapterExecutionContext { @@ -730,4 +736,106 @@ describe("mapFinalResultForTest", () => { expect(result.errorCode).toBe("hermes_gateway_run_failed"); expect(result.errorMessage).toBe("boom"); }); + + it("promotes provider-quota exhaustion in terminal failed events to a backoff-friendly result", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota", + status: "failed", + payload: { + status: "failed", + error: "Codex provider quota exhausted (429); retry after 4825s. Credentials still valid.", + }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.exitCode).toBe(1); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 4825 * 1000).toISOString()); + }); + + it("falls back to a 60s cool-down when the quota message omits an explicit retry-after", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota-no-hint", + status: "failed", + payload: { status: "failed", error: "HTTP 429: The usage limit has been reached" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); + }); + + it("leaves unrelated terminal failures untouched", () => { + const result = mapFinalResultForTest({ + terminal: { + runId: "run-other", + status: "failed", + payload: { status: "failed", error: "internal boom" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_run_failed"); + expect(result.errorFamily).toBeUndefined(); + expect(result.retryNotBefore).toBeUndefined(); + }); +}); + +describe("parseHermesRetryAfterHeader", () => { + const { parseHermesRetryAfterHeader } = __providerQuotaInternals; + + it("interprets delta-seconds values as an absolute future ISO datetime", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + expect(parseHermesRetryAfterHeader("120", now)).toBe(new Date(now + 120_000).toISOString()); + }); + + it("parses HTTP-date values into an ISO datetime", () => { + expect(parseHermesRetryAfterHeader("Fri, 04 Sep 2026 00:02:00 GMT")).toBe( + new Date("2026-09-04T00:02:00Z").toISOString(), + ); + }); + + it("returns null for empty, missing, or malformed values", () => { + expect(parseHermesRetryAfterHeader(null)).toBeNull(); + expect(parseHermesRetryAfterHeader(undefined)).toBeNull(); + expect(parseHermesRetryAfterHeader("")).toBeNull(); + expect(parseHermesRetryAfterHeader(" ")).toBeNull(); + expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull(); + }); +}); + +describe("detectProviderQuotaExhaustion", () => { + const { detectProviderQuotaExhaustion } = __providerQuotaInternals; + + it("returns null for messages that do not match a known quota signature", () => { + expect(detectProviderQuotaExhaustion(null)).toBeNull(); + expect(detectProviderQuotaExhaustion("")).toBeNull(); + expect(detectProviderQuotaExhaustion("random failure")).toBeNull(); + }); + + it("extracts an explicit retry-after from the Codex quota message", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + const result = detectProviderQuotaExhaustion( + "Codex provider quota exhausted (429); retry after 1653s. Credentials still valid.", + now, + ); + expect(result).toEqual({ + errorCode: "hermes_gateway_rate_limited", + errorFamily: "provider_quota", + retryNotBefore: new Date(now + 1653 * 1000).toISOString(), + }); + }); }); diff --git a/packages/adapters/hermes/src/gateway/server/execute.ts b/packages/adapters/hermes/src/gateway/server/execute.ts index fdb27ec6b6..03e81267e7 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.ts @@ -353,6 +353,73 @@ function classifyHttpError(status: number): { code: string; family: AdapterExecu return { code: "hermes_gateway_protocol_error", family: null }; } +// The Hermes gateway surfaces upstream provider rate-limit exhaustion by +// completing the run with status=failed and an error message like: +// "Codex provider quota exhausted (429); retry after 4825s. Credentials..." +// "HTTP 429: The usage limit has been reached" +// Without special handling the adapter emits errorCode=hermes_gateway_run_failed +// with no errorFamily / retryNotBefore, so the downstream heartbeat scheduler +// immediately fires another run — which hits the same 429 — producing a storm +// of failed runs during the entire cooldown window (~1h). We tag those +// terminal-event failures with errorFamily=provider_quota and a real +// retryNotBefore timestamp so the scheduler honours the backoff. +const CODEX_QUOTA_MESSAGE_RE = + /(?:codex\s+provider\s+)?quota\s+exhausted\s*\(?\s*429\s*\)?(?:[^0-9]*?retry\s*after\s+(\d+)\s*s)?/i; +const HTTP_429_MESSAGE_RE = /\bhttp\s*429\b|\b429\s*:/i; +const USAGE_LIMIT_MESSAGE_RE = /\busage[-_\s]?limit(?:\s+has\s+been)?\s+reached\b/i; +const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i; + +// Fallback cool-down (seconds) when the gateway did not include an explicit +// retry-after. Kept modest so a real reset-time reported by a later attempt +// can shorten it, but long enough to break the immediate hot-loop. +const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60; + +function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null { + if (raw === null || raw === undefined) return null; + const value = String(raw).trim(); + if (!value) return null; + // delta-seconds form (e.g. "120") + if (/^\d+$/.test(value)) { + const seconds = Number.parseInt(value, 10); + if (!Number.isFinite(seconds) || seconds < 0) return null; + return new Date(now + seconds * 1000).toISOString(); + } + // HTTP-date form + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) return null; + return new Date(parsed).toISOString(); +} + +function detectProviderQuotaExhaustion( + message: string | null | undefined, + now = Date.now(), +): { errorCode: string; errorFamily: AdapterExecutionResult["errorFamily"]; retryNotBefore: string | null } | null { + if (!message) return null; + const trimmed = String(message).trim(); + if (!trimmed) return null; + const isQuotaExhausted = + CODEX_QUOTA_MESSAGE_RE.test(trimmed) || + HTTP_429_MESSAGE_RE.test(trimmed) || + USAGE_LIMIT_MESSAGE_RE.test(trimmed); + if (!isQuotaExhausted) return null; + const explicit = CODEX_QUOTA_MESSAGE_RE.exec(trimmed); + const hint = RETRY_AFTER_HINT_RE.exec(trimmed); + const rawSeconds = (explicit && explicit[1]) ?? (hint && hint[1]) ?? null; + let seconds: number | null = null; + if (rawSeconds !== null) { + const parsed = Number.parseInt(rawSeconds, 10); + if (Number.isFinite(parsed) && parsed > 0) seconds = parsed; + } + if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC; + const retryNotBefore = new Date(now + seconds * 1000).toISOString(); + return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore }; +} + +export const __providerQuotaInternals = { + parseHermesRetryAfterHeader, + detectProviderQuotaExhaustion, +}; + function fetchFailureMessage(err: unknown): string { const message = err instanceof Error ? err.message : String(err); const cause = err instanceof Error ? (err as { cause?: unknown }).cause : null; @@ -380,7 +447,13 @@ async function fetchJson(input: RequestInfo | URL, init: RequestInit): Promise Date: Fri, 11 Sep 2026 18:42:14 +0100 Subject: [PATCH 2/2] fix(adapter-hermes-gateway): range-check retry timestamps and classify headerless 429s from the body MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two follow-ups to the provider-quota backoff change, from review: - A finite but out-of-range retry value (e.g. `Retry-After: 9999999999999999`, or the same number in a terminal "retry after Ns" hint) built an Invalid Date whose toISOString() threw. On the terminal-event path that exception escaped result mapping and rejected execute() instead of returning a handled failure. Serialise through a range guard: the header degrades to "no retry-after", the terminal hint degrades to the 60s fallback cool-down. - The direct HTTP 429 path fed the synthetic "Hermes gateway HTTP 429" message to the quota detector, so every headerless 429 — including the gateway merely throttling requests — was promoted to provider_quota and routed through provider-quota recovery. The detector now only sees the response body (plain text, or error/error.message/message/detail/text); a headerless 429 without an upstream quota signature stays transient_upstream with no retryNotBefore. Tests cover the oversized header and hint, the body extractor shapes, and execute() on headerless/gateway-throttling/upstream-quota/retry-after 429s. Co-Authored-By: Claude Fable 5.1 --- .../hermes/src/gateway/server/execute.test.ts | 117 ++++++++++++++++++ .../hermes/src/gateway/server/execute.ts | 54 ++++++-- 2 files changed, 163 insertions(+), 8 deletions(-) diff --git a/packages/adapters/hermes/src/gateway/server/execute.test.ts b/packages/adapters/hermes/src/gateway/server/execute.test.ts index a22bb5aabb..e5348609b1 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.test.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.test.ts @@ -777,6 +777,27 @@ describe("mapFinalResultForTest", () => { expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); }); + it("degrades an out-of-range retry hint in a terminal failed event to the fallback cool-down instead of throwing", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota-absurd", + status: "failed", + payload: { + status: "failed", + error: "Codex provider quota exhausted (429); retry after 9999999999999999s.", + }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); + }); + it("leaves unrelated terminal failures untouched", () => { const result = mapFinalResultForTest({ terminal: { @@ -815,6 +836,39 @@ describe("parseHermesRetryAfterHeader", () => { expect(parseHermesRetryAfterHeader(" ")).toBeNull(); expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull(); }); + + it("returns null instead of throwing for finite values beyond the Date range", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + expect(() => parseHermesRetryAfterHeader("9999999999999999", now)).not.toThrow(); + expect(parseHermesRetryAfterHeader("9999999999999999", now)).toBeNull(); + expect(parseHermesRetryAfterHeader("99999999999999999999999", now)).toBeNull(); + }); +}); + +describe("extractQuotaSignalFromBody", () => { + const { extractQuotaSignalFromBody } = __providerQuotaInternals; + + it("reads the upstream message from the shapes the gateway uses", () => { + expect(extractQuotaSignalFromBody("HTTP 429: The usage limit has been reached")).toBe( + "HTTP 429: The usage limit has been reached", + ); + expect(extractQuotaSignalFromBody({ text: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ error: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ error: { message: "quota exhausted (429)" } })).toBe( + "quota exhausted (429)", + ); + expect(extractQuotaSignalFromBody({ message: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + expect(extractQuotaSignalFromBody({ detail: "quota exhausted (429)" })).toBe("quota exhausted (429)"); + }); + + it("returns null for empty or unrelated bodies", () => { + expect(extractQuotaSignalFromBody(null)).toBeNull(); + expect(extractQuotaSignalFromBody(undefined)).toBeNull(); + expect(extractQuotaSignalFromBody("")).toBeNull(); + expect(extractQuotaSignalFromBody({})).toBeNull(); + expect(extractQuotaSignalFromBody([])).toBeNull(); + expect(extractQuotaSignalFromBody({ error: 42 })).toBeNull(); + }); }); describe("detectProviderQuotaExhaustion", () => { @@ -826,6 +880,17 @@ describe("detectProviderQuotaExhaustion", () => { expect(detectProviderQuotaExhaustion("random failure")).toBeNull(); }); + it("degrades an out-of-range retry hint to the fallback cool-down instead of throwing", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + const message = "Codex provider quota exhausted (429); retry after 9999999999999999s."; + expect(() => detectProviderQuotaExhaustion(message, now)).not.toThrow(); + expect(detectProviderQuotaExhaustion(message, now)).toEqual({ + errorCode: "hermes_gateway_rate_limited", + errorFamily: "provider_quota", + retryNotBefore: new Date(now + 60 * 1000).toISOString(), + }); + }); + it("extracts an explicit retry-after from the Codex quota message", () => { const now = new Date("2026-09-04T00:00:00Z").getTime(); const result = detectProviderQuotaExhaustion( @@ -839,3 +904,55 @@ describe("detectProviderQuotaExhaustion", () => { }); }); }); + +describe("execute: direct HTTP 429 classification", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + + function run429(init: { body?: string; headers?: Record }) { + vi.spyOn(Date, "now").mockReturnValue(now); + const fetchMock = vi.fn(async () => + new Response(init.body ?? "", { + status: 429, + headers: { "content-type": "application/json", ...(init.headers ?? {}) }, + }), + ); + vi.stubGlobal("fetch", fetchMock); + return execute(makeCtx({ apiBaseUrl: "http://127.0.0.1:8642", apiKey: "secret-key", timeoutSec: 5 })); + } + + it("keeps a headerless 429 with no upstream quota signal as transient gateway throttling", async () => { + const result = await run429({}); + expect(result.exitCode).toBe(1); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); + + it("keeps a headerless 429 whose body only describes gateway throttling as transient_upstream", async () => { + const result = await run429({ body: JSON.stringify({ error: "Too many requests to the gateway, slow down" }) }); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); + + it("promotes a headerless 429 to provider_quota when the body carries the upstream quota signature", async () => { + const result = await run429({ + body: JSON.stringify({ error: "Codex provider quota exhausted (429); retry after 120s. Credentials still valid." }), + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 120 * 1000).toISOString()); + }); + + it("honours the retry-after header on a 429 and leaves the family as transient_upstream", async () => { + const result = await run429({ headers: { "retry-after": "30" } }); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBe(new Date(now + 30 * 1000).toISOString()); + }); + + it("does not reject execute() when the retry-after header is beyond the Date range", async () => { + const result = await run429({ headers: { "retry-after": "9999999999999999" } }); + expect(result.exitCode).toBe(1); + expect(result.errorFamily).toBe("transient_upstream"); + expect(result.retryNotBefore).toBeNull(); + }); +}); diff --git a/packages/adapters/hermes/src/gateway/server/execute.ts b/packages/adapters/hermes/src/gateway/server/execute.ts index 03e81267e7..fdc02c2258 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.ts @@ -374,6 +374,18 @@ const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i; // can shorten it, but long enough to break the immediate hot-loop. const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60; +// ECMAScript Dates only cover ±8.64e15 ms around the epoch. A finite but +// out-of-range value (e.g. "Retry-After: 9999999999999999") builds an Invalid +// Date whose toISOString() throws, and on the terminal-event path that +// exception would escape result mapping and reject execute() instead of +// yielding a handled failure. Serialise through this guard instead. +const MAX_DATE_MS = 8_640_000_000_000_000; + +function toSafeIsoTimestamp(ms: number): string | null { + if (!Number.isFinite(ms) || Math.abs(ms) > MAX_DATE_MS) return null; + return new Date(ms).toISOString(); +} + function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null { if (raw === null || raw === undefined) return null; const value = String(raw).trim(); @@ -382,12 +394,30 @@ function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date. if (/^\d+$/.test(value)) { const seconds = Number.parseInt(value, 10); if (!Number.isFinite(seconds) || seconds < 0) return null; - return new Date(now + seconds * 1000).toISOString(); + return toSafeIsoTimestamp(now + seconds * 1000); } // HTTP-date form const parsed = Date.parse(value); if (!Number.isFinite(parsed)) return null; - return new Date(parsed).toISOString(); + return toSafeIsoTimestamp(parsed); +} + +// The direct HTTP 429 path only sees the synthetic "Hermes gateway HTTP 429" +// message, which says nothing about *who* is throttling. The upstream quota +// signal, when present, lives in the response body — either as a plain-text +// body (wrapped as { text }) or as an { error | message | detail } record. +function extractQuotaSignalFromBody(body: unknown): string | null { + if (typeof body === "string") return nonEmpty(body); + const record = asRecord(body); + if (!record) return null; + const nestedError = asRecord(record.error); + return ( + nonEmpty(record.error) ?? + nonEmpty(nestedError?.message) ?? + nonEmpty(record.message) ?? + nonEmpty(record.detail) ?? + nonEmpty(record.text) + ); } function detectProviderQuotaExhaustion( @@ -411,13 +441,18 @@ function detectProviderQuotaExhaustion( if (Number.isFinite(parsed) && parsed > 0) seconds = parsed; } if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC; - const retryNotBefore = new Date(now + seconds * 1000).toISOString(); + // An absurd retry hint must not turn into an exception (or a null backoff + // that re-enables the hot-loop): degrade to the fallback cool-down. + const retryNotBefore = + toSafeIsoTimestamp(now + seconds * 1000) ?? + toSafeIsoTimestamp(now + HERMES_GATEWAY_QUOTA_FALLBACK_SEC * 1000); return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore }; } export const __providerQuotaInternals = { parseHermesRetryAfterHeader, detectProviderQuotaExhaustion, + extractQuotaSignalFromBody, }; function fetchFailureMessage(err: unknown): string { @@ -840,16 +875,19 @@ function errorResult(err: unknown, redactText: TextRedactor = sanitizeSensitiveT const hermesError = err as HermesHttpError; const code = hermesError.code ?? "hermes_gateway_protocol_error"; const classified = hermesError.status ? classifyHttpError(hermesError.status) : null; - const rawMessage = err instanceof Error ? err.message : String(err); const errorMessage = code === "hermes_gateway_auth_failed" ? `${redactErrorMessage(err, redactText)}. Check adapterConfig.apiKey matches the Hermes API_SERVER_KEY for the running gateway.` : redactErrorMessage(err, redactText); - // On real HTTP 429s, upgrade the family to provider_quota (more specific than - // transient_upstream) when the message or body signals it, and synthesise - // retryNotBefore from the message if the header was missing. + // On real HTTP 429s without a retry-after header, upgrade the family to + // provider_quota (more specific than transient_upstream) only when the + // response *body* carries an upstream quota signature, and synthesise + // retryNotBefore from it. The synthetic "Hermes gateway HTTP 429" message + // must not be consulted: it matches the broad 429 matcher for every + // headerless 429, including the gateway merely throttling requests, which + // has to stay transient_upstream. const quotaOverride = hermesError.status === 429 && !hermesError.retryNotBefore - ? detectProviderQuotaExhaustion(rawMessage) + ? detectProviderQuotaExhaustion(extractQuotaSignalFromBody(hermesError.body)) : null; return { exitCode: 1,