From 411f9535edcc7468c4ac706ef05456da9ef49265 Mon Sep 17 00:00:00 2001 From: "Builder (NUBLO)" Date: Fri, 4 Sep 2026 05:59:17 +0000 Subject: [PATCH] fix(adapter-hermes-gateway): honor provider-quota backoff on terminal 429 events When the Hermes gateway completes a run with status=failed carrying an upstream provider quota-exhaustion message (e.g. Codex "quota exhausted (429); retry after 4825s", "HTTP 429", "usage limit reached"), the adapter emits errorCode=hermes_gateway_run_failed with no errorFamily or retryNotBefore. The heartbeat scheduler treats that as a generic failure and immediately fires the next run, which hits the same 429, producing a storm of failed runs during the entire cooldown window. This change: 1. Adds detectProviderQuotaExhaustion() that recognises the three known forms of the quota message. When matched, the result is upgraded to errorCode=hermes_gateway_rate_limited + errorFamily=provider_quota and a real retryNotBefore is derived from the seconds hint in the message (fallback: 60 s cool-down). 2. Adds parseHermesRetryAfterHeader() so the HTTP 429 path (which already reads Retry-After) normalises delta-seconds and HTTP-date values into an absolute ISO datetime. Passing the raw value "120" was making the downstream new Date(value) parse it as year 120 CE, so the scheduler saw a past timestamp and skipped the backoff. 3. Applies the same quota upgrade in errorResult() when a real HTTP 429 is received without a Retry-After header, so the scheduler still gets a family + retryNotBefore. Adds unit tests for mapFinalResultForTest, parseHermesRetryAfterHeader, and detectProviderQuotaExhaustion covering both explicit-retry-after and fallback code paths. --- .../hermes/src/gateway/server/execute.test.ts | 110 +++++++++++++++++- .../hermes/src/gateway/server/execute.ts | 106 ++++++++++++++++- 2 files changed, 209 insertions(+), 7 deletions(-) diff --git a/packages/adapters/hermes/src/gateway/server/execute.test.ts b/packages/adapters/hermes/src/gateway/server/execute.test.ts index e824cf2620..a22bb5aabb 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.test.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.test.ts @@ -1,6 +1,12 @@ import { describe, expect, it, vi, afterEach } from "vitest"; import type { AdapterExecutionContext } from "@paperclipai/adapter-utils"; -import { execute, mapFinalResultForTest, parseSseFramesForTest, resolveSessionKey } from "./execute.js"; +import { + __providerQuotaInternals, + execute, + mapFinalResultForTest, + parseSseFramesForTest, + resolveSessionKey, +} from "./execute.js"; import { testEnvironment } from "./test.js"; function makeCtx(config: Record): AdapterExecutionContext { @@ -730,4 +736,106 @@ describe("mapFinalResultForTest", () => { expect(result.errorCode).toBe("hermes_gateway_run_failed"); expect(result.errorMessage).toBe("boom"); }); + + it("promotes provider-quota exhaustion in terminal failed events to a backoff-friendly result", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota", + status: "failed", + payload: { + status: "failed", + error: "Codex provider quota exhausted (429); retry after 4825s. Credentials still valid.", + }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.exitCode).toBe(1); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 4825 * 1000).toISOString()); + }); + + it("falls back to a 60s cool-down when the quota message omits an explicit retry-after", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + vi.spyOn(Date, "now").mockReturnValue(now); + const result = mapFinalResultForTest({ + terminal: { + runId: "run-quota-no-hint", + status: "failed", + payload: { status: "failed", error: "HTTP 429: The usage limit has been reached" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_rate_limited"); + expect(result.errorFamily).toBe("provider_quota"); + expect(result.retryNotBefore).toBe(new Date(now + 60 * 1000).toISOString()); + }); + + it("leaves unrelated terminal failures untouched", () => { + const result = mapFinalResultForTest({ + terminal: { + runId: "run-other", + status: "failed", + payload: { status: "failed", error: "internal boom" }, + }, + outputChunks: [], + sessionKey: "session-key", + strategy: "issue", + }); + expect(result.errorCode).toBe("hermes_gateway_run_failed"); + expect(result.errorFamily).toBeUndefined(); + expect(result.retryNotBefore).toBeUndefined(); + }); +}); + +describe("parseHermesRetryAfterHeader", () => { + const { parseHermesRetryAfterHeader } = __providerQuotaInternals; + + it("interprets delta-seconds values as an absolute future ISO datetime", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + expect(parseHermesRetryAfterHeader("120", now)).toBe(new Date(now + 120_000).toISOString()); + }); + + it("parses HTTP-date values into an ISO datetime", () => { + expect(parseHermesRetryAfterHeader("Fri, 04 Sep 2026 00:02:00 GMT")).toBe( + new Date("2026-09-04T00:02:00Z").toISOString(), + ); + }); + + it("returns null for empty, missing, or malformed values", () => { + expect(parseHermesRetryAfterHeader(null)).toBeNull(); + expect(parseHermesRetryAfterHeader(undefined)).toBeNull(); + expect(parseHermesRetryAfterHeader("")).toBeNull(); + expect(parseHermesRetryAfterHeader(" ")).toBeNull(); + expect(parseHermesRetryAfterHeader("not-a-date")).toBeNull(); + }); +}); + +describe("detectProviderQuotaExhaustion", () => { + const { detectProviderQuotaExhaustion } = __providerQuotaInternals; + + it("returns null for messages that do not match a known quota signature", () => { + expect(detectProviderQuotaExhaustion(null)).toBeNull(); + expect(detectProviderQuotaExhaustion("")).toBeNull(); + expect(detectProviderQuotaExhaustion("random failure")).toBeNull(); + }); + + it("extracts an explicit retry-after from the Codex quota message", () => { + const now = new Date("2026-09-04T00:00:00Z").getTime(); + const result = detectProviderQuotaExhaustion( + "Codex provider quota exhausted (429); retry after 1653s. Credentials still valid.", + now, + ); + expect(result).toEqual({ + errorCode: "hermes_gateway_rate_limited", + errorFamily: "provider_quota", + retryNotBefore: new Date(now + 1653 * 1000).toISOString(), + }); + }); }); diff --git a/packages/adapters/hermes/src/gateway/server/execute.ts b/packages/adapters/hermes/src/gateway/server/execute.ts index fdb27ec6b6..03e81267e7 100644 --- a/packages/adapters/hermes/src/gateway/server/execute.ts +++ b/packages/adapters/hermes/src/gateway/server/execute.ts @@ -353,6 +353,73 @@ function classifyHttpError(status: number): { code: string; family: AdapterExecu return { code: "hermes_gateway_protocol_error", family: null }; } +// The Hermes gateway surfaces upstream provider rate-limit exhaustion by +// completing the run with status=failed and an error message like: +// "Codex provider quota exhausted (429); retry after 4825s. Credentials..." +// "HTTP 429: The usage limit has been reached" +// Without special handling the adapter emits errorCode=hermes_gateway_run_failed +// with no errorFamily / retryNotBefore, so the downstream heartbeat scheduler +// immediately fires another run — which hits the same 429 — producing a storm +// of failed runs during the entire cooldown window (~1h). We tag those +// terminal-event failures with errorFamily=provider_quota and a real +// retryNotBefore timestamp so the scheduler honours the backoff. +const CODEX_QUOTA_MESSAGE_RE = + /(?:codex\s+provider\s+)?quota\s+exhausted\s*\(?\s*429\s*\)?(?:[^0-9]*?retry\s*after\s+(\d+)\s*s)?/i; +const HTTP_429_MESSAGE_RE = /\bhttp\s*429\b|\b429\s*:/i; +const USAGE_LIMIT_MESSAGE_RE = /\busage[-_\s]?limit(?:\s+has\s+been)?\s+reached\b/i; +const RETRY_AFTER_HINT_RE = /retry[-_\s]?after[:\s]+(\d+)\s*s?\b/i; + +// Fallback cool-down (seconds) when the gateway did not include an explicit +// retry-after. Kept modest so a real reset-time reported by a later attempt +// can shorten it, but long enough to break the immediate hot-loop. +const HERMES_GATEWAY_QUOTA_FALLBACK_SEC = 60; + +function parseHermesRetryAfterHeader(raw: string | null | undefined, now = Date.now()): string | null { + if (raw === null || raw === undefined) return null; + const value = String(raw).trim(); + if (!value) return null; + // delta-seconds form (e.g. "120") + if (/^\d+$/.test(value)) { + const seconds = Number.parseInt(value, 10); + if (!Number.isFinite(seconds) || seconds < 0) return null; + return new Date(now + seconds * 1000).toISOString(); + } + // HTTP-date form + const parsed = Date.parse(value); + if (!Number.isFinite(parsed)) return null; + return new Date(parsed).toISOString(); +} + +function detectProviderQuotaExhaustion( + message: string | null | undefined, + now = Date.now(), +): { errorCode: string; errorFamily: AdapterExecutionResult["errorFamily"]; retryNotBefore: string | null } | null { + if (!message) return null; + const trimmed = String(message).trim(); + if (!trimmed) return null; + const isQuotaExhausted = + CODEX_QUOTA_MESSAGE_RE.test(trimmed) || + HTTP_429_MESSAGE_RE.test(trimmed) || + USAGE_LIMIT_MESSAGE_RE.test(trimmed); + if (!isQuotaExhausted) return null; + const explicit = CODEX_QUOTA_MESSAGE_RE.exec(trimmed); + const hint = RETRY_AFTER_HINT_RE.exec(trimmed); + const rawSeconds = (explicit && explicit[1]) ?? (hint && hint[1]) ?? null; + let seconds: number | null = null; + if (rawSeconds !== null) { + const parsed = Number.parseInt(rawSeconds, 10); + if (Number.isFinite(parsed) && parsed > 0) seconds = parsed; + } + if (seconds === null) seconds = HERMES_GATEWAY_QUOTA_FALLBACK_SEC; + const retryNotBefore = new Date(now + seconds * 1000).toISOString(); + return { errorCode: "hermes_gateway_rate_limited", errorFamily: "provider_quota", retryNotBefore }; +} + +export const __providerQuotaInternals = { + parseHermesRetryAfterHeader, + detectProviderQuotaExhaustion, +}; + function fetchFailureMessage(err: unknown): string { const message = err instanceof Error ? err.message : String(err); const cause = err instanceof Error ? (err as { cause?: unknown }).cause : null; @@ -380,7 +447,13 @@ async function fetchJson(input: RequestInfo | URL, init: RequestInit): Promise