Merge ba42f0d36f into c9e3bb7ca4

packages/adapter-utils/src/server-utils.test.ts

@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
 import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
-import { describe, expect, it } from "vitest";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { CONNECTION_INTENT_AGENT_GUIDANCE } from "@paperclipai/shared";
 import {
   readPaperclipRuntimeSkillEntries,
@@ -15,6 +15,7 @@ import {
   buildPaperclipEnv,
   buildRuntimeToolsEnv,
   DEFAULT_PAPERCLIP_AGENT_PROMPT_TEMPLATE,
+  fetchWithRetry,
   DEFAULT_PAPERCLIP_CONVERSATION_PROMPT_TEMPLATE,
   isPaperclipExternalChatContractTurn,
   isPaperclipExternalChatQuestionResponseTurn,
@@ -3798,3 +3799,192 @@ describe("runtime skill assignment boundaries", () => {
     }
   });
 });
+
+describe("fetchWithRetry", () => {
+  let originalFetch: typeof fetch;
+
+  beforeEach(() => {
+    originalFetch = globalThis.fetch;
+  });
+
+  afterEach(() => {
+    globalThis.fetch = originalFetch;
+    vi.restoreAllMocks();
+  });
+
+  function makeResponse(status: number, headers: Record<string, string> = {}) {
+    return new Response("", { status, headers });
+  }
+
+  it("returns the first successful response without retrying", async () => {
+    const fetchMock = vi.fn().mockResolvedValue(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await fetchWithRetry("https://example.test", {});
+
+    expect(res.status).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("retries on 429 and returns the eventual 200", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(makeResponse(429))
+      .mockResolvedValueOnce(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { baseDelayMs: 1, maxDelayMs: 1 },
+    );
+
+    expect(res.status).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("returns the last failed response after exhausting retries", async () => {
+    const fetchMock = vi.fn().mockResolvedValue(makeResponse(503));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { maxRetries: 2, baseDelayMs: 1, maxDelayMs: 1 },
+    );
+
+    expect(res.status).toBe(503);
+    expect(fetchMock).toHaveBeenCalledTimes(3);
+  });
+
+  it("does not retry on non-retryable status codes", async () => {
+    const fetchMock = vi.fn().mockResolvedValue(makeResponse(400));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { baseDelayMs: 1, maxDelayMs: 1 },
+    );
+
+    expect(res.status).toBe(400);
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it("invokes onRetry with attempt metadata before each wait", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(makeResponse(429, { "retry-after": "0" }))
+      .mockResolvedValueOnce(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const onRetry = vi.fn();
+
+    await fetchWithRetry(
+      "https://example.test",
+      {},
+      {
+        baseDelayMs: 1,
+        maxDelayMs: 1,
+        onRetry,
+      },
+    );
+
+    expect(onRetry).toHaveBeenCalledTimes(1);
+    expect(onRetry).toHaveBeenCalledWith(
+      expect.objectContaining({
+        attempt: 1,
+        maxRetries: 3,
+        status: 429,
+        delayMs: 0,
+        retryAfterHeader: "0",
+      }),
+    );
+  });
+
+  it("treats Retry-After: 0 as 'retry immediately' (delayMs: 0)", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(makeResponse(429, { "retry-after": "0" }))
+      .mockResolvedValueOnce(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const onRetry = vi.fn();
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { baseDelayMs: 60_000, maxDelayMs: 60_000, onRetry },
+    );
+
+    expect(res.status).toBe(200);
+    expect(onRetry).toHaveBeenCalledWith(
+      expect.objectContaining({ delayMs: 0, retryAfterHeader: "0" }),
+    );
+  });
+
+  it("treats custom retryableStatuses as retryable (e.g. 403)", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(makeResponse(403))
+      .mockResolvedValueOnce(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { baseDelayMs: 1, maxDelayMs: 1, retryableStatuses: [403, 429] },
+    );
+
+    expect(res.status).toBe(200);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it("caps Retry-After header value at maxDelayMs", async () => {
+    const fetchMock = vi
+      .fn()
+      .mockResolvedValueOnce(makeResponse(429, { "retry-after": "60" }))
+      .mockResolvedValueOnce(makeResponse(200));
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    const onRetry = vi.fn();
+    const start = Date.now();
+
+    const res = await fetchWithRetry(
+      "https://example.test",
+      {},
+      { baseDelayMs: 1, maxDelayMs: 5, onRetry },
+    );
+
+    const elapsed = Date.now() - start;
+
+    expect(res.status).toBe(200);
+    expect(onRetry).toHaveBeenCalledWith(
+      expect.objectContaining({ delayMs: 5 }),
+    );
+    expect(elapsed).toBeLessThan(1_000);
+  });
+
+  it("aborts the request when the per-attempt timeout fires", async () => {
+    const fetchMock = vi.fn((_url: string | URL, init?: RequestInit) => {
+      return new Promise<Response>((_resolve, reject) => {
+        const signal = init?.signal as AbortSignal | undefined;
+        if (signal) {
+          signal.addEventListener("abort", () => {
+            reject(new DOMException("aborted", "AbortError"));
+          });
+        }
+      });
+    });
+    globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+    await expect(
+      fetchWithRetry(
+        "https://example.test",
+        {},
+        { maxRetries: 0, timeoutMs: 5 },
+      ),
+    ).rejects.toMatchObject({ name: "AbortError" });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+});

packages/adapter-utils/src/server-utils.ts

@@ -4835,3 +4835,123 @@ export async function runChildProcess(
       .catch(reject);
   });
 }
+
+/**
+ * Options for `fetchWithRetry`.
+ */
+export interface FetchRetryOptions {
+  /** Maximum number of retry attempts (default: 3). 0 = no retries. */
+  maxRetries?: number;
+  /** Base delay in ms before first retry — doubles each attempt (default: 2000). */
+  baseDelayMs?: number;
+  /** Maximum delay cap in ms (default: 30000). */
+  maxDelayMs?: number;
+  /** HTTP status codes that trigger a retry (default: [429, 502, 503, 504]).
+   *  403 is NOT included by default — pass it explicitly for APIs like Copilot
+   *  that use 403 for TPM rate limits. */
+  retryableStatuses?: number[];
+  /** Optional callback for retry logging. Called before each retry wait. */
+  onRetry?: (info: {
+    attempt: number;
+    maxRetries: number;
+    status: number;
+    delayMs: number;
+    retryAfterHeader?: string | null;
+  }) => void | Promise<void>;
+  /** Request timeout in ms per attempt (default: no timeout). */
+  timeoutMs?: number;
+}
+
+const DEFAULT_RETRYABLE_STATUSES = [429, 502, 503, 504];
+
+/**
+ * Fetch with automatic retry on rate-limit / transient server errors.
+ *
+ * - Respects `Retry-After` header (seconds or HTTP-date) when present.
+ * - Falls back to exponential backoff: baseDelay * 2^attempt, capped at maxDelay.
+ * - Returns the final Response (successful or last failed attempt).
+ * - Throws only on network/abort errors, never on HTTP status codes.
+ *
+ * Adapters that call external HTTP APIs (e.g. provider model-discovery or
+ * token-introspection endpoints) should use this instead of bare `fetch()`
+ * to handle transient rate-limit and server errors gracefully.
+ */
+export async function fetchWithRetry(
+  url: string | URL,
+  init: RequestInit,
+  options: FetchRetryOptions = {},
+): Promise<Response> {
+  const {
+    maxRetries = 3,
+    baseDelayMs = 2000,
+    maxDelayMs = 30_000,
+    retryableStatuses = DEFAULT_RETRYABLE_STATUSES,
+    onRetry,
+    timeoutMs,
+  } = options;
+
+  let lastResponse: Response | undefined;
+
+  for (let attempt = 0; attempt <= maxRetries; attempt++) {
+    let controller: AbortController | undefined;
+    let timer: ReturnType<typeof setTimeout> | undefined;
+
+    if (timeoutMs && timeoutMs > 0) {
+      controller = new AbortController();
+      timer = setTimeout(() => controller!.abort(), timeoutMs);
+    }
+
+    const signals: AbortSignal[] = [];
+    if (controller?.signal) signals.push(controller.signal);
+    if (init.signal) signals.push(init.signal as AbortSignal);
+    const signal =
+      signals.length > 1
+        ? AbortSignal.any(signals)
+        : signals[0] ?? undefined;
+
+    try {
+      lastResponse = await fetch(url, { ...init, signal });
+    } finally {
+      if (timer) clearTimeout(timer);
+    }
+
+    if (lastResponse.ok) return lastResponse;
+
+    if (!retryableStatuses.includes(lastResponse.status) || attempt >= maxRetries) {
+      return lastResponse;
+    }
+
+    const retryAfter = lastResponse.headers.get("retry-after");
+    let delayMs = baseDelayMs * Math.pow(2, attempt);
+
+    if (retryAfter) {
+      const parsed = Number(retryAfter);
+      if (!Number.isNaN(parsed) && parsed >= 0) {
+        delayMs = parsed * 1000;
+      } else {
+        const date = new Date(retryAfter).getTime();
+        if (!Number.isNaN(date)) {
+          delayMs = Math.max(0, date - Date.now());
+        }
+      }
+    }
+
+    delayMs = Math.min(delayMs, maxDelayMs);
+
+    try { await lastResponse.text(); } catch { /* drain body to free connection */ }
+
+    if (onRetry) {
+      await onRetry({
+        attempt: attempt + 1,
+        maxRetries,
+        status: lastResponse.status,
+        delayMs,
+        retryAfterHeader: retryAfter,
+      });
+    }
+
+    await new Promise((resolve) => setTimeout(resolve, delayMs));
+  }
+
+  return lastResponse!;
+}