diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts index b0300d6d7d..a3140e9b64 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.test.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.test.ts @@ -475,6 +475,68 @@ describe("sandbox callback bridge", () => { } }); + it("recovers from transient queue polling failures and keeps relaying requests", async () => { + // A single reset or slow sandbox exec used to unwind the poll loop into its + // terminal catch, killing the relay for the rest of the run. The loop must + // instead back off, retry, and still deliver requests queued after the + // failure window. + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-transient-")); + cleanupDirs.push(rootDir); + + const queueDir = path.posix.join(rootDir, "queue"); + const directories = sandboxCallbackBridgeDirectories(queueDir); + const baseClient = createFileSystemSandboxCallbackBridgeQueueClient(); + let listCalls = 0; + const client: SandboxCallbackBridgeQueueClient = { + ...baseClient, + listJsonFiles: async (dirPath: string) => { + listCalls += 1; + if (listCalls <= 3) { + throw new Error("list requests failed: kex_exchange_identification: read: Connection reset by peer"); + } + return baseClient.listJsonFiles(dirPath); + }, + }; + + const seenPaths: string[] = []; + const worker = await startSandboxCallbackBridgeWorker({ + client, + queueDir, + authorizeRequest: async () => null, + handleRequest: async (request) => { + seenPaths.push(request.path); + return { + status: 200, + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ok: true }), + }; + }, + }); + cleanupFns.push(async () => { + await worker.stop(); + }); + + const requestId = "transient-recovery-1"; + await writeFile( + path.join(directories.requestsDir, `${requestId}.json`), + JSON.stringify({ + id: requestId, + method: "GET", + path: "/api/agents/me", + query: "", + headers: {}, + body: "", + }), + "utf8", + ); + + const responseFile = await waitForJsonFile(directories.responsesDir, 10_000); + const raw = await readFile(path.join(directories.responsesDir, responseFile), "utf8"); + expect(JSON.parse(raw)).toMatchObject({ id: requestId, status: 200 }); + expect(seenPaths).toEqual(["/api/agents/me"]); + expect(listCalls).toBeGreaterThan(3); + }); + it("keeps the queue-directory setup on the startup step but resets the poll loop store", async () => { // The worker starts inside the measured `bridge.paperclip` step. Its awaited // queue-directory setup is startup work, so a `makeDir` `sandbox.exec` span @@ -1510,7 +1572,7 @@ describe("sandbox callback bridge", () => { })}\n`; } - it("times out a stalled poll, writes a 503, and surfaces a run-level error", async () => { + it("times out a stalled poll, surfaces a run-level error, and recovers to deliver the request", async () => { const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-bridge-hang-")); cleanupDirs.push(rootDir); @@ -1526,9 +1588,11 @@ describe("sandbox callback bridge", () => { const client: SandboxCallbackBridgeQueueClient = { ...base, // The first poll never resolves — a silently unresponsive sandbox channel. - // The per-iteration timeout must convert the hang into a caught error. The - // request never reaches the handler, so the recovery path can safely 503 - // it. Later calls (the recovery's failPendingRequests) resolve. + // The per-iteration timeout must convert the hang into a caught error, and + // the loop must then back off and retry rather than die: the request never + // reached the handler, so the retry delivers the real response. A + // sustained outage is the watchdog's job (proven separately below), not a + // reason to fail a request one transient hang could still serve. listJsonFiles: async (dir) => { listCalls += 1; if (listCalls === 1) { @@ -1550,7 +1614,7 @@ describe("sandbox callback bridge", () => { const responseFile = await waitForJsonFile(directories.responsesDir, 3_000); const responseBody = await readFile(path.posix.join(directories.responsesDir, responseFile), "utf8"); - expect(JSON.parse(responseBody).status).toBe(503); + expect(JSON.parse(responseBody).status).toBe(200); expect(workerErrors.length).toBeGreaterThan(0); expect(workerErrors[0]).toContain("timed out"); @@ -2261,10 +2325,13 @@ describe("sandbox callback bridge", () => { }); it("retries a recovery 503 write that fails transiently, delivers the 503, and removes the request", async () => { - // The poll times out, so the recovery path aborts the queued request with a - // 503. The first 503 write fails, so the recovery must retry it inside the - // same pass. It then delivers the 503 and removes the request. The request is - // unclaimed, so its host mutation never ran; the 503 stays retry-safe. + // A request attempt times out (its first read hangs), so the loop's request + // catch runs the recovery pass, which aborts the queued request with a 503. + // The first 503 write fails, so the recovery must retry it inside the same + // pass. It then delivers the 503 and removes the request. The request is + // unclaimed, so its host mutation never ran; the 503 stays retry-safe. (A + // hung poll no longer triggers this pass — the loop backs off and retries + // the poll instead, and a sustained hang is the watchdog's job.) const waitFor = async (predicate: () => boolean, timeoutMs: number) => { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { @@ -2286,27 +2353,26 @@ describe("sandbox callback bridge", () => { requestBodies.set(requestPath, bridgeRequestJson("req-503-retry")); const responseWrites: Array<{ path: string; status: number; body: string }> = []; const requestRemovals: string[] = []; - let listCalls = 0; + let readCalls = 0; let writeAttempts = 0; const client: SandboxCallbackBridgeQueueClient = { makeDir: async () => {}, makeDirs: async () => {}, - // The first poll never resolves — a silently unresponsive sandbox channel. - // The per-iteration timeout converts the hang into a caught error, so the - // loop `catch` runs the recovery path. Later listings resolve, so the - // recovery enumerates and aborts the request. - listJsonFiles: async (dir) => { - if (dir !== directories.requestsDir) { - return []; - } - listCalls += 1; - if (listCalls === 1) { - return await new Promise(() => {}); - } - return [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort(); - }, + listJsonFiles: async (dir) => + dir === directories.requestsDir + ? [...requestBodies.keys()].map((entry) => path.posix.basename(entry)).sort() + : [], + // The first read never resolves — a silently unresponsive sandbox channel + // hit mid-request, before the handler claim. The per-iteration timeout + // converts the hang into a caught error, so the loop's request catch runs + // the recovery pass. The recovery's own read resolves, so it can build and + // deliver the 503. readTextFile: async (remotePath) => { + readCalls += 1; + if (readCalls === 1) { + return await new Promise(() => {}); + } const body = requestBodies.get(remotePath); if (body === undefined) { throw new Error(`missing request ${remotePath}`); diff --git a/packages/adapter-utils/src/sandbox-callback-bridge.ts b/packages/adapter-utils/src/sandbox-callback-bridge.ts index 5d8907ce04..b198d6d9cb 100644 --- a/packages/adapter-utils/src/sandbox-callback-bridge.ts +++ b/packages/adapter-utils/src/sandbox-callback-bridge.ts @@ -34,7 +34,8 @@ const DEFAULT_BRIDGE_MAX_DUPLEX_DECODER_BYTES = 8 * 1024 * 1024; // (PAPERCLIP_BRIDGE_RESPONSE_TIMEOUT_MS), so the host loop fails fast and writes // 503 responses before the in-sandbox client gives up. A silently unresponsive // sandbox channel makes a client call hang with no reject; this timeout turns -// that hang into a caught error, so the loop `catch` runs `failPendingRequests`. +// that hang into a caught error, so the poll loop can back off and retry while +// the watchdog below decides when to fail the queued requests. const DEFAULT_BRIDGE_ITERATION_TIMEOUT_MS = 10_000; // Watchdog backstop for a hang that the per-iteration timeout does not catch // (for example many slow-but-under-timeout calls, or a stall outside the awaited @@ -58,6 +59,12 @@ const MAX_BACKSTOP_WRITE_ATTEMPTS = 3; // The delay between two 504 backstop write attempts. It is short, so all retries // finish well under the in-sandbox 30s response deadline. const BACKSTOP_WRITE_RETRY_MS = 50; +// Backoff cap between poll-loop retries after a transient iteration failure. +// The cap keeps a recovering loop probing often enough to resume before the +// in-sandbox 30s response deadline strands queued callers, while the +// exponential ramp below it keeps a hard-down channel from burning an exec +// call every poll interval. +const MAX_TRANSIENT_ITERATION_BACKOFF_MS = 5_000; const REMOTE_WRITE_BASE64_CHUNK_SIZE = 32 * 1024; export const SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT = "paperclip-bridge-server.mjs"; const SANDBOX_EXEC_CHANNEL_ENV = "PAPERCLIP_SANDBOX_EXEC_CHANNEL"; @@ -1380,13 +1387,54 @@ export async function startSandboxCallbackBridgeWorker(input: { watchdogTimer.unref(); } try { + // Consecutive transient poll failures. A single failed list call — one + // reset or slow sandbox exec — must not end the relay for the rest of the + // run: the in-sandbox gateway keeps queueing requests, so a dead loop + // strands every later API call from the agent (its status writes then look + // like connection failures and the issue loses its disposition). Back off + // and retry the poll instead. The watchdog stays the escalation path for a + // sustained outage — it fires after `watchdogTimeoutMs` without a + // successful iteration and fails the queued requests fast, while this loop + // keeps probing for recovery. + let consecutivePollFailures = 0; while (true) { - const fileNames = await withTimeout( - input.client.listJsonFiles(directories.requestsDir), - iterationTimeoutMs, - "Sandbox callback bridge list requests", - ); - if (fileNames.length === 0) { + let fileNames: string[]; + try { + fileNames = await withTimeout( + input.client.listJsonFiles(directories.requestsDir), + iterationTimeoutMs, + "Sandbox callback bridge list requests", + ); + consecutivePollFailures = 0; + } catch (error) { + if (stopping) { + break; + } + consecutivePollFailures += 1; + const message = `${buildWorkerFailureMessage(error)} (transient poll failure ${consecutivePollFailures}; retrying)`; + if (consecutivePollFailures === 1) { + // Put the first failure of a streak on the run trace; later repeats + // only warn, so a flapping channel does not spam failed spans. + await surfaceRunError(new Error(message)); + } else { + console.warn(`[paperclip] ${message}`); + } + const backoffMs = Math.min( + pollIntervalMs * 2 ** consecutivePollFailures, + MAX_TRANSIENT_ITERATION_BACKOFF_MS, + ); + await new Promise((resolve) => setTimeout(resolve, backoffMs)); + continue; + } + // A file whose attempt is still in flight (or waiting on its 504 + // backstop) is not actionable: `processRequestFile` would skip it via + // the guard map. Treat an all-guarded listing like an empty one and + // sleep a poll interval. Re-listing immediately would spin the loop — + // an exec storm against a real sandbox channel, and with an in-memory + // client a pure-microtask loop that starves every timer in the process + // (including the guard's own backstop and abort timers). + const actionableFileNames = fileNames.filter((fileName) => !inFlightRequestGuards.has(fileName)); + if (actionableFileNames.length === 0) { lastSuccessfulIterationAt = Date.now(); if (stopping) { break; @@ -1394,7 +1442,7 @@ export async function startSandboxCallbackBridgeWorker(input: { await new Promise((resolve) => setTimeout(resolve, pollIntervalMs)); continue; } - for (const fileName of fileNames) { + for (const fileName of actionableFileNames) { if (stopping && Date.now() >= stopDeadline) break; inFlight += 1; try { @@ -1406,7 +1454,7 @@ export async function startSandboxCallbackBridgeWorker(input: { // `task.run` after it. Without a runner, the request runs under the // run parent with no wrapper span, exactly like the earlier behavior. // The per-iteration timeout wraps the whole request, so a hung - // request rejects and the loop `catch` runs `failPendingRequests`. + // request rejects and the catch below runs the recovery pass. await withTimeout( input.runtimeSpan ? input.runtimeSpan(CALLBACK_BRIDGE_RELAY_REQUEST_SPAN, () => @@ -1419,6 +1467,23 @@ export async function startSandboxCallbackBridgeWorker(input: { `Sandbox callback bridge process request ${fileName}`, ); lastSuccessfulIterationAt = Date.now(); + } catch (error) { + // A single request attempt failed or hung. Run the same recovery + // pass the loop previously died on — abort the in-flight handler + // (its 504 backstop keeps the caller from stranding) and 503 the + // unclaimed queued requests — but keep the loop alive afterward. A + // caller that sees the retry-safe 503 re-queues, and the recovered + // loop serves the retry; the old terminal catch left every later + // request to strand instead. + const message = buildWorkerFailureMessage(error); + await surfaceRunError(new Error(message)); + try { + await failPendingRequests(message, { abandonInFlight: true }); + } catch (failPendingError) { + console.warn( + `[paperclip] sandbox callback bridge failed to abort queued requests after a request failure: ${failPendingError instanceof Error ? failPendingError.message : String(failPendingError)}`, + ); + } } finally { inFlight -= 1; }