From af42cc89eca26cd6351f372e0d35443fbaf0c15f Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 10 Sep 2026 10:40:46 -0500 Subject: [PATCH] fix(adapter-utils): retry idempotent GitHub launcher staging --- doc/sandbox-work-folders.md | 7 ++ .../src/execution-target-sandbox.test.ts | 119 +++++++++++++++++- .../adapter-utils/src/execution-target.ts | 40 +++++- 3 files changed, 161 insertions(+), 5 deletions(-) diff --git a/doc/sandbox-work-folders.md b/doc/sandbox-work-folders.md index 8c10ed7672..e2124f19c8 100644 --- a/doc/sandbox-work-folders.md +++ b/doc/sandbox-work-folders.md @@ -254,6 +254,13 @@ either observes completion or fails visibly and retains the working copy for the next run's existing intent reconciliation. This does not make arbitrary sandbox commands or repository mutations retryable. +Host-owned GitHub launcher staging also retries transient transport failures up +to three times within a single 15-second deadline per file or permission step. +The same run-specific file is locked, hash-checked, and atomically replaced, so +a lost reply after a successful upload does not rewrite the file on retry. +Cancellation, script failures, and invalid responses stop setup. This retry is +limited to launcher preparation; it never replays an agent or Git command. + Repository checkpoints transfer up to sixteen distinct batch-readable blobs of at most 1 MiB concurrently, plus at most four larger streaming blobs. Transports without batched reads retain the four-stream limit. Identical files diff --git a/packages/adapter-utils/src/execution-target-sandbox.test.ts b/packages/adapter-utils/src/execution-target-sandbox.test.ts index 542dc16594..484303423f 100644 --- a/packages/adapter-utils/src/execution-target-sandbox.test.ts +++ b/packages/adapter-utils/src/execution-target-sandbox.test.ts @@ -5,7 +5,7 @@ import http2 from "node:http2"; import net from "node:net"; import { duplexPair, type Duplex } from "node:stream"; import { execFile, spawn } from "node:child_process"; -import { mkdir, mkdtemp, readdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -27,6 +27,7 @@ import { formatAdapterExecutionTimeoutStartLogLine, parseAdapterExecutionTarget, postedIssueCommentLogMarker, + prepareGitHubOperationLaunchers, resolveAdapterExecutionTargetTimeout, resolveAdapterExecutionTargetTimeoutSec, runAdapterExecutionTargetProcess, @@ -158,6 +159,122 @@ describe("sandbox adapter execution targets", () => { }; } + describe("GitHub launcher staging transport retries", () => { + const transient = () => Object.assign(new Error("Request failed with status code 502"), { + name: "JsonRpcCallError", code: -32002, + }); + async function fixture(execute: NonNullable["execute"]) { + const root = await mkdtemp(path.join(os.tmpdir(), "github-launcher-retry-")); + cleanupDirs.push(root); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", transport: "sandbox", providerKey: "test", environmentId: "env-1", + leaseId: "lease-1", remoteCwd: root, timeoutMs: 30_000, runner: { execute }, + }; + return { runId: "launcher-retry", target, cwd: root, env: {} }; + } + + it("hash-skips an accepted upload after its provider reply is lost", async () => { + const local = createLocalSandboxRunner(); + let firstMtime = 0; + const outputs: string[] = []; + const execute = vi.fn(async (request: Parameters[0]) => { + const result = await local.execute(request); + outputs.push(result.stdout); + if (outputs.length === 1) { + firstMtime = (await stat(path.join(request.cwd!, ".paperclip-runtime/github/launcher-retry/package.json"))).mtimeMs; + throw transient(); + } + return result; + }); + const input = await fixture(execute); + const env = await prepareGitHubOperationLaunchers(input); + expect(execute).toHaveBeenCalledTimes(11); + expect(JSON.parse(outputs[1].trim())).toEqual({ uploaded: false }); + expect((await stat(path.join(env.PAPERCLIP_GITHUB_LAUNCHER_DIR, "package.json"))).mtimeMs).toBe(firstMtime); + expect(JSON.parse(await readFile(path.join(env.PAPERCLIP_GITHUB_LAUNCHER_DIR, "package.json"), "utf8"))).toEqual({ type: "commonjs" }); + expect((await stat(path.join(env.PAPERCLIP_GITHUB_LAUNCHER_DIR, "git"))).mode & 0o777).toBe(0o700); + expect((await stat(env.GH_CONFIG_DIR)).isDirectory()).toBe(true); + }); + + it("safely completes permissions after their provider reply is lost", async () => { + const local = createLocalSandboxRunner(); + let lost = false; + const execute = vi.fn(async (request: Parameters[0]) => { + const result = await local.execute(request); + if (request.args?.[1]?.startsWith("chmod 700") && !lost) { + lost = true; + throw transient(); + } + return result; + }); + const env = await prepareGitHubOperationLaunchers(await fixture(execute)); + expect(lost).toBe(true); + expect(execute).toHaveBeenCalledTimes(11); + expect((await stat(path.join(env.PAPERCLIP_GITHUB_LAUNCHER_DIR, "gh"))).mode & 0o777).toBe(0o700); + expect((await stat(env.GH_CONFIG_DIR)).isDirectory()).toBe(true); + }); + + it("limits persistent transport failures to three attempts within one deadline", async () => { + const error = transient(); + const execute = vi.fn["execute"]>().mockRejectedValue(error); + await expect(prepareGitHubOperationLaunchers(await fixture(execute))).rejects.toBe(error); + expect(execute).toHaveBeenCalledTimes(3); + const budgets = execute.mock.calls.map(([request]) => request.timeoutMs!); + expect(budgets[0]).toBeLessThanOrEqual(15_000); + expect(budgets[1]).toBeLessThan(budgets[0]); + expect(budgets[2]).toBeLessThan(budgets[1]); + }); + + it.each([ + new Error("script failed: Request failed with status code 502"), + Object.assign(new Error("permission denied"), { code: "EACCES" }), + Object.assign(new Error("Request failed with status code 502"), { name: "JsonRpcCallError", code: -32602 }), + ])("does not replay unclassified errors: %s", async (error) => { + const execute = vi.fn["execute"]>().mockRejectedValue(error); + await expect(prepareGitHubOperationLaunchers(await fixture(execute))).rejects.toBe(error); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it("does not retry a completed shell failure containing an HTTP status", async () => { + const local = createLocalSandboxRunner(); + const execute = vi.fn(async (request: Parameters[0]) => { + const result = await local.execute({ ...request, command: "sh", args: ["-c", "echo 'Request failed with status code 502' >&2; exit 1"] }); + return result; + }); + await expect(prepareGitHubOperationLaunchers(await fixture(execute))).rejects.toThrow("502"); + expect(execute).toHaveBeenCalledTimes(1); + }); + + it("does not exceed the deadline after a failed attempt", async () => { + const error = transient(); + let now = 1_000; + const execute = vi.fn["execute"]>().mockImplementation(async () => { + now += 15_000; + throw error; + }); + const input = await fixture(execute); + const clock = vi.spyOn(Date, "now").mockImplementation(() => now); + try { + await expect(prepareGitHubOperationLaunchers(input)).rejects.toBe(error); + expect(execute).toHaveBeenCalledTimes(1); + } finally { clock.mockRestore(); } + }); + + it("stops retries when the run is cancelled", async () => { + let cancelled: Promise | undefined; + const execute = vi.fn["execute"]>().mockImplementation(async () => { + cancelled = cancelAdapterRunExecution("launcher-retry"); + throw transient(); + }); + const input = await fixture(execute); + beginAdapterRunCancellation(input.runId); + try { + await expect(prepareGitHubOperationLaunchers(input)).rejects.toMatchObject({ code: "ADAPTER_RUN_CANCELLED" }); + expect(execute).toHaveBeenCalledTimes(1); + } finally { finishAdapterRunCancellation(input.runId); await cancelled; } + }); + }); + async function readRuntimeTextFiles(rootDir: string): Promise { const entries = await readdir(rootDir, { withFileTypes: true }).catch(() => []); const contents: string[] = []; diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index b45cacb172..5b78a86a37 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1541,6 +1541,35 @@ export async function cleanupGitHubOperationLaunchers(input: GitHubLauncherLocat } } +// This retry boundary is deliberately private to host-owned launcher setup. +// A lost reply can follow a completed write: staging locks, verifies the hash, +// and atomically replaces the same run-specific file with identical bytes. +async function prepareGitHubLauncherWithRetry(runId: string, operation: (timeoutMs: number) => Promise): Promise { + const deadline = Date.now() + 15_000; + for (let attempt = 0; ; attempt++) { + throwIfAdapterRunCancelled(runId); + const remaining = deadline - Date.now(); + if (remaining <= 0) throw new Error("GitHub launcher preparation deadline exceeded"); + try { return await operation(remaining); } + catch (error) { + throwIfAdapterRunCancelled(runId); + const detail = error instanceof Error + ? error as Error & { code?: unknown; status?: unknown; statusCode?: unknown; response?: { status?: unknown } } + : null; + const transient = detail && ( + ["ECONNRESET", "EPIPE", "EAI_AGAIN", "ECONNABORTED"].includes(String(detail.code ?? "")) + || detail.message === "socket hang up" + || [detail.status, detail.statusCode, detail.response?.status].some((status) => [502, 503, 504].includes(status as number)) + || (detail.name === "JsonRpcCallError" && detail.code === -32002 + && /^Request failed with status code (502|503|504)(?:: Sandbox command requested here)?$/.test(detail.message)) + ); + const waitMs = 250 * (attempt + 1); + if (!transient || attempt >= 2 || Date.now() + waitMs >= deadline) throw error; + await new Promise((resolve) => setTimeout(resolve, waitMs)); + } + } +} + /** Stage token-free launchers next to the execution, not in shared global Git config. */ export async function prepareGitHubOperationLaunchers(input: { runId: string; target: AdapterExecutionTarget | null | undefined; cwd: string; env: Record; @@ -1563,15 +1592,18 @@ export async function prepareGitHubOperationLaunchers(input: { if (remote) { const runner = adapterExecutionTargetCommandRunner(remote); for (const [program, body] of Object.entries(files)) { - await syncRemoteTextFileWithHashSkip({ + await prepareGitHubLauncherWithRetry(input.runId, (timeoutMs) => syncRemoteTextFileWithHashSkip({ runner, remoteCwd: remote.remoteCwd, remoteDir: directory, remotePath: path.posix.join(directory, program), body, label: "GitHub operation launcher", action: "stage GitHub operation launcher", lockDir: path.posix.join(directory, `.${program}.lock`), - timeoutMs: 15_000, shellCommand: adapterExecutionTargetShellCommand(remote), - }); + timeoutMs, shellCommand: adapterExecutionTargetShellCommand(remote), + })); } - const permissions = await runner.execute({ command: "sh", args: ["-c", `chmod 700 ${shellQuote(directory)}/git ${shellQuote(directory)}/gh && mkdir -p ${shellQuote(configDirectory)}`], cwd: remote.remoteCwd, timeoutMs: 15_000 }); + const permissions = await prepareGitHubLauncherWithRetry(input.runId, (timeoutMs) => runner.execute({ + command: "sh", args: ["-c", `chmod 700 ${shellQuote(directory)}/git ${shellQuote(directory)}/gh && mkdir -p ${shellQuote(configDirectory)}`], + cwd: remote.remoteCwd, timeoutMs, + })); if (permissions.exitCode !== 0) throw new Error("Could not prepare managed GitHub launchers"); } else { await fs.mkdir(directory, { recursive: true, mode: 0o700 });