fix: tolerate transient sandbox Git credential transport delays

Honor the native duplex opt-in and align repeat-safe credential acquisition with the file bridge response budget, without retrying Git commands.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Dotta 2026-09-09 12:19:39 -05:00
parent 2d95eef2ba
commit ffb01ff3d5
8 changed files with 353 additions and 22 deletions

View File

@ -353,6 +353,16 @@ verification; a fallback within `runner.artifact.prepare` can transfer gigabytes
independently of task files. Acceptance must prove that a matching image uses
its installed pack instead of silently relying on that fallback.
Native and legacy Git credential callbacks honor the same experimental duplex
setting and provider capability gates. When streaming is disabled or unavailable,
the file bridge remains supported. Credential acquisition allows 35 seconds per
request so the bridge can return its response within its 30-second window.
Transient transport failures receive at most three attempts within a 75-second
overall budget; authorization denials and invalid responses are not retried.
Only credential acquisition is retried, before starting Git or `gh`; repository
operations are never replayed. Acceptance must exercise both transport paths
and record which one was actually selected.
Automated tests do not qualify a deployed runner image. Before merging, use a
new pinned staging stack with the branch's Cloud image and matching migrator.
The deployed harness must target that tenant URL without launching a local

View File

@ -3215,6 +3215,7 @@ describe("sandbox adapter execution targets", () => {
auth: string | null;
runId: string | null;
headers: Record<string, string>;
body: string;
}>;
close: () => Promise<void>;
}> {
@ -3224,8 +3225,11 @@ describe("sandbox adapter execution targets", () => {
auth: string | null;
runId: string | null;
headers: Record<string, string>;
body: string;
}> = [];
const server = createServer((req, res) => {
const server = createServer(async (req, res) => {
const chunks: Buffer[] = [];
for await (const chunk of req) chunks.push(Buffer.from(chunk));
const headers: Record<string, string> = {};
for (const [key, value] of Object.entries(req.headers)) {
if (typeof value === "string") headers[key] = value;
@ -3236,6 +3240,7 @@ describe("sandbox adapter execution targets", () => {
auth: req.headers.authorization ?? null,
runId: typeof req.headers["x-paperclip-run-id"] === "string" ? req.headers["x-paperclip-run-id"] : null,
headers,
body: Buffer.concat(chunks).toString("utf8"),
});
res.writeHead(200, { "content-type": "application/json" });
res.end(JSON.stringify({ ok: true }));
@ -3363,6 +3368,27 @@ describe("sandbox adapter execution targets", () => {
auth: "Bearer real-run-jwt",
runId: "run-http2",
});
// Native Git uses this same channel. Keep its runtime capability header
// while the host replaces bridge authentication and binds the run ID.
const credentials = await http2TestRequest(sessionRef.current!, {
method: "POST",
path: "/runtime-tools/github/credentials",
headers: {
authorization: `Bearer ${bridgeToken}`,
"x-paperclip-github-capability": "current-github-capability",
"content-type": "application/json",
},
body: "{}",
});
expect(credentials.status).toBe(200);
expect(api.requests[1]).toMatchObject({
method: "POST",
url: "/runtime-tools/github/credentials",
auth: "Bearer real-run-jwt",
runId: "run-http2",
headers: { "x-paperclip-github-capability": "current-github-capability" },
body: "{}",
});
} finally {
sessionRef.current?.close();
await bridge?.stop();

View File

@ -0,0 +1,109 @@
import vm from "node:vm";
import { describe, expect, it, vi } from "vitest";
import { githubLauncherSource } from "./github-launcher.js";
// Execute the exact self-contained acquisition code staged inside the launcher.
// A virtual clock exercises the real 35s/75s budgets without sleeping for them.
function acquisitionFixture() {
let now = 0;
const budgets: number[] = [];
const signals: Array<{ budget: number; aborted: boolean; reason?: Error }> = [];
const sleeps: number[] = [];
const fetch = vi.fn();
const source = githubLauncherSource();
const context = vm.createContext({
fetch, Set, SyntaxError,
Date: { now: () => now },
AbortSignal: { timeout: (ms: number) => { budgets.push(ms); const signal = { budget: ms, aborted: false }; signals.push(signal); return signal; } },
setTimeout: (callback: () => void, ms: number) => { sleeps.push(ms); now += ms; callback(); },
});
vm.runInContext(source.slice(source.indexOf("const credentialRequestTimeoutMs"), source.indexOf("async function main()")), context);
return {
fetch, budgets, sleeps, signals, advance: (ms: number) => { now += ms; },
acquire: () => vm.runInContext("acquireCredentials('http://bridge/runtime-tools/github/credentials', { authorization: 'Bearer bridge-token', 'x-paperclip-github-capability': 'run-capability' })", context) as Promise<unknown>,
};
}
function response(status = 200, result: unknown = { status: "available", env: { GH_TOKEN: "fixture-token" } }) {
return { ok: status === 200, status, body: { cancel: vi.fn().mockResolvedValue(undefined) }, json: vi.fn().mockResolvedValue(result) };
}
describe("staged Git credential acquisition budgets", () => {
it("lets a file-bridge response arrive after the old 10s deadline", async () => {
const fixture = acquisitionFixture();
fixture.fetch.mockImplementationOnce(async () => {
fixture.advance(11_000);
return response();
});
await expect(fixture.acquire()).resolves.toMatchObject({ status: "available" });
expect(fixture.budgets).toEqual([35_000]);
expect(fixture.fetch).toHaveBeenCalledTimes(1);
});
it("reacquires after a reset and 503 without changing the capability", async () => {
const fixture = acquisitionFixture();
const unavailable = response(503);
fixture.fetch.mockRejectedValueOnce(Object.assign(new TypeError("secret upstream URL"), { cause: { code: "ECONNRESET" } }))
.mockResolvedValueOnce(unavailable).mockResolvedValueOnce(response());
await expect(fixture.acquire()).resolves.toMatchObject({ env: { GH_TOKEN: "fixture-token" } });
expect(fixture.fetch).toHaveBeenCalledTimes(3);
expect(unavailable.body.cancel).toHaveBeenCalledOnce();
expect(fixture.sleeps).toEqual([250, 500]);
for (const [, input] of fixture.fetch.mock.calls) {
expect(input.headers).toEqual({ authorization: "Bearer bridge-token", "x-paperclip-github-capability": "run-capability" });
expect(input.body).toBe("{}");
}
});
it("exhausts three transient failures with a finite sanitized error", async () => {
const fixture = acquisitionFixture();
fixture.fetch.mockRejectedValue(Object.assign(new Error("secret body"), { name: "TimeoutError" }));
await expect(fixture.acquire()).rejects.toMatchObject({ credentialCategory: "timeout", message: "GitHub credential acquisition failed" });
expect(fixture.fetch).toHaveBeenCalledTimes(3);
});
it.each([401, 403, 400, 500])("does not retry permanent HTTP %s", async (status) => {
const fixture = acquisitionFixture();
fixture.fetch.mockResolvedValue(response(status));
await expect(fixture.acquire()).rejects.toMatchObject({ credentialCategory: status === 401 || status === 403 ? "denied" : "unavailable" });
expect(fixture.fetch).toHaveBeenCalledTimes(1);
});
it("does not retry malformed JSON or an unknown fetch error", async () => {
for (const error of [new SyntaxError("secret response"), new TypeError("unclassified fetch failure")]) {
const fixture = acquisitionFixture();
fixture.fetch.mockResolvedValue({ ...response(), json: async () => { throw error; } });
await expect(fixture.acquire()).rejects.toMatchObject({ credentialCategory: error instanceof SyntaxError ? "invalidresponse" : "unavailable" });
expect(fixture.fetch).toHaveBeenCalledTimes(1);
}
});
it("keeps the timeout active through body consumption and retries a body timeout", async () => {
const fixture = acquisitionFixture();
fixture.fetch.mockResolvedValueOnce({ ...response(), json: async () => {
fixture.advance(35_000);
fixture.signals[0].aborted = true;
fixture.signals[0].reason = Object.assign(new Error("deadline"), { name: "TimeoutError" });
throw Object.assign(new Error("body stalled"), { name: "AbortError" });
} }).mockResolvedValueOnce(response());
await expect(fixture.acquire()).resolves.toMatchObject({ status: "available" });
expect(fixture.budgets).toEqual([35_000, 35_000]);
});
it("caps mixed retries and steering waits at 75s and rejects late credentials", async () => {
const fixture = acquisitionFixture();
fixture.fetch.mockImplementationOnce(async () => { fixture.advance(35_000); throw Object.assign(new Error(), { name: "TimeoutError" }); })
.mockImplementationOnce(async () => { fixture.advance(35_000); return response(409); })
.mockImplementationOnce(async () => { fixture.advance(4_000); return response(); });
await expect(fixture.acquire()).rejects.toMatchObject({ credentialCategory: "timeout" });
expect(fixture.budgets).toEqual([35_000, 35_000, 3_750]);
expect(fixture.fetch).toHaveBeenCalledTimes(3);
});
it("bounds steering reconciliation to thirty requests", async () => {
const fixture = acquisitionFixture();
fixture.fetch.mockResolvedValue(response(409));
await expect(fixture.acquire()).rejects.toMatchObject({ credentialCategory: "unavailable" });
expect(fixture.fetch).toHaveBeenCalledTimes(30);
expect(fixture.sleeps).toHaveLength(29);
});
});

View File

@ -1,6 +1,6 @@
import { execFile } from "node:child_process";
import { createServer } from "node:http";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { mkdtemp, mkdir, writeFile, readFile, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
@ -100,4 +100,47 @@ process.stdout.write(JSON.stringify({identity, token:process.env.GH_TOKEN ?? nul
expect(env.GH_TOKEN).toBe("");
expect(env.GIT_AUTHOR_NAME).toBe("");
});
it("retries acquisition before spawning once, but never retries denied access or the Git command", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "paperclip-github-retry-"));
cleanups.push(() => rm(root, { recursive: true, force: true }));
const bin = path.join(root, "managed"), realBin = path.join(root, "real");
await mkdir(bin); await mkdir(realBin);
const countPath = path.join(root, "spawn-count");
await writeFile(path.join(bin, "gh"), githubLauncherSource(), { mode: 0o700 });
await writeFile(path.join(realBin, "gh"), `#!/usr/bin/env node
const fs = require('node:fs');
fs.appendFileSync(process.env.SPAWN_COUNT_PATH, 'spawn\\n');
process.stdout.write(process.env.GH_TOKEN || 'no-token');
process.exit(Number(process.env.CHILD_EXIT_CODE || '0'));
`, { mode: 0o700 });
let calls = 0, denied = false;
const server = createServer((req, res) => {
calls++;
expect(req.headers["x-paperclip-github-capability"]).toBe("fixture-capability");
if (denied) { res.writeHead(403); res.end("secret denial body"); return; }
if (calls === 1) { req.socket.destroy(); return; }
if (calls === 2) { res.writeHead(503); res.end("secret transient body"); return; }
res.setHeader("content-type", "application/json");
res.end(JSON.stringify({ status: "available", env: { GH_TOKEN: "current-fixture-token" } }));
});
await new Promise<void>(resolve => server.listen(0, "127.0.0.1", resolve));
cleanups.push(() => new Promise<void>((resolve, reject) => server.close(error => error ? reject(error) : resolve())));
const env = { ...process.env, ...githubBrokerEnvironment({}, {
url: `http://127.0.0.1:${(server.address() as { port: number }).port}`, token: "fixture-capability",
}), PATH: `${bin}:${realBin}:${process.env.PATH}`, SPAWN_COUNT_PATH: countPath };
expect((await exec(path.join(bin, "gh"), [], { env })).stdout).toBe("current-fixture-token");
expect(calls).toBe(3);
expect(await readFile(countPath, "utf8")).toBe("spawn\n");
denied = true;
await expect(exec(path.join(bin, "gh"), [], { env })).rejects.toMatchObject({
code: 1, stderr: "Paperclip: GitHub credential context unavailable (denied); retry this operation.\n",
});
expect(calls).toBe(4);
expect(await readFile(countPath, "utf8")).toBe("spawn\n");
denied = false;
await expect(exec(path.join(bin, "gh"), [], { env: { ...env, CHILD_EXIT_CODE: "17" } })).rejects.toMatchObject({ code: 17 });
expect(calls).toBe(5);
expect(await readFile(countPath, "utf8")).toBe("spawn\nspawn\n");
});
});

View File

@ -17,6 +17,77 @@ if (!['git', 'gh'].includes(program) || !executable) {
process.stderr.write('Paperclip: requested GitHub command is not installed.\n');
process.exit(127);
}
// The file gateway owns a 30s response window. Let it report its result before
// retrying only this repeat-safe acquisition; the Git command has not started.
const credentialRequestTimeoutMs = 35000;
const credentialAcquisitionTimeoutMs = 75000;
function credentialError(category) {
const error = new Error('GitHub credential acquisition failed');
error.credentialCategory = category;
return error;
}
function transientCredentialError(error) {
if (error && error.name === 'TimeoutError') return true;
const codes = new Set(['ECONNRESET', 'EPIPE', 'ETIMEDOUT', 'ECONNREFUSED', 'EAI_AGAIN',
'UND_ERR_SOCKET', 'UND_ERR_CONNECT_TIMEOUT', 'UND_ERR_HEADERS_TIMEOUT', 'UND_ERR_BODY_TIMEOUT']);
return Boolean(error && (codes.has(error.code) || (error.cause && codes.has(error.cause.code))));
}
async function acquireCredentials(url, headers) {
const deadline = Date.now() + credentialAcquisitionTimeoutMs;
let transientFailures = 0;
let conflicts = 0;
const remaining = () => {
const ms = deadline - Date.now();
if (ms <= 0) throw credentialError('timeout');
return ms;
};
const pause = async (ms) => {
await new Promise(resolve => setTimeout(resolve, Math.min(ms, remaining())));
remaining();
};
while (true) {
let response;
const requestSignal = AbortSignal.timeout(Math.min(credentialRequestTimeoutMs, remaining()));
try {
response = await fetch(url, {
method: 'POST', redirect: 'error',
signal: requestSignal,
headers, body: '{}',
});
if (response.ok) {
const result = await response.json();
remaining(); // A late credential must never start a command.
if (!result || !['available', 'absent', 'unavailable'].includes(result.status) ||
(result.status === 'available' && (!result.env || typeof result.env !== 'object' || Array.isArray(result.env)))) {
throw credentialError('invalidresponse');
}
return result;
}
} catch (error) {
if (error && error.credentialCategory) throw error;
const timedOut = requestSignal.aborted && requestSignal.reason?.name === 'TimeoutError';
if (timedOut) error = requestSignal.reason;
if (error instanceof SyntaxError) throw credentialError('invalidresponse');
if (!transientCredentialError(error)) throw credentialError('unavailable');
transientFailures++;
if (transientFailures >= 3) throw credentialError(error.name === 'TimeoutError' ? 'timeout' : 'unavailable');
await pause(250 * transientFailures);
continue;
}
// Discard credentials/error bodies without printing or retaining them.
await response.body?.cancel().catch(() => {});
if (response.status === 401 || response.status === 403) throw credentialError('denied');
if (response.status === 409) {
if (++conflicts >= 30) throw credentialError('unavailable');
await pause(1000);
continue;
}
if (![502, 503, 504].includes(response.status) || ++transientFailures >= 3) {
throw credentialError('unavailable');
}
await pause(250 * transientFailures);
}
}
async function main() {
let env = { ...process.env };
const configRoot = env.GH_CONFIG_DIR || os.tmpdir();
@ -40,22 +111,13 @@ async function main() {
GIT_CONFIG_KEY_3: 'core.askPass', GIT_CONFIG_VALUE_3: '',
});
const base = env.PAPERCLIP_API_URL || env.PAPERCLIP_GITHUB_BROKER_URL;
let response;
if (base && env.PAPERCLIP_GITHUB_BROKER_TOKEN) {
const url = base.replace(/\/+$/, '').replace(/\/api$/, '') + '/runtime-tools/github/credentials';
for (let attempt = 0; attempt < 30; attempt++) {
response = await fetch(url, {
method: 'POST', redirect: 'error', signal: AbortSignal.timeout(10000),
headers: { authorization: 'Bearer ' + (env.PAPERCLIP_GITHUB_BRIDGE_TOKEN || env.PAPERCLIP_API_KEY || env.PAPERCLIP_GITHUB_BROKER_TOKEN),
'x-paperclip-github-capability': env.PAPERCLIP_GITHUB_BROKER_TOKEN, 'content-type': 'application/json' },
body: '{}',
});
if (response.status !== 409) break;
await response.arrayBuffer();
await new Promise(resolve => setTimeout(resolve, 1000));
}
if (!response.ok) throw new Error('GitHub credential context unavailable; retry this operation');
const result = await response.json();
const result = await acquireCredentials(url, {
authorization: 'Bearer ' + (env.PAPERCLIP_GITHUB_BRIDGE_TOKEN || env.PAPERCLIP_API_KEY || env.PAPERCLIP_GITHUB_BROKER_TOKEN),
'x-paperclip-github-capability': env.PAPERCLIP_GITHUB_BROKER_TOKEN,
'content-type': 'application/json',
});
if (result.status === 'available') {
for (const [key, value] of Object.entries(result.env || {})) {
if (/^(GH_TOKEN|GITHUB_TOKEN|PAPERCLIP_GIT_TOKEN|GIT_TERMINAL_PROMPT|GIT_AUTHOR_(NAME|EMAIL)|GIT_COMMITTER_(NAME|EMAIL)|GIT_CONFIG_COUNT|GIT_CONFIG_(KEY|VALUE)_\d+)$/.test(key) && typeof value === 'string') env[key] = value;
@ -76,7 +138,12 @@ async function main() {
child.once('error', () => { process.stderr.write('Paperclip: GitHub command could not start.\n'); process.exitCode = 1; });
child.once('exit', (code, signal) => { process.exitCode = code === null ? 128 : code; });
}
main().catch(() => { process.stderr.write('Paperclip: GitHub credential context unavailable; retry this operation.\n'); process.exitCode = 1; });
main().catch((error) => {
const category = ['timeout', 'unavailable', 'denied', 'invalidresponse'].includes(error && error.credentialCategory)
? error.credentialCategory : 'unavailable';
process.stderr.write('Paperclip: GitHub credential context unavailable (' + category + '); retry this operation.\n');
process.exitCode = 1;
});
`;
}

View File

@ -0,0 +1,51 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { AdapterSandboxExecutionTarget } from "@paperclipai/adapter-utils/execution-target";
import { startNativeGitHubCallbackBridge } from "../services/native-github-bridge.js";
const mocks = vi.hoisted(() => ({ start: vi.fn() }));
vi.mock("@paperclipai/adapter-utils/execution-target", async (importOriginal) => ({
...await importOriginal<typeof import("@paperclipai/adapter-utils/execution-target")>(),
startAdapterExecutionTargetPaperclipBridge: mocks.start,
}));
afterEach(() => vi.clearAllMocks());
function target(enabled?: boolean): AdapterSandboxExecutionTarget {
return {
kind: "remote", transport: "sandbox", providerKey: "daytona",
environmentId: "environment", leaseId: "lease", remoteCwd: "/home/daytona/repos/primary",
timeoutMs: 30_000, runner: { execute: vi.fn() }, enableSandboxDuplexBridge: enabled,
};
}
describe("native GitHub callback transport", () => {
it.each([true, false, undefined])("preserves the target duplex opt-in %s", async (enabled) => {
const executionTarget = target(enabled);
const onLog = vi.fn();
const bridge = { env: { PAPERCLIP_API_URL: "http://127.0.0.1:8123" }, stop: vi.fn() };
mocks.start.mockResolvedValueOnce(bridge);
expect(await startNativeGitHubCallbackBridge({
runId: "current-run", target: executionTarget, hostApiToken: "current-run-capability", onLog,
})).toBe(bridge);
expect(mocks.start).toHaveBeenCalledWith({
runId: "current-run", target: executionTarget, hostApiToken: "current-run-capability", onLog,
runtimeRootDir: "/home/daytona/repos/primary/.paperclip-runtime/github/current-run",
adapterKey: "native-github", enableSandboxDuplexBridge: enabled === true,
duplexObservabilityRecorder: null,
});
});
it("forwards the operator-gated recorder without changing its lifetime", async () => {
const executionTarget = target(true);
const recorder: NonNullable<AdapterSandboxExecutionTarget["duplexObservabilityRecorder"]> = {
recordSpan: vi.fn(), incrementCounter: vi.fn(), emitEvent: vi.fn(),
};
executionTarget.duplexObservabilityRecorder = recorder;
await startNativeGitHubCallbackBridge({runId:"run",target:executionTarget,hostApiToken:"capability"});
expect(mocks.start.mock.calls[0][0].duplexObservabilityRecorder).toBe(recorder);
});
it("leaves local execution unchanged", async () => {
expect(await startNativeGitHubCallbackBridge({runId:"run",target:null,hostApiToken:"capability"})).toBeNull();
expect(mocks.start).not.toHaveBeenCalled();
});
});

View File

@ -1,6 +1,7 @@
import { initializeRunIdentity } from "./run-identity.js";
import { startNativeGitHubCallbackBridge } from "./native-github-bridge.js";
import { githubBrokerEnvironment } from "@paperclipai/adapter-utils/github-launcher";
import { cleanupGitHubOperationLaunchers, prepareGitHubOperationLaunchers, startAdapterExecutionTargetPaperclipBridge } from "@paperclipai/adapter-utils/execution-target";
import { cleanupGitHubOperationLaunchers, prepareGitHubOperationLaunchers } from "@paperclipai/adapter-utils/execution-target";
import fs from "node:fs/promises";
import { retainUnsavedWorkFolderLease, workFolderSandboxKey } from "./work-folder-retention.js";
import { findUnboundLegacyTaskWorkspace, hasLegacySandboxWorkspace } from "./legacy-sandbox-workspace.js";
@ -21436,14 +21437,12 @@ export function heartbeatService(
// Native Git/gh uses the same authenticated remote callback
// transport as managed adapters. A bridge failure must not make
// GitHub a prerequisite for otherwise unrelated native work.
let nativeGitHubBridge: Awaited<ReturnType<typeof startAdapterExecutionTargetPaperclipBridge>> = null;
let nativeGitHubBridge: Awaited<ReturnType<typeof startNativeGitHubCallbackBridge>> = null;
if (executionTarget?.kind === "remote" && adapterEnv.PAPERCLIP_GITHUB_BROKER_TOKEN) {
try {
nativeGitHubBridge = await startAdapterExecutionTargetPaperclipBridge({
nativeGitHubBridge = await startNativeGitHubCallbackBridge({
runId: run.id,
target: executionTarget,
runtimeRootDir: path.posix.join(executionTarget.remoteCwd, ".paperclip-runtime", "github", run.id),
adapterKey: "native-github",
hostApiToken: adapterEnv.PAPERCLIP_GITHUB_BROKER_TOKEN,
// Forward inside this API process. The public tenant origin
// requires a browser session and rejects runtime capabilities.

View File

@ -0,0 +1,26 @@
import path from "node:path";
import {
adapterExecutionTargetDuplexObservabilityRecorder,
adapterExecutionTargetEnablesSandboxDuplexBridge,
startAdapterExecutionTargetPaperclipBridge,
} from "@paperclipai/adapter-utils/execution-target";
type BridgeInput = Parameters<typeof startAdapterExecutionTargetPaperclipBridge>[0];
/** Native Git uses the same opt-in and capability gates as managed adapters. */
export async function startNativeGitHubCallbackBridge(
input: Pick<BridgeInput, "runId" | "target" | "hostApiToken" | "onLog">,
) {
const target = input.target;
if (target?.kind !== "remote") return null;
return startAdapterExecutionTargetPaperclipBridge({
...input,
target,
runtimeRootDir: path.posix.join(target.remoteCwd, ".paperclip-runtime", "github", input.runId),
adapterKey: "native-github",
// Without forwarding this opt-in, native Git silently selects remote-file
// polling even when the same sandbox uses HTTP/2 for its legacy adapters.
enableSandboxDuplexBridge: adapterExecutionTargetEnablesSandboxDuplexBridge(target),
duplexObservabilityRecorder: adapterExecutionTargetDuplexObservabilityRecorder(target),
});
}