import fs from "node:fs/promises"; import { execFile } from "node:child_process"; import { promisify } from "node:util"; import net from "node:net"; import os from "node:os"; import path from "node:path"; import { randomBytes, randomUUID } from "node:crypto"; import { githubLauncherSource } from "./github-launcher.js"; import type { SshRemoteExecutionSpec } from "./ssh.js"; import { prepareCommandManagedRuntime, type CommandManagedDuplexChannel, type CommandManagedRuntimeAsset, type CommandManagedRuntimeRunner, } from "./command-managed-runtime.js"; import { buildRemoteExecutionSessionIdentity, prepareRemoteManagedRuntime, remoteExecutionSessionMatches, } from "./remote-managed-runtime.js"; import type { AdditionalSourceStagingFailure, SandboxAdditionalSource, WorkspaceDurableSeedPaths, WorkspaceInboundMode, } from "./sandbox-managed-runtime.js"; import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js"; import type { DirectorySnapshot } from "./workspace-restore-merge.js"; export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js"; export type { AdditionalSourceStagingFailure, ReferencedSourceIgnoreResolution, SandboxAdditionalSource, } from "./sandbox-managed-runtime.js"; import { createCommandManagedSandboxCallbackBridgeQueueClient, createSandboxCallbackBridgeAsset, createSandboxCallbackBridgeToken, DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES, HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST, SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT, SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE, sandboxCallbackBridgeDirectories, startSandboxCallbackBridgeServer, startSandboxCallbackBridgeWorker, syncRemoteTextFileWithHashSkip, syncSandboxCallbackBridgeEntrypoint, } from "./sandbox-callback-bridge.js"; import { createHttp2BridgeServer, BridgeProcessCapacityError, type BridgeBodyReservation, type Http2BridgeForwardHandler, } from "./http2-bridge-server.js"; import { createSandboxRunLogTailFactory, type SandboxRunLogTailFactory, } from "./sandbox-run-log-stream.js"; import { DEFAULT_DUPLEX_BROKER_BUDGETS, DUPLEX_CHANNEL_LOST_ERROR_CODE, isSafeBridgeMethod, type DuplexBrokerRunDisposition, } from "./bridge-transport-contract.js"; import { decodeDuplexLine, DEFAULT_MAX_DUPLEX_FRAME_BYTES } from "./duplex-frame-codec.js"; import { createDuplexObservability, mapHttp2EventToDuplexLossReason, type DuplexFallbackReason, type DuplexLossReason, type DuplexObservabilityRecorder, type Http2TelemetryEventName, } from "./duplex-observability.js"; import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js"; import { ensureCommandResolvable, resolveCommandForLogs, runChildProcess, type RunProcessResult, type TerminalResultCleanupOptions, } from "./server-utils.js"; import { sanitizeRemoteExecutionEnv } from "./remote-execution-env.js"; import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js"; import { runWithRuntimeParent, type RuntimeSpanRunner, type StartupSpanContext, } from "./acpx-engine/startup-timing.js"; import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js"; import type { LocalProcessSandboxOptions } from "./local-process-sandbox.js"; import type { RunnerIngressEndpoint } from "./runner-connectivity.js"; export type { RuntimeProgressSink } from "./runtime-progress.js"; export function postedIssueCommentLogMarker( method: string, requestPath: string, status: number, body: Buffer | string, ) { if (method !== "POST" || !/^\/api\/issues\/[^/]+\/comments$/.test(requestPath) || status < 200 || status >= 300) { return null; } const bodyText = typeof body === "string" ? body : body.toString("utf8"); try { const parsed = JSON.parse(bodyText) as { id?: unknown }; return typeof parsed.id === "string" && parsed.id.length > 0 ? `comment id: ${parsed.id}\n` : null; } catch { return null; } } export type AdapterWorkspaceRealizationMode = "copy" | "in_place"; export interface AdapterWorkspacePathAlias { path: string; target: string; } export interface AdapterWorkspaceRealization { mode: AdapterWorkspaceRealizationMode; authoritativeRoot: string; pathAliases: AdapterWorkspacePathAlias[]; outboundRestorePaths: string[]; } interface AdapterExecutionTargetWorkspaceMetadata { workspaceRealization?: AdapterWorkspaceRealization | null; } export interface AdapterLocalExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata { kind: "local"; environmentId?: string | null; leaseId?: string | null; } export interface AdapterSshExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata { kind: "remote"; transport: "ssh"; environmentId?: string | null; leaseId?: string | null; remoteCwd: string; spec: SshRemoteExecutionSpec; } /** * Read-only snapshot of the effective execution capabilities for one * execution target — local, ssh, sandbox, or plugin. Each flag is the * resolved result of the provider's declaration, the live worker's verified * methods, and any narrowing from the config or lease. The host computes it * once and attaches it to the target; a consumer reads it but never changes * it, so every field is `readonly`. */ export interface EffectiveExecutionCapabilities { readonly reusableLeases: boolean; readonly nativeSyncIn: boolean; readonly nativeSyncOut: boolean; readonly persistentProcessSessions: boolean; readonly independentControlCommands: boolean; readonly incrementalSessionOutput: boolean; readonly concurrentSyncOperations: boolean; readonly duplexCommandStream: boolean; /** Provider can expose a private authenticated WebSocket endpoint for runnerd. */ readonly runnerWebSocketIngress: boolean; } /** * @deprecated Renamed to `EffectiveExecutionCapabilities`. This alias will * be removed in a later major release. */ export interface EffectiveSandboxCapabilities extends EffectiveExecutionCapabilities {} export interface SandboxLeaseAcquisition { outcome: "created" | "resumed" | "replacement"; providerLeaseId: string; previousProviderLeaseId?: string; reason?: "not_found" | "expired" | "identity_mismatch" | "resume_failed"; } export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata { kind: "remote"; transport: "sandbox"; providerKey?: string | null; /** * Read-only effective capability snapshot for this sandbox target. The host * resolves it from the provider declaration ∩ the verified worker methods ∩ * narrowing, then attaches it here. Absent when no snapshot was resolved. */ readonly effectiveCapabilities?: EffectiveExecutionCapabilities | null; /** * Per-run duplex bridge kill switch. The host stamps it on the same seam as * `effectiveCapabilities`. `true` selects the duplex transport only when the * capability `duplexCommandStream` is also `true`; any other value keeps the * file bridge. The value stays on the host and never enters the sandbox * environment. Absent means no grant. */ readonly enableSandboxDuplexBridge?: boolean; /** Host-owned lifecycle override for paperclip_runner in this environment. */ readonly runnerLifecyclePolicy?: | { mode: "per_turn"; idleTimeoutMs: null } | { mode: "warm"; idleTimeoutMs: number } | null; /** Whether this environment is configured to reuse its provider lease. */ readonly reusableLeaseConfigured?: boolean; /** Host-observed provenance for this exact sandbox acquisition. */ readonly sandboxLeaseAcquisition?: SandboxLeaseAcquisition | null; shellCommand?: "bash" | "sh" | null; environmentId?: string | null; leaseId?: string | null; remoteCwd: string; timeoutMs?: number | null; runner?: CommandManagedRuntimeRunner; /** Host-only provider operation. It is never serialized into the sandbox. */ getRunnerIngressEndpoint?: (input: { leaseId: string; port: number; path: string; }) => Promise; /** * Sandbox-backed adapter runs stream the agent CLI's stdout/stderr * incrementally via a log-tail loop beside the callback bridge instead of * waiting for the batched provider result. Streaming is ON by default; * set to `false` to explicitly opt out back to batch-at-end delivery. */ streamRunLogs?: boolean | null; /** * The injected duplex observability recorder for this run. The host attaches * it on the same seam as `runner`, so this live object stays on the host and * never enters the sandbox environment. The bridge binds it to the fixed * duplex observability surface. Absent means the safe no-op default. */ duplexObservabilityRecorder?: DuplexObservabilityRecorder | null; } export type AdapterExecutionTarget = | AdapterLocalExecutionTarget | AdapterSshExecutionTarget | AdapterSandboxExecutionTarget; export type AdapterRemoteExecutionSpec = SshRemoteExecutionSpec; // The adapter-facing managed-runtime asset type. Aliased to the sandbox/command // asset descriptor so the per-asset lifecycle contributions (`provision` / // `restore`) declared on the sandbox core are load-bearing all the way from the // adapter call site through to the sandbox runtime. The SSH transport consumes // the subset of fields it understands and ignores the rest. export type AdapterManagedRuntimeAsset = CommandManagedRuntimeAsset; export interface PreparedAdapterExecutionTargetRuntime { target: AdapterExecutionTarget; workspaceRemoteDir: string | null; runtimeRootDir: string | null; assetDirs: Record; /** * Remote directory of each additional (referenced) project that staged * successfully, keyed by `projectId`. Empty for a local target or when no * additional sources were requested. */ additionalSourceDirs: Record; /** * Each additional (referenced) project whose staging failed, paired with the * failure message. Empty for a local target, for a transport that does not * stage referenced projects, or when every requested project staged. */ additionalSourceFailures: AdditionalSourceStagingFailure[]; workspaceSyncSnapshot: { baseline: DirectorySnapshot; gitSnapshot: GitWorkspaceSnapshot | null; } | null; restoreWorkspace(onProgress?: RuntimeProgressSink): Promise; } export interface AdapterExecutionTargetProcessOptions { cwd: string; env: Record; stdin?: string; timeoutSec: number; graceSec: number; onLog: (stream: "stdout" | "stderr", chunk: string) => Promise; onRuntimeProgress?: RuntimeStatusSink; onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string }) => Promise; terminalResultCleanup?: TerminalResultCleanupOptions; /** * Sandbox-only: factory from the Paperclip bridge handle that streams the * CLI's stdout/stderr during the run. When provided, the batched provider * onLog is suppressed and incremental chunks flow through `onLog` instead. */ runLogTail?: SandboxRunLogTailFactory | null; /** * Sandbox-only: the atomic run-disposition settle from the Paperclip bridge * handle. When provided, `runAdapterExecutionTargetProcess` calls it once at * the clean-completion boundary of the process, synchronously and before the * run-log tail finishes. The call reads the disposition and marks the * host-observed orderly completion in one broker step, so a gateway exit * after the clean process completion cannot latch a false mid-run loss. A * control channel that died before the clean completion still fails the run * closed with the typed `duplex_channel_lost` code. The file bridge path * never sets it. */ settleRunDisposition?: (() => DuplexBrokerRunDisposition) | null; localProcessSandbox?: LocalProcessSandboxOptions | null; } export interface AdapterExecutionTargetShellOptions { cwd: string; env: Record; timeoutSec?: number; graceSec?: number; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; } export interface AdapterExecutionTargetPaperclipBridgeHandle { env: Record; /** * Present when the sandbox target opted into run-log streaming * (`streamRunLogs`). Create one handle per CLI attempt and pass it to * `runAdapterExecutionTargetProcess` via `options.runLogTail`. */ runLogTail?: SandboxRunLogTailFactory | null; /** * Read the terminal run disposition of the duplex control channel. It reports a * failure when the channel was lost before an orderly completion, and names the * typed loss reason. It reports a success for a healthy channel or a * normal-teardown loss. The file bridge path never sets it, so the method is * absent there. The caller reads it at the run-disposition seam to fail a run * whose control channel died mid-turn. */ readRunDisposition?(): DuplexBrokerRunDisposition; /** * Atomically read the run disposition and mark the host-observed orderly * completion in one broker step. The ACP lane calls it at the terminal * finalization boundary for a success-eligible completion, so no `await` can * separate the read from the mark and a teardown loss cannot slip in between. * A loss that already latched keeps the failure, because the broker no-ops the * mark after a latched loss. The file bridge path never sets it. */ settleRunDisposition?(): DuplexBrokerRunDisposition; /** * Mark the host-observed orderly completion of the agent turn on the broker's * ordered lifecycle. The caller marks it at the ACP terminal-finalization * boundary for a still-success-eligible completion, so a later teardown loss * cannot flip the run to a failure. A loss that already latched keeps the * failure, because the broker no-ops the mark after a latched loss. The file * bridge path never sets it, so the method is absent there. */ markOrderlyCompletion?(): void; /** * Register a listener for a newly latched terminal loss. The listener * fires at most once, and only for a loss that flips the disposition to * failed — never for a clean channel end that orders after a * host-observed orderly completion. Returns a function that unregisters * the listener. * * The caller uses this to abort an in-flight Agent Client Protocol turn * the moment the channel dies, instead of waiting for the turn to return * a terminal result on its own (a dead channel can leave a turn with * nothing to return). The file bridge path never sets it, so the method * is absent there. */ onLoss?(listener: (reason: DuplexLossReason) => void): () => void; stop(): Promise; } export interface AdapterExecutionTargetProcessSessionBridgeHandle { agentCommand: string; stop(): Promise; } export { sanitizeRemoteExecutionEnv } from "./remote-execution-env.js"; // 4-hour wall-clock backstop for sandbox-backed adapter runs. This is a // last-resort kill switch, not the primary hang detector: genuinely hung runs // are caught much earlier by the adapters' output-inactivity monitors (e.g. // codex-local's 7-minute monitor). The value intentionally matches the // recovery watchdog's ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS (4h) in // server/src/services/recovery/service.ts so healthy long runs are never // killed by the adapter before the watchdog would even consider them stuck. export const DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC = 14_400; function parseObject(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } function readString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } // Read a serialized effective-capability snapshot back into a full record. A // missing or non-boolean field reads as `false`, so a round-tripped target // never grants a capability that the snapshot did not carry. Returns null when // there is no object to read. function parseEffectiveExecutionCapabilities(value: unknown): EffectiveExecutionCapabilities | null { const parsed = parseObject(value); if (Object.keys(parsed).length === 0) return null; return { reusableLeases: parsed.reusableLeases === true, nativeSyncIn: parsed.nativeSyncIn === true, nativeSyncOut: parsed.nativeSyncOut === true, persistentProcessSessions: parsed.persistentProcessSessions === true, independentControlCommands: parsed.independentControlCommands === true, incrementalSessionOutput: parsed.incrementalSessionOutput === true, concurrentSyncOperations: parsed.concurrentSyncOperations === true, duplexCommandStream: parsed.duplexCommandStream === true, runnerWebSocketIngress: parsed.runnerWebSocketIngress === true, }; } function readStringMeta(parsed: Record, key: string): string | null { return readString(parsed[key]); } function resolveHostForUrl(rawHost: string): string { const host = rawHost.trim(); // Preserve the wildcard bind's address family: a server bound to 0.0.0.0 // accepts IPv4, so target the IPv4 loopback (and [::1] for ::) instead of // "localhost", which the resolver may map to the other family. if (host === "0.0.0.0") return "127.0.0.1"; if (host === "::") return "[::1]"; if (!host) return "localhost"; if (host.includes(":") && !host.startsWith("[") && !host.endsWith("]")) return `[${host}]`; return host; } function resolveDefaultPaperclipApiUrl(): string { const runtimeHost = resolveHostForUrl( process.env.PAPERCLIP_LISTEN_HOST ?? process.env.HOST ?? "localhost", ); // 3100 matches the default Paperclip dev server port when the runtime does not provide one. const runtimePort = process.env.PAPERCLIP_LISTEN_PORT ?? process.env.PORT ?? "3100"; return `http://${runtimeHost}:${runtimePort}`; } function isBridgeDebugEnabled(env: NodeJS.ProcessEnv): boolean { const value = env.PAPERCLIP_BRIDGE_DEBUG?.trim().toLowerCase(); return value === "1" || value === "true" || value === "yes"; } function isAdapterExecutionTargetInstance(value: unknown): value is AdapterExecutionTarget { const parsed = parseObject(value); if (parsed.kind === "local") return true; if (parsed.kind !== "remote") return false; if (parsed.transport === "ssh") return parseSshRemoteExecutionSpec(parseObject(parsed.spec)) !== null; if (parsed.transport !== "sandbox") return false; return readStringMeta(parsed, "remoteCwd") !== null; } export function adapterExecutionTargetToRemoteSpec( target: AdapterExecutionTarget | null | undefined, ): AdapterRemoteExecutionSpec | null { return target?.kind === "remote" && target.transport === "ssh" ? target.spec : null; } export function adapterExecutionTargetIsRemote( target: AdapterExecutionTarget | null | undefined, ): boolean { return target?.kind === "remote"; } export function adapterExecutionTargetUsesManagedHome( target: AdapterExecutionTarget | null | undefined, ): boolean { return target?.kind === "remote" && target.transport === "sandbox"; } /** * Read the per-run duplex bridge kill switch off a target. Only a sandbox * target with `enableSandboxDuplexBridge` set to `true` returns `true`. Every * other target and every other value returns `false`, so the caller fails * closed to the file bridge. */ export function adapterExecutionTargetEnablesSandboxDuplexBridge( target: AdapterExecutionTarget | null | undefined, ): boolean { return ( target?.kind === "remote" && target.transport === "sandbox" && target.enableSandboxDuplexBridge === true ); } /** * Read the injected duplex observability recorder off a target. Only a * sandbox target with a recorder attached returns it. Every other target * returns null, so the bridge falls back to the safe no-op recorder. */ export function adapterExecutionTargetDuplexObservabilityRecorder( target: AdapterExecutionTarget | null | undefined, ): DuplexObservabilityRecorder | null { return target?.kind === "remote" && target.transport === "sandbox" ? target.duplexObservabilityRecorder ?? null : null; } export function adapterExecutionTargetRemoteCwd( target: AdapterExecutionTarget | null | undefined, localCwd: string, ): string { return target?.kind === "remote" ? target.remoteCwd : localCwd; } export function overrideAdapterExecutionTargetRemoteCwd( target: AdapterExecutionTarget | null | undefined, remoteCwd: string | null | undefined, ): AdapterExecutionTarget | null | undefined { const nextRemoteCwd = remoteCwd?.trim(); if (!target || target.kind !== "remote" || !nextRemoteCwd) { return target; } if (target.remoteCwd === nextRemoteCwd) { return target; } if (target.transport === "ssh") { return { ...target, remoteCwd: nextRemoteCwd, spec: { ...target.spec, remoteCwd: nextRemoteCwd, }, }; } return { ...target, remoteCwd: nextRemoteCwd, }; } export function resolveAdapterExecutionTargetCwd( target: AdapterExecutionTarget | null | undefined, configuredCwd: string | null | undefined, localFallbackCwd: string, ): string { if (typeof configuredCwd === "string" && configuredCwd.trim().length > 0) { return configuredCwd; } return adapterExecutionTargetRemoteCwd(target, localFallbackCwd); } export function adapterExecutionTargetUsesPaperclipBridge( target: AdapterExecutionTarget | null | undefined, ): boolean { return target?.kind === "remote"; } export function describeAdapterExecutionTarget( target: AdapterExecutionTarget | null | undefined, ): string { if (!target || target.kind === "local") return "local environment"; if (target.transport === "ssh") { return `SSH environment ${target.spec.username}@${target.spec.host}:${target.spec.port}`; } return `sandbox environment${target.providerKey ? ` (${target.providerKey})` : ""}`; } export type AdapterExecutionTargetTimeoutSource = | "configured" | "sandbox_default" | "unlimited"; export interface AdapterExecutionTargetTimeoutResolution { /** Resolved wall-clock timeout in seconds; 0 means no adapter timeout. */ timeoutSec: number; /** Which knob produced the resolved value, for logs and error messages. */ source: AdapterExecutionTargetTimeoutSource; } export function resolveAdapterExecutionTargetTimeout( target: AdapterExecutionTarget | null | undefined, configuredTimeoutSec: number | null | undefined, ): AdapterExecutionTargetTimeoutResolution { if (typeof configuredTimeoutSec === "number" && Number.isFinite(configuredTimeoutSec)) { // Preserve fractional (sub-second) configured values instead of flooring: // adapters historically honored e.g. timeoutSec=0.5, and flooring would // silently turn it into "no timeout". if (configuredTimeoutSec > 0) { return { timeoutSec: configuredTimeoutSec, source: "configured" }; } // A negative timeoutSec is the explicit "no adapter wall-clock timeout" // opt-out, honored even on sandbox targets. Zero cannot carry that // meaning: the adapter config UI persists the schema default of 0 for // untouched fields, so timeoutSec=0 in stored config does not signal // operator intent and falls through to target defaults below. if (configuredTimeoutSec < 0) { return { timeoutSec: 0, source: "configured" }; } } // Local and SSH adapters preserve the historical "0 means no adapter // timeout" behavior. Sandbox-backed runs execute through provider RPCs // that usually apply their own shorter command defaults, so request an // explicit longer timeout for full adapter runs when the adapter leaves // timeoutSec unset. if (target?.kind === "remote" && target.transport === "sandbox") { return { timeoutSec: DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC, source: "sandbox_default" }; } return { timeoutSec: 0, source: "unlimited" }; } export function resolveAdapterExecutionTargetTimeoutSec( target: AdapterExecutionTarget | null | undefined, configuredTimeoutSec: number | null | undefined, ): number { return resolveAdapterExecutionTargetTimeout(target, configuredTimeoutSec).timeoutSec; } function describeAdapterExecutionTimeoutSource( source: AdapterExecutionTargetTimeoutSource, ): string { switch (source) { case "configured": return "configured via adapterConfig.timeoutSec"; case "sandbox_default": return "sandbox default"; case "unlimited": return "no adapter wall-clock timeout"; } } /** * Self-describing error message for when the adapter wall-clock execution * timeout kills a run. Names the timer that fired and the knob that controls * it so run failures never surface as a bare "Timed out". */ export function formatAdapterExecutionTimeoutErrorMessage( resolution: AdapterExecutionTargetTimeoutResolution, ): string { return ( `Run exceeded the adapter execution timeout ` + `(timeoutSec=${resolution.timeoutSec}, ${describeAdapterExecutionTimeoutSource(resolution.source)}). ` + `Set adapterConfig.timeoutSec to raise it.` ); } /** * One-line start-of-run statement of the effective wall-clock timeout and its * source. Callers prefix with `[paperclip] ` and append a newline. */ export function formatAdapterExecutionTimeoutStartLogLine( resolution: AdapterExecutionTargetTimeoutResolution, ): string { if (resolution.timeoutSec <= 0) { if (resolution.source === "configured") { return ( "Adapter execution timeout: none " + "(explicitly disabled via adapterConfig.timeoutSec; set it to a positive value to add one)." ); } return ( "Adapter execution timeout: none " + "(no adapter wall-clock timeout for this target; set adapterConfig.timeoutSec to add one)." ); } return ( `Adapter execution timeout: timeoutSec=${resolution.timeoutSec} ` + `(${describeAdapterExecutionTimeoutSource(resolution.source)}; set adapterConfig.timeoutSec to override).` ); } function requireSandboxRunner(target: AdapterSandboxExecutionTarget): CommandManagedRuntimeRunner { if (target.runner) return target.runner; throw new Error( "Sandbox execution target is missing its provider runtime runner. Sandbox commands must execute through the environment runtime.", ); } function preferredSandboxShell(target: AdapterSandboxExecutionTarget): "bash" | "sh" { return preferredShellForSandbox(target.shellCommand); } type AdapterCommandCapableExecutionTarget = AdapterSshExecutionTarget | AdapterSandboxExecutionTarget; // The Secure Shell command runner's own output buffer. This value used to // derive from the bridge body limit (`DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES // * 4`), so a bridge limit rise silently grew it too. It now stands on its // own local constant, independent of the bridge body limit, so a later // bridge limit change never resizes this buffer as a side effect. const SSH_COMMAND_MAX_BUFFER_BYTES = 1024 * 1024; function adapterExecutionTargetCommandRunner(target: AdapterCommandCapableExecutionTarget): CommandManagedRuntimeRunner { if (target.transport === "ssh") { return createSshCommandManagedRuntimeRunner({ spec: target.spec, defaultCwd: target.remoteCwd, maxBufferBytes: SSH_COMMAND_MAX_BUFFER_BYTES, }); } return requireSandboxRunner(target); } function adapterExecutionTargetShellCommand(target: AdapterCommandCapableExecutionTarget): "bash" | "sh" { return target.transport === "ssh" ? "sh" : preferredSandboxShell(target); } function adapterExecutionTargetTimeoutMs( target: AdapterCommandCapableExecutionTarget, ): number | null | undefined { return target.transport === "sandbox" ? target.timeoutMs : undefined; } export async function ensureAdapterExecutionTargetCommandResolvable( command: string, target: AdapterExecutionTarget | null | undefined, cwd: string, env: NodeJS.ProcessEnv, options: { installCommand?: string | null; timeoutSec?: number | null } = {}, ) { if (target?.kind === "remote" && target.transport === "sandbox") { await ensureSandboxCommandResolvable( command, target, sanitizeRemoteExecutionEnv(Object.fromEntries( Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"), )), options.installCommand?.trim() || null, options.timeoutSec, ); return; } await ensureCommandResolvable(command, cwd, env, { remoteExecution: adapterExecutionTargetToRemoteSpec(target), }); } async function probeSandboxCommandResolvable( command: string, target: AdapterSandboxExecutionTarget, env: Record, ): Promise<{ resolved: boolean; timedOut: boolean; stderr: string }> { const runner = requireSandboxRunner(target); const probeScript = `command -v ${shellQuote(command)}`; const result = await runner.execute({ command: "sh", args: ["-c", probeScript], cwd: target.remoteCwd, env, timeoutMs: target.timeoutMs ?? 15_000, }); return { resolved: !result.timedOut && (result.exitCode ?? 1) === 0, timedOut: result.timedOut, stderr: result.stderr.trim(), }; } async function ensureSandboxCommandResolvable( command: string, target: AdapterSandboxExecutionTarget, env: Record, installCommand: string | null, timeoutSec?: number | null, ): Promise { // Probe whether the binary is resolvable inside the sandbox. We previously // short-circuited this for sandbox targets, which let the caller report a // success message even when the CLI was missing from the image. Now we run // a real `command -v` through the same runner the hello probe will use, so // the first step honestly reflects whether the binary is on PATH. The // sandbox provider is responsible for sourcing login profiles (e2b mirrors // SSH's buildSshSpawnTarget) so this and the hello probe agree on PATH. let probe = await probeSandboxCommandResolvable(command, target, env); if (probe.resolved) return; if (probe.timedOut) { throw new Error(`Timed out checking command "${command}" on sandbox target.`); } // If the caller supplied an install command, attempt the install once via // the sandbox runner (which the sandbox provider wraps in a login shell) // and re-probe before reporting failure. This lets fresh sandbox leases // bring up the CLI before the resolvability gate, mirroring the test path. let installFailureDetail: string | null = null; if (installCommand) { const runner = requireSandboxRunner(target); const installTimeoutMs = typeof timeoutSec === "number" && Number.isFinite(timeoutSec) && timeoutSec > 0 ? Math.floor(timeoutSec * 1000) : target.timeoutMs ?? 300_000; try { const installResult = await runner.execute({ command: "sh", args: shellCommandArgs(installCommand), cwd: target.remoteCwd, env, timeoutMs: installTimeoutMs, }); if (installResult.timedOut) { installFailureDetail = `install command timed out: ${installCommand}`; } else if ((installResult.exitCode ?? 0) !== 0) { const tail = (text: string) => text.split(/\r?\n/).filter((line) => line.trim().length > 0).slice(-2).join(" | ").slice(0, 240); const reason = tail(installResult.stderr || installResult.stdout) || `exit ${installResult.exitCode ?? "?"}`; installFailureDetail = `install command exited ${installResult.exitCode ?? "?"}: ${reason}`; } } catch (err) { installFailureDetail = `install command threw: ${err instanceof Error ? err.message : String(err)}`; } probe = await probeSandboxCommandResolvable(command, target, env); if (probe.resolved) return; if (probe.timedOut) { throw new Error(`Timed out checking command "${command}" on sandbox target.`); } } const probeStderr = probe.stderr.length > 0 ? ` probe stderr: ${probe.stderr}` : ""; const installDetail = installFailureDetail ? `; ${installFailureDetail}` : ""; throw new Error( `Command "${command}" is not installed or not on PATH in the sandbox environment${installDetail}.${probeStderr}`, ); } export async function resolveAdapterExecutionTargetCommandForLogs( command: string, target: AdapterExecutionTarget | null | undefined, cwd: string, env: NodeJS.ProcessEnv, ): Promise { if (target?.kind === "remote" && target.transport === "sandbox") { return `sandbox://${target.providerKey ?? "provider"}/${target.leaseId ?? "lease"}/${target.remoteCwd} :: ${command}`; } return await resolveCommandForLogs(command, cwd, env, { remoteExecution: adapterExecutionTargetToRemoteSpec(target), }); } // Apply the run-disposition seam to one clean process result. Only a clean // completion is success-eligible: a timed-out, signalled, or non-zero-exit // result is already a failure, so the seam leaves it unchanged. This is the same // success-eligibility rule the ACP lane applies. For a success-eligible result // the seam settles the disposition in one atomic broker step: it reads the // disposition and marks the host-observed orderly completion together, so a // gateway exit after the clean completion cannot latch a false mid-run loss. A // duplex control channel that died before the clean completion fails the run // closed: the seam sets a non-zero exit code, the typed `duplex_channel_lost` // error code, and a stderr note that names only the typed loss reason. function applyRunDispositionSeam( result: RunProcessResult, settleRunDisposition: (() => DuplexBrokerRunDisposition) | null | undefined, ): RunProcessResult { const successEligible = result.exitCode === 0 && !result.timedOut && result.signal === null; if (!successEligible || !settleRunDisposition) return result; const disposition = settleRunDisposition(); if (!disposition.failed) return result; const lossReason = disposition.lossReason ?? "other"; const note = `[paperclip] The sandbox duplex control channel was lost (${lossReason}) before the run completed.\n`; const separator = result.stderr.length > 0 && !result.stderr.endsWith("\n") ? "\n" : ""; return { ...result, exitCode: 1, errorCode: DUPLEX_CHANNEL_LOST_ERROR_CODE, stderr: `${result.stderr}${separator}${note}`, }; } export async function runAdapterExecutionTargetProcess( runId: string, target: AdapterExecutionTarget | null | undefined, command: string, args: string[], options: AdapterExecutionTargetProcessOptions, ): Promise { if (target?.kind === "remote" && target.transport === "sandbox") { const runner = requireSandboxRunner(target); const env = sanitizeRemoteExecutionEnv(options.env); await options.onRuntimeProgress?.({ phase: "adapter_startup", message: "Starting adapter in environment", }); const runLogTail = options.runLogTail?.create() ?? null; let execCommand = command; let execArgs = args; if (runLogTail) { ({ command: execCommand, args: execArgs } = runLogTail.wrapCommand(command, args)); runLogTail.start(options.onLog); } try { const result = await runner.execute({ command: execCommand, args: execArgs, cwd: target.remoteCwd, env, stdin: options.stdin, timeoutMs: options.timeoutSec > 0 ? options.timeoutSec * 1000 : target.timeoutMs ?? undefined, // The tail loop already streams incremental chunks; suppress the // runner's end-of-run batched onLog to avoid duplicate log bytes. onLog: runLogTail ? undefined : options.onLog, onSpawn: options.onSpawn ? async (meta) => options.onSpawn?.({ ...meta, processGroupId: null }) : undefined, }); // Settle the duplex run disposition synchronously at the clean-completion // boundary, before the run-log tail finishes. The atomic settle marks the // host-observed orderly completion in one broker step, so a gateway exit // after the clean process completion cannot latch a false mid-run loss. A // control channel that died before this clean completion still fails the // run closed. const settled = applyRunDispositionSeam(result, options.settleRunDisposition); if (runLogTail) { await runLogTail.finish({ stdout: result.stdout, stderr: result.stderr }); } return settled; } catch (error) { if (runLogTail) { await runLogTail.abort(); } throw error; } } const env = target?.kind === "remote" && target.transport === "ssh" ? sanitizeRemoteExecutionEnv(options.env) : options.env; return await runChildProcess(runId, command, args, { cwd: options.cwd, env, stdin: options.stdin, timeoutSec: options.timeoutSec, graceSec: options.graceSec, onLog: options.onLog, onSpawn: options.onSpawn, terminalResultCleanup: options.terminalResultCleanup, localProcessSandbox: target?.kind === "local" || !target ? options.localProcessSandbox : null, remoteExecution: adapterExecutionTargetToRemoteSpec(target), }); } export async function runAdapterExecutionTargetShellCommand( runId: string, target: AdapterExecutionTarget | null | undefined, command: string, options: AdapterExecutionTargetShellOptions, ): Promise { const onLog = options.onLog ?? (async () => {}); if (target?.kind === "remote") { const startedAt = new Date().toISOString(); const env = sanitizeRemoteExecutionEnv(options.env); if (target.transport === "ssh") { try { // Pass the raw command — `runSshCommand` owns profile sourcing and // the outer shell wrapper. Wrapping again here would nest a second // shell after the explicit `env KEY=VAL` overrides, re-sourcing // login profiles AFTER the override and silently undoing any // identity var (NVM_DIR / PATH / etc.) that a profile re-exports. const result = await runSshCommand(target.spec, command, { env, timeoutMs: (options.timeoutSec ?? 15) * 1000, }); if (result.stdout) await onLog("stdout", result.stdout); if (result.stderr) await onLog("stderr", result.stderr); return { exitCode: 0, signal: null, timedOut: false, stdout: result.stdout, stderr: result.stderr, pid: null, startedAt, }; } catch (error) { const timedOutError = error as NodeJS.ErrnoException & { stdout?: string; stderr?: string; signal?: string | null; }; const stdout = timedOutError.stdout ?? ""; const stderr = timedOutError.stderr ?? ""; if (typeof timedOutError.code === "number") { if (stdout) await onLog("stdout", stdout); if (stderr) await onLog("stderr", stderr); return { exitCode: timedOutError.code, signal: timedOutError.signal ?? null, timedOut: false, stdout, stderr, pid: null, startedAt, }; } if (timedOutError.code !== "ETIMEDOUT") { throw error; } if (stdout) await onLog("stdout", stdout); if (stderr) await onLog("stderr", stderr); return { exitCode: null, signal: timedOutError.signal ?? null, timedOut: true, stdout, stderr, pid: null, startedAt, }; } } const shellCommand = preferredSandboxShell(target); return await requireSandboxRunner(target).execute({ command: shellCommand, args: shellCommandArgs(command), cwd: target.remoteCwd, env, timeoutMs: (options.timeoutSec ?? 15) * 1000, onLog, }); } return await runAdapterExecutionTargetProcess( runId, target, "sh", ["-lc", command], { cwd: options.cwd, env: options.env, timeoutSec: options.timeoutSec ?? 15, graceSec: options.graceSec ?? 5, onLog, }, ); } export interface AdapterSandboxInstallCommandCheck { code: string; level: "info" | "warn" | "error"; message: string; detail?: string; hint?: string; } // Best-effort run of an adapter-supplied install command on a sandbox target // before the resolvability + hello probe. Returns null for non-sandbox // targets so callers can no-op. Returns a structured check otherwise — never // throws — so the rest of the test still runs and reports the post-install // state honestly. Caller pushes the check into its result array; the test // report shows whether install was attempted and what came back. export async function maybeRunSandboxInstallCommand(input: { runId: string; target: AdapterExecutionTarget | null | undefined; adapterKey: string; installCommand: string; /** When provided, skip the install if `command -v ` succeeds. */ detectCommand?: string | null; env?: Record; timeoutSec?: number; }): Promise { const { target, adapterKey, installCommand } = input; if (!target || target.kind !== "remote" || target.transport !== "sandbox") { return null; } const trimmed = installCommand.trim(); if (trimmed.length === 0) return null; const code = `${adapterKey}_install_command_run`; // Skip install when the binary is already on PATH. Avoids running // network-dependent installers (e.g. `curl ... | bash`) on every test // probe when the CLI is preinstalled on the lease/template. const detectCommand = input.detectCommand?.trim(); if (detectCommand) { try { const probe = await runAdapterExecutionTargetShellCommand( input.runId, target, `command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`, { cwd: target.remoteCwd, env: input.env ?? {}, timeoutSec: 30, graceSec: 5, }, ); if (!probe.timedOut && probe.exitCode === 0) { return { code, level: "info", message: `${detectCommand} already on PATH; skipped install.`, }; } } catch { // Fall through to actually running the install — failure to probe // is not a reason to skip the install gate. } } let result; try { result = await runAdapterExecutionTargetShellCommand(input.runId, target, trimmed, { cwd: target.remoteCwd, env: input.env ?? {}, timeoutSec: input.timeoutSec ?? 240, graceSec: 10, }); } catch (err) { return { code, level: "warn", message: "Install command threw before completion.", detail: err instanceof Error ? err.message : String(err), }; } const tail = (text: string) => text.split(/\r?\n/).filter((line) => line.trim().length > 0).slice(-3).join(" | ").slice(0, 480); if (result.timedOut) { return { code, level: "warn", message: `Install command timed out: ${trimmed}`, detail: tail(result.stderr || result.stdout), }; } if ((result.exitCode ?? 1) === 0) { return { code, level: "info", message: `Install command ran: ${trimmed}`, ...(tail(result.stdout) ? { detail: tail(result.stdout) } : {}), }; } return { code, level: "warn", message: `Install command exited ${result.exitCode}: ${trimmed}`, detail: tail(result.stderr || result.stdout), }; } export async function readAdapterExecutionTargetHomeDir( runId: string, target: AdapterExecutionTarget | null | undefined, options: AdapterExecutionTargetShellOptions, ): Promise { const result = await runAdapterExecutionTargetShellCommand( runId, target, 'printf %s "$HOME"', options, ); const homeDir = result.stdout.trim(); return homeDir.length > 0 ? homeDir : null; } export async function ensureAdapterExecutionTargetRuntimeCommandInstalled(input: { runId: string; target: AdapterExecutionTarget | null | undefined; installCommand?: string | null; detectCommand?: string | null; cwd: string; env: Record; timeoutSec?: number; graceSec?: number; onLog?: AdapterExecutionTargetShellOptions["onLog"]; }): Promise { const installCommand = input.installCommand?.trim(); if (!installCommand || input.target?.kind !== "remote" || input.target.transport !== "sandbox") { return; } const detectCommand = input.detectCommand?.trim(); if (detectCommand) { const probe = await runAdapterExecutionTargetShellCommand( input.runId, input.target, `command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`, { cwd: input.cwd, env: input.env, timeoutSec: input.timeoutSec, graceSec: input.graceSec, }, ); if (!probe.timedOut && probe.exitCode === 0) { return; } } const result = await runAdapterExecutionTargetShellCommand( input.runId, input.target, installCommand, { cwd: input.cwd, env: input.env, timeoutSec: input.timeoutSec, graceSec: input.graceSec, onLog: input.onLog, }, ); // A failed or timed-out install is not necessarily fatal: the CLI may already // be on PATH from a previous lease's install, the template image, or another // path entry. Re-run the detect probe (when one is configured) so a transient // install failure does not abort the agent run when the binary is reachable. const installFailed = result.timedOut || (result.exitCode ?? 0) !== 0; if (!installFailed) { return; } if (detectCommand) { const recheck = await runAdapterExecutionTargetShellCommand( input.runId, input.target, `command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`, { cwd: input.cwd, env: input.env, timeoutSec: input.timeoutSec, graceSec: input.graceSec, }, ); if (!recheck.timedOut && recheck.exitCode === 0) { if (input.onLog) { const reason = result.timedOut ? "timed out" : `exited ${result.exitCode ?? "?"}`; await input.onLog( "stderr", `[paperclip] Install command ${reason} (${installCommand}) but ${detectCommand} is on PATH; continuing.\n`, ); } return; } } if (result.timedOut) { throw new Error(`Timed out while installing the adapter runtime command via: ${installCommand}`); } throw new Error(`Failed to install the adapter runtime command via: ${installCommand}`); } export async function ensureAdapterExecutionTargetFile( runId: string, target: AdapterExecutionTarget | null | undefined, filePath: string, options: AdapterExecutionTargetShellOptions, ): Promise { await runAdapterExecutionTargetShellCommand( runId, target, `mkdir -p ${shellQuote(path.posix.dirname(filePath))} && : > ${shellQuote(filePath)}`, options, ); } /** * Ensure a working directory exists (and is a directory) on the execution target. * * For local targets this delegates to the local `ensureAbsoluteDirectory` helper * (Node fs). For remote (SSH/sandbox) targets it shells out and runs * `mkdir -p` (when allowed) followed by a `[ -d ]` check so the result reflects * the directory state inside the environment, not on the Paperclip host. * * Throws an Error with a human-readable message on failure. */ export async function ensureAdapterExecutionTargetDirectory( runId: string, target: AdapterExecutionTarget | null | undefined, cwd: string, options: AdapterExecutionTargetShellOptions & { createIfMissing?: boolean }, ): Promise { const createIfMissing = options.createIfMissing ?? false; if (!target || target.kind === "local") { const { ensureAbsoluteDirectory } = await import("./server-utils.js"); await ensureAbsoluteDirectory(cwd, { createIfMissing }); return; } // Remote (SSH or sandbox): both expect POSIX absolute paths inside the env. if (!cwd.startsWith("/")) { throw new Error(`Working directory must be an absolute POSIX path on the remote target: "${cwd}"`); } const quoted = shellQuote(cwd); const script = createIfMissing ? `mkdir -p ${quoted} && [ -d ${quoted} ]` : `[ -d ${quoted} ]`; const result = await runAdapterExecutionTargetShellCommand(runId, target, script, { cwd: target.kind === "remote" ? target.remoteCwd : cwd, env: options.env, timeoutSec: options.timeoutSec ?? 15, graceSec: options.graceSec ?? 5, onLog: options.onLog, }); if (result.timedOut) { throw new Error(`Timed out checking working directory on remote target: "${cwd}"`); } if ((result.exitCode ?? 1) !== 0) { const detail = (result.stderr || result.stdout || "").trim(); if (createIfMissing) { throw new Error( `Could not create working directory "${cwd}" on remote target${detail ? `: ${detail}` : "."}`, ); } throw new Error( `Working directory does not exist on remote target: "${cwd}"${detail ? ` (${detail})` : ""}`, ); } } export function adapterExecutionTargetSessionIdentity( target: AdapterExecutionTarget | null | undefined, ): Record | null { if (!target || target.kind === "local") return null; if (target.transport === "ssh") return buildRemoteExecutionSessionIdentity(target.spec); return { transport: "sandbox", providerKey: target.providerKey ?? null, environmentId: target.environmentId ?? null, leaseId: target.leaseId ?? null, remoteCwd: target.remoteCwd, }; } export function adapterExecutionTargetSessionMatches( saved: unknown, target: AdapterExecutionTarget | null | undefined, ): boolean { if (!target || target.kind === "local") { return Object.keys(parseObject(saved)).length === 0; } if (target.transport === "ssh") return remoteExecutionSessionMatches(saved, target.spec); const current = adapterExecutionTargetSessionIdentity(target); const parsedSaved = parseObject(saved); return ( readStringMeta(parsedSaved, "transport") === current?.transport && readStringMeta(parsedSaved, "providerKey") === current?.providerKey && readStringMeta(parsedSaved, "environmentId") === current?.environmentId && readStringMeta(parsedSaved, "leaseId") === current?.leaseId && readStringMeta(parsedSaved, "remoteCwd") === current?.remoteCwd ); } export function parseAdapterExecutionTarget(value: unknown): AdapterExecutionTarget | null { const parsed = parseObject(value); const kind = readStringMeta(parsed, "kind"); if (kind === "local") { return { kind: "local", environmentId: readStringMeta(parsed, "environmentId"), leaseId: readStringMeta(parsed, "leaseId"), }; } if (kind === "remote" && readStringMeta(parsed, "transport") === "ssh") { const spec = parseSshRemoteExecutionSpec(parseObject(parsed.spec)); if (!spec) return null; return { kind: "remote", transport: "ssh", environmentId: readStringMeta(parsed, "environmentId"), leaseId: readStringMeta(parsed, "leaseId"), remoteCwd: spec.remoteCwd, spec, }; } if (kind === "remote" && readStringMeta(parsed, "transport") === "sandbox") { const remoteCwd = readStringMeta(parsed, "remoteCwd"); if (!remoteCwd) return null; const effectiveCapabilities = parseEffectiveExecutionCapabilities(parsed.effectiveCapabilities); return { kind: "remote", transport: "sandbox", providerKey: readStringMeta(parsed, "providerKey"), environmentId: readStringMeta(parsed, "environmentId"), leaseId: readStringMeta(parsed, "leaseId"), remoteCwd, timeoutMs: typeof parsed.timeoutMs === "number" ? parsed.timeoutMs : null, streamRunLogs: typeof parsed.streamRunLogs === "boolean" ? parsed.streamRunLogs : null, // Fail closed: only the literal `true` reads as a grant. An absent field // or any other value parses as no grant, so a round-trip never invents one. enableSandboxDuplexBridge: parsed.enableSandboxDuplexBridge === true, ...(effectiveCapabilities ? { effectiveCapabilities } : {}), }; } return null; } export function adapterExecutionTargetFromRemoteExecution( remoteExecution: unknown, metadata: Pick = {}, ): AdapterExecutionTarget | null { const parsed = parseObject(remoteExecution); const ssh = parseSshRemoteExecutionSpec(parsed); if (ssh) { return { kind: "remote", transport: "ssh", environmentId: metadata.environmentId ?? null, leaseId: metadata.leaseId ?? null, remoteCwd: ssh.remoteCwd, spec: ssh, }; } return null; } export function readAdapterExecutionTarget(input: { executionTarget?: unknown; legacyRemoteExecution?: unknown; }): AdapterExecutionTarget | null { if (isAdapterExecutionTargetInstance(input.executionTarget)) { return input.executionTarget; } return ( parseAdapterExecutionTarget(input.executionTarget) ?? adapterExecutionTargetFromRemoteExecution(input.legacyRemoteExecution) ); } export async function prepareAdapterExecutionTargetRuntime(input: { runId: string; target: AdapterExecutionTarget | null | undefined; adapterKey: string; workspaceLocalDir: string; timeoutSec?: number; workspaceRemoteDir?: string; syncWorkspace?: boolean; workspaceInboundMode?: WorkspaceInboundMode; workspaceDurableSeed?: WorkspaceDurableSeedPaths; workspaceBaseline?: DirectorySnapshot; workspaceGitSnapshot?: GitWorkspaceSnapshot | null; workspaceExclude?: string[]; preserveAbsentOnRestore?: string[]; assets?: AdapterManagedRuntimeAsset[]; /** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */ additionalSources?: SandboxAdditionalSource[]; installCommand?: string | null; /** When provided alongside `installCommand`, skip the install if the binary is already on PATH. */ detectCommand?: string | null; // Optional progress sink for the workspace/asset upload. The returned // `restoreWorkspace(onProgress?)` accepts its own sink for teardown. Both are // forwarded down to the transport so the sandbox/SSH children can attach byte // counters without further changes here. onProgress?: RuntimeProgressSink; onRuntimeProgress?: RuntimeStatusSink; // Optional host span runner for the workspace tarball build. Only the confined // sandbox lane uses it: it forwards the runner to prepareCommandManagedRuntime // so the host pack time rides one `pack` span under the `stage.sync` step. The // SSH and local lanes ignore it. The default is a no-op. runtimeSpan?: RuntimeSpanRunner; }): Promise { const target = input.target ?? { kind: "local" as const }; if (target.kind === "local") { return { target, workspaceRemoteDir: null, runtimeRootDir: null, assetDirs: {}, additionalSourceDirs: {}, additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace: async () => {}, }; } if (target.transport === "ssh") { const prepared = await prepareRemoteManagedRuntime({ spec: target.spec, runId: input.runId, adapterKey: input.adapterKey, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, assets: input.assets, additionalSources: input.additionalSources, onProgress: input.onProgress, }); return { target, workspaceRemoteDir: prepared.workspaceRemoteDir, runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, // The SSH transport does not stage referenced projects (it is out of scope), so it never // reports a per-project staging failure. additionalSourceFailures: [], workspaceSyncSnapshot: null, restoreWorkspace: prepared.restoreWorkspace, }; } const prepared = await prepareCommandManagedRuntime({ runner: requireSandboxRunner(target), spec: { providerKey: target.providerKey, shellCommand: target.shellCommand, leaseId: target.leaseId, remoteCwd: target.remoteCwd, timeoutMs: input.timeoutSec && input.timeoutSec > 0 ? input.timeoutSec * 1000 : target.timeoutMs, }, adapterKey: input.adapterKey, workspaceLocalDir: input.workspaceLocalDir, workspaceRemoteDir: input.workspaceRemoteDir, syncWorkspace: input.syncWorkspace, workspaceInboundMode: input.workspaceInboundMode, workspaceDurableSeed: input.workspaceDurableSeed, workspaceBaseline: input.workspaceBaseline, workspaceGitSnapshot: input.workspaceGitSnapshot, workspaceExclude: input.workspaceExclude, preserveAbsentOnRestore: input.preserveAbsentOnRestore, assets: input.assets, additionalSources: input.additionalSources, installCommand: input.installCommand, detectCommand: input.detectCommand, onProgress: input.onProgress, onRuntimeProgress: input.onRuntimeProgress, runtimeSpan: input.runtimeSpan, }); return { target, workspaceRemoteDir: prepared.workspaceRemoteDir, runtimeRootDir: prepared.runtimeRootDir, assetDirs: prepared.assetDirs, additionalSourceDirs: prepared.additionalSourceDirs, additionalSourceFailures: prepared.additionalSourceFailures, workspaceSyncSnapshot: prepared.workspaceSyncSnapshot, restoreWorkspace: prepared.restoreWorkspace, }; } export function runtimeAssetDir( prepared: Pick, key: string, fallbackRemoteCwd: string, ): string { return prepared.assetDirs[key] ?? path.posix.join(fallbackRemoteCwd, ".paperclip-runtime", key); } type GitHubLauncherLocation = { runId: string; target: AdapterExecutionTarget | null | undefined; }; function githubOperationLauncherDirectory(input: GitHubLauncherLocation): string { // Only controller-generated run IDs may name a removable directory. if (!/^[a-zA-Z0-9_-]+$/.test(input.runId)) throw new Error("Invalid GitHub launcher run ID"); return input.target?.kind === "remote" ? path.posix.join(input.target.remoteCwd, ".paperclip-runtime", "github", input.runId) : path.join(os.tmpdir(), "paperclip-github-runtime", input.runId); } /** Call only after execution settles, before releasing its remote environment lease. */ export async function cleanupGitHubOperationLaunchers(input: GitHubLauncherLocation): Promise { const directory = githubOperationLauncherDirectory(input); if (input.target?.kind === "remote") { const result = await adapterExecutionTargetCommandRunner(input.target).execute({ command: "sh", args: ["-c", `rm -rf -- ${shellQuote(directory)}`], cwd: input.target.remoteCwd, timeoutMs: 5_000, }); if (result.exitCode !== 0) throw new Error("Could not clean managed GitHub launchers"); } else { await fs.rm(directory, { recursive: true, force: true }); } } async function githubOperationLauncherBasePath( target: AdapterCommandCapableExecutionTarget | null, env: Record, ): Promise { if (!target) return env.PATH || process.env.PATH || "/usr/bin:/bin"; const configuredPath = sanitizeRemoteExecutionEnv(env).PATH; if (configuredPath !== undefined) return configuredPath; // The provider owns login/profile setup. Query its effective PATH before // staging BASH_ENV, rather than substituting the controller's toolchain or // a minimal PATH that hides legacy NVM/user-local agent installations. const result = await adapterExecutionTargetCommandRunner(target).execute({ command: "sh", args: ["-c", "printf '\\000%s\\000' \"$PATH\""], cwd: target.remoteCwd, timeoutMs: 15_000, }); // Frame the value so login banners cannot become executable search paths. const remotePath = result.stdout.match(/\0([^\0]+)\0/)?.[1]; if (result.timedOut || result.exitCode !== 0 || !remotePath) { throw new Error("Could not resolve remote PATH for managed GitHub launchers"); } return remotePath; } /** Read only execution-target Git context; never import the controller's credentials into SSH. */ export async function prepareGitHubExecutionEnvironment(input: { target: AdapterExecutionTarget | null | undefined; cwd: string; env: Record; hostCredentials: boolean; networkAccess: boolean; }): Promise> { const script = String.raw` const fs = require('node:fs'); const path = require('node:path'); const cp = require('node:child_process'); const env = {}; env.PAPERCLIP_RUNNER_NETWORK_ROOTS = JSON.stringify(['/etc/resolv.conf','/etc/hosts','/etc/nsswitch.conf','/etc/ssl/certs','/etc/ssl/cert.pem'].flatMap(p => { try { return [fs.realpathSync(p)]; } catch { return []; } })); if (process.argv[1] === 'host') { for (const [key, value] of Object.entries(process.env)) { if (/^(GH_TOKEN|GITHUB_TOKEN|GH_ENTERPRISE_TOKEN|GITHUB_ENTERPRISE_TOKEN|PAPERCLIP_GIT_TOKEN|GH_CONFIG_DIR|GIT_CONFIG_(GLOBAL|SYSTEM|NOSYSTEM|COUNT|KEY_\d+|VALUE_\d+)|GIT_(AUTHOR|COMMITTER)_(NAME|EMAIL)|GIT_ASKPASS|SSH_ASKPASS|SSH_AUTH_SOCK|GIT_SSH_COMMAND|GIT_SSH)$/.test(key)) env[key] = value; } env.PAPERCLIP_GITHUB_HOST_HOME = process.env.HOME || ''; env.GH_CONFIG_DIR ||= path.join(process.env.XDG_CONFIG_HOME || path.join(process.env.HOME || '', '.config'), 'gh'); } try { const top = cp.execFileSync('git', ['rev-parse', '--show-toplevel'], {encoding:'utf8',stdio:['ignore','pipe','ignore']}).trim(); if (fs.realpathSync(top) === fs.realpathSync(process.cwd())) { env.PAPERCLIP_GIT_METADATA_ROOTS = JSON.stringify(cp.execFileSync('git', ['rev-parse','--path-format=absolute','--git-common-dir','--git-dir'], {encoding:'utf8',stdio:['ignore','pipe','ignore']}).trim().split('\n').map(p => fs.realpathSync(p))); } } catch {} process.stdout.write("\0" + JSON.stringify(env) + "\0"); `; const args = ["-e", script, input.hostCredentials ? "host" : "managed"]; const remote = input.target?.kind === "remote" ? input.target : null; let discovered: Record; if (remote) { // A legacy SSH host may run a standalone agent binary without Node. Use // only the shell and Git, and emit bounded, NUL-framed environment records. const probe = String.raw` printf '\0PAPERCLIP_GIT_CONTEXT_V1\0' if [ "$1" = host ]; then for key in GH_TOKEN GITHUB_TOKEN GH_ENTERPRISE_TOKEN GITHUB_ENTERPRISE_TOKEN PAPERCLIP_GIT_TOKEN GH_CONFIG_DIR GIT_CONFIG_GLOBAL GIT_CONFIG_SYSTEM GIT_CONFIG_NOSYSTEM GIT_CONFIG_COUNT GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_ASKPASS SSH_ASKPASS SSH_AUTH_SOCK GIT_SSH_COMMAND GIT_SSH; do eval 'value=${"$"}{'"$key"'-}' [ -z "$value" ] || printf '%s\0%s\0' "$key" "$value" done index=0 while [ "$index" -lt 32 ]; do for prefix in GIT_CONFIG_KEY_ GIT_CONFIG_VALUE_; do key="$prefix$index" eval 'value=${"$"}{'"$key"'-}' [ -z "$value" ] || printf '%s\0%s\0' "$key" "$value" done index=$((index + 1)) done printf 'PAPERCLIP_GITHUB_HOST_HOME\0%s\0' "$HOME" printf 'GH_CONFIG_DIR\0%s\0' "${"$"}{GH_CONFIG_DIR:-${"$"}{XDG_CONFIG_HOME:-$HOME/.config}/gh}" fi for file in /etc/resolv.conf /etc/hosts /etc/nsswitch.conf /etc/ssl/certs /etc/ssl/cert.pem; do index=0 while [ -L "$file" ] && [ "$index" -lt 40 ]; do target=$(readlink "$file") || break case "$target" in /*) file="$target" ;; *) file="$(dirname "$file")/$target" ;; esac index=$((index + 1)) done if [ -e "$file" ]; then parent=$(cd "$(dirname "$file")" && pwd -P) || continue printf 'PAPERCLIP_RUNNER_NETWORK_ROOT\0%s\0' "$parent/$(basename "$file")" fi done cwd=$(pwd -P) top=$(git rev-parse --show-toplevel 2>/dev/null) || top= if [ -n "$top" ] && [ "$(cd "$top" && pwd -P)" = "$cwd" ]; then for kind in --git-common-dir --git-dir; do root=$(git rev-parse --path-format=absolute "$kind" 2>/dev/null) || continue root=$(cd "$root" && pwd -P) || continue printf 'PAPERCLIP_GIT_METADATA_ROOT\0%s\0' "$root" done fi printf '\0PAPERCLIP_GIT_CONTEXT_END\0' `; const result = await adapterExecutionTargetCommandRunner(remote).execute({ command: "sh", args: ["-c", probe, "paperclip-git-context", input.hostCredentials ? "host" : "managed"], // The caller's cwd belongs to the controller. Copied sandbox/SSH // workspaces can live at a different path on the execution target. cwd: remote.remoteCwd, timeoutMs: 15_000, }); if (result.exitCode !== 0) throw new Error("Could not read execution-target Git context"); const payload = result.stdout.split("\0PAPERCLIP_GIT_CONTEXT_V1\0")[1]?.split("\0PAPERCLIP_GIT_CONTEXT_END\0")[0]; if (payload === undefined) throw new Error("Could not read execution-target Git context"); discovered = {}; const records = payload.split("\0"); const roots: string[] = []; const networkRoots: string[] = []; for (let index = 0; index + 1 < records.length; index += 2) { const key = records[index]!; const value = records[index + 1]!; if (key === "PAPERCLIP_GIT_METADATA_ROOT") roots.push(value); else if (key === "PAPERCLIP_RUNNER_NETWORK_ROOT") networkRoots.push(value); else discovered[key] = value; } discovered.PAPERCLIP_GIT_METADATA_ROOTS = JSON.stringify([...new Set(roots)]); discovered.PAPERCLIP_RUNNER_NETWORK_ROOTS = JSON.stringify([...new Set(networkRoots)]); } else { const result = await promisify(execFile)(process.execPath, args, { cwd: input.cwd, timeout: 15_000, maxBuffer: 1024 * 1024 }); try { discovered = JSON.parse(result.stdout.split("\0")[1] ?? ""); } catch { throw new Error("Could not read execution-target Git context"); } } // Controller-derived roots and mode must not be replaced by agent bindings. return { ...discovered, ...input.env, ...(input.hostCredentials ? { PAPERCLIP_GITHUB_HOST_HOME: discovered.PAPERCLIP_GITHUB_HOST_HOME } : {}), PAPERCLIP_GIT_METADATA_ROOTS: discovered.PAPERCLIP_GIT_METADATA_ROOTS ?? "[]", PAPERCLIP_RUNNER_NETWORK_ROOTS: discovered.PAPERCLIP_RUNNER_NETWORK_ROOTS ?? "[]", PAPERCLIP_GITHUB_AUTH_MODE: input.hostCredentials ? "host" : "managed", PAPERCLIP_RUNNER_NETWORK_ACCESS: input.networkAccess ? "enabled" : "disabled", }; } /** Stage token-free launchers next to the execution, not in shared global Git config. */ export async function prepareGitHubOperationLaunchers(input: { runId: string; target: AdapterExecutionTarget | null | undefined; cwd: string; env: Record; }): Promise> { const remote = input.target?.kind === "remote" ? input.target : null; const directory = githubOperationLauncherDirectory(input); const configDirectory = path.posix.join(directory, "gh-config"); const basePath = await githubOperationLauncherBasePath(remote, input.env); const managedPath = basePath ? `${directory}:${basePath}` : directory; // Login shells may reorder PATH through /etc/profile or path_helper. Restore // the managed launchers after startup without loading a host user's profile. const profile = `export PATH=${shellQuote(managedPath)}\n`; const files: Record = Object.fromEntries([ ...["git", "gh"].map((name) => [name, githubLauncherSource()] as const), ...[".zshenv", ".zprofile", ".zshrc", ".bash_profile", ".bashrc", ".profile"].map((name) => [name, profile] as const), ]); if (remote) { const runner = adapterExecutionTargetCommandRunner(remote); for (const [program, body] of Object.entries(files)) { await syncRemoteTextFileWithHashSkip({ runner, remoteCwd: remote.remoteCwd, remoteDir: directory, remotePath: path.posix.join(directory, program), body, label: "GitHub operation launcher", action: "stage GitHub operation launcher", lockDir: path.posix.join(directory, `.${program}.lock`), timeoutMs: 15_000, shellCommand: adapterExecutionTargetShellCommand(remote), }); } const permissions = await runner.execute({ command: "sh", args: ["-c", `chmod 700 ${shellQuote(directory)}/git ${shellQuote(directory)}/gh && mkdir -p ${shellQuote(configDirectory)}`], cwd: remote.remoteCwd, timeoutMs: 15_000 }); if (permissions.exitCode !== 0) throw new Error("Could not prepare managed GitHub launchers"); } else { await fs.mkdir(directory, { recursive: true, mode: 0o700 }); await fs.mkdir(configDirectory, { recursive: true, mode: 0o700 }); for (const [program, body] of Object.entries(files)) await fs.writeFile(path.join(directory, program), body, { mode: 0o700 }); } return { ...input.env, PATH: managedPath, ZDOTDIR: directory, BASH_ENV: `${directory}/.bashrc`, GH_CONFIG_DIR: configDirectory, PAPERCLIP_GITHUB_LAUNCHER_DIR: directory }; } function buildBridgeResponseHeaders(response: Response): Record { const out: Record = {}; // Keep `x-paperclip-bridge-outcome` in this list. The host marks a // possibly-committed mutation with the `indeterminate` outcome. The in-sandbox // server reads that header to map the 504 to a terminal 409. If the forward // drops the header, the server keeps the retryable 504 and a caller that // retries 5xx can repeat a mutation that already committed. for (const key of ["content-type", "etag", "last-modified", "x-paperclip-bridge-outcome"]) { const value = response.headers.get(key); if (value && value.trim().length > 0) out[key] = value.trim(); } return out; } function buildBridgeForwardUrl(baseUrl: string, request: { path: string; query: string }): URL { const url = new URL(request.path, baseUrl); const query = request.query.trim(); url.search = query.startsWith("?") ? query.slice(1) : query; return url; } function bridgeResponseBodyLimitError(maxBodyBytes: number): Error { return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`); } /** * Read the forward response body into a `Buffer`, with no text decoding. The * per-request `maxBodyBytes` limit rejects a body larger than the configured * per-request ceiling. * * When the caller passes a `reservation`, this reserves each chunk's bytes * against it immediately after `reader.read()` yields the chunk, and before * `Buffer.from(value)` copies it — the allocation happens inside that * expression, so reserving only before the later `chunks.push` would let the * copy happen first. It also reserves the concatenated buffer's own byte * count before `Buffer.concat` allocates it: the chunk array and the * concatenated buffer are two separate live copies. A denied reservation * cancels the reader and throws {@link BridgeProcessCapacityError}, copying * no further chunk. This function never releases the reservation; the * stream owner that created it does, once the whole forward call settles. * * With no `reservation`, this function enforces only the one request's own * ceiling, exactly as it did before this parameter existed — the queue * transport calls it with no reservation, and its behavior must not change. */ async function readBridgeForwardResponseBody( response: Response, maxBodyBytes: number, reservation?: BridgeBodyReservation, ): Promise { const rawContentLength = response.headers.get("content-length"); if (rawContentLength) { const contentLength = Number.parseInt(rawContentLength, 10); if (Number.isFinite(contentLength) && contentLength > maxBodyBytes) { throw bridgeResponseBodyLimitError(maxBodyBytes); } } if (!response.body) { return Buffer.alloc(0); } const reader = response.body.getReader(); const chunks: Buffer[] = []; let totalBytes = 0; while (true) { const { done, value } = await reader.read(); if (done) break; if (!value) continue; const chunkBytes = value.byteLength; totalBytes += chunkBytes; if (totalBytes > maxBodyBytes) { await reader.cancel().catch(() => undefined); throw bridgeResponseBodyLimitError(maxBodyBytes); } if (reservation && !reservation.reserve(chunkBytes)) { await reader.cancel().catch(() => undefined); throw new BridgeProcessCapacityError(); } chunks.push(Buffer.from(value)); } if (reservation && !reservation.reserve(totalBytes)) { await reader.cancel().catch(() => undefined); throw new BridgeProcessCapacityError(); } return Buffer.concat(chunks, totalBytes); } const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs"; const PROCESS_SESSION_REMOTE_SCRIPT = "paperclip-process-session-remote.mjs"; // The streamed variant writes its output frames to stdout, so it rides a // separate remote path. A sandbox can hold both scripts without the content // hash-skip gate thrashing when a run switches output mode. const PROCESS_SESSION_REMOTE_STREAM_SCRIPT = "paperclip-process-session-remote-stream.mjs"; const PROCESS_SESSION_AUTH_TIMEOUT_MS = 5_000; // The bounded budget `stop()` waits for the wrapper's `shutdownAck` event // before it removes `sessionDir` unconditionally. The wrapper writes the // acknowledgement right after it arms its own kill timer, well before its // child actually exits, so this budget only needs to cover message delivery, // not the child's full shutdown. const DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS = 3_000; function jsonLine(value: unknown): string { return `${JSON.stringify(value)}\n`; } function splitJsonLines(buffer: string): { lines: string[]; rest: string } { const parts = buffer.split(/\n/); return { lines: parts.slice(0, -1), rest: parts.at(-1) ?? "" }; } async function writeProcessSessionProxyScript(dir: string, port: number, token: string): Promise { await fs.mkdir(dir, { recursive: true }); const proxyPath = path.join(dir, PROCESS_SESSION_PROXY_SCRIPT); await fs.writeFile(proxyPath, getProcessSessionProxySource({ port, token }), { mode: 0o700 }); return proxyPath; } // Content-hash-skip the process-session remote script write, mirroring the // sandbox callback bridge entrypoint sha256 gate. The script is a static // Paperclip-authored `.mjs` that only changes when the build changes, so on a // warm start (same sandbox, script already present) the single sha-gate exec // skips the ~3-exec base64 upload entirely. `syncRemoteTextFileWithHashSkip` // fails loud on a check error rather than silently re-uploading. async function syncProcessSessionRemoteScript(input: { runner: CommandManagedRuntimeRunner; remoteCwd: string; remoteScriptDir: string; remoteScriptPath: string; timeoutMs?: number | null; shellCommand?: "bash" | "sh" | null; outputToStdout?: boolean; }): Promise<{ uploaded: boolean }> { const { uploaded } = await syncRemoteTextFileWithHashSkip({ runner: input.runner, remoteCwd: input.remoteCwd, remoteDir: input.remoteScriptDir, remotePath: input.remoteScriptPath, body: getProcessSessionRemoteSource({ outputToStdout: input.outputToStdout === true }), label: "Process session remote script", action: "sync process session remote script", lockDir: path.posix.join(input.remoteScriptDir, ".paperclip-process-session-script.lock"), timeoutMs: input.timeoutMs, shellCommand: input.shellCommand, }); return { uploaded }; } async function readRemoteJsonFiles(input: { client: ReturnType; dir: string; }): Promise> { const names = await input.client.listJsonFiles(input.dir); const out: Array<{ name: string; body: string }> = []; for (const name of names) { const filePath = path.posix.join(input.dir, name); const body = await input.client.readTextFile(filePath); await input.client.remove(filePath).catch(() => undefined); out.push({ name, body }); } return out; } async function waitForLocalServerListen(server: net.Server): Promise { await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", () => { server.off("error", reject); resolve(); }); }); const address = server.address(); if (!address || typeof address === "string") { throw new Error("Process session bridge did not expose a TCP port."); } return address.port; } /** Span name that wraps the socket handler's one `writeTextFile` exec — one * outbound ACP message to the agent. */ const AGENT_SESSION_SEND_INPUT_SPAN = "sandbox.agentSession.sendInput"; /** Span name that wraps one 100 ms poll tick — `list`, then `read`+`remove` per * file found (`1 + 2n` execs). */ const AGENT_SESSION_POLL_OUTPUT_SPAN = "sandbox.agentSession.pollOutput"; export async function startAdapterExecutionTargetProcessSessionBridge(input: { runId: string; target: AdapterExecutionTarget | null | undefined; runtimeRootDir: string | null | undefined; adapterKey: string; command: string; args: string[]; cwd: string; // The launch env is consumed ONLY when building the base64 `commandPayload` // below — never during the env-INDEPENDENT dir/script setup. Accepting a // resolver (in addition to a plain object) lets a caller overlap that setup // with other work — e.g. starting the paperclip callback bridge — and hand the // merged env in right before the launch. env: Record | (() => Promise>); timeoutSec?: number | null; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; // Return the current-run parent-context token. The socket handlers and the // poll timer read it per unit of work and run under it, so their run-time // `sandbox.exec` spans parent to the live run span (`agent.turn` during the // turn, `task.run` otherwise). When it is absent, the work runs with an empty // store, exactly like the earlier `runWithoutActiveStep` behavior. getRuntimeParentContext?: () => StartupSpanContext | undefined; // Wrap each unit of run-time work in its own named span. The socket handler // uses it for `sandbox.agentSession.sendInput` and the poll timer for // `sandbox.agentSession.pollOutput`, so each unit's inner `sandbox.exec` spans // group under one wrapper span. When it is absent, the work runs under the run // parent with no wrapper span, exactly like the earlier behavior. runtimeSpan?: RuntimeSpanRunner; // Stream the agent output through the persistent session log stream instead of // the host output-file poll. When true, the bridge runs the wrapper as one // long-lived session command and reads its stdout frames from the stream, and // it does not start the 100 ms poll. Default OFF: the bridge keeps the poll. streamOutputViaSession?: boolean; }): Promise { if (!input.target || input.target.kind !== "remote" || input.target.transport !== "sandbox") { return null; } const target = input.target; const onLog = input.onLog ?? (async () => {}); const runner = requireSandboxRunner(target); // Run one unit of run-time work under its named wrapper span when a span // runner is injected. Without a runner, run the work under the current run // parent, so the inner `sandbox.exec` spans parent to the live run span, // exactly like the earlier behavior. const runRuntimeWork = (name: string, work: () => Promise): Promise => input.runtimeSpan ? input.runtimeSpan(name, work) : runWithRuntimeParent(input.getRuntimeParentContext?.(), work); const shellCommand = preferredSandboxShell(target); const timeoutMs = typeof input.timeoutSec === "number" && Number.isFinite(input.timeoutSec) && input.timeoutSec > 0 ? Math.trunc(input.timeoutSec * 1000) : target.timeoutMs ?? undefined; const bridgeRuntimeDir = path.posix.join( input.runtimeRootDir?.trim() || path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey), "process-sessions", ); const sessionId = randomUUID(); const sessionDir = path.posix.join(bridgeRuntimeDir, sessionId); const stdinDir = path.posix.join(sessionDir, "stdin"); const eventsDir = path.posix.join(sessionDir, "events"); // The streamed wrapper writes its frames to stdout and rides a separate remote // path, so a warm sandbox can hold both wrapper scripts without the content // hash-skip gate thrashing when a run switches output mode. const streamOutput = input.streamOutputViaSession === true; const remoteScriptPath = path.posix.join( bridgeRuntimeDir, streamOutput ? PROCESS_SESSION_REMOTE_STREAM_SCRIPT : PROCESS_SESSION_REMOTE_SCRIPT, ); const client = createCommandManagedSandboxCallbackBridgeQueueClient({ runner, remoteCwd: target.remoteCwd, timeoutMs, shellCommand, }); // The launch exec below re-creates stdinDir and eventsDir with one `mkdir -p`, // and the remote script also creates them on start. No reader touches the two // directories before the launch exec runs, so upfront makeDir execs are redundant. await syncProcessSessionRemoteScript({ runner, remoteCwd: target.remoteCwd, remoteScriptDir: bridgeRuntimeDir, remoteScriptPath, timeoutMs, shellCommand, outputToStdout: streamOutput, }); // Resolve the launch env AFTER the env-independent setup above, so a caller // can defer it until an upstream dependency (e.g. the paperclip bridge's env) // is ready without blocking the dir/script setup. const launchEnv = typeof input.env === "function" ? await input.env() : input.env; const commandPayload = Buffer.from(JSON.stringify({ command: input.command, args: input.args, cwd: input.cwd || target.remoteCwd, // The ACP engine has already projected this launch env from explicit // adapter/runtime inputs and registered contributions. Compare against an // empty inherited baseline so an explicit identity value (notably PATH) // is not reclassified as ambient merely because it equals the host value. env: sanitizeRemoteExecutionEnv(launchEnv, {}), }), "utf8").toString("base64"); // Legacy poll path: background the wrapper with `nohup` and read its output // event files with the host poll below. The streamed path launches the wrapper // as one foreground session command further down instead, so skip this. if (!streamOutput) { await onLog("stdout", `[paperclip] Starting ACP process session bridge in sandbox (${target.providerKey ?? "provider"}).\n`); const startResult = await runner.execute({ command: shellCommand, args: shellCommandArgs( [ `mkdir -p ${shellQuote(stdinDir)} ${shellQuote(eventsDir)}`, // I3: no numeric process identifier anywhere. Background the // wrapper and let it go; do not capture `$!`. `PAPERCLIP_PROCESS_SESSION_DIR=${shellQuote(sessionDir)} ` + `PAPERCLIP_PROCESS_SESSION_COMMAND_B64=${shellQuote(commandPayload)} ` + `nohup node ${shellQuote(remoteScriptPath)} >/dev/null 2>&1 < /dev/null &`, ].join("\n"), ), cwd: target.remoteCwd, env: { PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge", }, timeoutMs, // The wrapper launch is bridge plumbing. Keep it off the persistent // session so it never queues behind an in-run session command. bypassSession: true, }); if (startResult.timedOut || (startResult.exitCode ?? 1) !== 0) { throw new Error(`Failed to start sandbox ACP process session bridge: ${startResult.stderr || startResult.stdout}`); } } let socket: net.Socket | null = null; let stopping = false; // Resolves when `stop()` tears the bridge down. The streamed `sandbox.agentProcess` // span races its work against this, so the span ends at teardown at the latest // even when the remote process lingers, and never outlives the run root span. let signalStopped: () => void = () => {}; const stopped = new Promise((resolve) => { signalStopped = resolve; }); let stdinSeq = 0; // One promise chain per session that serializes the stdin file writes. Each // write is multi-exec on the command-managed client: prepare, append per 32 // KiB, then an atomic rename. The chain makes the rename for file N finish // before the write for file N+1 starts, so the files land in send order. // Without it the writes overlap. A small later chunk can then rename ahead of // a big earlier chunk, so the wrapper reads the stdin bytes out of order and // corrupts a large prompt on the stdin path. let stdinWriteChain: Promise = Promise.resolve(); let pollTimer: NodeJS.Timeout | null = null; const pendingRemoteEvents: Array<{ type?: string; stream?: "stdout" | "stderr"; data?: string; code?: number | null; signal?: string | null; message?: string; }> = []; const token = createSandboxCallbackBridgeToken(18); const proxyDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-process-session-proxy-")); // `stop()` waits on this promise, bounded, for the wrapper's `shutdownAck` // event. `deliverRemoteEvent` resolves it below and never forwards the // event further: it is a host-internal control ack, not part of the ACP // output stream. An event under `sessionDir` is untrusted telemetry: an // `exit` or `error` event is never treated as proof of shutdown, because // any process running under the sandbox can write one. Only `shutdownAck` // counts, and `stop()` also gives itself a dedicated reader for it below, // so a late `shutdownAck` still lands even after the long-lived poll has // stopped re-arming. let signalShutdownAcknowledged: () => void = () => {}; const shutdownAcknowledged = new Promise((resolve) => { signalShutdownAcknowledged = resolve; }); const writeRemoteEventToSocket = (event: (typeof pendingRemoteEvents)[number]) => { if (!socket) return false; socket.write(jsonLine(event)); if (event.type === "exit") { stopping = true; socket.end(); } else if (event.type === "error") { stopping = true; socket.destroy(); } return true; }; const deliverRemoteEvent = (event: (typeof pendingRemoteEvents)[number]) => { if (event.type === "shutdownAck") { signalShutdownAcknowledged(); return; } if (socket) { writeRemoteEventToSocket(event); return; } pendingRemoteEvents.push(event); if (event.type === "exit" || event.type === "error") { stopping = true; } }; const flushPendingRemoteEvents = () => { if (!socket) return; while (pendingRemoteEvents.length > 0 && socket) { const event = pendingRemoteEvents.shift(); if (event) writeRemoteEventToSocket(event); } }; const liveSockets = new Set(); // Register the per-connection socket handlers with no run parent context. // A stdin write from a socket handler is a run-time exec, not startup work. // The connection can open under `task.run` and receive stdin later, during an // `agent.turn`. So the handler must read the current-run parent at send time, // not at connect time. A connect-time read captures the parent that was live // when the socket opened, and every later exec span parents to that stale // parent. The `data` handler below reads the getter per message instead. const server = net.createServer((nextSocket) => { liveSockets.add(nextSocket); nextSocket.setEncoding("utf8"); nextSocket.on("error", () => undefined); let connectionBuffer = ""; let authenticated = false; // Connections own the session (and receive buffered process output) only // after presenting the bridge token; idle unauthenticated peers are dropped. const authTimer = setTimeout(() => { if (!authenticated) nextSocket.destroy(); }, PROCESS_SESSION_AUTH_TIMEOUT_MS); authTimer.unref?.(); nextSocket.on("close", () => { clearTimeout(authTimer); liveSockets.delete(nextSocket); }); nextSocket.on("data", (chunk) => { connectionBuffer += chunk; const split = splitJsonLines(connectionBuffer); connectionBuffer = split.rest; for (const line of split.lines) { if (!line.trim()) continue; let message: { token?: string; type?: string; data?: string }; try { message = JSON.parse(line) as { token?: string; type?: string; data?: string }; } catch { nextSocket.destroy(); return; } if (message.token !== token) { nextSocket.destroy(); return; } if (!authenticated) { if (socket) { nextSocket.destroy(); return; } authenticated = true; clearTimeout(authTimer); socket = nextSocket; flushPendingRemoteEvents(); } // Wrap one outbound ACP message to the agent in a // `sandbox.agentSession.sendInput` span, so its one `writeTextFile` exec // groups under one named span. The span runner reads the current-run // parent at send time: the live parent switches to `agent.turn` during // the turn and back to `task.run` after it. A message that is neither // `stdin` nor `stdinEnd` writes nothing, so it opens no span. const stdinPayload = message.type === "stdin" && typeof message.data === "string" ? { type: "stdin", data: message.data } : message.type === "stdinEnd" ? { type: "stdinEnd" } : null; if (stdinPayload) { stdinSeq += 1; const name = `${String(stdinSeq).padStart(12, "0")}.json`; const filePath = path.posix.join(stdinDir, name); // Chain this write after the previous one, so the atomic rename for // file N finishes before the write for file N+1 starts. Keep the // per-message `sandbox.agentSession.sendInput` span inside the chain. const write = stdinWriteChain.then(() => runRuntimeWork(AGENT_SESSION_SEND_INPUT_SPAN, () => client.writeTextFile(filePath, jsonLine(stdinPayload)), ), ); // The next message chains after this write on success or failure, so a // failed write never blocks the chain. This mirrors the wrapper // `writeChain` pattern for its event files. stdinWriteChain = write.then(() => undefined, () => undefined); // Keep the failure behavior: send one error line, then destroy the socket. write.catch((error) => { nextSocket.write(jsonLine({ type: "error", message: error instanceof Error ? error.message : String(error) })); nextSocket.destroy(); }); } } }); }); const poll = async () => { if (stopping) return; try { // Read every file this tick fetched before this loop decides whether to // keep polling. A `shutdownAck` can land in the same batch right after // an `exit` event; deliver it too, so this tick never drops an // already-fetched (and already-removed-from-disk) event. const events = await readRemoteJsonFiles({ client, dir: eventsDir }); for (const event of events) { const parsed = JSON.parse(event.body) as { type?: string; stream?: "stdout" | "stderr"; data?: string; code?: number | null; signal?: string | null; message?: string; }; deliverRemoteEvent(parsed); } } catch (error) { const message = error instanceof Error ? error.message : String(error); await onLog("stderr", `[paperclip] ACP process session bridge poll failed: ${message}\n`); deliverRemoteEvent({ type: "error", message }); return; } finally { if (!stopping) { schedulePoll(); } } }; // Schedule the long-lived poll timer. Wrap each 100 ms poll tick in a // `sandbox.agentSession.pollOutput` span, so the tick's `list` plus per-file // `read`/`remove` execs group under one named span. The poll loop reads remote // event files with run-time execs, not startup work, so the wrapper span and // its child execs parent to the live run span, not to the ended bridge step. // The span runner reads the run parent per tick, because the re-arm timer that // the poll body schedules opens a new tick span: the live parent switches to // `agent.turn` during the turn and back to `task.run` after it. const schedulePoll = () => { pollTimer = setTimeout(() => void runRuntimeWork(AGENT_SESSION_POLL_OUTPUT_SPAN, poll), 100); pollTimer.unref?.(); }; const port = await waitForLocalServerListen(server); const agentCommand = await writeProcessSessionProxyScript(proxyDir, port, token); if (streamOutput) { // Streamed output path. Run the wrapper as one long-lived session command; // its stdout carries newline-delimited JSON frames that reach the host // through the provider session log stream. Deliver each frame exactly once // by its monotonic `seq`, so a frame that arrives both live and in the final // result is not repeated. There is no host output-file poll here. let streamBuffer = ""; let lastSeq = 0; let sawTerminal = false; const deliverFrame = (frame: (typeof pendingRemoteEvents)[number] & { seq?: number }) => { if (typeof frame.seq === "number") { if (frame.seq <= lastSeq) return; lastSeq = frame.seq; } if (frame.type === "exit" || frame.type === "error") sawTerminal = true; deliverRemoteEvent(frame); }; const parseFrameLine = (line: string) => { if (!line.trim()) return; let frame: (typeof pendingRemoteEvents)[number] & { seq?: number }; try { frame = JSON.parse(line) as typeof frame; } catch { return; } deliverFrame(frame); }; // Live delivery: buffer partial lines across stream chunks, deliver each // complete frame line as it arrives. const ingestStreamChunk = (text: string) => { streamBuffer += text; const split = splitJsonLines(streamBuffer); streamBuffer = split.rest; for (const line of split.lines) parseFrameLine(line); }; // Terminal delivery (the defined fallback to the poll): the resolved result // carries the full wrapper stdout even when the live stream degraded to the // provider session-log poll. The text is complete and self-contained, so // re-parse it on its own; the `seq` guard drops every frame the live stream // already delivered. Drop any partial live line — its complete form is in the // full text. const ingestFinalText = (text: string) => { streamBuffer = ""; for (const line of text.split(/\n/)) parseFrameLine(line); }; const launchEnvForStream = typeof input.env === "function" ? await input.env() : input.env; const streamCommandPayload = Buffer.from(JSON.stringify({ command: input.command, args: input.args, cwd: input.cwd || target.remoteCwd, // Same provenance-clean contract as the polled payload above. Preserve // every explicit identity override even when it equals the host value. env: sanitizeRemoteExecutionEnv(launchEnvForStream, {}), }), "utf8").toString("base64"); await onLog( "stdout", `[paperclip] Starting streamed ACP process session bridge in sandbox (${target.providerKey ?? "provider"}).\n`, ); // Fire the long-lived command; do NOT await it here. `useSession` forces the // persistent session so the provider streams the wrapper stdout back through // `onLog`. On resolve, the terminal re-parse fills any frames the live stream // missed; on reject, deliver one error frame so the local proxy fails loud. // // Wrap the launch in a `sandbox.agentProcess` span. `runRuntimeWork` parents // it to the LIVE RUN root (`task.run` at launch time — no turn has started // yet), not to the ephemeral `bridge.process-session` bring-up step, and it // stays open for the whole process lifetime. The inner `sandbox.exec` nests // under it. Without the wrapper the raw exec's span inherits the ~2.28s // bring-up step as its parent and then dangles ~50s past it, overlapping // `agent.turn` — a child outliving its parent. As a run-scoped span it reads // instead as a resource that OVERLAPS the sibling `agent.turn`, which is the // correct shape (the persistent process hosts the turn; it is not a child of // it, and on multi-turn runs one process spans several turns). `runRuntimeWork` // is voided, not awaited, so bring-up never blocks on the long-lived command, // and it defaults to a no-op parent when no span runner is injected. // // The span is bounded to the bridge lifecycle: it ends when the command // settles OR when `stop()` runs, whichever comes first. `stop()` runs during // run teardown, before the caller ends the `task.run` root span, so the span // never outlives the run root even if the remote process lingers past // teardown (`execute` has no cancel, so a lingering process cannot be forced // to resolve). The command promise keeps running after the span ends so its // frame handlers still deliver; they no-op once `stopping` is set. void runRuntimeWork("sandbox.agentProcess", async () => { const commandSettled = (async () => { try { const result = await runner.execute({ command: shellCommand, args: shellCommandArgs(`node ${shellQuote(remoteScriptPath)}`), cwd: target.remoteCwd, env: { PAPERCLIP_PROCESS_SESSION_DIR: sessionDir, PAPERCLIP_PROCESS_SESSION_COMMAND_B64: streamCommandPayload, PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge", }, timeoutMs, useSession: true, onLog: async (stream, chunk) => { if (stream === "stdout") ingestStreamChunk(chunk); }, }); ingestFinalText(result.stdout); if (!sawTerminal && !stopping) { deliverRemoteEvent({ type: "exit", code: typeof result.exitCode === "number" ? result.exitCode : null, }); } } catch (error) { if (!stopping) { deliverRemoteEvent({ type: "error", message: error instanceof Error ? error.message : String(error), }); } } })(); await Promise.race([commandSettled, stopped]); }); } else { schedulePoll(); } // `stop()` cannot rely on the long-lived poll above to observe a late // `shutdownAck`: that poll stops re-arming as soon as it forwards a // terminal `exit`/`error` event, and `stop()` itself sets `stopping` on // its own first line. A normal completion's `shutdownAck` file, written a // moment after `exit`, can then land on disk after nobody reads the events // directory any more. Give `stop()` its own bounded reader that looks only // for `shutdownAck` and ignores every other event type, so the wait below // shortens on the wrapper's own proof of shutdown -- never on an `exit` or // `error` event, which any process running under `sessionDir` can forge. let stopReadingForShutdownAck = false; const readShutdownAckUntil = (deadlineEpochMs: number) => { if (stopReadingForShutdownAck) return; void (async () => { try { const events = await readRemoteJsonFiles({ client, dir: eventsDir }); if (stopReadingForShutdownAck) return; for (const event of events) { try { const parsed = JSON.parse(event.body) as { type?: string }; if (parsed.type === "shutdownAck") { signalShutdownAcknowledged(); return; } } catch { // Not readable JSON yet. It is not a `shutdownAck`; ignore it. } } } catch { // Best-effort: a read failure here is not proof of anything. } if (!stopReadingForShutdownAck && Date.now() < deadlineEpochMs) { const timer = setTimeout(() => readShutdownAckUntil(deadlineEpochMs), 100); timer.unref?.(); } })(); }; return { agentCommand, stop: async () => { stopping = true; // End the `sandbox.agentProcess` span now, before the caller ends the run // root span, even if the remote command has not resolved yet. signalStopped(); if (pollTimer) clearTimeout(pollTimer); for (const liveSocket of liveSockets) liveSocket.destroy(); await new Promise((resolve) => server.close(() => resolve())).catch(() => undefined); // Wait for every accepted stdin write before `stdinEnd`. The socket handler // fires each chunk write un-awaited through `stdinWriteChain`, so an earlier // chunk can still be pending here. Chain the `stdinEnd` write onto the same // per-session chain, so its file rename never finishes before an earlier // chunk. `stdinSeq` is stable now, because the sockets are destroyed and the // server is closed, so no new message can increment it. const stdinEndPath = path.posix.join( stdinDir, `${String(stdinSeq + 1).padStart(12, "0")}.json`, ); const stdinEndWrite = stdinWriteChain.then(() => client.writeTextFile(stdinEndPath, jsonLine({ type: "stdinEnd" })), ); stdinWriteChain = stdinEndWrite.then(() => undefined, () => undefined); await stdinEndWrite.catch(() => undefined); // The `shutdown` control message tells the wrapper to terminate itself // and its own child (I3: no operating-system signal and no process // identifier cross this boundary — only a file-queue message does). // Chain it onto the same per-session write order as `stdinEnd`, so its // file never lands before the earlier one. const shutdownPath = path.posix.join( stdinDir, `${String(stdinSeq + 2).padStart(12, "0")}.json`, ); const shutdownWrite = stdinWriteChain.then(() => client.writeTextFile(shutdownPath, jsonLine({ type: "shutdown" })), ); stdinWriteChain = shutdownWrite.then(() => undefined, () => undefined); await shutdownWrite.catch(() => undefined); // Wait a bounded budget for a hint that the wrapper stopped: only the // `shutdownAck` event counts; an `exit` or `error` event is untrusted // telemetry from inside the sandbox and never shortens this wait or // suppresses the warning below. `shutdownAck` itself is ALSO an // untrusted hint, not proof: any process that shares the sandbox can // write the same event under this session's event directory. It can // only shorten this wait and suppress the warning below; it never // gates, shortens, or replaces the unconditional removal further down. // What actually makes the wrapper's own termination deterministic is // the wrapper-side session-identity latch, not this event. let acknowledgedInTime = false; readShutdownAckUntil(Date.now() + DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS); await Promise.race([ shutdownAcknowledged.then(() => { acknowledgedInTime = true; }), new Promise((resolve) => { const budgetTimer = setTimeout(resolve, DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS); budgetTimer.unref?.(); }), ]); stopReadingForShutdownAck = true; if (!acknowledgedInTime) { await onLog( "stderr", `[paperclip] ACP process session wrapper did not acknowledge shutdown within ${DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS}ms; removing the session directory anyway.\n`, ).catch(() => undefined); } // Unconditional: this removal runs whether or not the wrapper // acknowledged, and whether or not any event (real or forged) arrived // under `sessionDir`. `stop()` runs during run teardown and must stay // non-fatal, so every step above is best-effort and this step never // throws. await client.remove(sessionDir).catch(() => undefined); await fs.rm(proxyDir, { recursive: true, force: true }).catch(() => undefined); }, }; } function getProcessSessionProxySource(input: { port: number; token: string }): string { return `#!/usr/bin/env node import net from "node:net"; const socket = net.createConnection({ host: "127.0.0.1", port: ${input.port} }); const token = ${JSON.stringify(input.token)}; let buffer = ""; let exiting = false; function send(message) { socket.write(JSON.stringify({ token, ...message }) + "\\n"); } socket.on("connect", () => send({ type: "hello" })); process.stdin.on("data", (chunk) => send({ type: "stdin", data: Buffer.from(chunk).toString("base64") })); process.stdin.on("end", () => send({ type: "stdinEnd" })); process.stdin.resume(); socket.setEncoding("utf8"); socket.on("data", (chunk) => { buffer += chunk; const parts = buffer.split(/\\n/); buffer = parts.pop() || ""; for (const line of parts) { if (!line.trim()) continue; const message = JSON.parse(line); if (message.type === "data") { const out = Buffer.from(message.data || "", "base64"); (message.stream === "stderr" ? process.stderr : process.stdout).write(out); } else if (message.type === "error") { process.stderr.write(String(message.message || "Process session bridge failed.") + "\\n"); exiting = true; process.exitCode = 1; socket.end(); } else if (message.type === "exit") { exiting = true; process.exitCode = typeof message.code === "number" ? message.code : 1; socket.end(); } } }); socket.on("close", () => { if (!exiting) process.exit(1); }); `; } export function getProcessSessionRemoteSource(input?: { outputToStdout?: boolean }): string { return input?.outputToStdout === true ? getProcessSessionRemoteStreamSource() : getProcessSessionRemoteEventFileSource(); } // The shared stdin drain. Both wrappers read newline-delimited stdin messages // from the stdin file queue and write them to the child, then end the child // stdin on `stdinEnd`. A write to a closed child stdin only emits an `error` // event, so the wrapper installs a no-op handler at the call site. const PROCESS_SESSION_STDIN_POLL_TAIL = `child.stdin.on("error", () => {}); // A stdin file can appear before the host finishes the write. An empty read is // the non-atomic-write window; a partial read makes JSON.parse throw. The // poller must not delete a file before it validates the content. So read and // parse each file first, and delete it only after a successful parse. The files // sort in send order. If an earlier file is not readable yet, stop the cycle and // keep the order: a later file (for example stdinEnd) must not run ahead of it. // Retry the earlier file on a later cycle. After the retry limit, drop the file // and write an error event, so a lost message fails loud, and let later files // run. const stdinMaxParseRetries = (() => { const raw = Number.parseInt(process.env.PAPERCLIP_PROCESS_SESSION_STDIN_MAX_RETRIES || "", 10); return Number.isFinite(raw) && raw > 0 ? raw : 100; })(); const stdinParseRetries = new Map(); // Track the next expected sequence number. The host writes the stdin files in // send order and pads the number to 12 digits, starting at 1. The files sort in // send order. When the smallest present number is greater than expected, an // earlier file has not appeared yet: a missing file, not an unreadable one. Hold // the send order and wait for it, bounded by the same retry budget as the // unreadable-file path. This turns a reordering into a loud error, never silent // corruption. let stdinExpectedSeq = 1; let stdinGapRetries = 0; // The bounded grace period between the SIGTERM and the SIGKILL a terminate() // call sends. A test can override it through the environment, so a stubborn // child does not force a slow test. const terminateGraceMs = (() => { const raw = Number.parseInt(process.env.PAPERCLIP_PROCESS_SESSION_TERMINATE_GRACE_MS || "", 10); return Number.isFinite(raw) && raw > 0 ? raw : 3000; })(); // I2: terminate() is the only function in this wrapper that calls // child.kill(). No child event handler and no sibling callback calls it. // terminate() is idempotent: a second call, or a first call after the child // already exited on its own, does nothing beyond what already ran. async function terminate() { if (terminated) return; terminated = true; shuttingDown = true; stdinClosed = true; child.stdin.end(); // A \`false\` return means the child's process handle is already gone (the // child exited before this call ran). ChildProcess#kill() is handle-scoped: // once Node clears the handle at reap, the method call above sends no // signal and never falls back to a stored process identifier (I3). Treat // \`false\` as a no-op and do not retry through a numeric identifier. const sentTerm = child.kill("SIGTERM"); if (sentTerm) { killTimer = setTimeout(() => { // Escalate on the same handle only (I2): the grace period expired, so // send SIGKILL through the same child handle, never a numeric // identifier and never a process-group signal. child.kill("SIGKILL"); }, terminateGraceMs); killTimer.unref?.(); } // This event is an untrusted latency hint, not proof. Any process that can // reach this session's event directory can write the same event type. It // can only shorten the host's shutdown wait and suppress the host's // timeout warning; it is never evidence that this wrapper's lifecycle // completed, and the host's cleanup never depends on it. The identity // latch below is what makes this wrapper's own termination deterministic. await writeEvent({ type: "shutdownAck" }); } // A sandbox peer can delete sessionDir and stdinDir, then recreate a // directory at the same pathname. A pathname does not prove identity: any // process that shares the sandbox can write it. So this wrapper captures the // OS-level identity of both paths once at startup, before the first poll // cycle, and checks it on every later cycle. // // The identity is the device number, the inode number, AND the inode's own // creation time. The device/inode pair alone is not enough: a filesystem can // reissue the exact inode number a just-removed directory held to the very // next directory created at the same path, with no attacker action needed // beyond the recreate the finding already describes. The creation time does // not have this gap: it is set fresh on every inode allocation, even when the // allocator reissues an old inode number, so a recreated directory always // carries a different creation time. The creation time alone is not enough // either, on a filesystem or kernel too old to report it, so this wrapper // keeps the device/inode pair as a second signal rather than relying on // either alone. Ordinary use of stdinDir (the host writing and this wrapper // deleting individual stdin files) changes that directory's OWN change time, // but never its creation time, so the creation time is safe to latch on // without producing a false positive on every stdin message. // // A filesystem or kernel that cannot report a real creation time does not // always report a value of zero. Node fails in one of two ways, and both are // grounded, not assumed: on Linux, when the statx() call finds no creation // time support, the kernel leaves the field unset and Node reports 0. On a // platform whose stat() call has no creation-time field at all, Node copies // the change time into the creation time instead. A 0 value fails open (any // recreated directory then matches on birthtimeMs alone), and a change-time // copy fails closed but far too often (it would move on every stdin file // this wrapper deletes). captureSessionIdentity() below proves the value is // usable before it trusts it, and fails closed on both known fallbacks. let sessionDirIdentity = null; let stdinDirIdentity = null; // The latch. Once set, it never clears. This replaces a counter that a // successful read reset to zero: an attacker who recreated the directory // before the counter reached its threshold kept the wrapper polling forever. // A latch has no threshold to race and no reset path. let identityLost = false; async function statPathIdentity(candidatePath) { const stats = await fs.lstat(candidatePath); if (stats.isSymbolicLink()) { const error = new Error("Refusing a symbolic link on a process session control path."); error.code = "EPAPERCLIP_SYMLINK"; throw error; } if (!stats.isDirectory()) { const error = new Error("A process session control path is not a directory."); error.code = "ENOTDIR"; throw error; } return { dev: stats.dev, ino: stats.ino, birthtimeMs: stats.birthtimeMs }; } function sameIdentity(left, right) { return left.dev === right.dev && left.ino === right.ino && left.birthtimeMs === right.birthtimeMs; } async function latchAndTerminate() { if (identityLost) return; identityLost = true; await terminate(); } let probeSeq = 0; // A probe file name that pollStdin() can never read as a stdin message: it // does not end in ".json", so the ".json" filter in pollStdin() skips it if // a poll cycle ever lists the directory during the probe's short window. function nextProbeFileName() { probeSeq += 1; return ".paperclip-birthtime-probe-" + process.pid + "-" + probeSeq; } // Proves a directory's reported birthtimeMs is a real creation time, not a // change-time copy. Creating and removing a file inside a directory changes // that directory's OWN change time but never its true creation time, so a // birthtimeMs that moves across the probe is a change-time copy. Returns // null when the value is proven real. Returns a stderr-ready reason string // on any failure (a detected copy, or a probe that cannot run at all, for // example a permission error or a pre-created probe path): either way the // caller must not trust the value. // // The open uses the "wx" flag: exclusive create, fail if the path exists. // A sandbox peer cannot pre-create the probe path as a symbolic link and // have this call follow it, because "wx" fails closed on an existing path // instead of following a link to it. // // Cleanup checks identity, not only ownership of the initial create. This // wrapper reads the probe file's identity, (dev, ino, ctimeMs), off the open // file descriptor itself (fstat), not off the path, so a peer that swaps the // path in the short gap after create cannot poison the identity this // wrapper trusts as its own. Right before removal, this wrapper reads the // path's identity again and removes it only when that identity still // matches. A same-sandbox peer that deletes the probe file and creates its // own entry at the same path in between leaves a different identity behind, // so this wrapper leaves that entry untouched instead of removing it. This // covers a peer's replacement file, a peer's replacement directory, and a // peer's replacement symbolic link alike, because all three change the // identity this wrapper reads back. The identity check includes ctimeMs, // not only (dev, ino): a filesystem can hand this call's freed inode number // straight back out to a peer's very next create at the same path, so // (dev, ino) alone can match a path this call no longer owns; ctimeMs resets // on every create, so a peer's replacement carries a different one even when // the inode number repeats. Node's filesystem API has no call that removes a // path only when its identity still matches an earlier read as one atomic // step, so a gap remains between this wrapper's final identity read and the // removal call itself. A peer that wins this gap can put any entry at the // probe path before the removal call runs. This can include a pre-existing // file the peer renames into place, not only a file the peer creates fresh. // The removal call then removes whatever entry sits at the probe path at // that moment. Two bounds still hold on that removal. The path always // stays under dirPath. If the entry is a symbolic link, the removal call // removes the link itself instead of following it to a different target. // A non-recursive removal call also fails if the entry is a directory. async function birthtimeSurvivesProbe(dirPath) { let before; try { before = (await fs.lstat(dirPath)).birthtimeMs; } catch { return "its reported creation time could not be read"; } const probePath = path.posix.join(dirPath, nextProbeFileName()); let handle; try { handle = await fs.open(probePath, "wx"); } catch { return "its probe file could not be created exclusively (the path may already exist)"; } // fstat on the open handle names the exact inode this call just created. // A path-based lstat here instead would be racy against a peer that swaps // the path in the gap between the create above and the stat: fstat has no // such gap, because a file descriptor keeps naming the inode it opened no // matter what a later swap does to the path. let ownedIdentity = null; try { const createdStats = await handle.stat(); // ctimeMs guards against inode reuse; see the function comment above. ownedIdentity = { dev: createdStats.dev, ino: createdStats.ino, ctimeMs: createdStats.ctimeMs }; } catch { ownedIdentity = null; } finally { await handle.close().catch(() => undefined); } if (!ownedIdentity) { // fstat on this call's own just-opened descriptor failed. This call then // has no verified identity for the probe file it created, so it must not // check or remove that file by path: a peer could already own the entry // at that path, and a path-based removal here could delete the peer's // entry instead of this call's own file. Fail closed right here instead // of falling through to the birthtime comparison below, so a failed // identity read can never let this probe report success. return "its own probe file's identity could not be read from the open file descriptor"; } // The one gap the fs API cannot close: this lstat and the removal below // are two separate calls, not one atomic "remove if identity still // matches" step. A peer that wins this narrow gap can put any entry at // the probe path, including a pre-existing file it renames into place, // and the removal call below removes whatever entry is there when it // runs. const currentStats = await fs.lstat(probePath).catch(() => null); const stillOwned = currentStats !== null && currentStats.dev === ownedIdentity.dev && currentStats.ino === ownedIdentity.ino && currentStats.ctimeMs === ownedIdentity.ctimeMs; if (stillOwned) { await fs.rm(probePath, { force: true }).catch(() => undefined); } let after; try { after = (await fs.lstat(dirPath)).birthtimeMs; } catch { return "its reported creation time could not be read"; } return before === after ? null : "its reported creation time changed after a probe write"; } async function refuseUnusableCreationTime(label, dirPath, reason) { process.stderr.write( "Refusing to trust the process session control path " + label + " (" + dirPath + "): " + reason + ". This filesystem or kernel gives no usable creation time. Terminating.\\n", ); await latchAndTerminate(); } // Runs once, before the first poll cycle, and before this wrapper captures // the identities it later checks on every cycle. A failed capture fails // closed: the wrapper has no verified identity to check on later cycles, so // it terminates now instead of polling a control path it never verified. // // This wrapper cannot assume stats.birthtimeMs is a real creation time. Node // can report a change-time copy as a creation time. That value fails closed // far too aggressively (it would move on every stdin file this wrapper // deletes), so this wrapper proves the value is not a copy with a probe before // it trusts it, run once here, before either directory's identity is captured. async function captureSessionIdentity() { try { const sessionProbeFailure = await birthtimeSurvivesProbe(sessionDir); if (sessionProbeFailure) { await refuseUnusableCreationTime("sessionDir", sessionDir, sessionProbeFailure); return; } const stdinProbeFailure = await birthtimeSurvivesProbe(stdinDir); if (stdinProbeFailure) { await refuseUnusableCreationTime("stdinDir", stdinDir, stdinProbeFailure); return; } const session = await statPathIdentity(sessionDir); const stdin = await statPathIdentity(stdinDir); sessionDirIdentity = session; stdinDirIdentity = stdin; } catch (error) { process.stderr.write( "Failed to capture the process session identity: " + (error instanceof Error ? error.message : String(error)) + ". Terminating.\\n", ); await latchAndTerminate(); } } // Runs on every poll cycle, before the wrapper reads stdinDir. Terminate and // latch on any proof the control path is no longer the one this wrapper // captured at startup (a missing path, a path that is no longer a directory, // a symbolic link, or a directory whose identity changed), AND on every // other lstat failure. A permission error is not transient here: a sandbox // peer can deny search permission on the control directory without removing // it, and treating that as transient would leave the wrapper and its child // alive forever. The error code below only picks the stderr message, so an // operator can still tell a removed directory from a permission error; it // never decides whether to latch. // // Contrast readStdinDirNames() right below, whose catch block stays narrow // on purpose: readdir() opens a directory descriptor, so it can fail with a // genuinely transient error under descriptor exhaustion, and latching there // would kill live sessions under load. lstat() opens no descriptor, and this // function already runs before every call to readStdinDirNames(), so a // permission error latches here before readdir() is ever reached. async function verifySessionIdentity() { if (identityLost) return false; try { const session = await statPathIdentity(sessionDir); const stdin = await statPathIdentity(stdinDir); if (!sameIdentity(session, sessionDirIdentity) || !sameIdentity(stdin, stdinDirIdentity)) { await latchAndTerminate(); return false; } return true; } catch (error) { const code = error && typeof error === "object" ? error.code : undefined; const reason = code === "ENOENT" ? "the control path no longer exists" : code === "ENOTDIR" ? "the control path is no longer a directory" : code === "EPAPERCLIP_SYMLINK" ? "the control path is now a symbolic link" : "lstat failed" + (code ? " with " + code : ""); process.stderr.write("Latching on a lost process session identity: " + reason + ". Terminating.\\n"); await latchAndTerminate(); return false; } } // This catch block stays narrow on purpose: see the comment above // verifySessionIdentity() for why a permission error here is treated as // transient while the same error latches there. async function readStdinDirNames() { if (!(await verifySessionIdentity())) return []; try { return await fs.readdir(stdinDir); } catch (error) { const code = error && typeof error === "object" ? error.code : undefined; if (code === "ENOENT" || code === "ENOTDIR") { await latchAndTerminate(); } return []; } } async function pollStdin() { while (!shuttingDown) { const entries = (await readStdinDirNames()).filter((name) => name.endsWith(".json")).sort(); for (const name of entries) { if (shuttingDown) break; const entrySeq = Number.parseInt(name, 10); // Hold the send order when an earlier file has not appeared. Do not consume // this later file: wait for the missing file on a later cycle, bounded by // the retry budget. After the budget, fail loud and advance past the gap, // so the present file can run. if (Number.isFinite(entrySeq) && entrySeq > stdinExpectedSeq) { stdinGapRetries += 1; if (stdinGapRetries < stdinMaxParseRetries) { break; } await writeEvent({ type: "error", message: "Advanced past missing stdin files " + stdinExpectedSeq + " to " + (entrySeq - 1) + " after " + stdinMaxParseRetries + " retries.", }); stdinGapRetries = 0; stdinExpectedSeq = entrySeq; } const file = path.posix.join(stdinDir, name); let message; try { // Hardening (I3): open with O_NOFOLLOW where the platform defines it, // so a control-path symbolic link swapped in after the directory // check fails the read instead of following it. const readFlag = typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW : "r"; const raw = await fs.readFile(file, { encoding: "utf8", flag: readFlag }); // An empty read means the content is not on disk yet. Treat it the same // as a parse failure: keep the file and retry on a later cycle. if (!raw) throw new Error("stdin file is empty"); message = JSON.parse(raw); } catch (error) { const retries = (stdinParseRetries.get(name) || 0) + 1; if (retries >= stdinMaxParseRetries) { // The retry limit is reached. Drop the file and write an error event, // so the lost message fails loud. The file is resolved now, so let the // loop go on to the next entry. stdinParseRetries.delete(name); await fs.rm(file, { force: true }).catch(() => undefined); await writeEvent({ type: "error", message: "Dropped unreadable stdin file after " + stdinMaxParseRetries + " retries: " + name + ": " + (error instanceof Error ? error.message : String(error)), }); // The file is resolved (dropped). Advance the expected number and reset // the gap budget, then let the loop go on to the next entry. if (Number.isFinite(entrySeq)) stdinExpectedSeq = entrySeq + 1; stdinGapRetries = 0; continue; } // The file is not readable yet and is not past the retry limit. Keep it // and stop this cycle to hold the send order. A later file (for example // stdinEnd) must not run before this earlier file. A later cycle reads // from the start again. stdinParseRetries.set(name, retries); break; } // The parse succeeded, so the content is complete. Delete the file first, // then act on the message. A later cycle never re-reads a handled file. stdinParseRetries.delete(name); await fs.rm(file, { force: true }).catch(() => undefined); // The file is handled. Advance the expected number and reset the gap // budget, so the next expected file starts fresh. if (Number.isFinite(entrySeq)) stdinExpectedSeq = entrySeq + 1; stdinGapRetries = 0; if (message.type === "stdin" && typeof message.data === "string") { if (!stdinClosed) child.stdin.write(Buffer.from(message.data, "base64")); } else if (message.type === "stdinEnd") { stdinClosed = true; child.stdin.end(); break; } else if (message.type === "shutdown") { await terminate(); break; } } if (!shuttingDown) await new Promise((resolve) => setTimeout(resolve, 50)); } } await captureSessionIdentity(); void pollStdin().catch((error) => void writeEvent({ type: "error", message: error instanceof Error ? error.message : String(error) })); `; // Streamed variant: the wrapper writes each output frame as one newline- // delimited JSON line to its stdout. The host runs this wrapper as one // long-lived session command and reads the frames from the session log stream, // so there is no host output-file poll. Each frame carries a monotonic `seq`, // so the host delivers every frame exactly once whether it arrives live or in // the final result. The wrapper exits when the child closes, so the session // command settles and the session shell (the subshell wrap around it) survives. function getProcessSessionRemoteStreamSource(): string { return `import { spawn } from "node:child_process"; import { promises as fs, constants as fsConstants } from "node:fs"; import path from "node:path"; const sessionDir = process.env.PAPERCLIP_PROCESS_SESSION_DIR; const commandPayload = process.env.PAPERCLIP_PROCESS_SESSION_COMMAND_B64; if (!sessionDir || !commandPayload) throw new Error("Missing process session bridge env."); const stdinDir = path.posix.join(sessionDir, "stdin"); let seq = 0; let stdinClosed = false; let shuttingDown = false; let terminated = false; let killTimer = null; const config = JSON.parse(Buffer.from(commandPayload, "base64").toString("utf8")); await fs.mkdir(stdinDir, { recursive: true }); // One newline-delimited JSON frame per event. Node keeps process.stdout writes // ordered, and the base64 payload holds no newline, so each frame is one line. function writeEvent(event) { seq += 1; process.stdout.write(JSON.stringify({ seq, ...event }) + "\\n"); } // Hardening (I3): refuse a symbolic link on a control path before this // wrapper reads or writes through it. A symbolic link here could let another // sandbox process redirect the wrapper's file I/O outside the session tree. async function isSymbolicLink(candidatePath) { try { const stats = await fs.lstat(candidatePath); return stats.isSymbolicLink(); } catch { return false; } } if ((await isSymbolicLink(sessionDir)) || (await isSymbolicLink(stdinDir))) { await writeEvent({ type: "error", message: "Refusing a symbolic link on a process session control path." }); process.exitCode = 1; process.exit(1); } // Hardening (I3, not containment): the wrapper's own launch env carries the // session dir and the command payload. Scrub both keys before they reach the // spawned child, so the child never inherits a path to its own control files. const childEnv = { ...process.env, ...(config.env || {}) }; delete childEnv.PAPERCLIP_PROCESS_SESSION_DIR; delete childEnv.PAPERCLIP_PROCESS_SESSION_COMMAND_B64; // I1: exactly one child process per emitted wrapper. Do not add a second // tracked child handle. const child = spawn(config.command, Array.isArray(config.args) ? config.args : [], { cwd: config.cwd || process.cwd(), env: childEnv, stdio: ["pipe", "pipe", "pipe"], }); child.stdout.on("data", (chunk) => writeEvent({ type: "data", stream: "stdout", data: Buffer.from(chunk).toString("base64") })); child.stderr.on("data", (chunk) => writeEvent({ type: "data", stream: "stderr", data: Buffer.from(chunk).toString("base64") })); child.on("error", (error) => writeEvent({ type: "error", message: error.message })); // "close" (not "exit") so stdout/stderr fully drain before the exit frame. // Queue the exit frame first, then run terminate(), so the exit frame always // lands even when the child closes on its own, with no stdinEnd and no // shutdown message ever received. writeEvent() only queues an asynchronous // write. terminate()'s own synchronous work (ending the child's stdin and // sending SIGTERM) already runs in this same handler by the time the exit // frame becomes readable on disk. terminate() is idempotent and its // child.kill() call here is always a no-op (I2): the child's process handle // is already gone by the time "close" fires. An error frame carries no such // guarantee: child.on("error", ...) below does not call terminate(), and // neither does the poll loop's own error writes, so those fire while the // wrapper and its child are still fully alive. child.on("close", (code, signal) => { writeEvent({ type: "exit", code, signal }); process.exitCode = typeof code === "number" ? code : 1; void terminate(); }); ${PROCESS_SESSION_STDIN_POLL_TAIL}`; } function getProcessSessionRemoteEventFileSource(): string { return `import { spawn } from "node:child_process"; import { promises as fs, constants as fsConstants } from "node:fs"; import path from "node:path"; const sessionDir = process.env.PAPERCLIP_PROCESS_SESSION_DIR; const commandPayload = process.env.PAPERCLIP_PROCESS_SESSION_COMMAND_B64; if (!sessionDir || !commandPayload) throw new Error("Missing process session bridge env."); const stdinDir = path.posix.join(sessionDir, "stdin"); const eventsDir = path.posix.join(sessionDir, "events"); let seq = 0; let stdinClosed = false; let shuttingDown = false; let terminated = false; let killTimer = null; const config = JSON.parse(Buffer.from(commandPayload, "base64").toString("utf8")); await fs.mkdir(stdinDir, { recursive: true }); await fs.mkdir(eventsDir, { recursive: true }); let writeChain = Promise.resolve(); function writeEvent(event) { seq += 1; const file = path.posix.join(eventsDir, String(seq).padStart(12, "0") + ".json"); const write = writeChain.then(async () => { await fs.writeFile(file + ".tmp", JSON.stringify(event) + "\\n", "utf8"); await fs.rename(file + ".tmp", file); }); writeChain = write.catch(() => undefined); return write; } // Hardening (I3): refuse a symbolic link on a control path before this // wrapper reads or writes through it. A symbolic link here could let another // sandbox process redirect the wrapper's file I/O outside the session tree. async function isSymbolicLink(candidatePath) { try { const stats = await fs.lstat(candidatePath); return stats.isSymbolicLink(); } catch { return false; } } if ((await isSymbolicLink(sessionDir)) || (await isSymbolicLink(stdinDir))) { await writeEvent({ type: "error", message: "Refusing a symbolic link on a process session control path." }); process.exitCode = 1; process.exit(1); } // Hardening (I3, not containment): the wrapper's own launch env carries the // session dir and the command payload. Scrub both keys before they reach the // spawned child, so the child never inherits a path to its own control files. const childEnv = { ...process.env, ...(config.env || {}) }; delete childEnv.PAPERCLIP_PROCESS_SESSION_DIR; delete childEnv.PAPERCLIP_PROCESS_SESSION_COMMAND_B64; // I1: exactly one child process per emitted wrapper. Do not add a second // tracked child handle. const child = spawn(config.command, Array.isArray(config.args) ? config.args : [], { cwd: config.cwd || process.cwd(), env: childEnv, stdio: ["pipe", "pipe", "pipe"], }); child.stdout.on("data", (chunk) => void writeEvent({ type: "data", stream: "stdout", data: Buffer.from(chunk).toString("base64") })); child.stderr.on("data", (chunk) => void writeEvent({ type: "data", stream: "stderr", data: Buffer.from(chunk).toString("base64") })); child.on("error", (error) => void writeEvent({ type: "error", message: error.message })); // "close" (not "exit") so stdout/stderr fully drain before the exit event; // the write chain then guarantees the exit file lands after every data file. // Queue the exit event first, then run terminate(), so the poll loop ends // even when the child closes on its own, with no stdinEnd and no shutdown // message ever received. writeEvent() only queues an asynchronous write. // terminate()'s own synchronous work (ending the child's stdin and sending // SIGTERM) already runs in this same handler by the time the exit file // becomes readable on disk. terminate() is idempotent and its child.kill() // call here is always a no-op (I2): the child's process handle is already // gone by the time "close" fires. An error event carries no such guarantee: // child.on("error", ...) below does not call terminate(), and neither does // the poll loop's own error writes, so those fire while the wrapper and its // child are still fully alive. child.on("close", (code, signal) => { void writeEvent({ type: "exit", code, signal }); process.exitCode = typeof code === "number" ? code : 1; void terminate(); }); ${PROCESS_SESSION_STDIN_POLL_TAIL}`; } /** The default deadline for the duplex readiness handshake, in milliseconds. */ const DEFAULT_DUPLEX_READINESS_TIMEOUT_MS = 10_000; /** The default bounded budget to close a partial duplex channel, in milliseconds. */ const DEFAULT_DUPLEX_CLEANUP_BUDGET_MS = 2_000; /** * Reserve a loopback port the host assigns to the duplex gateway. The host binds * an ephemeral listener on `127.0.0.1`, reads the port the operating system * chose, then closes the listener. The number is very likely free when the * gateway binds it a moment later. The gateway binds exactly this port or exits * nonzero, so a taken port fails closed to the file bridge and never steers the * endpoint. */ async function reserveHostAssignedLoopbackPort(): Promise { return new Promise((resolve, reject) => { const probe = net.createServer(); probe.once("error", reject); probe.listen(0, "127.0.0.1", () => { const address = probe.address(); if (!address || typeof address === "string") { probe.close(() => reject(new Error("Could not reserve a loopback port for the duplex gateway."))); return; } const reserved = address.port; probe.close(() => resolve(reserved)); }); }); } /** * Build the argument vector that launches the duplex gateway in the sandbox. The * host passes the assigned port and the per-open nonce only through the launch * environment, so the argument vector sets them as environment assignments in * front of the node command. No addressing data comes from the channel. * * The script uses `exec env NAME=value ... command`. A POSIX shell accepts an * environment-assignment prefix only on a plain command, never on `exec`. The * form `exec NAME=value command` exits with status 127. The `env` utility carries * the assignments, and `exec` still replaces the shell with the gateway process, * so the gateway keeps the process slot and the assigned environment. */ export function buildDuplexGatewayLaunchArgv(input: { shellCommand: "bash" | "sh"; remoteEntrypoint: string; nodeCommand?: string | null; env: Record; }): string[] { const assignments = Object.entries(input.env) .map(([key, value]) => `${key}=${shellQuote(value)}`) .join(" "); const nodeCommand = input.nodeCommand?.trim() || "node"; const script = `exec env ${assignments} ${shellQuote(nodeCommand)} ${shellQuote(input.remoteEntrypoint)}`; return [input.shellCommand, ...shellCommandArgs(script)]; } /** The reason the duplex readiness handshake did not pass. */ type DuplexReadinessFailure = "protocol_contamination" | "nonce_mismatch" | "channel_exit" | "timeout"; /** The outcome of the duplex readiness handshake. */ type DuplexReadinessResult = | { ok: true } | { ok: false; reason: DuplexReadinessFailure }; /** * Map a readiness failure to the fixed fallback reason. Each readiness failure * maps to exactly one reason from the closed telemetry set, so the fallback * counter and the transport event carry only an approved value. */ function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFallbackReason { switch (reason) { case "protocol_contamination": return "contaminated"; case "nonce_mismatch": return "ready_nonce_mismatch"; case "timeout": return "ready_timeout"; case "channel_exit": return "ready_invalid"; } } /** * The fixed marker the worker manager puts in a route-busy rejection. The manager * defines the constant; this module matches the fixed string, because the two * layers ship in separate packages and share no import. The marker names the * process-scoped route ceiling, so it carries no route, query, body, or token. */ const DUPLEX_ROUTE_BUSY_ERROR_MARKER = "DUPLEX_CHANNEL_ROUTE_BUSY"; /** * Report whether the caught open error is a route-busy rejection. The host maps it * to the `route_busy` fallback stage, so a full process-scoped route ceiling never * folds into a generic open failure. */ function isDuplexRouteBusyError(error: unknown): boolean { return error instanceof Error && error.message.includes(DUPLEX_ROUTE_BUSY_ERROR_MARKER); } /** * The cap on the pre-READY readiness buffer, in bytes. The gate reads untrusted * bytes before the READY line arrives, so it bounds the buffer. The cap is the * codec frame-size bound plus one line of margin, so a legitimate maximum-size * READY frame still fits. Past the cap with no newline, the stream cannot be a * valid READY frame, so the gate finishes with protocol contamination. */ const DUPLEX_READINESS_BUFFER_CAP_BYTES = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096; // --------------------------------------------------------------------------- // http2_v1: the client connection preface scan and the run disposition latch. // --------------------------------------------------------------------------- /** * The HTTP/2 client connection preface: 24 octets, `PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n` * (RFC 9113, Section 3.4). A Node `http2.performServerHandshake` call needs a * `Duplex` whose readable side starts at this exact sequence; one extra * leading byte makes the preface invalid, and the server reports a * `PROTOCOL_ERROR`. The server does not skip a leading byte and does not * search for the sequence, so the host finds the offset itself before it * hands the channel to the server. */ const HTTP2_CLIENT_CONNECTION_PREFACE = Buffer.from( "505249202a20485454502f322e300d0a0d0a534d0d0a0d0a", "hex", ); /** The one shared empty buffer. The preface scan starts and resets its two * retained buffers to it. */ const HTTP2_PREFACE_EMPTY_BUFFER = Buffer.alloc(0); // The count of bytes the preface scan's substring search examines, in bytes. // A search that always starts from the beginning of the retained buffer // re-examines the whole buffer on every fragment, so this count grows // quadratically in the number of fragments. `findPrefaceFrom` instead // resumes from where the prior search left off, so this count stays linear // in the bytes received. A test reads this count to prove the search work // does not regress to the quadratic shape. Production code never reads this // count. let http2PrefaceScanSearchUnits = 0; // The count of bytes the pre-preface scan buffer copies while it grows its // backing storage. A one-copy-per-fragment append copies the whole retained // buffer on every fragment, so this count grows quadratically in the number // of fragments. The doubling-growth approach copies only on a reallocation, // so this count stays linear in the bytes received. A test reads this count // to prove the growth work does not regress to the quadratic shape. // Production code never reads this count. let http2PrefaceScanBufferGrowthCopyUnits = 0; // The same count as `http2PrefaceScanBufferGrowthCopyUnits`, for the // post-preface replay buffer instead of the pre-preface scan buffer. let http2PrefaceReplayBufferGrowthCopyUnits = 0; /** * A byte buffer that grows its backing storage by doubling its capacity, * instead of copying the whole retained buffer on every appended fragment. A * sender that trickles input in many small fragments would otherwise force * one full copy of the whole retained buffer per fragment: with the buffer * growing toward its cap, that is quadratic work in the number of fragments. * Doubling the backing storage's capacity only when the current capacity * runs out reallocates and copies a logarithmic number of times, so the * total copy work stays linear in the bytes received. `countGrowthCopy` * receives the number of bytes each reallocation copies, so a test can add * these up and prove the growth work stays linear. */ function createGrowableByteBuffer(countGrowthCopy: (copiedBytes: number) => void): { view: () => Buffer; length: () => number; append: (chunk: Uint8Array) => void; reset: () => void; } { let used: Buffer = HTTP2_PREFACE_EMPTY_BUFFER; let storage: Buffer = HTTP2_PREFACE_EMPTY_BUFFER; return { view: () => used, length: () => used.length, append: (chunk: Uint8Array): void => { const usedLength = used.length; const neededLength = usedLength + chunk.byteLength; if (neededLength > storage.length) { let nextCapacity = storage.length === 0 ? chunk.byteLength : storage.length * 2; while (nextCapacity < neededLength) { nextCapacity *= 2; } const grown = Buffer.allocUnsafe(nextCapacity); storage.copy(grown, 0, 0, usedLength); countGrowthCopy(usedLength); storage = grown; } storage.set(chunk, usedLength); used = storage.subarray(0, neededLength); }, reset: (): void => { used = HTTP2_PREFACE_EMPTY_BUFFER; storage = HTTP2_PREFACE_EMPTY_BUFFER; }, }; } /** * Find the client connection preface in `buffer` at or after index `from` * and return its offset, or -1. This counts the real scan distance for a * test: from `from` up to the found preface's end, or to the end of the * buffer when it finds none. The count stays linear in the bytes received * when the caller advances `from` to just short of the buffer's end on every * miss, instead of always searching from the start of the buffer. */ function findPrefaceFrom(buffer: Buffer, from: number): number { const offset = buffer.indexOf(HTTP2_CLIENT_CONNECTION_PREFACE, from); const scannedTo = offset === -1 ? buffer.length : offset + HTTP2_CLIENT_CONNECTION_PREFACE.length; http2PrefaceScanSearchUnits += Math.max(scannedTo - from, 0); return offset; } /** * Wrap the readiness gate's broker channel so its `onData` delivers no byte * until the client connection preface appears, then delivers every byte from * the preface onward. * * The wrapped channel opens its scan window only on the bytes the readiness * gate already retained after it accepted the READY line (constraint: the * scan window opens only after the gate accepts the nonce). Nothing before * that line ever reaches this scan, because the gate itself discards the * whole pre-READY buffer on acceptance. The scan buffers at most * {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}; past that bound with no preface * found, it calls `onMissing` exactly one time and stops buffering, so the * caller can abort the open and fall back to `queue_v1`. The function holds * no prologue byte count: it always scans for the fixed 24-octet sequence, * never a length. * * The bytes that follow the found preface, before the HTTP/2 server binds a * downstream listener, land in `pendingAfterPreface`. This buffer holds * untrusted bytes on the same footing as the scan buffer, so it carries the * same {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap. A chunk that would pass * the cap fails closed: the function drops the buffer and stops the channel. * The caller reads {@link replayOverflowed} after the preface settles and, on * `true`, treats the open the same as a missing preface. */ function createHttp2PrefaceScanningChannel( channel: CommandManagedDuplexChannel, options: { capBytes: number; onFound: () => void; onMissing: () => void; }, ): { channel: CommandManagedDuplexChannel; replayOverflowed: () => boolean; disposeScanBuffer: () => void; } { // The pre-preface scan buffer. `scanBuf.append` grows its backing storage // by doubling, instead of copying the whole retained buffer on every // fragment — see {@link createGrowableByteBuffer}. `scanSearchFrom` is the // first index the next search must examine: `findPrefaceFrom` advances it // to just short of the buffer's end on every miss, so a fragmented preface // is found without a full rescan of the retained buffer on every fragment. const scanBuf = createGrowableByteBuffer((copiedBytes) => { http2PrefaceScanBufferGrowthCopyUnits += copiedBytes; }); let scanSearchFrom = 0; let sawPreface = false; let failed = false; let downstream: ((chunk: Uint8Array) => void) | null = null; // Bytes found after the preface before a downstream listener attaches. The // wrapped channel replays them on attach, the same pattern the readiness // gate itself uses for its own post-READY replay buffer. This also grows // by doubling, for the same reason as `scanBuf`: fragmented post-preface // input must not force a full copy of the retained buffer per fragment. const pendingAfterPreface = createGrowableByteBuffer((copiedBytes) => { http2PrefaceReplayBufferGrowthCopyUnits += copiedBytes; }); let replayOverflow = false; // Drop the pending buffer and stop the channel. The caller reads // `replayOverflowed()` after the preface settles and falls back the same // way it does for a missing preface. function overflowAndStop(): void { replayOverflow = true; pendingAfterPreface.reset(); channel.stop(); } function deliver(chunk: Buffer): void { if (downstream) { downstream(chunk); return; } if (replayOverflow) return; if (pendingAfterPreface.length() + chunk.byteLength > options.capBytes) { overflowAndStop(); return; } pendingAfterPreface.append(chunk); } channel.onData((chunk) => { if (failed || replayOverflow) return; if (sawPreface) { deliver(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); return; } const rawChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); // Reject the chunk on its prospective length before it grows the scan // buffer, the same way `deliver` bounds `pendingAfterPreface`. A single // oversized chunk — or a chunk that tips an already-large buffer past // the cap — must fail closed here, before `scanBuf.append` performs the // allocation. Checking the cap only after the append still bounds the // retained buffer, but it lets one untrusted chunk force an allocation // as large as the chunk itself, unbounded by `capBytes`. if (scanBuf.length() + rawChunk.byteLength > options.capBytes) { failed = true; scanBuf.reset(); scanSearchFrom = 0; options.onMissing(); return; } scanBuf.append(rawChunk); const scanBuffer = scanBuf.view(); const offset = findPrefaceFrom(scanBuffer, scanSearchFrom); if (offset === -1) { // No match yet. Resume the next search just short of the buffer's // end, keeping back an overlap of one octet less than the preface // length, so a preface split across this fragment and the next one is // still found. Each byte enters that overlap window a bounded number // of times, so the total search work stays linear in the bytes // received, not quadratic in the number of fragments. scanSearchFrom = Math.max( 0, scanBuffer.length - (HTTP2_CLIENT_CONNECTION_PREFACE.length - 1), ); return; } sawPreface = true; options.onFound(); const fromPreface = Buffer.from(scanBuffer.subarray(offset)); scanBuf.reset(); scanSearchFrom = 0; deliver(fromPreface); }); return { channel: { write: (data: Uint8Array) => channel.write(data), onData: (listener: (chunk: Uint8Array) => void) => { downstream = listener; if (pendingAfterPreface.length() > 0) { // Copy the exact retained bytes instead of handing the listener // the growable buffer's backing view. That backing storage can // run ahead of the bytes in use (the doubling growth in // `pendingAfterPreface.append` over-provisions it), so a raw view // would keep the whole over-provisioned allocation alive for as // long as the listener holds its reference. const replay = Buffer.from(pendingAfterPreface.view()); pendingAfterPreface.reset(); listener(replay); } }, onExit: (listener: (exit: { exitCode: number | null }) => void) => channel.onExit(listener), stop: () => channel.stop(), close: () => channel.close(), }, replayOverflowed: () => replayOverflow, /** * Drop the scan buffer, for a caller-side terminal path this function * itself never reaches — the bound readiness timeout elapsing while the * scan is still searching, with no preface found and no cap refusal of * its own. A call after the preface already matched, or after the cap * already failed the scan closed, is a no-op: both paths already reset * the scan buffer themselves. */ disposeScanBuffer: (): void => { if (sawPreface || failed) return; failed = true; scanBuf.reset(); scanSearchFrom = 0; }, }; } /** The terminal outcome of the preface scan: either the client preface * appeared inside the bounded readiness buffer, or it did not. */ type Http2PrefaceScanResult = "found" | "missing"; /** * Wait for {@link createHttp2PrefaceScanningChannel} to settle: either the * preface appears, or the scan passes the bound with no match, or the bound * readiness timeout elapses first. Reapplying the readiness timeout here * keeps one configured value for both the READY-line wait and this * immediately-following preface wait; the task adds no new timeout setting. * Returns the scanning channel alongside the settled result, so the caller * binds the HTTP/2 server to it only on a `found` result. On a `found` * result, the caller must still read `replayOverflowed()`: the post-preface * buffer can overflow its cap after the preface settles as `found` and * before the caller binds a downstream listener. */ function scanForHttp2ClientPreface( channel: CommandManagedDuplexChannel, options: { capBytes: number; timeoutMs: number }, ): { scanned: CommandManagedDuplexChannel; settled: Promise; replayOverflowed: () => boolean; } { let resolveSettled!: (result: Http2PrefaceScanResult) => void; let settledOnce = false; const settled = new Promise((resolve) => { resolveSettled = resolve; }); // Reassigned to the real function once `createHttp2PrefaceScanningChannel` // returns, below. `settle` can only actually run after that point (the // timer fires later, and `onMissing`/`onFound` fire from inside the // channel's own `onData`, which registers after this call), so the // placeholder never runs for real. let disposeScanBuffer: () => void = () => {}; const settle = (result: Http2PrefaceScanResult): void => { if (settledOnce) return; settledOnce = true; clearTimeout(timer); // The bound readiness timeout can elapse while the scan still searches, // with no preface found and no cap refusal of its own. That path holds // no other cleanup, so drop the scan buffer here. A `found` result, or a // `missing` result the scan itself already failed closed, is a no-op // inside `disposeScanBuffer`. if (result === "missing") disposeScanBuffer(); resolveSettled(result); }; const timer = setTimeout(() => settle("missing"), options.timeoutMs); timer.unref?.(); const scan = createHttp2PrefaceScanningChannel(channel, { capBytes: options.capBytes, onFound: () => settle("found"), onMissing: () => settle("missing"), }); disposeScanBuffer = scan.disposeScanBuffer; return { scanned: scan.channel, settled, replayOverflowed: scan.replayOverflowed }; } /** * Test-only surface for {@link scanForHttp2ClientPreface}. A test drives the * post-preface replay cap across every terminal path without the whole * bridge. Production code never reads this export. */ export const __http2PrefaceScanTesting = { scanForHttp2ClientPreface: ( channel: CommandManagedDuplexChannel, options: { capBytes: number; timeoutMs: number }, ) => scanForHttp2ClientPreface(channel, options), readScanSearchUnits: (): number => http2PrefaceScanSearchUnits, resetScanSearchUnits: (): void => { http2PrefaceScanSearchUnits = 0; }, readScanBufferGrowthCopyUnits: (): number => http2PrefaceScanBufferGrowthCopyUnits, resetScanBufferGrowthCopyUnits: (): void => { http2PrefaceScanBufferGrowthCopyUnits = 0; }, readReplayBufferGrowthCopyUnits: (): number => http2PrefaceReplayBufferGrowthCopyUnits, resetReplayBufferGrowthCopyUnits: (): void => { http2PrefaceReplayBufferGrowthCopyUnits = 0; }, }; /** * The terminal run disposition for the `http2_v1` path, in the same shape as * {@link DuplexBrokerRunDisposition}. A `failed` disposition means a terminal * loss ordered before an orderly completion, so the run must not report * success. */ interface Http2RunDispositionLatch { readonly disposition: DuplexBrokerRunDisposition; /** * Record a terminal loss. Returns `true` when the call flipped the * disposition to failed; returns `false` when a loss or an orderly * completion already latched, so the run already has its terminal result. * The first recorded loss latches — a later call never overrides it. */ recordLoss(reason: DuplexLossReason): boolean; /** Mark the host-observed orderly completion of the agent turn. A loss that * already latched keeps the failure. */ markOrderlyCompletion(): void; /** Atomically mark the orderly completion and read the disposition. */ settleRunDisposition(): DuplexBrokerRunDisposition; /** * Register a listener that fires once, only on the call to `recordLoss` * that actually latches a new terminal loss. Returns a function that * unregisters the listener. */ onLoss(listener: (reason: DuplexLossReason) => void): () => void; } function createHttp2RunDispositionLatch(): Http2RunDispositionLatch { let lossOrdered = false; let lossReason: DuplexLossReason | null = null; let completionOrdered = false; let lossListener: ((reason: DuplexLossReason) => void) | null = null; const markOrderlyCompletion = (): void => { if (completionOrdered || lossOrdered) return; completionOrdered = true; }; return { get disposition(): DuplexBrokerRunDisposition { return { failed: lossOrdered, lossReason }; }, recordLoss(reason: DuplexLossReason): boolean { if (lossOrdered || completionOrdered) return false; lossOrdered = true; lossReason = reason; lossListener?.(reason); return true; }, markOrderlyCompletion, settleRunDisposition(): DuplexBrokerRunDisposition { markOrderlyCompletion(); return { failed: lossOrdered, lossReason }; }, onLoss(listener: (reason: DuplexLossReason) => void): () => void { lossListener = listener; return () => { if (lossListener === listener) lossListener = null; }; }, }; } /** * Create the run-log directory on the sandbox before the tail starts. The file * bridge worker creates this directory on the file path. The duplex path starts * no worker, so the host creates the directory here. The tail then reads a real * directory from its first tick. * * This step is best effort. The broker already serves the duplex transport when * the host reaches it, and the tail wrap command runs its own `mkdir -p` as a * backstop. So a create failure must not tear down a working duplex transport; * the host swallows it and still builds the tail. The log line names no raw * error, so no raw error rides a log line here. */ async function ensureSandboxRunLogDirectory(input: { runner: CommandManagedRuntimeRunner; remoteCwd: string; logsDir: string; shellCommand: "bash" | "sh"; timeoutMs: number | null | undefined; }): Promise { try { await input.runner.execute({ command: input.shellCommand, args: shellCommandArgs(`mkdir -p ${shellQuote(input.logsDir)}`), cwd: input.remoteCwd, timeoutMs: input.timeoutMs ?? undefined, }); } catch { // Best effort: the tail wrap command creates the directory before it writes. } } /** * The duplex readiness gate. The gate owns the single data listener and the * single exit listener of the channel while the host waits for a valid READY * frame. It resolves the handshake, then hands the channel to the broker. * * The gate authenticates readiness with the nonce and the strict READY schema, * not with the line position. A PTY channel echoes the launch wrapper line before * it sets raw mode, so the first line is often not the READY frame. The gate skips * each pre-READY line that does not decode as a READY frame, then accepts the first * line that decodes as a READY frame with the matching nonce. A line that decodes * as a READY frame with a wrong nonce fails the handshake with a nonce mismatch. An * early exit or a timeout also fails the handshake. The gate never dispatches a * request; the broker does that after readiness passes. * * The skipped bytes stay in the capped buffer, so the O(1) cap and the readiness * timeout still bound the wait. The gate enforces the cap on every pre-READY path * and before it decodes a candidate line, so an over-cap prefix never reaches READY * acceptance. The cap check has priority over READY acceptance on every path. */ // The count of the pre-READY newline-scan work, in bytes. Each search adds the // number of bytes it can read. A test reads this count to prove the scan work // stays linear in the bytes received. Production code never reads this count. let duplexReadinessNewlineScanUnits = 0; // The count of bytes `appendReadinessBytes` copies while it grows the pre-READY // buffer's backing storage. A one-copy-per-fragment approach copies the whole // retained buffer on every fragment, so this count grows quadratically in the // number of fragments. The doubling-growth approach copies only on a // reallocation, so this count stays linear in the bytes received. A test reads // this count to prove the growth work does not regress to the quadratic shape. // Production code never reads this count. let duplexReadinessBufferGrowthCopyUnits = 0; /** The newline byte. The readiness buffer is a byte buffer, not a string. */ const READINESS_NEWLINE_BYTE = 0x0a; /** The opening-brace byte. The bracketed-paste retry scans for it, not a string. */ const READINESS_OPEN_BRACE_BYTE = 0x7b; /** The one shared empty buffer. The gate starts and resets `buffer` to it. */ const READINESS_EMPTY_BUFFER = Buffer.alloc(0); /** * Find the first newline in `buffer` at or after index `from` and return its * index, or -1. `Buffer#indexOf` reads the bytes from `from` up to the newline * it finds, or to the end of the buffer when it finds none. This helper counts * that real scan distance for a test. The count stays linear in the bytes * received when the caller advances `from` past each newline it consumes. */ function findNewlineFrom(buffer: Buffer, from: number): number { const newlineIndex = buffer.indexOf(READINESS_NEWLINE_BYTE, from); const scanned = newlineIndex === -1 ? buffer.length - from : newlineIndex - from + 1; duplexReadinessNewlineScanUnits += scanned; return newlineIndex; } /** * Test-only surface for the pre-READY readiness gate. A test reads the scan * count to prove the newline search work stays linear in the bytes received. * Production code does not use this object. */ export const __duplexReadinessTesting = { readNewlineScanUnits: (): number => duplexReadinessNewlineScanUnits, resetNewlineScanUnits: (): void => { duplexReadinessNewlineScanUnits = 0; }, readBufferGrowthCopyUnits: (): number => duplexReadinessBufferGrowthCopyUnits, resetBufferGrowthCopyUnits: (): void => { duplexReadinessBufferGrowthCopyUnits = 0; }, // Build one readiness gate over a supplied channel, so a test can drive the // readiness-replay cap lifecycle across every terminal path without the whole // bridge. Production code never reads this factory. createReadinessGate: (channel: CommandManagedDuplexChannel, options: { nonce: string; timeoutMs: number }) => createDuplexReadinessGate(channel, options), }; interface DuplexReadinessGate { /** Resolves with the handshake outcome. It never rejects. */ readonly ready: Promise; /** * The channel view the broker consumes after readiness passes. It replays the * bytes that followed the READY frame, then forwards each later chunk and the * exit. The gate keeps one real data listener, so the broker never double-binds * the channel. */ readonly brokerChannel: CommandManagedDuplexChannel; /** * Report whether a post-READY pre-bind chunk tipped the pending replay buffer * past {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}. On such an overflow the gate * drops the pending replay buffer and stops the channel. The caller reads this * after `ready` resolves `ok`, and before it binds the broker. */ replayOverflowed(): boolean; /** * Drop the pending replay buffer. The caller runs this on a terminal path that * abandons the pending replay without a broker handoff: a readiness failure, a * replay overflow, or a broker-construction failure. The normal handoff already * drops the buffer inside `brokerChannel.onData`, so a later call here is a * no-op. */ disposePendingReplay(): void; /** * Test-only. Report the length of the retained pre-READY buffer, in bytes. A * test reads this to prove the gate drops the pre-READY buffer on READY * acceptance, so the process does not retain the sandbox-controlled prefix. * Production code does not read this. */ retainedReadinessBufferLength(): number; } function createDuplexReadinessGate( channel: CommandManagedDuplexChannel, options: { nonce: string; timeoutMs: number; }, ): DuplexReadinessGate { let settled = false; let readyOk = false; // The gate sets this when a post-READY pre-bind chunk tips the pending replay // buffer past the cap. On that overflow the gate drops the pending buffer and // stops the channel. The caller reads it through `replayOverflowed` and selects // the file bridge. let replayOverflow = false; // The raw bytes the host reads before the READY frame completes. `buffer` is // append-only and always a zero-copy view over the used prefix of `storage`, // so the O(1) cap check on `buffer.length` stays valid. let buffer: Buffer = READINESS_EMPTY_BUFFER; // The backing storage for `buffer`. `appendReadinessBytes` grows this by // doubling its capacity, instead of copying the whole retained buffer on // every fragment. See `appendReadinessBytes` for why this bounds the total // copy work. let storage: Buffer = READINESS_EMPTY_BUFFER; // The start index of the current line in `buffer`. A leading blank line // advances this cursor past its newline without a buffer copy. let lineStart = 0; // The next index to search for a newline. The gate scans from here, so each // byte is read at most one time for the newline search. let scanFrom = 0; /** * Append `chunk` to the pre-READY buffer without copying the bytes already * retained. A sender that trickles the handshake in many small fragments * (a slow socket, a byte-at-a-time PTY echo) would otherwise force one full * copy of the whole retained buffer per fragment: with `buffer` growing * toward the {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap, that is * quadratic work in the number of fragments. This instead grows `storage` * by doubling its capacity only when the current capacity runs out, so the * backing store reallocates and copies a logarithmic number of times, and * each append copies only the incoming chunk. The used length still reads * in O(1) through `buffer.length`, so every cap check and slice below stays * unchanged. */ function appendReadinessBytes(chunk: Uint8Array): void { const usedLength = buffer.length; const neededLength = usedLength + chunk.byteLength; if (neededLength > storage.length) { let nextCapacity = storage.length === 0 ? chunk.byteLength : storage.length * 2; while (nextCapacity < neededLength) { nextCapacity *= 2; } const grown = Buffer.allocUnsafe(nextCapacity); storage.copy(grown, 0, 0, usedLength); duplexReadinessBufferGrowthCopyUnits += usedLength; storage = grown; } storage.set(chunk, usedLength); buffer = storage.subarray(0, neededLength); } // The bytes that followed the READY frame, held until the broker binds. let pending: Uint8Array = READINESS_EMPTY_BUFFER; // The exit that arrived after READY but before the broker bound, if any. let pendingExit: { exitCode: number | null } | null = null; let dataSink: ((chunk: Uint8Array) => void) | null = null; let exitSink: ((exit: { exitCode: number | null }) => void) | null = null; let resolveReady!: (result: DuplexReadinessResult) => void; const ready = new Promise((resolve) => { resolveReady = resolve; }); const timer = setTimeout(() => finish({ ok: false, reason: "timeout" }), options.timeoutMs); timer.unref?.(); function finish(result: DuplexReadinessResult): void { if (settled) return; settled = true; clearTimeout(timer); if (result.ok) readyOk = true; resolveReady(result); } channel.onData((chunk) => { if (dataSink) { dataSink(chunk); return; } if (readyOk) { // READY already passed; hold the bytes until the broker binds. Bound this // buffer directly against {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}, the // same way the preface-scan gate's own post-preface replay buffer bounds // itself: a worker that keeps sending bytes after READY, faster than the // broker can bind, cannot grow this buffer past the cap. A chunk that // would pass the cap fails closed: the gate drops the pending buffer, // stops the channel, and sets the overflow flag. The caller reads the // flag and selects the file bridge, because `ready` already resolved // before this synchronous post-READY chunk arrived. if (pending.length + chunk.byteLength > DUPLEX_READINESS_BUFFER_CAP_BYTES) { replayOverflow = true; pending = READINESS_EMPTY_BUFFER; channel.stop(); return; } // Copy a first chunk instead of aliasing the caller's `Uint8Array`, so a // channel that reuses its delivered buffer across calls cannot corrupt the // bytes this gate holds for the broker replay. pending = pending.length === 0 ? Buffer.from(chunk) : Buffer.concat([pending, chunk]); return; } if (settled) { // The handshake already failed. The channel is untrusted and can keep // sending bytes until the host closes it. Drop them, so a failed handshake // never grows the buffer after the gate settles. return; } // Append the new bytes and continue the newline search from `scanFrom`, the // first index not yet examined. Each byte is read at most one time for the // search, and `appendReadinessBytes` copies at most the incoming chunk, so // the total work stays linear in the bytes received, not in the number of // fragments they arrive in. appendReadinessBytes(chunk); for (;;) { const newlineIndex = findNewlineFrom(buffer, scanFrom); if (newlineIndex === -1) { // No complete line yet. The whole buffer up to the end is now scanned. scanFrom = buffer.length; // The gate reads untrusted bytes, so bound the pre-READY buffer. Past // the cap with no complete READY line, the stream cannot be a valid // READY frame, so finish with protocol contamination. The retained // skipped lines count against the cap; that is acceptable and fail-closed. // // `buffer` is a byte buffer, so `buffer.length` is the exact byte count, // read in O(1). if (buffer.length > DUPLEX_READINESS_BUFFER_CAP_BYTES) { finish({ ok: false, reason: "protocol_contamination" }); } return; } if (newlineIndex === lineStart) { // Skip a blank line, the same as the frame decoder. Advance the line-start // cursor past the newline without a buffer copy, then search the next line // from the position after the newline. Enforce the cap after the advance, // so a blank-line flood past the cap fails closed before READY acceptance. lineStart = newlineIndex + 1; scanFrom = newlineIndex + 1; if (lineStart > DUPLEX_READINESS_BUFFER_CAP_BYTES) { finish({ ok: false, reason: "protocol_contamination" }); return; } continue; } // A complete non-blank candidate line spans `[lineStart, newlineIndex)`. // Enforce the cap on the line's end offset before the decode, so an over-cap // prefix never reaches READY acceptance on a completed line. if (newlineIndex + 1 > DUPLEX_READINESS_BUFFER_CAP_BYTES) { finish({ ok: false, reason: "protocol_contamination" }); return; } // Slice only the single candidate line for the decode. const line = buffer.slice(lineStart, newlineIndex); let decoded = decodeDuplexLine(line); if (!decoded.ok) { // The whole line did not decode. A terminal can put bytes in front of the // gateway's first frame on the same line, with no newline between them: a // shell with bracketed paste enabled writes its disable sequence and a // bare carriage return (`ESC [ ? 2 0 0 4 l CR`) immediately before the // child's first output. So retry the decode from the first `{` in the // line, which is where a frame can start. // // This does not weaken the handshake. The retry still runs the same // strict decode over the remainder of the line, and readiness still // authenticates on the nonce below, so a prefix cannot forge a frame or // smuggle a second one — it can only be discarded. const braceIndex = line.indexOf(READINESS_OPEN_BRACE_BYTE); if (braceIndex > 0) { decoded = decodeDuplexLine(line.slice(braceIndex)); } } if (decoded.ok && decoded.frame.type === "ready") { // A line that decodes as a READY frame authenticates by the nonce. A wrong // nonce fails the handshake; the matching nonce passes it. if (decoded.frame.nonce !== options.nonce) { finish({ ok: false, reason: "nonce_mismatch" }); return; } // The bytes that follow the READY line become the replay buffer for the // broker. // // Copy the suffix instead of slicing it off `buffer`. `buffer` is a view // over `storage`, and `storage`'s capacity can run ahead of the bytes in // use (the doubling growth in `appendReadinessBytes` over-provisions it). // A slice would keep that whole over-provisioned allocation alive for as // long as the broker replay holds its reference. The copy is exactly // `suffix.length` bytes, one time, not a per-fragment cost. const suffix = Buffer.from(buffer.subarray(newlineIndex + 1)); // Drop the original pre-READY buffer and its backing storage now. The // gate keeps only the retained suffix as `pending`. This clear also // covers the broker handoff and the replay disposal. Both run later and // read no buffer bytes. buffer = READINESS_EMPTY_BUFFER; storage = READINESS_EMPTY_BUFFER; pending = suffix; finish({ ok: true }); return; } // The line does not decode as a READY frame. A PTY echo line or any other // pre-READY noise reaches here. Skip it and keep scanning; the nonce and the // strict schema, not the line position, authenticate readiness. Enforce the // cap after the advance, so a noise-line flood past the cap fails closed. lineStart = newlineIndex + 1; scanFrom = newlineIndex + 1; if (lineStart > DUPLEX_READINESS_BUFFER_CAP_BYTES) { finish({ ok: false, reason: "protocol_contamination" }); return; } } }); channel.onExit((exit) => { if (exitSink) { exitSink(exit); return; } if (readyOk) { // The channel exited after READY but before the broker bound. Hold the // exit so the broker still learns of the loss. pendingExit = exit; return; } finish({ ok: false, reason: "channel_exit" }); }); const brokerChannel: CommandManagedDuplexChannel = { write: (data: Uint8Array) => channel.write(data), onData: (listener: (chunk: Uint8Array) => void) => { dataSink = listener; if (pending.length > 0) { const replay = pending; pending = READINESS_EMPTY_BUFFER; listener(replay); return; } }, onExit: (listener: (exit: { exitCode: number | null }) => void) => { exitSink = listener; if (pendingExit) { const exit = pendingExit; pendingExit = null; listener(exit); } }, stop: () => channel.stop(), close: () => channel.close(), }; return { ready, brokerChannel, replayOverflowed: () => replayOverflow, disposePendingReplay: () => { pending = READINESS_EMPTY_BUFFER; }, retainedReadinessBufferLength: () => buffer.length, }; } /** * Close a partial duplex channel inside a bounded budget, then stop the child. * The host runs this on any readiness failure, so a failed duplex attempt leaves * no live provider session before the fallback to the file bridge. */ async function closeDuplexChannelWithinBudget( channel: CommandManagedDuplexChannel, budgetMs: number, ): Promise { try { let timer: ReturnType | undefined; const budget = new Promise((resolve) => { timer = setTimeout(resolve, budgetMs); timer.unref?.(); }); await Promise.race([channel.close().catch(() => undefined), budget]); if (timer !== undefined) clearTimeout(timer); } catch { // Best effort: the stop below still removes the child. } finally { try { channel.stop(); } catch { // The channel is already gone; nothing more to do. } } } export async function startAdapterExecutionTargetPaperclipBridge(input: { runId: string; target: AdapterExecutionTarget | null | undefined; runtimeRootDir: string | null | undefined; adapterKey: string; timeoutSec?: number | null; hostApiToken: string | null | undefined; hostApiUrl?: string | null; onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; maxBodyBytes?: number | null; // The deadline for one forward call, in milliseconds. This is the inner budget // of the duplex broker's nested budget set. The default is the forward budget // in `DEFAULT_DUPLEX_BROKER_BUDGETS` (30 s), so the current behavior does not // change when the caller sets no option. forwardTimeoutMs?: number | null; // The first strict gate for the duplex transport. The host selects duplex only // when this is exactly `true` and the resolved capability // `duplexCommandStream` is exactly `true`. Any other value of either gate // selects the file bridge. The caller reads this from the experimental instance // setting `enableSandboxDuplexBridge`. The default is the file bridge. // HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. enableSandboxDuplexBridge?: boolean | null; // The deadline for the duplex readiness handshake, in milliseconds. On a // timeout the host closes the partial channel and selects the file bridge. The // default is `DEFAULT_DUPLEX_READINESS_TIMEOUT_MS`. duplexReadinessTimeoutMs?: number | null; // Return the current-run parent-context token. The factory threads it into the // callback bridge worker, which reads it per request so each request // `sandbox.exec` span parents to the live run span. When it is absent, the // request work runs with an empty store, exactly like the earlier behavior. getRuntimeParentContext?: () => StartupSpanContext | undefined; // Wrap each callback request in a `sandbox.callbackBridge.relayRequest` span. // The factory threads it into the worker, which uses it per request so each // request's execs group under one wrapper span. When it is absent, the request // work runs under the run parent with no wrapper span. runtimeSpan?: RuntimeSpanRunner; // The injected recorder for the fixed duplex observability surface. The factory // binds it to a provider-scoped telemetry facade, which records the channel-open // span, the request span, the guarded counters, and the transport event. The // default is a no-op recorder, so the surface stays inert until the host injects // a real recorder. duplexObservabilityRecorder?: DuplexObservabilityRecorder | null; }): Promise { if (!adapterExecutionTargetUsesPaperclipBridge(input.target)) { return null; } if (!input.target || input.target.kind !== "remote") { return null; } const target = input.target; const onLog = input.onLog ?? (async () => {}); const hostApiToken = input.hostApiToken?.trim() ?? ""; if (hostApiToken.length === 0) { throw new Error("Sandbox bridge mode requires a host-side Paperclip API token."); } // The forward budget for one relayed request. It stays at the broker's default // forward budget (30 s) when the caller sets no option, so current behavior // does not change. const forwardTimeoutMs = input.forwardTimeoutMs ?? DEFAULT_DUPLEX_BROKER_BUDGETS.forwardTimeoutMs; const runtimeRootDir = input.runtimeRootDir?.trim().length ? input.runtimeRootDir.trim() : path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey); const bridgeRuntimeDir = path.posix.join(runtimeRootDir, "paperclip-bridge"); const queueDir = path.posix.join(bridgeRuntimeDir, "queue"); const assetRemoteDir = path.posix.join(bridgeRuntimeDir, "server"); const bridgeToken = createSandboxCallbackBridgeToken(); const maxBodyBytes = typeof input.maxBodyBytes === "number" && Number.isFinite(input.maxBodyBytes) && input.maxBodyBytes > 0 ? Math.trunc(input.maxBodyBytes) : DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES; // The bridge worker runs inside the same process that serves the Paperclip // API, so forwarded sandbox calls must target the LOCAL listen origin. The // PAPERCLIP_RUNTIME_API_URL / PAPERCLIP_API_URL exports now prefer a // configured public base URL, which is the origin browsers and external // agents use; routing this in-process loopback hop through the network edge // breaks deployments whose public origin sits behind a session-gated proxy // (every forwarded agent API call is rejected at the edge). Server boot // exports PAPERCLIP_LISTEN_HOST / PAPERCLIP_LISTEN_PORT before any run // executes, and resolveDefaultPaperclipApiUrl() maps wildcard listen hosts // to the loopback address of the same family (0.0.0.0 -> 127.0.0.1, // :: -> [::1]), so the fallback is always loopback-reachable. // input.hostApiUrl stays available as an explicit override seam. const hostApiUrl = input.hostApiUrl?.trim() || resolveDefaultPaperclipApiUrl(); const shellCommand = adapterExecutionTargetShellCommand(target); const runner = adapterExecutionTargetCommandRunner(target); const bridgeTimeoutMs = typeof input.timeoutSec === "number" && Number.isFinite(input.timeoutSec) && input.timeoutSec > 0 ? Math.trunc(input.timeoutSec * 1000) : adapterExecutionTargetTimeoutMs(target); await onLog( "stdout", `[paperclip] Starting sandbox callback bridge for ${input.adapterKey} in ${bridgeRuntimeDir}.\n`, ); const bridgeAsset = await createSandboxCallbackBridgeAsset(); // The provider-scoped telemetry facade for the fixed duplex observability // surface. It maps the raw provider key through the allowlist one time, so no // raw plugin key reaches a span, a counter, or the event. The default recorder // is a no-op, so the facade is inert until the host injects a real recorder. const duplexProviderKey = "providerKey" in target ? target.providerKey ?? undefined : undefined; const duplexObservability = createDuplexObservability({ recorder: input.duplexObservabilityRecorder ?? undefined, providerKey: duplexProviderKey, // http2_v1 is the one active non-file transport now; every non-file // record this facade produces stamps the `http2` transport value. transport: "http2", }); // PAPERCLIP_BRIDGE_DEBUG opts into verbose stdout logs of every bridge proxy // request/response. The query string is logged verbatim, so callers who pass // auth tokens or other sensitive values as query parameters should be aware // those values appear in the host process's stdout when this flag is enabled. // Only intended for active debugging in trusted environments. const bridgeDebugEnabled = isBridgeDebugEnabled(process.env); // One forward of a relayed sandbox request onto the existing Paperclip API // path. The forward applies the real host token and the signed run id, so the // token replacement and the run attribution stay in one place for both the // file bridge and the duplex broker. The sandbox request carries only the // bridge token; the real agent token never leaves the host. const forwardBridgeRequest = async ( request: { method: string; path: string; query: string; headers: Record; /** The file bridge passes the whole request body here as one string. * The HTTP/2 bridge passes it as the raw `Buffer` it read off the wire. */ body?: string | Buffer; }, signal?: AbortSignal, options?: { suppressDebugLog?: boolean; /** * The caller's stream reservation owner, if it has one. The HTTP/2 * bridge passes the stream's own owner here, so the response body copy * reserves against the same ceiling the request body copy already * reserved against. The queue transport passes no owner, so its * response-body read enforces only the per-request size ceiling, exactly * as it did before this option existed. */ reservation?: BridgeBodyReservation; }, ): Promise<{ status: number; headers: Record; body: Buffer }> => { const method = request.method.trim().toUpperCase() || "GET"; // The per-request debug log prints the method, the path, and the query. The // duplex path suppresses it, so no route or query rides a log line there. The // file path keeps the existing behavior. const emitDebugLog = bridgeDebugEnabled && options?.suppressDebugLog !== true; if (emitDebugLog) { await onLog( "stdout", `[paperclip] Bridge proxy ${method} ${request.path}${request.query ? `?${request.query}` : ""}\n`, ); } const headers = new Headers(); for (const [key, value] of Object.entries(request.headers)) { if (value.trim().length === 0) continue; headers.set(key, value); } headers.set("authorization", `Bearer ${hostApiToken}`); headers.set("x-paperclip-run-id", input.runId); // Abort the forward when the caller aborts the request (its per-iteration // timeout or watchdog fired, or the broker's forward budget ended), or after // the forward budget here, whichever comes first. const timeoutSignal = AbortSignal.timeout(forwardTimeoutMs); const forwardSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal; // Build the request-body init. A GET or a HEAD carries no body. The file // bridge passes the whole body as one string; the HTTP/2 bridge passes it // as a raw `Buffer`. Undici accepts a `Buffer` request body directly (a // `Buffer` is an `ArrayBufferView`), so neither shape needs a conversion. // The cast below only bridges a `BodyInit` typing gap: the DOM library // type this project's ambient `RequestInit` resolves to excludes a // `Buffer`, though Undici accepts one at runtime. const forwardInit: RequestInit = { method, headers, signal: forwardSignal, }; if (method !== "GET" && method !== "HEAD" && request.body !== undefined) { forwardInit.body = request.body as BodyInit; } const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), forwardInit); if (emitDebugLog) { await onLog( "stdout", `[paperclip] Bridge proxy response ${response.status} for ${method} ${request.path}${request.query ? `?${request.query}` : ""}\n`, ); } // The host delivered response headers, so the response-body read starts after // the host processed the request. A later response-body read failure (a body // over the size limit, or a stream read error) needs a classification by // method safety. A safe method (GET, HEAD, OPTIONS, TRACE) never changes host // state, so a retry cannot double-apply a mutation and the failure stays // retryable. A mutating or otherwise unsafe method may have committed on the // host, so a retryable status is unsafe: it makes the caller repeat the // request with a new request id outside the broker deduplication set, and the // host applies the mutation twice. For an unsafe method the code returns a // non-retryable 504 and marks the outcome indeterminate, exactly like an // aborted in-flight forward. The in-sandbox server maps the indeterminate 504 // to a non-retryable 409 for both the file bridge and the duplex broker. let responseBody: Buffer; try { responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes, options?.reservation); } catch (error) { // A denied reservation is retryable capacity pressure for a safe // method, not a body-read fault: rethrow it before the method-safety // classification below runs, so it reaches the HTTP/2 bridge's own // capacity-denial catch (which answers the retryable 503 and settles // the stream) instead of this function turning it into a 502. For an // unsafe (mutating) method, the host has already delivered response // headers by this point, so it may already have committed the // mutation. Rethrowing there too would let the retryable 503 reach a // caller that repeats the request, applying the mutation twice. An // unsafe method's capacity denial falls through to the same // non-retryable indeterminate 504 any other response-body read fault // gets below. if (error instanceof BridgeProcessCapacityError && isSafeBridgeMethod(method)) throw error; if (isSafeBridgeMethod(method)) { // The method is safe, so a retry cannot double-apply a mutation. Return a // retryable 502 with no indeterminate marker, so the gateway passes it // through as a retryable status. return { status: 502, headers: { "content-type": "application/json" }, body: Buffer.from( JSON.stringify({ error: error instanceof Error ? error.message : String(error), }), "utf8", ), }; } return { status: 504, headers: { "content-type": "application/json", "x-paperclip-bridge-outcome": "indeterminate", }, body: Buffer.from( JSON.stringify({ error: error instanceof Error ? error.message : String(error), outcome: "indeterminate", retryable: false, }), "utf8", ), }; } const commentMarker = postedIssueCommentLogMarker(method, request.path, response.status, responseBody); if (commentMarker) await onLog("stdout", commentMarker); return { status: response.status, headers: buildBridgeResponseHeaders(response), body: responseBody, }; }; // Two strict gates guard the duplex transport. Select duplex only when the // experimental setting is exactly `true`, the resolved capability // `duplexCommandStream` is exactly `true`, and the runner exposes the duplex // channel. Any other value of either gate selects the file bridge below. // HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback. const duplexRequested = input.enableSandboxDuplexBridge === true; const capabilityGranted = "effectiveCapabilities" in target && target.effectiveCapabilities?.duplexCommandStream === true; const openDuplexChannel = runner.openDuplexChannel?.bind(runner); // Record the pre-attempt fallback for a file-bridge selection that opens no // channel. `gate_off` marks the kill switch off; `capability_absent` marks the // capability or the runner method absent. A later channel-open failure records // its own fallback through the channel-open attempt below. if (!duplexRequested) { duplexObservability.recordFallback("gate_off"); } else if (!capabilityGranted || typeof openDuplexChannel !== "function") { duplexObservability.recordFallback("capability_absent"); } if (duplexRequested && capabilityGranted && typeof openDuplexChannel === "function") { // Begin the channel-open attempt. The block reports exactly one terminal: // `ready` on success, or `fallback(reason)` on an open or a readiness failure. const duplexChannelOpen = duplexObservability.startChannelOpen(); const readinessTimeoutMs = typeof input.duplexReadinessTimeoutMs === "number" && Number.isFinite(input.duplexReadinessTimeoutMs) && input.duplexReadinessTimeoutMs > 0 ? Math.trunc(input.duplexReadinessTimeoutMs) : DEFAULT_DUPLEX_READINESS_TIMEOUT_MS; // The host assigns the loopback port before it opens the channel. It passes // the port and one random per-open nonce to the gateway only through the // launch environment. It builds the sandbox-facing origin from its own // stored port. No field of any channel frame contributes to the endpoint. // // Provider-boundary rationale (a security-review condition): the Daytona SDK // surface the plugin uses (`sandbox.process` sessions, PTY, and exec; // `sandbox.fs`) exposes no listener identity that the provider control plane // binds to a launched process. No channel-supplied address can be attested, // so this design uses zero channel-supplied addressing data. The only actor // that can pre-bind the host-assigned loopback port before the agent starts // is the provider itself, which already delivers the agent launch // environment. The gateway bind-or-exit contract fails closed. This is an // accepted provider-untrusted residual, not endpoint authentication: the // nonce is a liveness signal, because a compromised channel can read the // launch environment; the endpoint stays safe because it never derives from // the channel. const assignedPort = await reserveHostAssignedLoopbackPort(); const nonce = randomBytes(16).toString("hex"); const sandboxOrigin = `http://127.0.0.1:${assignedPort}`; let channel: CommandManagedDuplexChannel | null = null; // The open runs two stages under one try: the entrypoint sync, then the // channel open. The stage names the exact open-failure outcome in the catch. let openStage: "entrypoint_sync" | "channel_open" = "entrypoint_sync"; try { const assetSync = await syncSandboxCallbackBridgeEntrypoint({ runner, remoteCwd: target.remoteCwd, assetRemoteDir, bridgeAsset, timeoutMs: bridgeTimeoutMs, shellCommand, }); const gatewayEnv: Record = { PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE, PAPERCLIP_BRIDGE_TOKEN: bridgeToken, PAPERCLIP_BRIDGE_HOST: "127.0.0.1", PAPERCLIP_BRIDGE_PORT: String(assignedPort), PAPERCLIP_BRIDGE_NONCE: nonce, PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes), }; const command = buildDuplexGatewayLaunchArgv({ shellCommand, remoteEntrypoint: assetSync.remoteEntrypoint, env: gatewayEnv, }); openStage = "channel_open"; channel = await openDuplexChannel({ command }); } catch (error) { // The channel never opened, so no request could carry the bridge token. // The open call is the last statement of the try, so `channel` is still // null here; there is nothing to close. Fall through to the file bridge // below. Bind and keep the caught error, so the fallback names the exact // open-failure stage: a full process-scoped route ceiling is `route_busy`; // otherwise the stage is the entrypoint sync or the channel open. The log // line names only the fixed stage enum, so no raw provider error rides a log // line on the duplex path. const reason: DuplexFallbackReason = isDuplexRouteBusyError(error) ? "route_busy" : openStage === "entrypoint_sync" ? "entrypoint_sync_failed" : "channel_open_failed"; duplexChannelOpen.fallback(reason); await onLog( "stderr", `[paperclip] Could not open the sandbox duplex channel (${reason}). Using the file bridge.\n`, ); channel = null; } if (channel) { const gate = createDuplexReadinessGate(channel, { nonce, timeoutMs: readinessTimeoutMs, }); const readiness = await gate.ready; if (!readiness.ok) { // Fail closed. Close the partial channel inside a bounded budget, then // select the file bridge. The broker never started, so no request that // carries the bridge token reached the channel or any endpoint. The reason // is a fixed enum, so it rides the log line and the fallback telemetry // with no raw value. gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback(duplexReadinessFallbackReason(readiness.reason)); await onLog( "stderr", `[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`, ); } else { // Readiness passed. The gate retained every byte that followed the // accepted READY line. Scan those retained bytes for the HTTP/2 // client connection preface — the scan window opens only now, after // the gate accepted the nonce — and start the HTTP/2 session at that // offset, inclusive. A missing preface inside the bounded readiness // buffer aborts the open; the run falls back to `queue_v1` exactly // one time (accepted security fix 6). const openedChannel = channel; const prefaceScan = scanForHttp2ClientPreface(gate.brokerChannel, { capBytes: DUPLEX_READINESS_BUFFER_CAP_BYTES, timeoutMs: readinessTimeoutMs, }); const prefaceResult = await prefaceScan.settled; if (prefaceResult === "missing") { // Fail closed, the same shape as a readiness failure: close the // partial channel inside the cleanup budget, then select the file // bridge. No HTTP/2 server ever bound to this channel, so no // request reached it or any endpoint. gate.disposePendingReplay(); await closeDuplexChannelWithinBudget(openedChannel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); duplexChannelOpen.fallback("preface_missing"); await onLog( "stderr", "[paperclip] Sandbox HTTP/2 client preface did not appear inside the bounded readiness buffer (preface_missing). Using the file bridge.\n", ); } else { // The run disposition latch for the http2_v1 path, in the same // shape the retired duplex_v1 broker exposed. A loss ordered before // an orderly completion reports a failure with the typed loss // reason; every other state reports a success. const dispositionLatch = createHttp2RunDispositionLatch(); // Set before the first await inside the forward handler, so a loss // that lands mid-request still classifies as `post_dispatch`. let anyStreamDispatched = false; const recordHttp2Loss = (event: Http2TelemetryEventName): void => { const reason = mapHttp2EventToDuplexLossReason(event); const disposedNow = dispositionLatch.recordLoss(reason); if (!disposedNow && (reason === "provider_exit" || reason === "transport_closed")) { // A clean channel end that orders after a host-observed orderly // completion is a normal teardown, not a loss: the run already // completed. Emit no loss telemetry and no log line for it, the // same policy the retired duplex_v1 broker applied. return; } const lossClass = anyStreamDispatched ? "post_dispatch" : "pre_dispatch"; duplexObservability.recordLoss(lossClass, reason); void onLog("stderr", `[paperclip] Sandbox HTTP/2 channel lost (${reason}). The run fails.\n`); }; // The forward handler applies the real host token and the run id // through the existing `forwardBridgeRequest` — the same function // the file bridge uses. The route allowlist, the header allowlist, // and the per-request debug-log suppression already ran inside // `createHttp2BridgeServer`'s own stream handler before this call. // It records the request span with the same latency-and-outcome // shape the retired duplex_v1 broker recorded: `ok` for any // delivered host response (any status), `error` only when the // forward call itself throws. const http2ForwardRequest: Http2BridgeForwardHandler = async (request) => { anyStreamDispatched = true; const dispatchStartMs = Date.now(); try { const result = await forwardBridgeRequest( { method: request.method, path: request.pathname, query: request.query, headers: request.headers, body: request.body, }, request.signal, { suppressDebugLog: true, reservation: request.reservation }, ); duplexObservability.recordRequest({ latencyMs: Date.now() - dispatchStartMs, outcome: "ok" }); return { status: result.status, headers: result.headers, body: result.body }; } catch (error) { duplexObservability.recordRequest({ latencyMs: Date.now() - dispatchStartMs, outcome: "error" }); throw error; } }; const http2Server = createHttp2BridgeServer({ bridgeToken, forwardRequest: http2ForwardRequest, routes: HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST, // The same resolved limit the launch environment hands the // sandbox-side gateway (`PAPERCLIP_BRIDGE_MAX_BODY_BYTES`, // below), so the host check and the gateway check enforce one // value instead of the host silently falling back to the // package default. maxBodyBytes, onGoaway: () => recordHttp2Loss("session_goaway"), onSessionError: () => recordHttp2Loss("session_error"), }); // Combine the loss hook with the forward-to-Duplex handoff inside // one `onExit` registration: the channel primitive holds exactly // one listener slot, and `bindChannel` below registers the one that // ends the wrapped `Duplex`. // // This object's `stop()` is also the real sandbox-side effect of // the post-bind read backpressure bound `bindChannel` applies // (`wrapDuplexChannelAsNodeDuplex` in `http2-bridge-server.ts`): // this raw provider channel exposes no pause, so once the bounded // read queue there overflows, it calls `stop()` through this exact // chain, down to `prefaceScan.scanned.stop()` and on to the real // channel, instead of letting sandbox-controlled bytes grow host // memory with no bound. const channelForHttp2Server: CommandManagedDuplexChannel = { write: (data: Uint8Array) => prefaceScan.scanned.write(data), onData: (listener: (chunk: Uint8Array) => void) => prefaceScan.scanned.onData(listener), onExit: (listener: (exit: { exitCode: number | null }) => void) => { prefaceScan.scanned.onExit((exit) => { recordHttp2Loss("channel_exit"); listener(exit); }); }, stop: () => prefaceScan.scanned.stop(), close: () => prefaceScan.scanned.close(), }; const boundDuplex = http2Server.bindChannel(channelForHttp2Server); // Also catches the `Duplex` this wrapper destroys when the bounded // read backpressure queue overflows post-bind, so that loss still // reaches `recordHttp2Loss` the same way any other write fault does. boundDuplex.on("error", () => recordHttp2Loss("write_error")); duplexChannelOpen.ready(); await onLog( "stdout", "[paperclip] Sandbox HTTP/2 transport ready; serving the host-assigned origin.\n", ); // Stream run logs on the http2 path with the same gate and the same // log line as the file path. The http2 path starts no file-bridge // worker, so create the log directory before the tail starts. let duplexRunLogTail: SandboxRunLogTailFactory | null = null; if (target.transport === "sandbox" && target.streamRunLogs !== false) { const duplexLogsDir = sandboxCallbackBridgeDirectories(queueDir).logsDir; await ensureSandboxRunLogDirectory({ runner, remoteCwd: target.remoteCwd, logsDir: duplexLogsDir, shellCommand, timeoutMs: bridgeTimeoutMs, }); duplexRunLogTail = createSandboxRunLogTailFactory({ runner, remoteCwd: target.remoteCwd, logsDir: duplexLogsDir, shellCommand, }); await onLog("stdout", "[paperclip] Sandbox run log streaming enabled for this run.\n"); } return { env: { PAPERCLIP_API_URL: sandboxOrigin, PAPERCLIP_API_KEY: bridgeToken, PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE, }, runLogTail: duplexRunLogTail, readRunDisposition: (): DuplexBrokerRunDisposition => dispositionLatch.disposition, // Atomically read the latch and mark the orderly completion for the // ACP success-eligible terminal, so no await separates the read from // the mark and a teardown loss cannot slip in between. settleRunDisposition: (): DuplexBrokerRunDisposition => dispositionLatch.settleRunDisposition(), markOrderlyCompletion: (): void => dispositionLatch.markOrderlyCompletion(), onLoss: (listener: (reason: DuplexLossReason) => void): (() => void) => dispositionLatch.onLoss(listener), stop: async () => { // Close the HTTP/2 server's sessions, then the channel, before // lease release, so no live provider session remains when the // caller releases the lease. await http2Server.close(); await closeDuplexChannelWithinBudget(openedChannel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS); await bridgeAsset.cleanup(); }, }; } } } } let server: Awaited> | null = null; let worker: Awaited> | null = null; try { const client = createCommandManagedSandboxCallbackBridgeQueueClient({ runner, remoteCwd: target.remoteCwd, timeoutMs: bridgeTimeoutMs, shellCommand, }); // `startSandboxCallbackBridgeWorker` keeps its awaited queue-directory // setup on the active `bridge.paperclip` step, and runs each request under // the run parent context (see `runWithRuntimeParent` inside that function). // So the startup `mkdir` execs stay parented to the step, and every later // request `sandbox.exec` span parents to the live run span. worker = await startSandboxCallbackBridgeWorker({ client, queueDir, maxBodyBytes, getRuntimeParentContext: input.getRuntimeParentContext, runtimeSpan: input.runtimeSpan, // The queue transport writes the response body to a text file, so this // is the one place the forward path decodes the response `Buffer` to a // UTF-8 string. The queue's own on-wire behavior does not change. handleRequest: async (request, options) => { const result = await forwardBridgeRequest(request, options?.signal); return { status: result.status, headers: result.headers, body: result.body.toString("utf8") }; }, }); server = await startSandboxCallbackBridgeServer({ runner, remoteCwd: target.remoteCwd, assetRemoteDir, queueDir, bridgeToken, bridgeAsset, timeoutMs: bridgeTimeoutMs, maxBodyBytes, shellCommand, }); } catch (error) { await Promise.allSettled([ server?.stop(), worker?.stop(), bridgeAsset.cleanup(), ]); throw error; } let runLogTail: SandboxRunLogTailFactory | null = null; if (target.transport === "sandbox" && target.streamRunLogs !== false) { runLogTail = createSandboxRunLogTailFactory({ runner, remoteCwd: target.remoteCwd, logsDir: sandboxCallbackBridgeDirectories(queueDir).logsDir, shellCommand, }); await onLog("stdout", "[paperclip] Sandbox run log streaming enabled for this run.\n"); } return { env: { PAPERCLIP_API_URL: server.baseUrl, PAPERCLIP_API_KEY: bridgeToken, PAPERCLIP_API_BRIDGE_MODE: "queue_v1", PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir, }, runLogTail, stop: async () => { await Promise.allSettled([ server?.stop(), ]); await Promise.allSettled([ worker?.stop(), bridgeAsset.cleanup(), ]); }, }; }