4797 lines
207 KiB
TypeScript
4797 lines
207 KiB
TypeScript
import fs from "node:fs/promises";
|
|
import { execFile } from "node:child_process";
|
|
import { promisify } from "node:util";
|
|
import net from "node:net";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { randomBytes, randomUUID } from "node:crypto";
|
|
import { githubLauncherSource } from "./github-launcher.js";
|
|
import type { SshRemoteExecutionSpec } from "./ssh.js";
|
|
import {
|
|
prepareCommandManagedRuntime,
|
|
type CommandManagedDuplexChannel,
|
|
type CommandManagedRuntimeAsset,
|
|
type CommandManagedRuntimeRunner,
|
|
} from "./command-managed-runtime.js";
|
|
import {
|
|
buildRemoteExecutionSessionIdentity,
|
|
prepareRemoteManagedRuntime,
|
|
remoteExecutionSessionMatches,
|
|
} from "./remote-managed-runtime.js";
|
|
import type {
|
|
AdditionalSourceStagingFailure,
|
|
SandboxAdditionalSource,
|
|
WorkspaceDurableSeedPaths,
|
|
WorkspaceInboundMode,
|
|
} from "./sandbox-managed-runtime.js";
|
|
import type { GitWorkspaceSnapshot } from "./git-workspace-sync.js";
|
|
import type { DirectorySnapshot } from "./workspace-restore-merge.js";
|
|
export { resolveReferencedSourceIgnore } from "./sandbox-managed-runtime.js";
|
|
export type {
|
|
AdditionalSourceStagingFailure,
|
|
ReferencedSourceIgnoreResolution,
|
|
SandboxAdditionalSource,
|
|
} from "./sandbox-managed-runtime.js";
|
|
import {
|
|
createCommandManagedSandboxCallbackBridgeQueueClient,
|
|
createSandboxCallbackBridgeAsset,
|
|
createSandboxCallbackBridgeToken,
|
|
DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES,
|
|
HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST,
|
|
SANDBOX_CALLBACK_BRIDGE_ENTRYPOINT,
|
|
SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE,
|
|
sandboxCallbackBridgeDirectories,
|
|
startSandboxCallbackBridgeServer,
|
|
startSandboxCallbackBridgeWorker,
|
|
syncRemoteTextFileWithHashSkip,
|
|
syncSandboxCallbackBridgeEntrypoint,
|
|
} from "./sandbox-callback-bridge.js";
|
|
import {
|
|
createHttp2BridgeServer,
|
|
BridgeProcessCapacityError,
|
|
type BridgeBodyReservation,
|
|
type Http2BridgeForwardHandler,
|
|
} from "./http2-bridge-server.js";
|
|
import {
|
|
createSandboxRunLogTailFactory,
|
|
type SandboxRunLogTailFactory,
|
|
} from "./sandbox-run-log-stream.js";
|
|
import {
|
|
DEFAULT_DUPLEX_BROKER_BUDGETS,
|
|
DUPLEX_CHANNEL_LOST_ERROR_CODE,
|
|
isSafeBridgeMethod,
|
|
type DuplexBrokerRunDisposition,
|
|
} from "./bridge-transport-contract.js";
|
|
import { decodeDuplexLine, DEFAULT_MAX_DUPLEX_FRAME_BYTES } from "./duplex-frame-codec.js";
|
|
import {
|
|
createDuplexObservability,
|
|
mapHttp2EventToDuplexLossReason,
|
|
type DuplexFallbackReason,
|
|
type DuplexLossReason,
|
|
type DuplexObservabilityRecorder,
|
|
type Http2TelemetryEventName,
|
|
} from "./duplex-observability.js";
|
|
import { createSshCommandManagedRuntimeRunner, parseSshRemoteExecutionSpec, runSshCommand, shellQuote } from "./ssh.js";
|
|
import {
|
|
ensureCommandResolvable,
|
|
resolveCommandForLogs,
|
|
runChildProcess,
|
|
type RunProcessResult,
|
|
type TerminalResultCleanupOptions,
|
|
} from "./server-utils.js";
|
|
import { sanitizeRemoteExecutionEnv } from "./remote-execution-env.js";
|
|
import { preferredShellForSandbox, shellCommandArgs } from "./sandbox-shell.js";
|
|
import {
|
|
runWithRuntimeParent,
|
|
type RuntimeSpanRunner,
|
|
type StartupSpanContext,
|
|
} from "./acpx-engine/startup-timing.js";
|
|
import type { RuntimeProgressSink, RuntimeStatusSink } from "./runtime-progress.js";
|
|
import type { LocalProcessSandboxOptions } from "./local-process-sandbox.js";
|
|
import type { RunnerIngressEndpoint } from "./runner-connectivity.js";
|
|
|
|
export type { RuntimeProgressSink } from "./runtime-progress.js";
|
|
|
|
export function postedIssueCommentLogMarker(
|
|
method: string,
|
|
requestPath: string,
|
|
status: number,
|
|
body: Buffer | string,
|
|
) {
|
|
if (method !== "POST" || !/^\/api\/issues\/[^/]+\/comments$/.test(requestPath) || status < 200 || status >= 300) {
|
|
return null;
|
|
}
|
|
const bodyText = typeof body === "string" ? body : body.toString("utf8");
|
|
try {
|
|
const parsed = JSON.parse(bodyText) as { id?: unknown };
|
|
return typeof parsed.id === "string" && parsed.id.length > 0 ? `comment id: ${parsed.id}\n` : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export type AdapterWorkspaceRealizationMode = "copy" | "in_place";
|
|
|
|
export interface AdapterWorkspacePathAlias {
|
|
path: string;
|
|
target: string;
|
|
}
|
|
|
|
export interface AdapterWorkspaceRealization {
|
|
mode: AdapterWorkspaceRealizationMode;
|
|
authoritativeRoot: string;
|
|
pathAliases: AdapterWorkspacePathAlias[];
|
|
outboundRestorePaths: string[];
|
|
}
|
|
|
|
interface AdapterExecutionTargetWorkspaceMetadata {
|
|
workspaceRealization?: AdapterWorkspaceRealization | null;
|
|
}
|
|
|
|
export interface AdapterLocalExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata {
|
|
kind: "local";
|
|
environmentId?: string | null;
|
|
leaseId?: string | null;
|
|
}
|
|
|
|
export interface AdapterSshExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata {
|
|
kind: "remote";
|
|
transport: "ssh";
|
|
environmentId?: string | null;
|
|
leaseId?: string | null;
|
|
remoteCwd: string;
|
|
spec: SshRemoteExecutionSpec;
|
|
}
|
|
|
|
/**
|
|
* Read-only snapshot of the effective execution capabilities for one
|
|
* execution target — local, ssh, sandbox, or plugin. Each flag is the
|
|
* resolved result of the provider's declaration, the live worker's verified
|
|
* methods, and any narrowing from the config or lease. The host computes it
|
|
* once and attaches it to the target; a consumer reads it but never changes
|
|
* it, so every field is `readonly`.
|
|
*/
|
|
export interface EffectiveExecutionCapabilities {
|
|
readonly reusableLeases: boolean;
|
|
readonly nativeSyncIn: boolean;
|
|
readonly nativeSyncOut: boolean;
|
|
readonly persistentProcessSessions: boolean;
|
|
readonly independentControlCommands: boolean;
|
|
readonly incrementalSessionOutput: boolean;
|
|
readonly concurrentSyncOperations: boolean;
|
|
readonly duplexCommandStream: boolean;
|
|
/** Provider can expose a private authenticated WebSocket endpoint for runnerd. */
|
|
readonly runnerWebSocketIngress: boolean;
|
|
}
|
|
|
|
/**
|
|
* @deprecated Renamed to `EffectiveExecutionCapabilities`. This alias will
|
|
* be removed in a later major release.
|
|
*/
|
|
export interface EffectiveSandboxCapabilities extends EffectiveExecutionCapabilities {}
|
|
|
|
export interface SandboxLeaseAcquisition {
|
|
outcome: "created" | "resumed" | "replacement";
|
|
providerLeaseId: string;
|
|
previousProviderLeaseId?: string;
|
|
reason?: "not_found" | "expired" | "identity_mismatch" | "resume_failed";
|
|
}
|
|
|
|
export interface AdapterSandboxExecutionTarget extends AdapterExecutionTargetWorkspaceMetadata {
|
|
kind: "remote";
|
|
transport: "sandbox";
|
|
providerKey?: string | null;
|
|
/**
|
|
* Read-only effective capability snapshot for this sandbox target. The host
|
|
* resolves it from the provider declaration ∩ the verified worker methods ∩
|
|
* narrowing, then attaches it here. Absent when no snapshot was resolved.
|
|
*/
|
|
readonly effectiveCapabilities?: EffectiveExecutionCapabilities | null;
|
|
/**
|
|
* Per-run duplex bridge kill switch. The host stamps it on the same seam as
|
|
* `effectiveCapabilities`. `true` selects the duplex transport only when the
|
|
* capability `duplexCommandStream` is also `true`; any other value keeps the
|
|
* file bridge. The value stays on the host and never enters the sandbox
|
|
* environment. Absent means no grant.
|
|
*/
|
|
readonly enableSandboxDuplexBridge?: boolean;
|
|
/** Host-owned lifecycle override for paperclip_runner in this environment. */
|
|
readonly runnerLifecyclePolicy?:
|
|
| { mode: "per_turn"; idleTimeoutMs: null }
|
|
| { mode: "warm"; idleTimeoutMs: number }
|
|
| null;
|
|
/** Whether this environment is configured to reuse its provider lease. */
|
|
readonly reusableLeaseConfigured?: boolean;
|
|
/** Host-observed provenance for this exact sandbox acquisition. */
|
|
readonly sandboxLeaseAcquisition?: SandboxLeaseAcquisition | null;
|
|
shellCommand?: "bash" | "sh" | null;
|
|
environmentId?: string | null;
|
|
leaseId?: string | null;
|
|
remoteCwd: string;
|
|
timeoutMs?: number | null;
|
|
runner?: CommandManagedRuntimeRunner;
|
|
/** Host-only provider operation. It is never serialized into the sandbox. */
|
|
getRunnerIngressEndpoint?: (input: {
|
|
leaseId: string;
|
|
port: number;
|
|
path: string;
|
|
}) => Promise<RunnerIngressEndpoint>;
|
|
/**
|
|
* Sandbox-backed adapter runs stream the agent CLI's stdout/stderr
|
|
* incrementally via a log-tail loop beside the callback bridge instead of
|
|
* waiting for the batched provider result. Streaming is ON by default;
|
|
* set to `false` to explicitly opt out back to batch-at-end delivery.
|
|
*/
|
|
streamRunLogs?: boolean | null;
|
|
/**
|
|
* The injected duplex observability recorder for this run. The host attaches
|
|
* it on the same seam as `runner`, so this live object stays on the host and
|
|
* never enters the sandbox environment. The bridge binds it to the fixed
|
|
* duplex observability surface. Absent means the safe no-op default.
|
|
*/
|
|
duplexObservabilityRecorder?: DuplexObservabilityRecorder | null;
|
|
}
|
|
|
|
export type AdapterExecutionTarget =
|
|
| AdapterLocalExecutionTarget
|
|
| AdapterSshExecutionTarget
|
|
| AdapterSandboxExecutionTarget;
|
|
|
|
export type AdapterRemoteExecutionSpec = SshRemoteExecutionSpec;
|
|
|
|
// The adapter-facing managed-runtime asset type. Aliased to the sandbox/command
|
|
// asset descriptor so the per-asset lifecycle contributions (`provision` /
|
|
// `restore`) declared on the sandbox core are load-bearing all the way from the
|
|
// adapter call site through to the sandbox runtime. The SSH transport consumes
|
|
// the subset of fields it understands and ignores the rest.
|
|
export type AdapterManagedRuntimeAsset = CommandManagedRuntimeAsset;
|
|
|
|
export interface PreparedAdapterExecutionTargetRuntime {
|
|
target: AdapterExecutionTarget;
|
|
workspaceRemoteDir: string | null;
|
|
runtimeRootDir: string | null;
|
|
assetDirs: Record<string, string>;
|
|
/**
|
|
* Remote directory of each additional (referenced) project that staged
|
|
* successfully, keyed by `projectId`. Empty for a local target or when no
|
|
* additional sources were requested.
|
|
*/
|
|
additionalSourceDirs: Record<string, string>;
|
|
/**
|
|
* Each additional (referenced) project whose staging failed, paired with the
|
|
* failure message. Empty for a local target, for a transport that does not
|
|
* stage referenced projects, or when every requested project staged.
|
|
*/
|
|
additionalSourceFailures: AdditionalSourceStagingFailure[];
|
|
workspaceSyncSnapshot: {
|
|
baseline: DirectorySnapshot;
|
|
gitSnapshot: GitWorkspaceSnapshot | null;
|
|
} | null;
|
|
restoreWorkspace(onProgress?: RuntimeProgressSink): Promise<void>;
|
|
}
|
|
|
|
export interface AdapterExecutionTargetProcessOptions {
|
|
cwd: string;
|
|
env: Record<string, string>;
|
|
stdin?: string;
|
|
timeoutSec: number;
|
|
graceSec: number;
|
|
onLog: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
|
onRuntimeProgress?: RuntimeStatusSink;
|
|
onSpawn?: (meta: { pid: number; processGroupId: number | null; startedAt: string }) => Promise<void>;
|
|
terminalResultCleanup?: TerminalResultCleanupOptions;
|
|
/**
|
|
* Sandbox-only: factory from the Paperclip bridge handle that streams the
|
|
* CLI's stdout/stderr during the run. When provided, the batched provider
|
|
* onLog is suppressed and incremental chunks flow through `onLog` instead.
|
|
*/
|
|
runLogTail?: SandboxRunLogTailFactory | null;
|
|
/**
|
|
* Sandbox-only: the atomic run-disposition settle from the Paperclip bridge
|
|
* handle. When provided, `runAdapterExecutionTargetProcess` calls it once at
|
|
* the clean-completion boundary of the process, synchronously and before the
|
|
* run-log tail finishes. The call reads the disposition and marks the
|
|
* host-observed orderly completion in one broker step, so a gateway exit
|
|
* after the clean process completion cannot latch a false mid-run loss. A
|
|
* control channel that died before the clean completion still fails the run
|
|
* closed with the typed `duplex_channel_lost` code. The file bridge path
|
|
* never sets it.
|
|
*/
|
|
settleRunDisposition?: (() => DuplexBrokerRunDisposition) | null;
|
|
localProcessSandbox?: LocalProcessSandboxOptions | null;
|
|
}
|
|
|
|
export interface AdapterExecutionTargetShellOptions {
|
|
cwd: string;
|
|
env: Record<string, string>;
|
|
timeoutSec?: number;
|
|
graceSec?: number;
|
|
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
|
}
|
|
|
|
export interface AdapterExecutionTargetPaperclipBridgeHandle {
|
|
env: Record<string, string>;
|
|
/**
|
|
* Present when the sandbox target opted into run-log streaming
|
|
* (`streamRunLogs`). Create one handle per CLI attempt and pass it to
|
|
* `runAdapterExecutionTargetProcess` via `options.runLogTail`.
|
|
*/
|
|
runLogTail?: SandboxRunLogTailFactory | null;
|
|
/**
|
|
* Read the terminal run disposition of the duplex control channel. It reports a
|
|
* failure when the channel was lost before an orderly completion, and names the
|
|
* typed loss reason. It reports a success for a healthy channel or a
|
|
* normal-teardown loss. The file bridge path never sets it, so the method is
|
|
* absent there. The caller reads it at the run-disposition seam to fail a run
|
|
* whose control channel died mid-turn.
|
|
*/
|
|
readRunDisposition?(): DuplexBrokerRunDisposition;
|
|
/**
|
|
* Atomically read the run disposition and mark the host-observed orderly
|
|
* completion in one broker step. The ACP lane calls it at the terminal
|
|
* finalization boundary for a success-eligible completion, so no `await` can
|
|
* separate the read from the mark and a teardown loss cannot slip in between.
|
|
* A loss that already latched keeps the failure, because the broker no-ops the
|
|
* mark after a latched loss. The file bridge path never sets it.
|
|
*/
|
|
settleRunDisposition?(): DuplexBrokerRunDisposition;
|
|
/**
|
|
* Mark the host-observed orderly completion of the agent turn on the broker's
|
|
* ordered lifecycle. The caller marks it at the ACP terminal-finalization
|
|
* boundary for a still-success-eligible completion, so a later teardown loss
|
|
* cannot flip the run to a failure. A loss that already latched keeps the
|
|
* failure, because the broker no-ops the mark after a latched loss. The file
|
|
* bridge path never sets it, so the method is absent there.
|
|
*/
|
|
markOrderlyCompletion?(): void;
|
|
/**
|
|
* Register a listener for a newly latched terminal loss. The listener
|
|
* fires at most once, and only for a loss that flips the disposition to
|
|
* failed — never for a clean channel end that orders after a
|
|
* host-observed orderly completion. Returns a function that unregisters
|
|
* the listener.
|
|
*
|
|
* The caller uses this to abort an in-flight Agent Client Protocol turn
|
|
* the moment the channel dies, instead of waiting for the turn to return
|
|
* a terminal result on its own (a dead channel can leave a turn with
|
|
* nothing to return). The file bridge path never sets it, so the method
|
|
* is absent there.
|
|
*/
|
|
onLoss?(listener: (reason: DuplexLossReason) => void): () => void;
|
|
stop(): Promise<void>;
|
|
}
|
|
|
|
export interface AdapterExecutionTargetProcessSessionBridgeHandle {
|
|
agentCommand: string;
|
|
stop(): Promise<void>;
|
|
}
|
|
|
|
export { sanitizeRemoteExecutionEnv } from "./remote-execution-env.js";
|
|
|
|
// 4-hour wall-clock backstop for sandbox-backed adapter runs. This is a
|
|
// last-resort kill switch, not the primary hang detector: genuinely hung runs
|
|
// are caught much earlier by the adapters' output-inactivity monitors (e.g.
|
|
// codex-local's 7-minute monitor). The value intentionally matches the
|
|
// recovery watchdog's ACTIVE_RUN_OUTPUT_CRITICAL_THRESHOLD_MS (4h) in
|
|
// server/src/services/recovery/service.ts so healthy long runs are never
|
|
// killed by the adapter before the watchdog would even consider them stuck.
|
|
export const DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC = 14_400;
|
|
|
|
function parseObject(value: unknown): Record<string, unknown> {
|
|
return value && typeof value === "object" && !Array.isArray(value)
|
|
? (value as Record<string, unknown>)
|
|
: {};
|
|
}
|
|
|
|
function readString(value: unknown): string | null {
|
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
}
|
|
|
|
// Read a serialized effective-capability snapshot back into a full record. A
|
|
// missing or non-boolean field reads as `false`, so a round-tripped target
|
|
// never grants a capability that the snapshot did not carry. Returns null when
|
|
// there is no object to read.
|
|
function parseEffectiveExecutionCapabilities(value: unknown): EffectiveExecutionCapabilities | null {
|
|
const parsed = parseObject(value);
|
|
if (Object.keys(parsed).length === 0) return null;
|
|
return {
|
|
reusableLeases: parsed.reusableLeases === true,
|
|
nativeSyncIn: parsed.nativeSyncIn === true,
|
|
nativeSyncOut: parsed.nativeSyncOut === true,
|
|
persistentProcessSessions: parsed.persistentProcessSessions === true,
|
|
independentControlCommands: parsed.independentControlCommands === true,
|
|
incrementalSessionOutput: parsed.incrementalSessionOutput === true,
|
|
concurrentSyncOperations: parsed.concurrentSyncOperations === true,
|
|
duplexCommandStream: parsed.duplexCommandStream === true,
|
|
runnerWebSocketIngress: parsed.runnerWebSocketIngress === true,
|
|
};
|
|
}
|
|
|
|
function readStringMeta(parsed: Record<string, unknown>, key: string): string | null {
|
|
return readString(parsed[key]);
|
|
}
|
|
|
|
function resolveHostForUrl(rawHost: string): string {
|
|
const host = rawHost.trim();
|
|
// Preserve the wildcard bind's address family: a server bound to 0.0.0.0
|
|
// accepts IPv4, so target the IPv4 loopback (and [::1] for ::) instead of
|
|
// "localhost", which the resolver may map to the other family.
|
|
if (host === "0.0.0.0") return "127.0.0.1";
|
|
if (host === "::") return "[::1]";
|
|
if (!host) return "localhost";
|
|
if (host.includes(":") && !host.startsWith("[") && !host.endsWith("]")) return `[${host}]`;
|
|
return host;
|
|
}
|
|
|
|
function resolveDefaultPaperclipApiUrl(): string {
|
|
const runtimeHost = resolveHostForUrl(
|
|
process.env.PAPERCLIP_LISTEN_HOST ?? process.env.HOST ?? "localhost",
|
|
);
|
|
// 3100 matches the default Paperclip dev server port when the runtime does not provide one.
|
|
const runtimePort = process.env.PAPERCLIP_LISTEN_PORT ?? process.env.PORT ?? "3100";
|
|
return `http://${runtimeHost}:${runtimePort}`;
|
|
}
|
|
|
|
function isBridgeDebugEnabled(env: NodeJS.ProcessEnv): boolean {
|
|
const value = env.PAPERCLIP_BRIDGE_DEBUG?.trim().toLowerCase();
|
|
return value === "1" || value === "true" || value === "yes";
|
|
}
|
|
|
|
function isAdapterExecutionTargetInstance(value: unknown): value is AdapterExecutionTarget {
|
|
const parsed = parseObject(value);
|
|
if (parsed.kind === "local") return true;
|
|
if (parsed.kind !== "remote") return false;
|
|
if (parsed.transport === "ssh") return parseSshRemoteExecutionSpec(parseObject(parsed.spec)) !== null;
|
|
if (parsed.transport !== "sandbox") return false;
|
|
return readStringMeta(parsed, "remoteCwd") !== null;
|
|
}
|
|
|
|
export function adapterExecutionTargetToRemoteSpec(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): AdapterRemoteExecutionSpec | null {
|
|
return target?.kind === "remote" && target.transport === "ssh" ? target.spec : null;
|
|
}
|
|
|
|
export function adapterExecutionTargetIsRemote(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): boolean {
|
|
return target?.kind === "remote";
|
|
}
|
|
|
|
export function adapterExecutionTargetUsesManagedHome(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): boolean {
|
|
return target?.kind === "remote" && target.transport === "sandbox";
|
|
}
|
|
|
|
/**
|
|
* Read the per-run duplex bridge kill switch off a target. Only a sandbox
|
|
* target with `enableSandboxDuplexBridge` set to `true` returns `true`. Every
|
|
* other target and every other value returns `false`, so the caller fails
|
|
* closed to the file bridge.
|
|
*/
|
|
export function adapterExecutionTargetEnablesSandboxDuplexBridge(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): boolean {
|
|
return (
|
|
target?.kind === "remote" &&
|
|
target.transport === "sandbox" &&
|
|
target.enableSandboxDuplexBridge === true
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Read the injected duplex observability recorder off a target. Only a
|
|
* sandbox target with a recorder attached returns it. Every other target
|
|
* returns null, so the bridge falls back to the safe no-op recorder.
|
|
*/
|
|
export function adapterExecutionTargetDuplexObservabilityRecorder(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): DuplexObservabilityRecorder | null {
|
|
return target?.kind === "remote" && target.transport === "sandbox"
|
|
? target.duplexObservabilityRecorder ?? null
|
|
: null;
|
|
}
|
|
|
|
|
|
export function adapterExecutionTargetRemoteCwd(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
localCwd: string,
|
|
): string {
|
|
return target?.kind === "remote" ? target.remoteCwd : localCwd;
|
|
}
|
|
|
|
export function overrideAdapterExecutionTargetRemoteCwd(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
remoteCwd: string | null | undefined,
|
|
): AdapterExecutionTarget | null | undefined {
|
|
const nextRemoteCwd = remoteCwd?.trim();
|
|
if (!target || target.kind !== "remote" || !nextRemoteCwd) {
|
|
return target;
|
|
}
|
|
if (target.remoteCwd === nextRemoteCwd) {
|
|
return target;
|
|
}
|
|
if (target.transport === "ssh") {
|
|
return {
|
|
...target,
|
|
remoteCwd: nextRemoteCwd,
|
|
spec: {
|
|
...target.spec,
|
|
remoteCwd: nextRemoteCwd,
|
|
},
|
|
};
|
|
}
|
|
return {
|
|
...target,
|
|
remoteCwd: nextRemoteCwd,
|
|
};
|
|
}
|
|
|
|
export function resolveAdapterExecutionTargetCwd(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
configuredCwd: string | null | undefined,
|
|
localFallbackCwd: string,
|
|
): string {
|
|
if (typeof configuredCwd === "string" && configuredCwd.trim().length > 0) {
|
|
return configuredCwd;
|
|
}
|
|
return adapterExecutionTargetRemoteCwd(target, localFallbackCwd);
|
|
}
|
|
|
|
export function adapterExecutionTargetUsesPaperclipBridge(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): boolean {
|
|
return target?.kind === "remote";
|
|
}
|
|
|
|
export function describeAdapterExecutionTarget(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): string {
|
|
if (!target || target.kind === "local") return "local environment";
|
|
if (target.transport === "ssh") {
|
|
return `SSH environment ${target.spec.username}@${target.spec.host}:${target.spec.port}`;
|
|
}
|
|
return `sandbox environment${target.providerKey ? ` (${target.providerKey})` : ""}`;
|
|
}
|
|
|
|
export type AdapterExecutionTargetTimeoutSource =
|
|
| "configured"
|
|
| "sandbox_default"
|
|
| "unlimited";
|
|
|
|
export interface AdapterExecutionTargetTimeoutResolution {
|
|
/** Resolved wall-clock timeout in seconds; 0 means no adapter timeout. */
|
|
timeoutSec: number;
|
|
/** Which knob produced the resolved value, for logs and error messages. */
|
|
source: AdapterExecutionTargetTimeoutSource;
|
|
}
|
|
|
|
export function resolveAdapterExecutionTargetTimeout(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
configuredTimeoutSec: number | null | undefined,
|
|
): AdapterExecutionTargetTimeoutResolution {
|
|
if (typeof configuredTimeoutSec === "number" && Number.isFinite(configuredTimeoutSec)) {
|
|
// Preserve fractional (sub-second) configured values instead of flooring:
|
|
// adapters historically honored e.g. timeoutSec=0.5, and flooring would
|
|
// silently turn it into "no timeout".
|
|
if (configuredTimeoutSec > 0) {
|
|
return { timeoutSec: configuredTimeoutSec, source: "configured" };
|
|
}
|
|
// A negative timeoutSec is the explicit "no adapter wall-clock timeout"
|
|
// opt-out, honored even on sandbox targets. Zero cannot carry that
|
|
// meaning: the adapter config UI persists the schema default of 0 for
|
|
// untouched fields, so timeoutSec=0 in stored config does not signal
|
|
// operator intent and falls through to target defaults below.
|
|
if (configuredTimeoutSec < 0) {
|
|
return { timeoutSec: 0, source: "configured" };
|
|
}
|
|
}
|
|
// Local and SSH adapters preserve the historical "0 means no adapter
|
|
// timeout" behavior. Sandbox-backed runs execute through provider RPCs
|
|
// that usually apply their own shorter command defaults, so request an
|
|
// explicit longer timeout for full adapter runs when the adapter leaves
|
|
// timeoutSec unset.
|
|
if (target?.kind === "remote" && target.transport === "sandbox") {
|
|
return { timeoutSec: DEFAULT_REMOTE_SANDBOX_ADAPTER_TIMEOUT_SEC, source: "sandbox_default" };
|
|
}
|
|
return { timeoutSec: 0, source: "unlimited" };
|
|
}
|
|
|
|
export function resolveAdapterExecutionTargetTimeoutSec(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
configuredTimeoutSec: number | null | undefined,
|
|
): number {
|
|
return resolveAdapterExecutionTargetTimeout(target, configuredTimeoutSec).timeoutSec;
|
|
}
|
|
|
|
function describeAdapterExecutionTimeoutSource(
|
|
source: AdapterExecutionTargetTimeoutSource,
|
|
): string {
|
|
switch (source) {
|
|
case "configured":
|
|
return "configured via adapterConfig.timeoutSec";
|
|
case "sandbox_default":
|
|
return "sandbox default";
|
|
case "unlimited":
|
|
return "no adapter wall-clock timeout";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Self-describing error message for when the adapter wall-clock execution
|
|
* timeout kills a run. Names the timer that fired and the knob that controls
|
|
* it so run failures never surface as a bare "Timed out".
|
|
*/
|
|
export function formatAdapterExecutionTimeoutErrorMessage(
|
|
resolution: AdapterExecutionTargetTimeoutResolution,
|
|
): string {
|
|
return (
|
|
`Run exceeded the adapter execution timeout ` +
|
|
`(timeoutSec=${resolution.timeoutSec}, ${describeAdapterExecutionTimeoutSource(resolution.source)}). ` +
|
|
`Set adapterConfig.timeoutSec to raise it.`
|
|
);
|
|
}
|
|
|
|
/**
|
|
* One-line start-of-run statement of the effective wall-clock timeout and its
|
|
* source. Callers prefix with `[paperclip] ` and append a newline.
|
|
*/
|
|
export function formatAdapterExecutionTimeoutStartLogLine(
|
|
resolution: AdapterExecutionTargetTimeoutResolution,
|
|
): string {
|
|
if (resolution.timeoutSec <= 0) {
|
|
if (resolution.source === "configured") {
|
|
return (
|
|
"Adapter execution timeout: none " +
|
|
"(explicitly disabled via adapterConfig.timeoutSec; set it to a positive value to add one)."
|
|
);
|
|
}
|
|
return (
|
|
"Adapter execution timeout: none " +
|
|
"(no adapter wall-clock timeout for this target; set adapterConfig.timeoutSec to add one)."
|
|
);
|
|
}
|
|
return (
|
|
`Adapter execution timeout: timeoutSec=${resolution.timeoutSec} ` +
|
|
`(${describeAdapterExecutionTimeoutSource(resolution.source)}; set adapterConfig.timeoutSec to override).`
|
|
);
|
|
}
|
|
|
|
function requireSandboxRunner(target: AdapterSandboxExecutionTarget): CommandManagedRuntimeRunner {
|
|
if (target.runner) return target.runner;
|
|
throw new Error(
|
|
"Sandbox execution target is missing its provider runtime runner. Sandbox commands must execute through the environment runtime.",
|
|
);
|
|
}
|
|
|
|
function preferredSandboxShell(target: AdapterSandboxExecutionTarget): "bash" | "sh" {
|
|
return preferredShellForSandbox(target.shellCommand);
|
|
}
|
|
|
|
type AdapterCommandCapableExecutionTarget = AdapterSshExecutionTarget | AdapterSandboxExecutionTarget;
|
|
|
|
// The Secure Shell command runner's own output buffer. This value used to
|
|
// derive from the bridge body limit (`DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES
|
|
// * 4`), so a bridge limit rise silently grew it too. It now stands on its
|
|
// own local constant, independent of the bridge body limit, so a later
|
|
// bridge limit change never resizes this buffer as a side effect.
|
|
const SSH_COMMAND_MAX_BUFFER_BYTES = 1024 * 1024;
|
|
|
|
function adapterExecutionTargetCommandRunner(target: AdapterCommandCapableExecutionTarget): CommandManagedRuntimeRunner {
|
|
if (target.transport === "ssh") {
|
|
return createSshCommandManagedRuntimeRunner({
|
|
spec: target.spec,
|
|
defaultCwd: target.remoteCwd,
|
|
maxBufferBytes: SSH_COMMAND_MAX_BUFFER_BYTES,
|
|
});
|
|
}
|
|
return requireSandboxRunner(target);
|
|
}
|
|
|
|
function adapterExecutionTargetShellCommand(target: AdapterCommandCapableExecutionTarget): "bash" | "sh" {
|
|
return target.transport === "ssh" ? "sh" : preferredSandboxShell(target);
|
|
}
|
|
|
|
function adapterExecutionTargetTimeoutMs(
|
|
target: AdapterCommandCapableExecutionTarget,
|
|
): number | null | undefined {
|
|
return target.transport === "sandbox" ? target.timeoutMs : undefined;
|
|
}
|
|
|
|
export async function ensureAdapterExecutionTargetCommandResolvable(
|
|
command: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
cwd: string,
|
|
env: NodeJS.ProcessEnv,
|
|
options: { installCommand?: string | null; timeoutSec?: number | null } = {},
|
|
) {
|
|
if (target?.kind === "remote" && target.transport === "sandbox") {
|
|
await ensureSandboxCommandResolvable(
|
|
command,
|
|
target,
|
|
sanitizeRemoteExecutionEnv(Object.fromEntries(
|
|
Object.entries(env).filter((entry): entry is [string, string] => typeof entry[1] === "string"),
|
|
)),
|
|
options.installCommand?.trim() || null,
|
|
options.timeoutSec,
|
|
);
|
|
return;
|
|
}
|
|
await ensureCommandResolvable(command, cwd, env, {
|
|
remoteExecution: adapterExecutionTargetToRemoteSpec(target),
|
|
});
|
|
}
|
|
|
|
async function probeSandboxCommandResolvable(
|
|
command: string,
|
|
target: AdapterSandboxExecutionTarget,
|
|
env: Record<string, string>,
|
|
): Promise<{ resolved: boolean; timedOut: boolean; stderr: string }> {
|
|
const runner = requireSandboxRunner(target);
|
|
const probeScript = `command -v ${shellQuote(command)}`;
|
|
const result = await runner.execute({
|
|
command: "sh",
|
|
args: ["-c", probeScript],
|
|
cwd: target.remoteCwd,
|
|
env,
|
|
timeoutMs: target.timeoutMs ?? 15_000,
|
|
});
|
|
return {
|
|
resolved: !result.timedOut && (result.exitCode ?? 1) === 0,
|
|
timedOut: result.timedOut,
|
|
stderr: result.stderr.trim(),
|
|
};
|
|
}
|
|
|
|
async function ensureSandboxCommandResolvable(
|
|
command: string,
|
|
target: AdapterSandboxExecutionTarget,
|
|
env: Record<string, string>,
|
|
installCommand: string | null,
|
|
timeoutSec?: number | null,
|
|
): Promise<void> {
|
|
// Probe whether the binary is resolvable inside the sandbox. We previously
|
|
// short-circuited this for sandbox targets, which let the caller report a
|
|
// success message even when the CLI was missing from the image. Now we run
|
|
// a real `command -v` through the same runner the hello probe will use, so
|
|
// the first step honestly reflects whether the binary is on PATH. The
|
|
// sandbox provider is responsible for sourcing login profiles (e2b mirrors
|
|
// SSH's buildSshSpawnTarget) so this and the hello probe agree on PATH.
|
|
let probe = await probeSandboxCommandResolvable(command, target, env);
|
|
if (probe.resolved) return;
|
|
if (probe.timedOut) {
|
|
throw new Error(`Timed out checking command "${command}" on sandbox target.`);
|
|
}
|
|
|
|
// If the caller supplied an install command, attempt the install once via
|
|
// the sandbox runner (which the sandbox provider wraps in a login shell)
|
|
// and re-probe before reporting failure. This lets fresh sandbox leases
|
|
// bring up the CLI before the resolvability gate, mirroring the test path.
|
|
let installFailureDetail: string | null = null;
|
|
if (installCommand) {
|
|
const runner = requireSandboxRunner(target);
|
|
const installTimeoutMs =
|
|
typeof timeoutSec === "number" && Number.isFinite(timeoutSec) && timeoutSec > 0
|
|
? Math.floor(timeoutSec * 1000)
|
|
: target.timeoutMs ?? 300_000;
|
|
try {
|
|
const installResult = await runner.execute({
|
|
command: "sh",
|
|
args: shellCommandArgs(installCommand),
|
|
cwd: target.remoteCwd,
|
|
env,
|
|
timeoutMs: installTimeoutMs,
|
|
});
|
|
if (installResult.timedOut) {
|
|
installFailureDetail = `install command timed out: ${installCommand}`;
|
|
} else if ((installResult.exitCode ?? 0) !== 0) {
|
|
const tail = (text: string) =>
|
|
text.split(/\r?\n/).filter((line) => line.trim().length > 0).slice(-2).join(" | ").slice(0, 240);
|
|
const reason = tail(installResult.stderr || installResult.stdout) || `exit ${installResult.exitCode ?? "?"}`;
|
|
installFailureDetail = `install command exited ${installResult.exitCode ?? "?"}: ${reason}`;
|
|
}
|
|
} catch (err) {
|
|
installFailureDetail = `install command threw: ${err instanceof Error ? err.message : String(err)}`;
|
|
}
|
|
probe = await probeSandboxCommandResolvable(command, target, env);
|
|
if (probe.resolved) return;
|
|
if (probe.timedOut) {
|
|
throw new Error(`Timed out checking command "${command}" on sandbox target.`);
|
|
}
|
|
}
|
|
|
|
const probeStderr = probe.stderr.length > 0 ? ` probe stderr: ${probe.stderr}` : "";
|
|
const installDetail = installFailureDetail ? `; ${installFailureDetail}` : "";
|
|
throw new Error(
|
|
`Command "${command}" is not installed or not on PATH in the sandbox environment${installDetail}.${probeStderr}`,
|
|
);
|
|
}
|
|
|
|
export async function resolveAdapterExecutionTargetCommandForLogs(
|
|
command: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
cwd: string,
|
|
env: NodeJS.ProcessEnv,
|
|
): Promise<string> {
|
|
if (target?.kind === "remote" && target.transport === "sandbox") {
|
|
return `sandbox://${target.providerKey ?? "provider"}/${target.leaseId ?? "lease"}/${target.remoteCwd} :: ${command}`;
|
|
}
|
|
return await resolveCommandForLogs(command, cwd, env, {
|
|
remoteExecution: adapterExecutionTargetToRemoteSpec(target),
|
|
});
|
|
}
|
|
|
|
// Apply the run-disposition seam to one clean process result. Only a clean
|
|
// completion is success-eligible: a timed-out, signalled, or non-zero-exit
|
|
// result is already a failure, so the seam leaves it unchanged. This is the same
|
|
// success-eligibility rule the ACP lane applies. For a success-eligible result
|
|
// the seam settles the disposition in one atomic broker step: it reads the
|
|
// disposition and marks the host-observed orderly completion together, so a
|
|
// gateway exit after the clean completion cannot latch a false mid-run loss. A
|
|
// duplex control channel that died before the clean completion fails the run
|
|
// closed: the seam sets a non-zero exit code, the typed `duplex_channel_lost`
|
|
// error code, and a stderr note that names only the typed loss reason.
|
|
function applyRunDispositionSeam(
|
|
result: RunProcessResult,
|
|
settleRunDisposition: (() => DuplexBrokerRunDisposition) | null | undefined,
|
|
): RunProcessResult {
|
|
const successEligible = result.exitCode === 0 && !result.timedOut && result.signal === null;
|
|
if (!successEligible || !settleRunDisposition) return result;
|
|
const disposition = settleRunDisposition();
|
|
if (!disposition.failed) return result;
|
|
const lossReason = disposition.lossReason ?? "other";
|
|
const note = `[paperclip] The sandbox duplex control channel was lost (${lossReason}) before the run completed.\n`;
|
|
const separator = result.stderr.length > 0 && !result.stderr.endsWith("\n") ? "\n" : "";
|
|
return {
|
|
...result,
|
|
exitCode: 1,
|
|
errorCode: DUPLEX_CHANNEL_LOST_ERROR_CODE,
|
|
stderr: `${result.stderr}${separator}${note}`,
|
|
};
|
|
}
|
|
|
|
export async function runAdapterExecutionTargetProcess(
|
|
runId: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
command: string,
|
|
args: string[],
|
|
options: AdapterExecutionTargetProcessOptions,
|
|
): Promise<RunProcessResult> {
|
|
if (target?.kind === "remote" && target.transport === "sandbox") {
|
|
const runner = requireSandboxRunner(target);
|
|
const env = sanitizeRemoteExecutionEnv(options.env);
|
|
await options.onRuntimeProgress?.({
|
|
phase: "adapter_startup",
|
|
message: "Starting adapter in environment",
|
|
});
|
|
const runLogTail = options.runLogTail?.create() ?? null;
|
|
let execCommand = command;
|
|
let execArgs = args;
|
|
if (runLogTail) {
|
|
({ command: execCommand, args: execArgs } = runLogTail.wrapCommand(command, args));
|
|
runLogTail.start(options.onLog);
|
|
}
|
|
try {
|
|
const result = await runner.execute({
|
|
command: execCommand,
|
|
args: execArgs,
|
|
cwd: target.remoteCwd,
|
|
env,
|
|
stdin: options.stdin,
|
|
timeoutMs: options.timeoutSec > 0 ? options.timeoutSec * 1000 : target.timeoutMs ?? undefined,
|
|
// The tail loop already streams incremental chunks; suppress the
|
|
// runner's end-of-run batched onLog to avoid duplicate log bytes.
|
|
onLog: runLogTail ? undefined : options.onLog,
|
|
onSpawn: options.onSpawn
|
|
? async (meta) => options.onSpawn?.({ ...meta, processGroupId: null })
|
|
: undefined,
|
|
});
|
|
// Settle the duplex run disposition synchronously at the clean-completion
|
|
// boundary, before the run-log tail finishes. The atomic settle marks the
|
|
// host-observed orderly completion in one broker step, so a gateway exit
|
|
// after the clean process completion cannot latch a false mid-run loss. A
|
|
// control channel that died before this clean completion still fails the
|
|
// run closed.
|
|
const settled = applyRunDispositionSeam(result, options.settleRunDisposition);
|
|
if (runLogTail) {
|
|
await runLogTail.finish({ stdout: result.stdout, stderr: result.stderr });
|
|
}
|
|
return settled;
|
|
} catch (error) {
|
|
if (runLogTail) {
|
|
await runLogTail.abort();
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
const env =
|
|
target?.kind === "remote" && target.transport === "ssh"
|
|
? sanitizeRemoteExecutionEnv(options.env)
|
|
: options.env;
|
|
|
|
return await runChildProcess(runId, command, args, {
|
|
cwd: options.cwd,
|
|
env,
|
|
stdin: options.stdin,
|
|
timeoutSec: options.timeoutSec,
|
|
graceSec: options.graceSec,
|
|
onLog: options.onLog,
|
|
onSpawn: options.onSpawn,
|
|
terminalResultCleanup: options.terminalResultCleanup,
|
|
localProcessSandbox: target?.kind === "local" || !target ? options.localProcessSandbox : null,
|
|
remoteExecution: adapterExecutionTargetToRemoteSpec(target),
|
|
});
|
|
}
|
|
|
|
export async function runAdapterExecutionTargetShellCommand(
|
|
runId: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
command: string,
|
|
options: AdapterExecutionTargetShellOptions,
|
|
): Promise<RunProcessResult> {
|
|
const onLog = options.onLog ?? (async () => {});
|
|
if (target?.kind === "remote") {
|
|
const startedAt = new Date().toISOString();
|
|
const env = sanitizeRemoteExecutionEnv(options.env);
|
|
if (target.transport === "ssh") {
|
|
try {
|
|
// Pass the raw command — `runSshCommand` owns profile sourcing and
|
|
// the outer shell wrapper. Wrapping again here would nest a second
|
|
// shell after the explicit `env KEY=VAL` overrides, re-sourcing
|
|
// login profiles AFTER the override and silently undoing any
|
|
// identity var (NVM_DIR / PATH / etc.) that a profile re-exports.
|
|
const result = await runSshCommand(target.spec, command, {
|
|
env,
|
|
timeoutMs: (options.timeoutSec ?? 15) * 1000,
|
|
});
|
|
if (result.stdout) await onLog("stdout", result.stdout);
|
|
if (result.stderr) await onLog("stderr", result.stderr);
|
|
return {
|
|
exitCode: 0,
|
|
signal: null,
|
|
timedOut: false,
|
|
stdout: result.stdout,
|
|
stderr: result.stderr,
|
|
pid: null,
|
|
startedAt,
|
|
};
|
|
} catch (error) {
|
|
const timedOutError = error as NodeJS.ErrnoException & {
|
|
stdout?: string;
|
|
stderr?: string;
|
|
signal?: string | null;
|
|
};
|
|
const stdout = timedOutError.stdout ?? "";
|
|
const stderr = timedOutError.stderr ?? "";
|
|
if (typeof timedOutError.code === "number") {
|
|
if (stdout) await onLog("stdout", stdout);
|
|
if (stderr) await onLog("stderr", stderr);
|
|
return {
|
|
exitCode: timedOutError.code,
|
|
signal: timedOutError.signal ?? null,
|
|
timedOut: false,
|
|
stdout,
|
|
stderr,
|
|
pid: null,
|
|
startedAt,
|
|
};
|
|
}
|
|
if (timedOutError.code !== "ETIMEDOUT") {
|
|
throw error;
|
|
}
|
|
if (stdout) await onLog("stdout", stdout);
|
|
if (stderr) await onLog("stderr", stderr);
|
|
return {
|
|
exitCode: null,
|
|
signal: timedOutError.signal ?? null,
|
|
timedOut: true,
|
|
stdout,
|
|
stderr,
|
|
pid: null,
|
|
startedAt,
|
|
};
|
|
}
|
|
}
|
|
|
|
const shellCommand = preferredSandboxShell(target);
|
|
return await requireSandboxRunner(target).execute({
|
|
command: shellCommand,
|
|
args: shellCommandArgs(command),
|
|
cwd: target.remoteCwd,
|
|
env,
|
|
timeoutMs: (options.timeoutSec ?? 15) * 1000,
|
|
onLog,
|
|
});
|
|
}
|
|
|
|
return await runAdapterExecutionTargetProcess(
|
|
runId,
|
|
target,
|
|
"sh",
|
|
["-lc", command],
|
|
{
|
|
cwd: options.cwd,
|
|
env: options.env,
|
|
timeoutSec: options.timeoutSec ?? 15,
|
|
graceSec: options.graceSec ?? 5,
|
|
onLog,
|
|
},
|
|
);
|
|
}
|
|
|
|
export interface AdapterSandboxInstallCommandCheck {
|
|
code: string;
|
|
level: "info" | "warn" | "error";
|
|
message: string;
|
|
detail?: string;
|
|
hint?: string;
|
|
}
|
|
|
|
// Best-effort run of an adapter-supplied install command on a sandbox target
|
|
// before the resolvability + hello probe. Returns null for non-sandbox
|
|
// targets so callers can no-op. Returns a structured check otherwise — never
|
|
// throws — so the rest of the test still runs and reports the post-install
|
|
// state honestly. Caller pushes the check into its result array; the test
|
|
// report shows whether install was attempted and what came back.
|
|
export async function maybeRunSandboxInstallCommand(input: {
|
|
runId: string;
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
adapterKey: string;
|
|
installCommand: string;
|
|
/** When provided, skip the install if `command -v <detectCommand>` succeeds. */
|
|
detectCommand?: string | null;
|
|
env?: Record<string, string>;
|
|
timeoutSec?: number;
|
|
}): Promise<AdapterSandboxInstallCommandCheck | null> {
|
|
const { target, adapterKey, installCommand } = input;
|
|
if (!target || target.kind !== "remote" || target.transport !== "sandbox") {
|
|
return null;
|
|
}
|
|
const trimmed = installCommand.trim();
|
|
if (trimmed.length === 0) return null;
|
|
|
|
const code = `${adapterKey}_install_command_run`;
|
|
|
|
// Skip install when the binary is already on PATH. Avoids running
|
|
// network-dependent installers (e.g. `curl ... | bash`) on every test
|
|
// probe when the CLI is preinstalled on the lease/template.
|
|
const detectCommand = input.detectCommand?.trim();
|
|
if (detectCommand) {
|
|
try {
|
|
const probe = await runAdapterExecutionTargetShellCommand(
|
|
input.runId,
|
|
target,
|
|
`command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`,
|
|
{
|
|
cwd: target.remoteCwd,
|
|
env: input.env ?? {},
|
|
timeoutSec: 30,
|
|
graceSec: 5,
|
|
},
|
|
);
|
|
if (!probe.timedOut && probe.exitCode === 0) {
|
|
return {
|
|
code,
|
|
level: "info",
|
|
message: `${detectCommand} already on PATH; skipped install.`,
|
|
};
|
|
}
|
|
} catch {
|
|
// Fall through to actually running the install — failure to probe
|
|
// is not a reason to skip the install gate.
|
|
}
|
|
}
|
|
|
|
let result;
|
|
try {
|
|
result = await runAdapterExecutionTargetShellCommand(input.runId, target, trimmed, {
|
|
cwd: target.remoteCwd,
|
|
env: input.env ?? {},
|
|
timeoutSec: input.timeoutSec ?? 240,
|
|
graceSec: 10,
|
|
});
|
|
} catch (err) {
|
|
return {
|
|
code,
|
|
level: "warn",
|
|
message: "Install command threw before completion.",
|
|
detail: err instanceof Error ? err.message : String(err),
|
|
};
|
|
}
|
|
const tail = (text: string) =>
|
|
text.split(/\r?\n/).filter((line) => line.trim().length > 0).slice(-3).join(" | ").slice(0, 480);
|
|
if (result.timedOut) {
|
|
return {
|
|
code,
|
|
level: "warn",
|
|
message: `Install command timed out: ${trimmed}`,
|
|
detail: tail(result.stderr || result.stdout),
|
|
};
|
|
}
|
|
if ((result.exitCode ?? 1) === 0) {
|
|
return {
|
|
code,
|
|
level: "info",
|
|
message: `Install command ran: ${trimmed}`,
|
|
...(tail(result.stdout) ? { detail: tail(result.stdout) } : {}),
|
|
};
|
|
}
|
|
return {
|
|
code,
|
|
level: "warn",
|
|
message: `Install command exited ${result.exitCode}: ${trimmed}`,
|
|
detail: tail(result.stderr || result.stdout),
|
|
};
|
|
}
|
|
|
|
export async function readAdapterExecutionTargetHomeDir(
|
|
runId: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
options: AdapterExecutionTargetShellOptions,
|
|
): Promise<string | null> {
|
|
const result = await runAdapterExecutionTargetShellCommand(
|
|
runId,
|
|
target,
|
|
'printf %s "$HOME"',
|
|
options,
|
|
);
|
|
const homeDir = result.stdout.trim();
|
|
return homeDir.length > 0 ? homeDir : null;
|
|
}
|
|
|
|
export async function ensureAdapterExecutionTargetRuntimeCommandInstalled(input: {
|
|
runId: string;
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
installCommand?: string | null;
|
|
detectCommand?: string | null;
|
|
cwd: string;
|
|
env: Record<string, string>;
|
|
timeoutSec?: number;
|
|
graceSec?: number;
|
|
onLog?: AdapterExecutionTargetShellOptions["onLog"];
|
|
}): Promise<void> {
|
|
const installCommand = input.installCommand?.trim();
|
|
if (!installCommand || input.target?.kind !== "remote" || input.target.transport !== "sandbox") {
|
|
return;
|
|
}
|
|
|
|
const detectCommand = input.detectCommand?.trim();
|
|
if (detectCommand) {
|
|
const probe = await runAdapterExecutionTargetShellCommand(
|
|
input.runId,
|
|
input.target,
|
|
`command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`,
|
|
{
|
|
cwd: input.cwd,
|
|
env: input.env,
|
|
timeoutSec: input.timeoutSec,
|
|
graceSec: input.graceSec,
|
|
},
|
|
);
|
|
if (!probe.timedOut && probe.exitCode === 0) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
const result = await runAdapterExecutionTargetShellCommand(
|
|
input.runId,
|
|
input.target,
|
|
installCommand,
|
|
{
|
|
cwd: input.cwd,
|
|
env: input.env,
|
|
timeoutSec: input.timeoutSec,
|
|
graceSec: input.graceSec,
|
|
onLog: input.onLog,
|
|
},
|
|
);
|
|
|
|
// A failed or timed-out install is not necessarily fatal: the CLI may already
|
|
// be on PATH from a previous lease's install, the template image, or another
|
|
// path entry. Re-run the detect probe (when one is configured) so a transient
|
|
// install failure does not abort the agent run when the binary is reachable.
|
|
const installFailed = result.timedOut || (result.exitCode ?? 0) !== 0;
|
|
if (!installFailed) {
|
|
return;
|
|
}
|
|
if (detectCommand) {
|
|
const recheck = await runAdapterExecutionTargetShellCommand(
|
|
input.runId,
|
|
input.target,
|
|
`command -v ${shellQuote(detectCommand)} >/dev/null 2>&1`,
|
|
{
|
|
cwd: input.cwd,
|
|
env: input.env,
|
|
timeoutSec: input.timeoutSec,
|
|
graceSec: input.graceSec,
|
|
},
|
|
);
|
|
if (!recheck.timedOut && recheck.exitCode === 0) {
|
|
if (input.onLog) {
|
|
const reason = result.timedOut ? "timed out" : `exited ${result.exitCode ?? "?"}`;
|
|
await input.onLog(
|
|
"stderr",
|
|
`[paperclip] Install command ${reason} (${installCommand}) but ${detectCommand} is on PATH; continuing.\n`,
|
|
);
|
|
}
|
|
return;
|
|
}
|
|
}
|
|
|
|
if (result.timedOut) {
|
|
throw new Error(`Timed out while installing the adapter runtime command via: ${installCommand}`);
|
|
}
|
|
throw new Error(`Failed to install the adapter runtime command via: ${installCommand}`);
|
|
}
|
|
|
|
export async function ensureAdapterExecutionTargetFile(
|
|
runId: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
filePath: string,
|
|
options: AdapterExecutionTargetShellOptions,
|
|
): Promise<void> {
|
|
await runAdapterExecutionTargetShellCommand(
|
|
runId,
|
|
target,
|
|
`mkdir -p ${shellQuote(path.posix.dirname(filePath))} && : > ${shellQuote(filePath)}`,
|
|
options,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Ensure a working directory exists (and is a directory) on the execution target.
|
|
*
|
|
* For local targets this delegates to the local `ensureAbsoluteDirectory` helper
|
|
* (Node fs). For remote (SSH/sandbox) targets it shells out and runs
|
|
* `mkdir -p` (when allowed) followed by a `[ -d ]` check so the result reflects
|
|
* the directory state inside the environment, not on the Paperclip host.
|
|
*
|
|
* Throws an Error with a human-readable message on failure.
|
|
*/
|
|
export async function ensureAdapterExecutionTargetDirectory(
|
|
runId: string,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
cwd: string,
|
|
options: AdapterExecutionTargetShellOptions & { createIfMissing?: boolean },
|
|
): Promise<void> {
|
|
const createIfMissing = options.createIfMissing ?? false;
|
|
|
|
if (!target || target.kind === "local") {
|
|
const { ensureAbsoluteDirectory } = await import("./server-utils.js");
|
|
await ensureAbsoluteDirectory(cwd, { createIfMissing });
|
|
return;
|
|
}
|
|
|
|
// Remote (SSH or sandbox): both expect POSIX absolute paths inside the env.
|
|
if (!cwd.startsWith("/")) {
|
|
throw new Error(`Working directory must be an absolute POSIX path on the remote target: "${cwd}"`);
|
|
}
|
|
|
|
const quoted = shellQuote(cwd);
|
|
const script = createIfMissing
|
|
? `mkdir -p ${quoted} && [ -d ${quoted} ]`
|
|
: `[ -d ${quoted} ]`;
|
|
|
|
const result = await runAdapterExecutionTargetShellCommand(runId, target, script, {
|
|
cwd: target.kind === "remote" ? target.remoteCwd : cwd,
|
|
env: options.env,
|
|
timeoutSec: options.timeoutSec ?? 15,
|
|
graceSec: options.graceSec ?? 5,
|
|
onLog: options.onLog,
|
|
});
|
|
|
|
if (result.timedOut) {
|
|
throw new Error(`Timed out checking working directory on remote target: "${cwd}"`);
|
|
}
|
|
if ((result.exitCode ?? 1) !== 0) {
|
|
const detail = (result.stderr || result.stdout || "").trim();
|
|
if (createIfMissing) {
|
|
throw new Error(
|
|
`Could not create working directory "${cwd}" on remote target${detail ? `: ${detail}` : "."}`,
|
|
);
|
|
}
|
|
throw new Error(
|
|
`Working directory does not exist on remote target: "${cwd}"${detail ? ` (${detail})` : ""}`,
|
|
);
|
|
}
|
|
}
|
|
|
|
export function adapterExecutionTargetSessionIdentity(
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): Record<string, unknown> | null {
|
|
if (!target || target.kind === "local") return null;
|
|
if (target.transport === "ssh") return buildRemoteExecutionSessionIdentity(target.spec);
|
|
return {
|
|
transport: "sandbox",
|
|
providerKey: target.providerKey ?? null,
|
|
environmentId: target.environmentId ?? null,
|
|
leaseId: target.leaseId ?? null,
|
|
remoteCwd: target.remoteCwd,
|
|
};
|
|
}
|
|
|
|
export function adapterExecutionTargetSessionMatches(
|
|
saved: unknown,
|
|
target: AdapterExecutionTarget | null | undefined,
|
|
): boolean {
|
|
if (!target || target.kind === "local") {
|
|
return Object.keys(parseObject(saved)).length === 0;
|
|
}
|
|
if (target.transport === "ssh") return remoteExecutionSessionMatches(saved, target.spec);
|
|
const current = adapterExecutionTargetSessionIdentity(target);
|
|
const parsedSaved = parseObject(saved);
|
|
return (
|
|
readStringMeta(parsedSaved, "transport") === current?.transport &&
|
|
readStringMeta(parsedSaved, "providerKey") === current?.providerKey &&
|
|
readStringMeta(parsedSaved, "environmentId") === current?.environmentId &&
|
|
readStringMeta(parsedSaved, "leaseId") === current?.leaseId &&
|
|
readStringMeta(parsedSaved, "remoteCwd") === current?.remoteCwd
|
|
);
|
|
}
|
|
|
|
export function parseAdapterExecutionTarget(value: unknown): AdapterExecutionTarget | null {
|
|
const parsed = parseObject(value);
|
|
const kind = readStringMeta(parsed, "kind");
|
|
|
|
if (kind === "local") {
|
|
return {
|
|
kind: "local",
|
|
environmentId: readStringMeta(parsed, "environmentId"),
|
|
leaseId: readStringMeta(parsed, "leaseId"),
|
|
};
|
|
}
|
|
|
|
if (kind === "remote" && readStringMeta(parsed, "transport") === "ssh") {
|
|
const spec = parseSshRemoteExecutionSpec(parseObject(parsed.spec));
|
|
if (!spec) return null;
|
|
return {
|
|
kind: "remote",
|
|
transport: "ssh",
|
|
environmentId: readStringMeta(parsed, "environmentId"),
|
|
leaseId: readStringMeta(parsed, "leaseId"),
|
|
remoteCwd: spec.remoteCwd,
|
|
spec,
|
|
};
|
|
}
|
|
|
|
if (kind === "remote" && readStringMeta(parsed, "transport") === "sandbox") {
|
|
const remoteCwd = readStringMeta(parsed, "remoteCwd");
|
|
if (!remoteCwd) return null;
|
|
const effectiveCapabilities = parseEffectiveExecutionCapabilities(parsed.effectiveCapabilities);
|
|
return {
|
|
kind: "remote",
|
|
transport: "sandbox",
|
|
providerKey: readStringMeta(parsed, "providerKey"),
|
|
environmentId: readStringMeta(parsed, "environmentId"),
|
|
leaseId: readStringMeta(parsed, "leaseId"),
|
|
remoteCwd,
|
|
timeoutMs: typeof parsed.timeoutMs === "number" ? parsed.timeoutMs : null,
|
|
streamRunLogs: typeof parsed.streamRunLogs === "boolean" ? parsed.streamRunLogs : null,
|
|
// Fail closed: only the literal `true` reads as a grant. An absent field
|
|
// or any other value parses as no grant, so a round-trip never invents one.
|
|
enableSandboxDuplexBridge: parsed.enableSandboxDuplexBridge === true,
|
|
...(effectiveCapabilities ? { effectiveCapabilities } : {}),
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
export function adapterExecutionTargetFromRemoteExecution(
|
|
remoteExecution: unknown,
|
|
metadata: Pick<AdapterLocalExecutionTarget, "environmentId" | "leaseId"> = {},
|
|
): AdapterExecutionTarget | null {
|
|
const parsed = parseObject(remoteExecution);
|
|
const ssh = parseSshRemoteExecutionSpec(parsed);
|
|
if (ssh) {
|
|
return {
|
|
kind: "remote",
|
|
transport: "ssh",
|
|
environmentId: metadata.environmentId ?? null,
|
|
leaseId: metadata.leaseId ?? null,
|
|
remoteCwd: ssh.remoteCwd,
|
|
spec: ssh,
|
|
};
|
|
}
|
|
|
|
return null;
|
|
}
|
|
|
|
export function readAdapterExecutionTarget(input: {
|
|
executionTarget?: unknown;
|
|
legacyRemoteExecution?: unknown;
|
|
}): AdapterExecutionTarget | null {
|
|
if (isAdapterExecutionTargetInstance(input.executionTarget)) {
|
|
return input.executionTarget;
|
|
}
|
|
return (
|
|
parseAdapterExecutionTarget(input.executionTarget) ??
|
|
adapterExecutionTargetFromRemoteExecution(input.legacyRemoteExecution)
|
|
);
|
|
}
|
|
|
|
export async function prepareAdapterExecutionTargetRuntime(input: {
|
|
runId: string;
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
adapterKey: string;
|
|
workspaceLocalDir: string;
|
|
timeoutSec?: number;
|
|
workspaceRemoteDir?: string;
|
|
syncWorkspace?: boolean;
|
|
workspaceInboundMode?: WorkspaceInboundMode;
|
|
workspaceDurableSeed?: WorkspaceDurableSeedPaths;
|
|
workspaceBaseline?: DirectorySnapshot;
|
|
workspaceGitSnapshot?: GitWorkspaceSnapshot | null;
|
|
workspaceExclude?: string[];
|
|
preserveAbsentOnRestore?: string[];
|
|
assets?: AdapterManagedRuntimeAsset[];
|
|
/** Referenced (additional) projects to stage into the sandbox as plain, read-only trees. */
|
|
additionalSources?: SandboxAdditionalSource[];
|
|
installCommand?: string | null;
|
|
/** When provided alongside `installCommand`, skip the install if the binary is already on PATH. */
|
|
detectCommand?: string | null;
|
|
// Optional progress sink for the workspace/asset upload. The returned
|
|
// `restoreWorkspace(onProgress?)` accepts its own sink for teardown. Both are
|
|
// forwarded down to the transport so the sandbox/SSH children can attach byte
|
|
// counters without further changes here.
|
|
onProgress?: RuntimeProgressSink;
|
|
onRuntimeProgress?: RuntimeStatusSink;
|
|
// Optional host span runner for the workspace tarball build. Only the confined
|
|
// sandbox lane uses it: it forwards the runner to prepareCommandManagedRuntime
|
|
// so the host pack time rides one `pack` span under the `stage.sync` step. The
|
|
// SSH and local lanes ignore it. The default is a no-op.
|
|
runtimeSpan?: RuntimeSpanRunner;
|
|
}): Promise<PreparedAdapterExecutionTargetRuntime> {
|
|
const target = input.target ?? { kind: "local" as const };
|
|
if (target.kind === "local") {
|
|
return {
|
|
target,
|
|
workspaceRemoteDir: null,
|
|
runtimeRootDir: null,
|
|
assetDirs: {},
|
|
additionalSourceDirs: {},
|
|
additionalSourceFailures: [],
|
|
workspaceSyncSnapshot: null,
|
|
restoreWorkspace: async () => {},
|
|
};
|
|
}
|
|
|
|
if (target.transport === "ssh") {
|
|
const prepared = await prepareRemoteManagedRuntime({
|
|
spec: target.spec,
|
|
runId: input.runId,
|
|
adapterKey: input.adapterKey,
|
|
workspaceLocalDir: input.workspaceLocalDir,
|
|
workspaceRemoteDir: input.workspaceRemoteDir,
|
|
syncWorkspace: input.syncWorkspace,
|
|
assets: input.assets,
|
|
additionalSources: input.additionalSources,
|
|
onProgress: input.onProgress,
|
|
});
|
|
return {
|
|
target,
|
|
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
|
runtimeRootDir: prepared.runtimeRootDir,
|
|
assetDirs: prepared.assetDirs,
|
|
additionalSourceDirs: prepared.additionalSourceDirs,
|
|
// The SSH transport does not stage referenced projects (it is out of scope), so it never
|
|
// reports a per-project staging failure.
|
|
additionalSourceFailures: [],
|
|
workspaceSyncSnapshot: null,
|
|
restoreWorkspace: prepared.restoreWorkspace,
|
|
};
|
|
}
|
|
|
|
const prepared = await prepareCommandManagedRuntime({
|
|
runner: requireSandboxRunner(target),
|
|
spec: {
|
|
providerKey: target.providerKey,
|
|
shellCommand: target.shellCommand,
|
|
leaseId: target.leaseId,
|
|
remoteCwd: target.remoteCwd,
|
|
timeoutMs:
|
|
input.timeoutSec && input.timeoutSec > 0
|
|
? input.timeoutSec * 1000
|
|
: target.timeoutMs,
|
|
},
|
|
adapterKey: input.adapterKey,
|
|
workspaceLocalDir: input.workspaceLocalDir,
|
|
workspaceRemoteDir: input.workspaceRemoteDir,
|
|
syncWorkspace: input.syncWorkspace,
|
|
workspaceInboundMode: input.workspaceInboundMode,
|
|
workspaceDurableSeed: input.workspaceDurableSeed,
|
|
workspaceBaseline: input.workspaceBaseline,
|
|
workspaceGitSnapshot: input.workspaceGitSnapshot,
|
|
workspaceExclude: input.workspaceExclude,
|
|
preserveAbsentOnRestore: input.preserveAbsentOnRestore,
|
|
assets: input.assets,
|
|
additionalSources: input.additionalSources,
|
|
installCommand: input.installCommand,
|
|
detectCommand: input.detectCommand,
|
|
onProgress: input.onProgress,
|
|
onRuntimeProgress: input.onRuntimeProgress,
|
|
runtimeSpan: input.runtimeSpan,
|
|
});
|
|
return {
|
|
target,
|
|
workspaceRemoteDir: prepared.workspaceRemoteDir,
|
|
runtimeRootDir: prepared.runtimeRootDir,
|
|
assetDirs: prepared.assetDirs,
|
|
additionalSourceDirs: prepared.additionalSourceDirs,
|
|
additionalSourceFailures: prepared.additionalSourceFailures,
|
|
workspaceSyncSnapshot: prepared.workspaceSyncSnapshot,
|
|
restoreWorkspace: prepared.restoreWorkspace,
|
|
};
|
|
}
|
|
|
|
export function runtimeAssetDir(
|
|
prepared: Pick<PreparedAdapterExecutionTargetRuntime, "assetDirs">,
|
|
key: string,
|
|
fallbackRemoteCwd: string,
|
|
): string {
|
|
return prepared.assetDirs[key] ?? path.posix.join(fallbackRemoteCwd, ".paperclip-runtime", key);
|
|
}
|
|
|
|
type GitHubLauncherLocation = {
|
|
runId: string; target: AdapterExecutionTarget | null | undefined;
|
|
};
|
|
|
|
function githubOperationLauncherDirectory(input: GitHubLauncherLocation): string {
|
|
// Only controller-generated run IDs may name a removable directory.
|
|
if (!/^[a-zA-Z0-9_-]+$/.test(input.runId)) throw new Error("Invalid GitHub launcher run ID");
|
|
return input.target?.kind === "remote"
|
|
? path.posix.join(input.target.remoteCwd, ".paperclip-runtime", "github", input.runId)
|
|
: path.join(os.tmpdir(), "paperclip-github-runtime", input.runId);
|
|
}
|
|
|
|
/** Call only after execution settles, before releasing its remote environment lease. */
|
|
export async function cleanupGitHubOperationLaunchers(input: GitHubLauncherLocation): Promise<void> {
|
|
const directory = githubOperationLauncherDirectory(input);
|
|
if (input.target?.kind === "remote") {
|
|
const result = await adapterExecutionTargetCommandRunner(input.target).execute({
|
|
command: "sh", args: ["-c", `rm -rf -- ${shellQuote(directory)}`],
|
|
cwd: input.target.remoteCwd, timeoutMs: 5_000,
|
|
});
|
|
if (result.exitCode !== 0) throw new Error("Could not clean managed GitHub launchers");
|
|
} else {
|
|
await fs.rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
async function githubOperationLauncherBasePath(
|
|
target: AdapterCommandCapableExecutionTarget | null,
|
|
env: Record<string, string>,
|
|
): Promise<string> {
|
|
if (!target) return env.PATH || process.env.PATH || "/usr/bin:/bin";
|
|
const configuredPath = sanitizeRemoteExecutionEnv(env).PATH;
|
|
if (configuredPath !== undefined) return configuredPath;
|
|
|
|
// The provider owns login/profile setup. Query its effective PATH before
|
|
// staging BASH_ENV, rather than substituting the controller's toolchain or
|
|
// a minimal PATH that hides legacy NVM/user-local agent installations.
|
|
const result = await adapterExecutionTargetCommandRunner(target).execute({
|
|
command: "sh",
|
|
args: ["-c", "printf '\\000%s\\000' \"$PATH\""],
|
|
cwd: target.remoteCwd,
|
|
timeoutMs: 15_000,
|
|
});
|
|
// Frame the value so login banners cannot become executable search paths.
|
|
const remotePath = result.stdout.match(/\0([^\0]+)\0/)?.[1];
|
|
if (result.timedOut || result.exitCode !== 0 || !remotePath) {
|
|
throw new Error("Could not resolve remote PATH for managed GitHub launchers");
|
|
}
|
|
return remotePath;
|
|
}
|
|
|
|
/** Read only execution-target Git context; never import the controller's credentials into SSH. */
|
|
export async function prepareGitHubExecutionEnvironment(input: {
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
cwd: string;
|
|
env: Record<string, string>;
|
|
hostCredentials: boolean;
|
|
networkAccess: boolean;
|
|
}): Promise<Record<string, string>> {
|
|
const script = String.raw`
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
const cp = require('node:child_process');
|
|
const env = {};
|
|
env.PAPERCLIP_RUNNER_NETWORK_ROOTS = JSON.stringify(['/etc/resolv.conf','/etc/hosts','/etc/nsswitch.conf','/etc/ssl/certs','/etc/ssl/cert.pem'].flatMap(p => { try { return [fs.realpathSync(p)]; } catch { return []; } }));
|
|
if (process.argv[1] === 'host') {
|
|
for (const [key, value] of Object.entries(process.env)) {
|
|
if (/^(GH_TOKEN|GITHUB_TOKEN|GH_ENTERPRISE_TOKEN|GITHUB_ENTERPRISE_TOKEN|PAPERCLIP_GIT_TOKEN|GH_CONFIG_DIR|GIT_CONFIG_(GLOBAL|SYSTEM|NOSYSTEM|COUNT|KEY_\d+|VALUE_\d+)|GIT_(AUTHOR|COMMITTER)_(NAME|EMAIL)|GIT_ASKPASS|SSH_ASKPASS|SSH_AUTH_SOCK|GIT_SSH_COMMAND|GIT_SSH)$/.test(key)) env[key] = value;
|
|
}
|
|
env.PAPERCLIP_GITHUB_HOST_HOME = process.env.HOME || '';
|
|
env.GH_CONFIG_DIR ||= path.join(process.env.XDG_CONFIG_HOME || path.join(process.env.HOME || '', '.config'), 'gh');
|
|
}
|
|
try {
|
|
const top = cp.execFileSync('git', ['rev-parse', '--show-toplevel'], {encoding:'utf8',stdio:['ignore','pipe','ignore']}).trim();
|
|
if (fs.realpathSync(top) === fs.realpathSync(process.cwd())) {
|
|
env.PAPERCLIP_GIT_METADATA_ROOTS = JSON.stringify(cp.execFileSync('git', ['rev-parse','--path-format=absolute','--git-common-dir','--git-dir'], {encoding:'utf8',stdio:['ignore','pipe','ignore']}).trim().split('\n').map(p => fs.realpathSync(p)));
|
|
}
|
|
} catch {}
|
|
process.stdout.write("\0" + JSON.stringify(env) + "\0");
|
|
`;
|
|
const args = ["-e", script, input.hostCredentials ? "host" : "managed"];
|
|
const remote = input.target?.kind === "remote" ? input.target : null;
|
|
let discovered: Record<string, string>;
|
|
if (remote) {
|
|
// A legacy SSH host may run a standalone agent binary without Node. Use
|
|
// only the shell and Git, and emit bounded, NUL-framed environment records.
|
|
const probe = String.raw`
|
|
printf '\0PAPERCLIP_GIT_CONTEXT_V1\0'
|
|
if [ "$1" = host ]; then
|
|
for key in GH_TOKEN GITHUB_TOKEN GH_ENTERPRISE_TOKEN GITHUB_ENTERPRISE_TOKEN PAPERCLIP_GIT_TOKEN GH_CONFIG_DIR GIT_CONFIG_GLOBAL GIT_CONFIG_SYSTEM GIT_CONFIG_NOSYSTEM GIT_CONFIG_COUNT GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL GIT_ASKPASS SSH_ASKPASS SSH_AUTH_SOCK GIT_SSH_COMMAND GIT_SSH; do
|
|
eval 'value=${"$"}{'"$key"'-}'
|
|
[ -z "$value" ] || printf '%s\0%s\0' "$key" "$value"
|
|
done
|
|
index=0
|
|
while [ "$index" -lt 32 ]; do
|
|
for prefix in GIT_CONFIG_KEY_ GIT_CONFIG_VALUE_; do
|
|
key="$prefix$index"
|
|
eval 'value=${"$"}{'"$key"'-}'
|
|
[ -z "$value" ] || printf '%s\0%s\0' "$key" "$value"
|
|
done
|
|
index=$((index + 1))
|
|
done
|
|
printf 'PAPERCLIP_GITHUB_HOST_HOME\0%s\0' "$HOME"
|
|
printf 'GH_CONFIG_DIR\0%s\0' "${"$"}{GH_CONFIG_DIR:-${"$"}{XDG_CONFIG_HOME:-$HOME/.config}/gh}"
|
|
fi
|
|
for file in /etc/resolv.conf /etc/hosts /etc/nsswitch.conf /etc/ssl/certs /etc/ssl/cert.pem; do
|
|
index=0
|
|
while [ -L "$file" ] && [ "$index" -lt 40 ]; do
|
|
target=$(readlink "$file") || break
|
|
case "$target" in /*) file="$target" ;; *) file="$(dirname "$file")/$target" ;; esac
|
|
index=$((index + 1))
|
|
done
|
|
if [ -e "$file" ]; then
|
|
parent=$(cd "$(dirname "$file")" && pwd -P) || continue
|
|
printf 'PAPERCLIP_RUNNER_NETWORK_ROOT\0%s\0' "$parent/$(basename "$file")"
|
|
fi
|
|
done
|
|
cwd=$(pwd -P)
|
|
top=$(git rev-parse --show-toplevel 2>/dev/null) || top=
|
|
if [ -n "$top" ] && [ "$(cd "$top" && pwd -P)" = "$cwd" ]; then
|
|
for kind in --git-common-dir --git-dir; do
|
|
root=$(git rev-parse --path-format=absolute "$kind" 2>/dev/null) || continue
|
|
root=$(cd "$root" && pwd -P) || continue
|
|
printf 'PAPERCLIP_GIT_METADATA_ROOT\0%s\0' "$root"
|
|
done
|
|
fi
|
|
printf '\0PAPERCLIP_GIT_CONTEXT_END\0'
|
|
`;
|
|
const result = await adapterExecutionTargetCommandRunner(remote).execute({
|
|
command: "sh", args: ["-c", probe, "paperclip-git-context", input.hostCredentials ? "host" : "managed"],
|
|
// The caller's cwd belongs to the controller. Copied sandbox/SSH
|
|
// workspaces can live at a different path on the execution target.
|
|
cwd: remote.remoteCwd, timeoutMs: 15_000,
|
|
});
|
|
if (result.exitCode !== 0) throw new Error("Could not read execution-target Git context");
|
|
const payload = result.stdout.split("\0PAPERCLIP_GIT_CONTEXT_V1\0")[1]?.split("\0PAPERCLIP_GIT_CONTEXT_END\0")[0];
|
|
if (payload === undefined) throw new Error("Could not read execution-target Git context");
|
|
discovered = {};
|
|
const records = payload.split("\0");
|
|
const roots: string[] = [];
|
|
const networkRoots: string[] = [];
|
|
for (let index = 0; index + 1 < records.length; index += 2) {
|
|
const key = records[index]!;
|
|
const value = records[index + 1]!;
|
|
if (key === "PAPERCLIP_GIT_METADATA_ROOT") roots.push(value);
|
|
else if (key === "PAPERCLIP_RUNNER_NETWORK_ROOT") networkRoots.push(value);
|
|
else discovered[key] = value;
|
|
}
|
|
discovered.PAPERCLIP_GIT_METADATA_ROOTS = JSON.stringify([...new Set(roots)]);
|
|
discovered.PAPERCLIP_RUNNER_NETWORK_ROOTS = JSON.stringify([...new Set(networkRoots)]);
|
|
} else {
|
|
const result = await promisify(execFile)(process.execPath, args, { cwd: input.cwd, timeout: 15_000, maxBuffer: 1024 * 1024 });
|
|
try { discovered = JSON.parse(result.stdout.split("\0")[1] ?? ""); }
|
|
catch { throw new Error("Could not read execution-target Git context"); }
|
|
}
|
|
// Controller-derived roots and mode must not be replaced by agent bindings.
|
|
return { ...discovered, ...input.env,
|
|
...(input.hostCredentials ? { PAPERCLIP_GITHUB_HOST_HOME: discovered.PAPERCLIP_GITHUB_HOST_HOME } : {}),
|
|
PAPERCLIP_GIT_METADATA_ROOTS: discovered.PAPERCLIP_GIT_METADATA_ROOTS ?? "[]",
|
|
PAPERCLIP_RUNNER_NETWORK_ROOTS: discovered.PAPERCLIP_RUNNER_NETWORK_ROOTS ?? "[]",
|
|
PAPERCLIP_GITHUB_AUTH_MODE: input.hostCredentials ? "host" : "managed",
|
|
PAPERCLIP_RUNNER_NETWORK_ACCESS: input.networkAccess ? "enabled" : "disabled",
|
|
};
|
|
}
|
|
|
|
/** Stage token-free launchers next to the execution, not in shared global Git config. */
|
|
export async function prepareGitHubOperationLaunchers(input: {
|
|
runId: string; target: AdapterExecutionTarget | null | undefined; cwd: string; env: Record<string, string>;
|
|
}): Promise<Record<string, string>> {
|
|
const remote = input.target?.kind === "remote" ? input.target : null;
|
|
const directory = githubOperationLauncherDirectory(input);
|
|
const configDirectory = path.posix.join(directory, "gh-config");
|
|
const basePath = await githubOperationLauncherBasePath(remote, input.env);
|
|
const managedPath = basePath ? `${directory}:${basePath}` : directory;
|
|
// Login shells may reorder PATH through /etc/profile or path_helper. Restore
|
|
// the managed launchers after startup without loading a host user's profile.
|
|
const profile = `export PATH=${shellQuote(managedPath)}\n`;
|
|
const files: Record<string, string> = Object.fromEntries([
|
|
...["git", "gh"].map((name) => [name, githubLauncherSource()] as const),
|
|
...[".zshenv", ".zprofile", ".zshrc", ".bash_profile", ".bashrc", ".profile"].map((name) => [name, profile] as const),
|
|
]);
|
|
if (remote) {
|
|
const runner = adapterExecutionTargetCommandRunner(remote);
|
|
for (const [program, body] of Object.entries(files)) {
|
|
await syncRemoteTextFileWithHashSkip({
|
|
runner, remoteCwd: remote.remoteCwd, remoteDir: directory,
|
|
remotePath: path.posix.join(directory, program), body,
|
|
label: "GitHub operation launcher", action: "stage GitHub operation launcher",
|
|
lockDir: path.posix.join(directory, `.${program}.lock`),
|
|
timeoutMs: 15_000, shellCommand: adapterExecutionTargetShellCommand(remote),
|
|
});
|
|
}
|
|
const permissions = await runner.execute({ command: "sh", args: ["-c", `chmod 700 ${shellQuote(directory)}/git ${shellQuote(directory)}/gh && mkdir -p ${shellQuote(configDirectory)}`], cwd: remote.remoteCwd, timeoutMs: 15_000 });
|
|
if (permissions.exitCode !== 0) throw new Error("Could not prepare managed GitHub launchers");
|
|
} else {
|
|
await fs.mkdir(directory, { recursive: true, mode: 0o700 });
|
|
await fs.mkdir(configDirectory, { recursive: true, mode: 0o700 });
|
|
for (const [program, body] of Object.entries(files)) await fs.writeFile(path.join(directory, program), body, { mode: 0o700 });
|
|
}
|
|
return { ...input.env, PATH: managedPath, ZDOTDIR: directory, BASH_ENV: `${directory}/.bashrc`,
|
|
GH_CONFIG_DIR: configDirectory, PAPERCLIP_GITHUB_LAUNCHER_DIR: directory };
|
|
}
|
|
|
|
function buildBridgeResponseHeaders(response: Response): Record<string, string> {
|
|
const out: Record<string, string> = {};
|
|
// Keep `x-paperclip-bridge-outcome` in this list. The host marks a
|
|
// possibly-committed mutation with the `indeterminate` outcome. The in-sandbox
|
|
// server reads that header to map the 504 to a terminal 409. If the forward
|
|
// drops the header, the server keeps the retryable 504 and a caller that
|
|
// retries 5xx can repeat a mutation that already committed.
|
|
for (const key of ["content-type", "etag", "last-modified", "x-paperclip-bridge-outcome"]) {
|
|
const value = response.headers.get(key);
|
|
if (value && value.trim().length > 0) out[key] = value.trim();
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function buildBridgeForwardUrl(baseUrl: string, request: { path: string; query: string }): URL {
|
|
const url = new URL(request.path, baseUrl);
|
|
const query = request.query.trim();
|
|
url.search = query.startsWith("?") ? query.slice(1) : query;
|
|
return url;
|
|
}
|
|
|
|
function bridgeResponseBodyLimitError(maxBodyBytes: number): Error {
|
|
return new Error(`Bridge response body exceeded the configured size limit of ${maxBodyBytes} bytes.`);
|
|
}
|
|
|
|
/**
|
|
* Read the forward response body into a `Buffer`, with no text decoding. The
|
|
* per-request `maxBodyBytes` limit rejects a body larger than the configured
|
|
* per-request ceiling.
|
|
*
|
|
* When the caller passes a `reservation`, this reserves each chunk's bytes
|
|
* against it immediately after `reader.read()` yields the chunk, and before
|
|
* `Buffer.from(value)` copies it — the allocation happens inside that
|
|
* expression, so reserving only before the later `chunks.push` would let the
|
|
* copy happen first. It also reserves the concatenated buffer's own byte
|
|
* count before `Buffer.concat` allocates it: the chunk array and the
|
|
* concatenated buffer are two separate live copies. A denied reservation
|
|
* cancels the reader and throws {@link BridgeProcessCapacityError}, copying
|
|
* no further chunk. This function never releases the reservation; the
|
|
* stream owner that created it does, once the whole forward call settles.
|
|
*
|
|
* With no `reservation`, this function enforces only the one request's own
|
|
* ceiling, exactly as it did before this parameter existed — the queue
|
|
* transport calls it with no reservation, and its behavior must not change.
|
|
*/
|
|
async function readBridgeForwardResponseBody(
|
|
response: Response,
|
|
maxBodyBytes: number,
|
|
reservation?: BridgeBodyReservation,
|
|
): Promise<Buffer> {
|
|
const rawContentLength = response.headers.get("content-length");
|
|
if (rawContentLength) {
|
|
const contentLength = Number.parseInt(rawContentLength, 10);
|
|
if (Number.isFinite(contentLength) && contentLength > maxBodyBytes) {
|
|
throw bridgeResponseBodyLimitError(maxBodyBytes);
|
|
}
|
|
}
|
|
|
|
if (!response.body) {
|
|
return Buffer.alloc(0);
|
|
}
|
|
|
|
const reader = response.body.getReader();
|
|
const chunks: Buffer[] = [];
|
|
let totalBytes = 0;
|
|
while (true) {
|
|
const { done, value } = await reader.read();
|
|
if (done) break;
|
|
if (!value) continue;
|
|
const chunkBytes = value.byteLength;
|
|
totalBytes += chunkBytes;
|
|
if (totalBytes > maxBodyBytes) {
|
|
await reader.cancel().catch(() => undefined);
|
|
throw bridgeResponseBodyLimitError(maxBodyBytes);
|
|
}
|
|
if (reservation && !reservation.reserve(chunkBytes)) {
|
|
await reader.cancel().catch(() => undefined);
|
|
throw new BridgeProcessCapacityError();
|
|
}
|
|
chunks.push(Buffer.from(value));
|
|
}
|
|
if (reservation && !reservation.reserve(totalBytes)) {
|
|
await reader.cancel().catch(() => undefined);
|
|
throw new BridgeProcessCapacityError();
|
|
}
|
|
return Buffer.concat(chunks, totalBytes);
|
|
}
|
|
|
|
const PROCESS_SESSION_PROXY_SCRIPT = "paperclip-process-session-proxy.mjs";
|
|
const PROCESS_SESSION_REMOTE_SCRIPT = "paperclip-process-session-remote.mjs";
|
|
// The streamed variant writes its output frames to stdout, so it rides a
|
|
// separate remote path. A sandbox can hold both scripts without the content
|
|
// hash-skip gate thrashing when a run switches output mode.
|
|
const PROCESS_SESSION_REMOTE_STREAM_SCRIPT = "paperclip-process-session-remote-stream.mjs";
|
|
const PROCESS_SESSION_AUTH_TIMEOUT_MS = 5_000;
|
|
// The bounded budget `stop()` waits for the wrapper's `shutdownAck` event
|
|
// before it removes `sessionDir` unconditionally. The wrapper writes the
|
|
// acknowledgement right after it arms its own kill timer, well before its
|
|
// child actually exits, so this budget only needs to cover message delivery,
|
|
// not the child's full shutdown.
|
|
const DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS = 3_000;
|
|
|
|
function jsonLine(value: unknown): string {
|
|
return `${JSON.stringify(value)}\n`;
|
|
}
|
|
|
|
function splitJsonLines(buffer: string): { lines: string[]; rest: string } {
|
|
const parts = buffer.split(/\n/);
|
|
return { lines: parts.slice(0, -1), rest: parts.at(-1) ?? "" };
|
|
}
|
|
|
|
async function writeProcessSessionProxyScript(dir: string, port: number, token: string): Promise<string> {
|
|
await fs.mkdir(dir, { recursive: true });
|
|
const proxyPath = path.join(dir, PROCESS_SESSION_PROXY_SCRIPT);
|
|
await fs.writeFile(proxyPath, getProcessSessionProxySource({ port, token }), { mode: 0o700 });
|
|
return proxyPath;
|
|
}
|
|
|
|
// Content-hash-skip the process-session remote script write, mirroring the
|
|
// sandbox callback bridge entrypoint sha256 gate. The script is a static
|
|
// Paperclip-authored `.mjs` that only changes when the build changes, so on a
|
|
// warm start (same sandbox, script already present) the single sha-gate exec
|
|
// skips the ~3-exec base64 upload entirely. `syncRemoteTextFileWithHashSkip`
|
|
// fails loud on a check error rather than silently re-uploading.
|
|
async function syncProcessSessionRemoteScript(input: {
|
|
runner: CommandManagedRuntimeRunner;
|
|
remoteCwd: string;
|
|
remoteScriptDir: string;
|
|
remoteScriptPath: string;
|
|
timeoutMs?: number | null;
|
|
shellCommand?: "bash" | "sh" | null;
|
|
outputToStdout?: boolean;
|
|
}): Promise<{ uploaded: boolean }> {
|
|
const { uploaded } = await syncRemoteTextFileWithHashSkip({
|
|
runner: input.runner,
|
|
remoteCwd: input.remoteCwd,
|
|
remoteDir: input.remoteScriptDir,
|
|
remotePath: input.remoteScriptPath,
|
|
body: getProcessSessionRemoteSource({ outputToStdout: input.outputToStdout === true }),
|
|
label: "Process session remote script",
|
|
action: "sync process session remote script",
|
|
lockDir: path.posix.join(input.remoteScriptDir, ".paperclip-process-session-script.lock"),
|
|
timeoutMs: input.timeoutMs,
|
|
shellCommand: input.shellCommand,
|
|
});
|
|
return { uploaded };
|
|
}
|
|
|
|
async function readRemoteJsonFiles(input: {
|
|
client: ReturnType<typeof createCommandManagedSandboxCallbackBridgeQueueClient>;
|
|
dir: string;
|
|
}): Promise<Array<{ name: string; body: string }>> {
|
|
const names = await input.client.listJsonFiles(input.dir);
|
|
const out: Array<{ name: string; body: string }> = [];
|
|
for (const name of names) {
|
|
const filePath = path.posix.join(input.dir, name);
|
|
const body = await input.client.readTextFile(filePath);
|
|
await input.client.remove(filePath).catch(() => undefined);
|
|
out.push({ name, body });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
async function waitForLocalServerListen(server: net.Server): Promise<number> {
|
|
await new Promise<void>((resolve, reject) => {
|
|
server.once("error", reject);
|
|
server.listen(0, "127.0.0.1", () => {
|
|
server.off("error", reject);
|
|
resolve();
|
|
});
|
|
});
|
|
const address = server.address();
|
|
if (!address || typeof address === "string") {
|
|
throw new Error("Process session bridge did not expose a TCP port.");
|
|
}
|
|
return address.port;
|
|
}
|
|
|
|
/** Span name that wraps the socket handler's one `writeTextFile` exec — one
|
|
* outbound ACP message to the agent. */
|
|
const AGENT_SESSION_SEND_INPUT_SPAN = "sandbox.agentSession.sendInput";
|
|
|
|
/** Span name that wraps one 100 ms poll tick — `list`, then `read`+`remove` per
|
|
* file found (`1 + 2n` execs). */
|
|
const AGENT_SESSION_POLL_OUTPUT_SPAN = "sandbox.agentSession.pollOutput";
|
|
|
|
export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
|
runId: string;
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
runtimeRootDir: string | null | undefined;
|
|
adapterKey: string;
|
|
command: string;
|
|
args: string[];
|
|
cwd: string;
|
|
// The launch env is consumed ONLY when building the base64 `commandPayload`
|
|
// below — never during the env-INDEPENDENT dir/script setup. Accepting a
|
|
// resolver (in addition to a plain object) lets a caller overlap that setup
|
|
// with other work — e.g. starting the paperclip callback bridge — and hand the
|
|
// merged env in right before the launch.
|
|
env: Record<string, string> | (() => Promise<Record<string, string>>);
|
|
timeoutSec?: number | null;
|
|
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
|
// Return the current-run parent-context token. The socket handlers and the
|
|
// poll timer read it per unit of work and run under it, so their run-time
|
|
// `sandbox.exec` spans parent to the live run span (`agent.turn` during the
|
|
// turn, `task.run` otherwise). When it is absent, the work runs with an empty
|
|
// store, exactly like the earlier `runWithoutActiveStep` behavior.
|
|
getRuntimeParentContext?: () => StartupSpanContext | undefined;
|
|
// Wrap each unit of run-time work in its own named span. The socket handler
|
|
// uses it for `sandbox.agentSession.sendInput` and the poll timer for
|
|
// `sandbox.agentSession.pollOutput`, so each unit's inner `sandbox.exec` spans
|
|
// group under one wrapper span. When it is absent, the work runs under the run
|
|
// parent with no wrapper span, exactly like the earlier behavior.
|
|
runtimeSpan?: RuntimeSpanRunner;
|
|
// Stream the agent output through the persistent session log stream instead of
|
|
// the host output-file poll. When true, the bridge runs the wrapper as one
|
|
// long-lived session command and reads its stdout frames from the stream, and
|
|
// it does not start the 100 ms poll. Default OFF: the bridge keeps the poll.
|
|
streamOutputViaSession?: boolean;
|
|
}): Promise<AdapterExecutionTargetProcessSessionBridgeHandle | null> {
|
|
if (!input.target || input.target.kind !== "remote" || input.target.transport !== "sandbox") {
|
|
return null;
|
|
}
|
|
|
|
const target = input.target;
|
|
const onLog = input.onLog ?? (async () => {});
|
|
const runner = requireSandboxRunner(target);
|
|
// Run one unit of run-time work under its named wrapper span when a span
|
|
// runner is injected. Without a runner, run the work under the current run
|
|
// parent, so the inner `sandbox.exec` spans parent to the live run span,
|
|
// exactly like the earlier behavior.
|
|
const runRuntimeWork = <T>(name: string, work: () => Promise<T>): Promise<T> =>
|
|
input.runtimeSpan
|
|
? input.runtimeSpan(name, work)
|
|
: runWithRuntimeParent(input.getRuntimeParentContext?.(), work);
|
|
const shellCommand = preferredSandboxShell(target);
|
|
const timeoutMs =
|
|
typeof input.timeoutSec === "number" && Number.isFinite(input.timeoutSec) && input.timeoutSec > 0
|
|
? Math.trunc(input.timeoutSec * 1000)
|
|
: target.timeoutMs ?? undefined;
|
|
const bridgeRuntimeDir = path.posix.join(
|
|
input.runtimeRootDir?.trim() || path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey),
|
|
"process-sessions",
|
|
);
|
|
const sessionId = randomUUID();
|
|
const sessionDir = path.posix.join(bridgeRuntimeDir, sessionId);
|
|
const stdinDir = path.posix.join(sessionDir, "stdin");
|
|
const eventsDir = path.posix.join(sessionDir, "events");
|
|
// The streamed wrapper writes its frames to stdout and rides a separate remote
|
|
// path, so a warm sandbox can hold both wrapper scripts without the content
|
|
// hash-skip gate thrashing when a run switches output mode.
|
|
const streamOutput = input.streamOutputViaSession === true;
|
|
const remoteScriptPath = path.posix.join(
|
|
bridgeRuntimeDir,
|
|
streamOutput ? PROCESS_SESSION_REMOTE_STREAM_SCRIPT : PROCESS_SESSION_REMOTE_SCRIPT,
|
|
);
|
|
const client = createCommandManagedSandboxCallbackBridgeQueueClient({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
timeoutMs,
|
|
shellCommand,
|
|
});
|
|
|
|
// The launch exec below re-creates stdinDir and eventsDir with one `mkdir -p`,
|
|
// and the remote script also creates them on start. No reader touches the two
|
|
// directories before the launch exec runs, so upfront makeDir execs are redundant.
|
|
await syncProcessSessionRemoteScript({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
remoteScriptDir: bridgeRuntimeDir,
|
|
remoteScriptPath,
|
|
timeoutMs,
|
|
shellCommand,
|
|
outputToStdout: streamOutput,
|
|
});
|
|
|
|
// Resolve the launch env AFTER the env-independent setup above, so a caller
|
|
// can defer it until an upstream dependency (e.g. the paperclip bridge's env)
|
|
// is ready without blocking the dir/script setup.
|
|
const launchEnv = typeof input.env === "function" ? await input.env() : input.env;
|
|
const commandPayload = Buffer.from(JSON.stringify({
|
|
command: input.command,
|
|
args: input.args,
|
|
cwd: input.cwd || target.remoteCwd,
|
|
// The ACP engine has already projected this launch env from explicit
|
|
// adapter/runtime inputs and registered contributions. Compare against an
|
|
// empty inherited baseline so an explicit identity value (notably PATH)
|
|
// is not reclassified as ambient merely because it equals the host value.
|
|
env: sanitizeRemoteExecutionEnv(launchEnv, {}),
|
|
}), "utf8").toString("base64");
|
|
|
|
// Legacy poll path: background the wrapper with `nohup` and read its output
|
|
// event files with the host poll below. The streamed path launches the wrapper
|
|
// as one foreground session command further down instead, so skip this.
|
|
if (!streamOutput) {
|
|
await onLog("stdout", `[paperclip] Starting ACP process session bridge in sandbox (${target.providerKey ?? "provider"}).\n`);
|
|
const startResult = await runner.execute({
|
|
command: shellCommand,
|
|
args: shellCommandArgs(
|
|
[
|
|
`mkdir -p ${shellQuote(stdinDir)} ${shellQuote(eventsDir)}`,
|
|
// I3: no numeric process identifier anywhere. Background the
|
|
// wrapper and let it go; do not capture `$!`.
|
|
`PAPERCLIP_PROCESS_SESSION_DIR=${shellQuote(sessionDir)} ` +
|
|
`PAPERCLIP_PROCESS_SESSION_COMMAND_B64=${shellQuote(commandPayload)} ` +
|
|
`nohup node ${shellQuote(remoteScriptPath)} >/dev/null 2>&1 < /dev/null &`,
|
|
].join("\n"),
|
|
),
|
|
cwd: target.remoteCwd,
|
|
env: {
|
|
PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
|
|
},
|
|
timeoutMs,
|
|
// The wrapper launch is bridge plumbing. Keep it off the persistent
|
|
// session so it never queues behind an in-run session command.
|
|
bypassSession: true,
|
|
});
|
|
if (startResult.timedOut || (startResult.exitCode ?? 1) !== 0) {
|
|
throw new Error(`Failed to start sandbox ACP process session bridge: ${startResult.stderr || startResult.stdout}`);
|
|
}
|
|
}
|
|
|
|
let socket: net.Socket | null = null;
|
|
let stopping = false;
|
|
// Resolves when `stop()` tears the bridge down. The streamed `sandbox.agentProcess`
|
|
// span races its work against this, so the span ends at teardown at the latest
|
|
// even when the remote process lingers, and never outlives the run root span.
|
|
let signalStopped: () => void = () => {};
|
|
const stopped = new Promise<void>((resolve) => {
|
|
signalStopped = resolve;
|
|
});
|
|
let stdinSeq = 0;
|
|
// One promise chain per session that serializes the stdin file writes. Each
|
|
// write is multi-exec on the command-managed client: prepare, append per 32
|
|
// KiB, then an atomic rename. The chain makes the rename for file N finish
|
|
// before the write for file N+1 starts, so the files land in send order.
|
|
// Without it the writes overlap. A small later chunk can then rename ahead of
|
|
// a big earlier chunk, so the wrapper reads the stdin bytes out of order and
|
|
// corrupts a large prompt on the stdin path.
|
|
let stdinWriteChain: Promise<void> = Promise.resolve();
|
|
let pollTimer: NodeJS.Timeout | null = null;
|
|
const pendingRemoteEvents: Array<{
|
|
type?: string;
|
|
stream?: "stdout" | "stderr";
|
|
data?: string;
|
|
code?: number | null;
|
|
signal?: string | null;
|
|
message?: string;
|
|
}> = [];
|
|
const token = createSandboxCallbackBridgeToken(18);
|
|
const proxyDir = await fs.mkdtemp(path.join(os.tmpdir(), "paperclip-process-session-proxy-"));
|
|
// `stop()` waits on this promise, bounded, for the wrapper's `shutdownAck`
|
|
// event. `deliverRemoteEvent` resolves it below and never forwards the
|
|
// event further: it is a host-internal control ack, not part of the ACP
|
|
// output stream. An event under `sessionDir` is untrusted telemetry: an
|
|
// `exit` or `error` event is never treated as proof of shutdown, because
|
|
// any process running under the sandbox can write one. Only `shutdownAck`
|
|
// counts, and `stop()` also gives itself a dedicated reader for it below,
|
|
// so a late `shutdownAck` still lands even after the long-lived poll has
|
|
// stopped re-arming.
|
|
let signalShutdownAcknowledged: () => void = () => {};
|
|
const shutdownAcknowledged = new Promise<void>((resolve) => {
|
|
signalShutdownAcknowledged = resolve;
|
|
});
|
|
|
|
const writeRemoteEventToSocket = (event: (typeof pendingRemoteEvents)[number]) => {
|
|
if (!socket) return false;
|
|
socket.write(jsonLine(event));
|
|
if (event.type === "exit") {
|
|
stopping = true;
|
|
socket.end();
|
|
} else if (event.type === "error") {
|
|
stopping = true;
|
|
socket.destroy();
|
|
}
|
|
return true;
|
|
};
|
|
|
|
const deliverRemoteEvent = (event: (typeof pendingRemoteEvents)[number]) => {
|
|
if (event.type === "shutdownAck") {
|
|
signalShutdownAcknowledged();
|
|
return;
|
|
}
|
|
if (socket) {
|
|
writeRemoteEventToSocket(event);
|
|
return;
|
|
}
|
|
pendingRemoteEvents.push(event);
|
|
if (event.type === "exit" || event.type === "error") {
|
|
stopping = true;
|
|
}
|
|
};
|
|
|
|
const flushPendingRemoteEvents = () => {
|
|
if (!socket) return;
|
|
while (pendingRemoteEvents.length > 0 && socket) {
|
|
const event = pendingRemoteEvents.shift();
|
|
if (event) writeRemoteEventToSocket(event);
|
|
}
|
|
};
|
|
|
|
const liveSockets = new Set<net.Socket>();
|
|
// Register the per-connection socket handlers with no run parent context.
|
|
// A stdin write from a socket handler is a run-time exec, not startup work.
|
|
// The connection can open under `task.run` and receive stdin later, during an
|
|
// `agent.turn`. So the handler must read the current-run parent at send time,
|
|
// not at connect time. A connect-time read captures the parent that was live
|
|
// when the socket opened, and every later exec span parents to that stale
|
|
// parent. The `data` handler below reads the getter per message instead.
|
|
const server = net.createServer((nextSocket) => {
|
|
liveSockets.add(nextSocket);
|
|
nextSocket.setEncoding("utf8");
|
|
nextSocket.on("error", () => undefined);
|
|
let connectionBuffer = "";
|
|
let authenticated = false;
|
|
// Connections own the session (and receive buffered process output) only
|
|
// after presenting the bridge token; idle unauthenticated peers are dropped.
|
|
const authTimer = setTimeout(() => {
|
|
if (!authenticated) nextSocket.destroy();
|
|
}, PROCESS_SESSION_AUTH_TIMEOUT_MS);
|
|
authTimer.unref?.();
|
|
nextSocket.on("close", () => {
|
|
clearTimeout(authTimer);
|
|
liveSockets.delete(nextSocket);
|
|
});
|
|
nextSocket.on("data", (chunk) => {
|
|
connectionBuffer += chunk;
|
|
const split = splitJsonLines(connectionBuffer);
|
|
connectionBuffer = split.rest;
|
|
for (const line of split.lines) {
|
|
if (!line.trim()) continue;
|
|
let message: { token?: string; type?: string; data?: string };
|
|
try {
|
|
message = JSON.parse(line) as { token?: string; type?: string; data?: string };
|
|
} catch {
|
|
nextSocket.destroy();
|
|
return;
|
|
}
|
|
if (message.token !== token) {
|
|
nextSocket.destroy();
|
|
return;
|
|
}
|
|
if (!authenticated) {
|
|
if (socket) {
|
|
nextSocket.destroy();
|
|
return;
|
|
}
|
|
authenticated = true;
|
|
clearTimeout(authTimer);
|
|
socket = nextSocket;
|
|
flushPendingRemoteEvents();
|
|
}
|
|
// Wrap one outbound ACP message to the agent in a
|
|
// `sandbox.agentSession.sendInput` span, so its one `writeTextFile` exec
|
|
// groups under one named span. The span runner reads the current-run
|
|
// parent at send time: the live parent switches to `agent.turn` during
|
|
// the turn and back to `task.run` after it. A message that is neither
|
|
// `stdin` nor `stdinEnd` writes nothing, so it opens no span.
|
|
const stdinPayload =
|
|
message.type === "stdin" && typeof message.data === "string"
|
|
? { type: "stdin", data: message.data }
|
|
: message.type === "stdinEnd"
|
|
? { type: "stdinEnd" }
|
|
: null;
|
|
if (stdinPayload) {
|
|
stdinSeq += 1;
|
|
const name = `${String(stdinSeq).padStart(12, "0")}.json`;
|
|
const filePath = path.posix.join(stdinDir, name);
|
|
// Chain this write after the previous one, so the atomic rename for
|
|
// file N finishes before the write for file N+1 starts. Keep the
|
|
// per-message `sandbox.agentSession.sendInput` span inside the chain.
|
|
const write = stdinWriteChain.then(() =>
|
|
runRuntimeWork(AGENT_SESSION_SEND_INPUT_SPAN, () =>
|
|
client.writeTextFile(filePath, jsonLine(stdinPayload)),
|
|
),
|
|
);
|
|
// The next message chains after this write on success or failure, so a
|
|
// failed write never blocks the chain. This mirrors the wrapper
|
|
// `writeChain` pattern for its event files.
|
|
stdinWriteChain = write.then(() => undefined, () => undefined);
|
|
// Keep the failure behavior: send one error line, then destroy the socket.
|
|
write.catch((error) => {
|
|
nextSocket.write(jsonLine({ type: "error", message: error instanceof Error ? error.message : String(error) }));
|
|
nextSocket.destroy();
|
|
});
|
|
}
|
|
}
|
|
});
|
|
});
|
|
|
|
const poll = async () => {
|
|
if (stopping) return;
|
|
try {
|
|
// Read every file this tick fetched before this loop decides whether to
|
|
// keep polling. A `shutdownAck` can land in the same batch right after
|
|
// an `exit` event; deliver it too, so this tick never drops an
|
|
// already-fetched (and already-removed-from-disk) event.
|
|
const events = await readRemoteJsonFiles({ client, dir: eventsDir });
|
|
for (const event of events) {
|
|
const parsed = JSON.parse(event.body) as {
|
|
type?: string;
|
|
stream?: "stdout" | "stderr";
|
|
data?: string;
|
|
code?: number | null;
|
|
signal?: string | null;
|
|
message?: string;
|
|
};
|
|
deliverRemoteEvent(parsed);
|
|
}
|
|
} catch (error) {
|
|
const message = error instanceof Error ? error.message : String(error);
|
|
await onLog("stderr", `[paperclip] ACP process session bridge poll failed: ${message}\n`);
|
|
deliverRemoteEvent({ type: "error", message });
|
|
return;
|
|
} finally {
|
|
if (!stopping) {
|
|
schedulePoll();
|
|
}
|
|
}
|
|
};
|
|
|
|
// Schedule the long-lived poll timer. Wrap each 100 ms poll tick in a
|
|
// `sandbox.agentSession.pollOutput` span, so the tick's `list` plus per-file
|
|
// `read`/`remove` execs group under one named span. The poll loop reads remote
|
|
// event files with run-time execs, not startup work, so the wrapper span and
|
|
// its child execs parent to the live run span, not to the ended bridge step.
|
|
// The span runner reads the run parent per tick, because the re-arm timer that
|
|
// the poll body schedules opens a new tick span: the live parent switches to
|
|
// `agent.turn` during the turn and back to `task.run` after it.
|
|
const schedulePoll = () => {
|
|
pollTimer = setTimeout(() => void runRuntimeWork(AGENT_SESSION_POLL_OUTPUT_SPAN, poll), 100);
|
|
pollTimer.unref?.();
|
|
};
|
|
|
|
const port = await waitForLocalServerListen(server);
|
|
const agentCommand = await writeProcessSessionProxyScript(proxyDir, port, token);
|
|
|
|
if (streamOutput) {
|
|
// Streamed output path. Run the wrapper as one long-lived session command;
|
|
// its stdout carries newline-delimited JSON frames that reach the host
|
|
// through the provider session log stream. Deliver each frame exactly once
|
|
// by its monotonic `seq`, so a frame that arrives both live and in the final
|
|
// result is not repeated. There is no host output-file poll here.
|
|
let streamBuffer = "";
|
|
let lastSeq = 0;
|
|
let sawTerminal = false;
|
|
const deliverFrame = (frame: (typeof pendingRemoteEvents)[number] & { seq?: number }) => {
|
|
if (typeof frame.seq === "number") {
|
|
if (frame.seq <= lastSeq) return;
|
|
lastSeq = frame.seq;
|
|
}
|
|
if (frame.type === "exit" || frame.type === "error") sawTerminal = true;
|
|
deliverRemoteEvent(frame);
|
|
};
|
|
const parseFrameLine = (line: string) => {
|
|
if (!line.trim()) return;
|
|
let frame: (typeof pendingRemoteEvents)[number] & { seq?: number };
|
|
try {
|
|
frame = JSON.parse(line) as typeof frame;
|
|
} catch {
|
|
return;
|
|
}
|
|
deliverFrame(frame);
|
|
};
|
|
// Live delivery: buffer partial lines across stream chunks, deliver each
|
|
// complete frame line as it arrives.
|
|
const ingestStreamChunk = (text: string) => {
|
|
streamBuffer += text;
|
|
const split = splitJsonLines(streamBuffer);
|
|
streamBuffer = split.rest;
|
|
for (const line of split.lines) parseFrameLine(line);
|
|
};
|
|
// Terminal delivery (the defined fallback to the poll): the resolved result
|
|
// carries the full wrapper stdout even when the live stream degraded to the
|
|
// provider session-log poll. The text is complete and self-contained, so
|
|
// re-parse it on its own; the `seq` guard drops every frame the live stream
|
|
// already delivered. Drop any partial live line — its complete form is in the
|
|
// full text.
|
|
const ingestFinalText = (text: string) => {
|
|
streamBuffer = "";
|
|
for (const line of text.split(/\n/)) parseFrameLine(line);
|
|
};
|
|
|
|
const launchEnvForStream =
|
|
typeof input.env === "function" ? await input.env() : input.env;
|
|
const streamCommandPayload = Buffer.from(JSON.stringify({
|
|
command: input.command,
|
|
args: input.args,
|
|
cwd: input.cwd || target.remoteCwd,
|
|
// Same provenance-clean contract as the polled payload above. Preserve
|
|
// every explicit identity override even when it equals the host value.
|
|
env: sanitizeRemoteExecutionEnv(launchEnvForStream, {}),
|
|
}), "utf8").toString("base64");
|
|
await onLog(
|
|
"stdout",
|
|
`[paperclip] Starting streamed ACP process session bridge in sandbox (${target.providerKey ?? "provider"}).\n`,
|
|
);
|
|
// Fire the long-lived command; do NOT await it here. `useSession` forces the
|
|
// persistent session so the provider streams the wrapper stdout back through
|
|
// `onLog`. On resolve, the terminal re-parse fills any frames the live stream
|
|
// missed; on reject, deliver one error frame so the local proxy fails loud.
|
|
//
|
|
// Wrap the launch in a `sandbox.agentProcess` span. `runRuntimeWork` parents
|
|
// it to the LIVE RUN root (`task.run` at launch time — no turn has started
|
|
// yet), not to the ephemeral `bridge.process-session` bring-up step, and it
|
|
// stays open for the whole process lifetime. The inner `sandbox.exec` nests
|
|
// under it. Without the wrapper the raw exec's span inherits the ~2.28s
|
|
// bring-up step as its parent and then dangles ~50s past it, overlapping
|
|
// `agent.turn` — a child outliving its parent. As a run-scoped span it reads
|
|
// instead as a resource that OVERLAPS the sibling `agent.turn`, which is the
|
|
// correct shape (the persistent process hosts the turn; it is not a child of
|
|
// it, and on multi-turn runs one process spans several turns). `runRuntimeWork`
|
|
// is voided, not awaited, so bring-up never blocks on the long-lived command,
|
|
// and it defaults to a no-op parent when no span runner is injected.
|
|
//
|
|
// The span is bounded to the bridge lifecycle: it ends when the command
|
|
// settles OR when `stop()` runs, whichever comes first. `stop()` runs during
|
|
// run teardown, before the caller ends the `task.run` root span, so the span
|
|
// never outlives the run root even if the remote process lingers past
|
|
// teardown (`execute` has no cancel, so a lingering process cannot be forced
|
|
// to resolve). The command promise keeps running after the span ends so its
|
|
// frame handlers still deliver; they no-op once `stopping` is set.
|
|
void runRuntimeWork("sandbox.agentProcess", async () => {
|
|
const commandSettled = (async () => {
|
|
try {
|
|
const result = await runner.execute({
|
|
command: shellCommand,
|
|
args: shellCommandArgs(`node ${shellQuote(remoteScriptPath)}`),
|
|
cwd: target.remoteCwd,
|
|
env: {
|
|
PAPERCLIP_PROCESS_SESSION_DIR: sessionDir,
|
|
PAPERCLIP_PROCESS_SESSION_COMMAND_B64: streamCommandPayload,
|
|
PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
|
|
},
|
|
timeoutMs,
|
|
useSession: true,
|
|
onLog: async (stream, chunk) => {
|
|
if (stream === "stdout") ingestStreamChunk(chunk);
|
|
},
|
|
});
|
|
ingestFinalText(result.stdout);
|
|
if (!sawTerminal && !stopping) {
|
|
deliverRemoteEvent({
|
|
type: "exit",
|
|
code: typeof result.exitCode === "number" ? result.exitCode : null,
|
|
});
|
|
}
|
|
} catch (error) {
|
|
if (!stopping) {
|
|
deliverRemoteEvent({
|
|
type: "error",
|
|
message: error instanceof Error ? error.message : String(error),
|
|
});
|
|
}
|
|
}
|
|
})();
|
|
await Promise.race([commandSettled, stopped]);
|
|
});
|
|
} else {
|
|
schedulePoll();
|
|
}
|
|
|
|
// `stop()` cannot rely on the long-lived poll above to observe a late
|
|
// `shutdownAck`: that poll stops re-arming as soon as it forwards a
|
|
// terminal `exit`/`error` event, and `stop()` itself sets `stopping` on
|
|
// its own first line. A normal completion's `shutdownAck` file, written a
|
|
// moment after `exit`, can then land on disk after nobody reads the events
|
|
// directory any more. Give `stop()` its own bounded reader that looks only
|
|
// for `shutdownAck` and ignores every other event type, so the wait below
|
|
// shortens on the wrapper's own proof of shutdown -- never on an `exit` or
|
|
// `error` event, which any process running under `sessionDir` can forge.
|
|
let stopReadingForShutdownAck = false;
|
|
const readShutdownAckUntil = (deadlineEpochMs: number) => {
|
|
if (stopReadingForShutdownAck) return;
|
|
void (async () => {
|
|
try {
|
|
const events = await readRemoteJsonFiles({ client, dir: eventsDir });
|
|
if (stopReadingForShutdownAck) return;
|
|
for (const event of events) {
|
|
try {
|
|
const parsed = JSON.parse(event.body) as { type?: string };
|
|
if (parsed.type === "shutdownAck") {
|
|
signalShutdownAcknowledged();
|
|
return;
|
|
}
|
|
} catch {
|
|
// Not readable JSON yet. It is not a `shutdownAck`; ignore it.
|
|
}
|
|
}
|
|
} catch {
|
|
// Best-effort: a read failure here is not proof of anything.
|
|
}
|
|
if (!stopReadingForShutdownAck && Date.now() < deadlineEpochMs) {
|
|
const timer = setTimeout(() => readShutdownAckUntil(deadlineEpochMs), 100);
|
|
timer.unref?.();
|
|
}
|
|
})();
|
|
};
|
|
|
|
return {
|
|
agentCommand,
|
|
stop: async () => {
|
|
stopping = true;
|
|
// End the `sandbox.agentProcess` span now, before the caller ends the run
|
|
// root span, even if the remote command has not resolved yet.
|
|
signalStopped();
|
|
if (pollTimer) clearTimeout(pollTimer);
|
|
for (const liveSocket of liveSockets) liveSocket.destroy();
|
|
await new Promise<void>((resolve) => server.close(() => resolve())).catch(() => undefined);
|
|
// Wait for every accepted stdin write before `stdinEnd`. The socket handler
|
|
// fires each chunk write un-awaited through `stdinWriteChain`, so an earlier
|
|
// chunk can still be pending here. Chain the `stdinEnd` write onto the same
|
|
// per-session chain, so its file rename never finishes before an earlier
|
|
// chunk. `stdinSeq` is stable now, because the sockets are destroyed and the
|
|
// server is closed, so no new message can increment it.
|
|
const stdinEndPath = path.posix.join(
|
|
stdinDir,
|
|
`${String(stdinSeq + 1).padStart(12, "0")}.json`,
|
|
);
|
|
const stdinEndWrite = stdinWriteChain.then(() =>
|
|
client.writeTextFile(stdinEndPath, jsonLine({ type: "stdinEnd" })),
|
|
);
|
|
stdinWriteChain = stdinEndWrite.then(() => undefined, () => undefined);
|
|
await stdinEndWrite.catch(() => undefined);
|
|
// The `shutdown` control message tells the wrapper to terminate itself
|
|
// and its own child (I3: no operating-system signal and no process
|
|
// identifier cross this boundary — only a file-queue message does).
|
|
// Chain it onto the same per-session write order as `stdinEnd`, so its
|
|
// file never lands before the earlier one.
|
|
const shutdownPath = path.posix.join(
|
|
stdinDir,
|
|
`${String(stdinSeq + 2).padStart(12, "0")}.json`,
|
|
);
|
|
const shutdownWrite = stdinWriteChain.then(() =>
|
|
client.writeTextFile(shutdownPath, jsonLine({ type: "shutdown" })),
|
|
);
|
|
stdinWriteChain = shutdownWrite.then(() => undefined, () => undefined);
|
|
await shutdownWrite.catch(() => undefined);
|
|
// Wait a bounded budget for a hint that the wrapper stopped: only the
|
|
// `shutdownAck` event counts; an `exit` or `error` event is untrusted
|
|
// telemetry from inside the sandbox and never shortens this wait or
|
|
// suppresses the warning below. `shutdownAck` itself is ALSO an
|
|
// untrusted hint, not proof: any process that shares the sandbox can
|
|
// write the same event under this session's event directory. It can
|
|
// only shorten this wait and suppress the warning below; it never
|
|
// gates, shortens, or replaces the unconditional removal further down.
|
|
// What actually makes the wrapper's own termination deterministic is
|
|
// the wrapper-side session-identity latch, not this event.
|
|
let acknowledgedInTime = false;
|
|
readShutdownAckUntil(Date.now() + DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS);
|
|
await Promise.race([
|
|
shutdownAcknowledged.then(() => {
|
|
acknowledgedInTime = true;
|
|
}),
|
|
new Promise<void>((resolve) => {
|
|
const budgetTimer = setTimeout(resolve, DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS);
|
|
budgetTimer.unref?.();
|
|
}),
|
|
]);
|
|
stopReadingForShutdownAck = true;
|
|
if (!acknowledgedInTime) {
|
|
await onLog(
|
|
"stderr",
|
|
`[paperclip] ACP process session wrapper did not acknowledge shutdown within ${DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS}ms; removing the session directory anyway.\n`,
|
|
).catch(() => undefined);
|
|
}
|
|
// Unconditional: this removal runs whether or not the wrapper
|
|
// acknowledged, and whether or not any event (real or forged) arrived
|
|
// under `sessionDir`. `stop()` runs during run teardown and must stay
|
|
// non-fatal, so every step above is best-effort and this step never
|
|
// throws.
|
|
await client.remove(sessionDir).catch(() => undefined);
|
|
await fs.rm(proxyDir, { recursive: true, force: true }).catch(() => undefined);
|
|
},
|
|
};
|
|
}
|
|
|
|
function getProcessSessionProxySource(input: { port: number; token: string }): string {
|
|
return `#!/usr/bin/env node
|
|
import net from "node:net";
|
|
|
|
const socket = net.createConnection({ host: "127.0.0.1", port: ${input.port} });
|
|
const token = ${JSON.stringify(input.token)};
|
|
let buffer = "";
|
|
let exiting = false;
|
|
|
|
function send(message) {
|
|
socket.write(JSON.stringify({ token, ...message }) + "\\n");
|
|
}
|
|
|
|
socket.on("connect", () => send({ type: "hello" }));
|
|
process.stdin.on("data", (chunk) => send({ type: "stdin", data: Buffer.from(chunk).toString("base64") }));
|
|
process.stdin.on("end", () => send({ type: "stdinEnd" }));
|
|
process.stdin.resume();
|
|
|
|
socket.setEncoding("utf8");
|
|
socket.on("data", (chunk) => {
|
|
buffer += chunk;
|
|
const parts = buffer.split(/\\n/);
|
|
buffer = parts.pop() || "";
|
|
for (const line of parts) {
|
|
if (!line.trim()) continue;
|
|
const message = JSON.parse(line);
|
|
if (message.type === "data") {
|
|
const out = Buffer.from(message.data || "", "base64");
|
|
(message.stream === "stderr" ? process.stderr : process.stdout).write(out);
|
|
} else if (message.type === "error") {
|
|
process.stderr.write(String(message.message || "Process session bridge failed.") + "\\n");
|
|
exiting = true;
|
|
process.exitCode = 1;
|
|
socket.end();
|
|
} else if (message.type === "exit") {
|
|
exiting = true;
|
|
process.exitCode = typeof message.code === "number" ? message.code : 1;
|
|
socket.end();
|
|
}
|
|
}
|
|
});
|
|
socket.on("close", () => {
|
|
if (!exiting) process.exit(1);
|
|
});
|
|
`;
|
|
}
|
|
|
|
export function getProcessSessionRemoteSource(input?: { outputToStdout?: boolean }): string {
|
|
return input?.outputToStdout === true
|
|
? getProcessSessionRemoteStreamSource()
|
|
: getProcessSessionRemoteEventFileSource();
|
|
}
|
|
|
|
// The shared stdin drain. Both wrappers read newline-delimited stdin messages
|
|
// from the stdin file queue and write them to the child, then end the child
|
|
// stdin on `stdinEnd`. A write to a closed child stdin only emits an `error`
|
|
// event, so the wrapper installs a no-op handler at the call site.
|
|
const PROCESS_SESSION_STDIN_POLL_TAIL = `child.stdin.on("error", () => {});
|
|
|
|
// A stdin file can appear before the host finishes the write. An empty read is
|
|
// the non-atomic-write window; a partial read makes JSON.parse throw. The
|
|
// poller must not delete a file before it validates the content. So read and
|
|
// parse each file first, and delete it only after a successful parse. The files
|
|
// sort in send order. If an earlier file is not readable yet, stop the cycle and
|
|
// keep the order: a later file (for example stdinEnd) must not run ahead of it.
|
|
// Retry the earlier file on a later cycle. After the retry limit, drop the file
|
|
// and write an error event, so a lost message fails loud, and let later files
|
|
// run.
|
|
const stdinMaxParseRetries = (() => {
|
|
const raw = Number.parseInt(process.env.PAPERCLIP_PROCESS_SESSION_STDIN_MAX_RETRIES || "", 10);
|
|
return Number.isFinite(raw) && raw > 0 ? raw : 100;
|
|
})();
|
|
const stdinParseRetries = new Map();
|
|
// Track the next expected sequence number. The host writes the stdin files in
|
|
// send order and pads the number to 12 digits, starting at 1. The files sort in
|
|
// send order. When the smallest present number is greater than expected, an
|
|
// earlier file has not appeared yet: a missing file, not an unreadable one. Hold
|
|
// the send order and wait for it, bounded by the same retry budget as the
|
|
// unreadable-file path. This turns a reordering into a loud error, never silent
|
|
// corruption.
|
|
let stdinExpectedSeq = 1;
|
|
let stdinGapRetries = 0;
|
|
|
|
// The bounded grace period between the SIGTERM and the SIGKILL a terminate()
|
|
// call sends. A test can override it through the environment, so a stubborn
|
|
// child does not force a slow test.
|
|
const terminateGraceMs = (() => {
|
|
const raw = Number.parseInt(process.env.PAPERCLIP_PROCESS_SESSION_TERMINATE_GRACE_MS || "", 10);
|
|
return Number.isFinite(raw) && raw > 0 ? raw : 3000;
|
|
})();
|
|
|
|
// I2: terminate() is the only function in this wrapper that calls
|
|
// child.kill(). No child event handler and no sibling callback calls it.
|
|
// terminate() is idempotent: a second call, or a first call after the child
|
|
// already exited on its own, does nothing beyond what already ran.
|
|
async function terminate() {
|
|
if (terminated) return;
|
|
terminated = true;
|
|
shuttingDown = true;
|
|
stdinClosed = true;
|
|
child.stdin.end();
|
|
// A \`false\` return means the child's process handle is already gone (the
|
|
// child exited before this call ran). ChildProcess#kill() is handle-scoped:
|
|
// once Node clears the handle at reap, the method call above sends no
|
|
// signal and never falls back to a stored process identifier (I3). Treat
|
|
// \`false\` as a no-op and do not retry through a numeric identifier.
|
|
const sentTerm = child.kill("SIGTERM");
|
|
if (sentTerm) {
|
|
killTimer = setTimeout(() => {
|
|
// Escalate on the same handle only (I2): the grace period expired, so
|
|
// send SIGKILL through the same child handle, never a numeric
|
|
// identifier and never a process-group signal.
|
|
child.kill("SIGKILL");
|
|
}, terminateGraceMs);
|
|
killTimer.unref?.();
|
|
}
|
|
// This event is an untrusted latency hint, not proof. Any process that can
|
|
// reach this session's event directory can write the same event type. It
|
|
// can only shorten the host's shutdown wait and suppress the host's
|
|
// timeout warning; it is never evidence that this wrapper's lifecycle
|
|
// completed, and the host's cleanup never depends on it. The identity
|
|
// latch below is what makes this wrapper's own termination deterministic.
|
|
await writeEvent({ type: "shutdownAck" });
|
|
}
|
|
|
|
// A sandbox peer can delete sessionDir and stdinDir, then recreate a
|
|
// directory at the same pathname. A pathname does not prove identity: any
|
|
// process that shares the sandbox can write it. So this wrapper captures the
|
|
// OS-level identity of both paths once at startup, before the first poll
|
|
// cycle, and checks it on every later cycle.
|
|
//
|
|
// The identity is the device number, the inode number, AND the inode's own
|
|
// creation time. The device/inode pair alone is not enough: a filesystem can
|
|
// reissue the exact inode number a just-removed directory held to the very
|
|
// next directory created at the same path, with no attacker action needed
|
|
// beyond the recreate the finding already describes. The creation time does
|
|
// not have this gap: it is set fresh on every inode allocation, even when the
|
|
// allocator reissues an old inode number, so a recreated directory always
|
|
// carries a different creation time. The creation time alone is not enough
|
|
// either, on a filesystem or kernel too old to report it, so this wrapper
|
|
// keeps the device/inode pair as a second signal rather than relying on
|
|
// either alone. Ordinary use of stdinDir (the host writing and this wrapper
|
|
// deleting individual stdin files) changes that directory's OWN change time,
|
|
// but never its creation time, so the creation time is safe to latch on
|
|
// without producing a false positive on every stdin message.
|
|
//
|
|
// A filesystem or kernel that cannot report a real creation time does not
|
|
// always report a value of zero. Node fails in one of two ways, and both are
|
|
// grounded, not assumed: on Linux, when the statx() call finds no creation
|
|
// time support, the kernel leaves the field unset and Node reports 0. On a
|
|
// platform whose stat() call has no creation-time field at all, Node copies
|
|
// the change time into the creation time instead. A 0 value fails open (any
|
|
// recreated directory then matches on birthtimeMs alone), and a change-time
|
|
// copy fails closed but far too often (it would move on every stdin file
|
|
// this wrapper deletes). captureSessionIdentity() below proves the value is
|
|
// usable before it trusts it, and fails closed on both known fallbacks.
|
|
let sessionDirIdentity = null;
|
|
let stdinDirIdentity = null;
|
|
// The latch. Once set, it never clears. This replaces a counter that a
|
|
// successful read reset to zero: an attacker who recreated the directory
|
|
// before the counter reached its threshold kept the wrapper polling forever.
|
|
// A latch has no threshold to race and no reset path.
|
|
let identityLost = false;
|
|
|
|
async function statPathIdentity(candidatePath) {
|
|
const stats = await fs.lstat(candidatePath);
|
|
if (stats.isSymbolicLink()) {
|
|
const error = new Error("Refusing a symbolic link on a process session control path.");
|
|
error.code = "EPAPERCLIP_SYMLINK";
|
|
throw error;
|
|
}
|
|
if (!stats.isDirectory()) {
|
|
const error = new Error("A process session control path is not a directory.");
|
|
error.code = "ENOTDIR";
|
|
throw error;
|
|
}
|
|
return { dev: stats.dev, ino: stats.ino, birthtimeMs: stats.birthtimeMs };
|
|
}
|
|
|
|
function sameIdentity(left, right) {
|
|
return left.dev === right.dev && left.ino === right.ino && left.birthtimeMs === right.birthtimeMs;
|
|
}
|
|
|
|
async function latchAndTerminate() {
|
|
if (identityLost) return;
|
|
identityLost = true;
|
|
await terminate();
|
|
}
|
|
|
|
let probeSeq = 0;
|
|
|
|
// A probe file name that pollStdin() can never read as a stdin message: it
|
|
// does not end in ".json", so the ".json" filter in pollStdin() skips it if
|
|
// a poll cycle ever lists the directory during the probe's short window.
|
|
function nextProbeFileName() {
|
|
probeSeq += 1;
|
|
return ".paperclip-birthtime-probe-" + process.pid + "-" + probeSeq;
|
|
}
|
|
|
|
// Proves a directory's reported birthtimeMs is a real creation time, not a
|
|
// change-time copy. Creating and removing a file inside a directory changes
|
|
// that directory's OWN change time but never its true creation time, so a
|
|
// birthtimeMs that moves across the probe is a change-time copy. Returns
|
|
// null when the value is proven real. Returns a stderr-ready reason string
|
|
// on any failure (a detected copy, or a probe that cannot run at all, for
|
|
// example a permission error or a pre-created probe path): either way the
|
|
// caller must not trust the value.
|
|
//
|
|
// The open uses the "wx" flag: exclusive create, fail if the path exists.
|
|
// A sandbox peer cannot pre-create the probe path as a symbolic link and
|
|
// have this call follow it, because "wx" fails closed on an existing path
|
|
// instead of following a link to it.
|
|
//
|
|
// Cleanup checks identity, not only ownership of the initial create. This
|
|
// wrapper reads the probe file's identity, (dev, ino, ctimeMs), off the open
|
|
// file descriptor itself (fstat), not off the path, so a peer that swaps the
|
|
// path in the short gap after create cannot poison the identity this
|
|
// wrapper trusts as its own. Right before removal, this wrapper reads the
|
|
// path's identity again and removes it only when that identity still
|
|
// matches. A same-sandbox peer that deletes the probe file and creates its
|
|
// own entry at the same path in between leaves a different identity behind,
|
|
// so this wrapper leaves that entry untouched instead of removing it. This
|
|
// covers a peer's replacement file, a peer's replacement directory, and a
|
|
// peer's replacement symbolic link alike, because all three change the
|
|
// identity this wrapper reads back. The identity check includes ctimeMs,
|
|
// not only (dev, ino): a filesystem can hand this call's freed inode number
|
|
// straight back out to a peer's very next create at the same path, so
|
|
// (dev, ino) alone can match a path this call no longer owns; ctimeMs resets
|
|
// on every create, so a peer's replacement carries a different one even when
|
|
// the inode number repeats. Node's filesystem API has no call that removes a
|
|
// path only when its identity still matches an earlier read as one atomic
|
|
// step, so a gap remains between this wrapper's final identity read and the
|
|
// removal call itself. A peer that wins this gap can put any entry at the
|
|
// probe path before the removal call runs. This can include a pre-existing
|
|
// file the peer renames into place, not only a file the peer creates fresh.
|
|
// The removal call then removes whatever entry sits at the probe path at
|
|
// that moment. Two bounds still hold on that removal. The path always
|
|
// stays under dirPath. If the entry is a symbolic link, the removal call
|
|
// removes the link itself instead of following it to a different target.
|
|
// A non-recursive removal call also fails if the entry is a directory.
|
|
async function birthtimeSurvivesProbe(dirPath) {
|
|
let before;
|
|
try {
|
|
before = (await fs.lstat(dirPath)).birthtimeMs;
|
|
} catch {
|
|
return "its reported creation time could not be read";
|
|
}
|
|
const probePath = path.posix.join(dirPath, nextProbeFileName());
|
|
let handle;
|
|
try {
|
|
handle = await fs.open(probePath, "wx");
|
|
} catch {
|
|
return "its probe file could not be created exclusively (the path may already exist)";
|
|
}
|
|
// fstat on the open handle names the exact inode this call just created.
|
|
// A path-based lstat here instead would be racy against a peer that swaps
|
|
// the path in the gap between the create above and the stat: fstat has no
|
|
// such gap, because a file descriptor keeps naming the inode it opened no
|
|
// matter what a later swap does to the path.
|
|
let ownedIdentity = null;
|
|
try {
|
|
const createdStats = await handle.stat();
|
|
// ctimeMs guards against inode reuse; see the function comment above.
|
|
ownedIdentity = { dev: createdStats.dev, ino: createdStats.ino, ctimeMs: createdStats.ctimeMs };
|
|
} catch {
|
|
ownedIdentity = null;
|
|
} finally {
|
|
await handle.close().catch(() => undefined);
|
|
}
|
|
if (!ownedIdentity) {
|
|
// fstat on this call's own just-opened descriptor failed. This call then
|
|
// has no verified identity for the probe file it created, so it must not
|
|
// check or remove that file by path: a peer could already own the entry
|
|
// at that path, and a path-based removal here could delete the peer's
|
|
// entry instead of this call's own file. Fail closed right here instead
|
|
// of falling through to the birthtime comparison below, so a failed
|
|
// identity read can never let this probe report success.
|
|
return "its own probe file's identity could not be read from the open file descriptor";
|
|
}
|
|
// The one gap the fs API cannot close: this lstat and the removal below
|
|
// are two separate calls, not one atomic "remove if identity still
|
|
// matches" step. A peer that wins this narrow gap can put any entry at
|
|
// the probe path, including a pre-existing file it renames into place,
|
|
// and the removal call below removes whatever entry is there when it
|
|
// runs.
|
|
const currentStats = await fs.lstat(probePath).catch(() => null);
|
|
const stillOwned =
|
|
currentStats !== null &&
|
|
currentStats.dev === ownedIdentity.dev &&
|
|
currentStats.ino === ownedIdentity.ino &&
|
|
currentStats.ctimeMs === ownedIdentity.ctimeMs;
|
|
if (stillOwned) {
|
|
await fs.rm(probePath, { force: true }).catch(() => undefined);
|
|
}
|
|
let after;
|
|
try {
|
|
after = (await fs.lstat(dirPath)).birthtimeMs;
|
|
} catch {
|
|
return "its reported creation time could not be read";
|
|
}
|
|
return before === after ? null : "its reported creation time changed after a probe write";
|
|
}
|
|
|
|
async function refuseUnusableCreationTime(label, dirPath, reason) {
|
|
process.stderr.write(
|
|
"Refusing to trust the process session control path " + label + " (" + dirPath + "): " + reason +
|
|
". This filesystem or kernel gives no usable creation time. Terminating.\\n",
|
|
);
|
|
await latchAndTerminate();
|
|
}
|
|
|
|
// Runs once, before the first poll cycle, and before this wrapper captures
|
|
// the identities it later checks on every cycle. A failed capture fails
|
|
// closed: the wrapper has no verified identity to check on later cycles, so
|
|
// it terminates now instead of polling a control path it never verified.
|
|
//
|
|
// This wrapper cannot assume stats.birthtimeMs is a real creation time. Node
|
|
// can report a change-time copy as a creation time. That value fails closed
|
|
// far too aggressively (it would move on every stdin file this wrapper
|
|
// deletes), so this wrapper proves the value is not a copy with a probe before
|
|
// it trusts it, run once here, before either directory's identity is captured.
|
|
async function captureSessionIdentity() {
|
|
try {
|
|
const sessionProbeFailure = await birthtimeSurvivesProbe(sessionDir);
|
|
if (sessionProbeFailure) {
|
|
await refuseUnusableCreationTime("sessionDir", sessionDir, sessionProbeFailure);
|
|
return;
|
|
}
|
|
const stdinProbeFailure = await birthtimeSurvivesProbe(stdinDir);
|
|
if (stdinProbeFailure) {
|
|
await refuseUnusableCreationTime("stdinDir", stdinDir, stdinProbeFailure);
|
|
return;
|
|
}
|
|
const session = await statPathIdentity(sessionDir);
|
|
const stdin = await statPathIdentity(stdinDir);
|
|
sessionDirIdentity = session;
|
|
stdinDirIdentity = stdin;
|
|
} catch (error) {
|
|
process.stderr.write(
|
|
"Failed to capture the process session identity: " +
|
|
(error instanceof Error ? error.message : String(error)) + ". Terminating.\\n",
|
|
);
|
|
await latchAndTerminate();
|
|
}
|
|
}
|
|
|
|
// Runs on every poll cycle, before the wrapper reads stdinDir. Terminate and
|
|
// latch on any proof the control path is no longer the one this wrapper
|
|
// captured at startup (a missing path, a path that is no longer a directory,
|
|
// a symbolic link, or a directory whose identity changed), AND on every
|
|
// other lstat failure. A permission error is not transient here: a sandbox
|
|
// peer can deny search permission on the control directory without removing
|
|
// it, and treating that as transient would leave the wrapper and its child
|
|
// alive forever. The error code below only picks the stderr message, so an
|
|
// operator can still tell a removed directory from a permission error; it
|
|
// never decides whether to latch.
|
|
//
|
|
// Contrast readStdinDirNames() right below, whose catch block stays narrow
|
|
// on purpose: readdir() opens a directory descriptor, so it can fail with a
|
|
// genuinely transient error under descriptor exhaustion, and latching there
|
|
// would kill live sessions under load. lstat() opens no descriptor, and this
|
|
// function already runs before every call to readStdinDirNames(), so a
|
|
// permission error latches here before readdir() is ever reached.
|
|
async function verifySessionIdentity() {
|
|
if (identityLost) return false;
|
|
try {
|
|
const session = await statPathIdentity(sessionDir);
|
|
const stdin = await statPathIdentity(stdinDir);
|
|
if (!sameIdentity(session, sessionDirIdentity) || !sameIdentity(stdin, stdinDirIdentity)) {
|
|
await latchAndTerminate();
|
|
return false;
|
|
}
|
|
return true;
|
|
} catch (error) {
|
|
const code = error && typeof error === "object" ? error.code : undefined;
|
|
const reason =
|
|
code === "ENOENT"
|
|
? "the control path no longer exists"
|
|
: code === "ENOTDIR"
|
|
? "the control path is no longer a directory"
|
|
: code === "EPAPERCLIP_SYMLINK"
|
|
? "the control path is now a symbolic link"
|
|
: "lstat failed" + (code ? " with " + code : "");
|
|
process.stderr.write("Latching on a lost process session identity: " + reason + ". Terminating.\\n");
|
|
await latchAndTerminate();
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// This catch block stays narrow on purpose: see the comment above
|
|
// verifySessionIdentity() for why a permission error here is treated as
|
|
// transient while the same error latches there.
|
|
async function readStdinDirNames() {
|
|
if (!(await verifySessionIdentity())) return [];
|
|
try {
|
|
return await fs.readdir(stdinDir);
|
|
} catch (error) {
|
|
const code = error && typeof error === "object" ? error.code : undefined;
|
|
if (code === "ENOENT" || code === "ENOTDIR") {
|
|
await latchAndTerminate();
|
|
}
|
|
return [];
|
|
}
|
|
}
|
|
|
|
async function pollStdin() {
|
|
while (!shuttingDown) {
|
|
const entries = (await readStdinDirNames()).filter((name) => name.endsWith(".json")).sort();
|
|
for (const name of entries) {
|
|
if (shuttingDown) break;
|
|
const entrySeq = Number.parseInt(name, 10);
|
|
// Hold the send order when an earlier file has not appeared. Do not consume
|
|
// this later file: wait for the missing file on a later cycle, bounded by
|
|
// the retry budget. After the budget, fail loud and advance past the gap,
|
|
// so the present file can run.
|
|
if (Number.isFinite(entrySeq) && entrySeq > stdinExpectedSeq) {
|
|
stdinGapRetries += 1;
|
|
if (stdinGapRetries < stdinMaxParseRetries) {
|
|
break;
|
|
}
|
|
await writeEvent({
|
|
type: "error",
|
|
message:
|
|
"Advanced past missing stdin files " + stdinExpectedSeq + " to " + (entrySeq - 1) +
|
|
" after " + stdinMaxParseRetries + " retries.",
|
|
});
|
|
stdinGapRetries = 0;
|
|
stdinExpectedSeq = entrySeq;
|
|
}
|
|
const file = path.posix.join(stdinDir, name);
|
|
let message;
|
|
try {
|
|
// Hardening (I3): open with O_NOFOLLOW where the platform defines it,
|
|
// so a control-path symbolic link swapped in after the directory
|
|
// check fails the read instead of following it.
|
|
const readFlag =
|
|
typeof fsConstants.O_NOFOLLOW === "number" ? fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW : "r";
|
|
const raw = await fs.readFile(file, { encoding: "utf8", flag: readFlag });
|
|
// An empty read means the content is not on disk yet. Treat it the same
|
|
// as a parse failure: keep the file and retry on a later cycle.
|
|
if (!raw) throw new Error("stdin file is empty");
|
|
message = JSON.parse(raw);
|
|
} catch (error) {
|
|
const retries = (stdinParseRetries.get(name) || 0) + 1;
|
|
if (retries >= stdinMaxParseRetries) {
|
|
// The retry limit is reached. Drop the file and write an error event,
|
|
// so the lost message fails loud. The file is resolved now, so let the
|
|
// loop go on to the next entry.
|
|
stdinParseRetries.delete(name);
|
|
await fs.rm(file, { force: true }).catch(() => undefined);
|
|
await writeEvent({
|
|
type: "error",
|
|
message:
|
|
"Dropped unreadable stdin file after " + stdinMaxParseRetries + " retries: " + name + ": " +
|
|
(error instanceof Error ? error.message : String(error)),
|
|
});
|
|
// The file is resolved (dropped). Advance the expected number and reset
|
|
// the gap budget, then let the loop go on to the next entry.
|
|
if (Number.isFinite(entrySeq)) stdinExpectedSeq = entrySeq + 1;
|
|
stdinGapRetries = 0;
|
|
continue;
|
|
}
|
|
// The file is not readable yet and is not past the retry limit. Keep it
|
|
// and stop this cycle to hold the send order. A later file (for example
|
|
// stdinEnd) must not run before this earlier file. A later cycle reads
|
|
// from the start again.
|
|
stdinParseRetries.set(name, retries);
|
|
break;
|
|
}
|
|
// The parse succeeded, so the content is complete. Delete the file first,
|
|
// then act on the message. A later cycle never re-reads a handled file.
|
|
stdinParseRetries.delete(name);
|
|
await fs.rm(file, { force: true }).catch(() => undefined);
|
|
// The file is handled. Advance the expected number and reset the gap
|
|
// budget, so the next expected file starts fresh.
|
|
if (Number.isFinite(entrySeq)) stdinExpectedSeq = entrySeq + 1;
|
|
stdinGapRetries = 0;
|
|
if (message.type === "stdin" && typeof message.data === "string") {
|
|
if (!stdinClosed) child.stdin.write(Buffer.from(message.data, "base64"));
|
|
} else if (message.type === "stdinEnd") {
|
|
stdinClosed = true;
|
|
child.stdin.end();
|
|
break;
|
|
} else if (message.type === "shutdown") {
|
|
await terminate();
|
|
break;
|
|
}
|
|
}
|
|
if (!shuttingDown) await new Promise((resolve) => setTimeout(resolve, 50));
|
|
}
|
|
}
|
|
|
|
await captureSessionIdentity();
|
|
|
|
void pollStdin().catch((error) => void writeEvent({ type: "error", message: error instanceof Error ? error.message : String(error) }));
|
|
`;
|
|
|
|
// Streamed variant: the wrapper writes each output frame as one newline-
|
|
// delimited JSON line to its stdout. The host runs this wrapper as one
|
|
// long-lived session command and reads the frames from the session log stream,
|
|
// so there is no host output-file poll. Each frame carries a monotonic `seq`,
|
|
// so the host delivers every frame exactly once whether it arrives live or in
|
|
// the final result. The wrapper exits when the child closes, so the session
|
|
// command settles and the session shell (the subshell wrap around it) survives.
|
|
function getProcessSessionRemoteStreamSource(): string {
|
|
return `import { spawn } from "node:child_process";
|
|
import { promises as fs, constants as fsConstants } from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const sessionDir = process.env.PAPERCLIP_PROCESS_SESSION_DIR;
|
|
const commandPayload = process.env.PAPERCLIP_PROCESS_SESSION_COMMAND_B64;
|
|
if (!sessionDir || !commandPayload) throw new Error("Missing process session bridge env.");
|
|
|
|
const stdinDir = path.posix.join(sessionDir, "stdin");
|
|
let seq = 0;
|
|
let stdinClosed = false;
|
|
let shuttingDown = false;
|
|
let terminated = false;
|
|
let killTimer = null;
|
|
|
|
const config = JSON.parse(Buffer.from(commandPayload, "base64").toString("utf8"));
|
|
await fs.mkdir(stdinDir, { recursive: true });
|
|
|
|
// One newline-delimited JSON frame per event. Node keeps process.stdout writes
|
|
// ordered, and the base64 payload holds no newline, so each frame is one line.
|
|
function writeEvent(event) {
|
|
seq += 1;
|
|
process.stdout.write(JSON.stringify({ seq, ...event }) + "\\n");
|
|
}
|
|
|
|
// Hardening (I3): refuse a symbolic link on a control path before this
|
|
// wrapper reads or writes through it. A symbolic link here could let another
|
|
// sandbox process redirect the wrapper's file I/O outside the session tree.
|
|
async function isSymbolicLink(candidatePath) {
|
|
try {
|
|
const stats = await fs.lstat(candidatePath);
|
|
return stats.isSymbolicLink();
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if ((await isSymbolicLink(sessionDir)) || (await isSymbolicLink(stdinDir))) {
|
|
await writeEvent({ type: "error", message: "Refusing a symbolic link on a process session control path." });
|
|
process.exitCode = 1;
|
|
process.exit(1);
|
|
}
|
|
|
|
// Hardening (I3, not containment): the wrapper's own launch env carries the
|
|
// session dir and the command payload. Scrub both keys before they reach the
|
|
// spawned child, so the child never inherits a path to its own control files.
|
|
const childEnv = { ...process.env, ...(config.env || {}) };
|
|
delete childEnv.PAPERCLIP_PROCESS_SESSION_DIR;
|
|
delete childEnv.PAPERCLIP_PROCESS_SESSION_COMMAND_B64;
|
|
|
|
// I1: exactly one child process per emitted wrapper. Do not add a second
|
|
// tracked child handle.
|
|
const child = spawn(config.command, Array.isArray(config.args) ? config.args : [], {
|
|
cwd: config.cwd || process.cwd(),
|
|
env: childEnv,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
});
|
|
|
|
child.stdout.on("data", (chunk) => writeEvent({ type: "data", stream: "stdout", data: Buffer.from(chunk).toString("base64") }));
|
|
child.stderr.on("data", (chunk) => writeEvent({ type: "data", stream: "stderr", data: Buffer.from(chunk).toString("base64") }));
|
|
child.on("error", (error) => writeEvent({ type: "error", message: error.message }));
|
|
// "close" (not "exit") so stdout/stderr fully drain before the exit frame.
|
|
// Queue the exit frame first, then run terminate(), so the exit frame always
|
|
// lands even when the child closes on its own, with no stdinEnd and no
|
|
// shutdown message ever received. writeEvent() only queues an asynchronous
|
|
// write. terminate()'s own synchronous work (ending the child's stdin and
|
|
// sending SIGTERM) already runs in this same handler by the time the exit
|
|
// frame becomes readable on disk. terminate() is idempotent and its
|
|
// child.kill() call here is always a no-op (I2): the child's process handle
|
|
// is already gone by the time "close" fires. An error frame carries no such
|
|
// guarantee: child.on("error", ...) below does not call terminate(), and
|
|
// neither does the poll loop's own error writes, so those fire while the
|
|
// wrapper and its child are still fully alive.
|
|
child.on("close", (code, signal) => {
|
|
writeEvent({ type: "exit", code, signal });
|
|
process.exitCode = typeof code === "number" ? code : 1;
|
|
void terminate();
|
|
});
|
|
|
|
${PROCESS_SESSION_STDIN_POLL_TAIL}`;
|
|
}
|
|
|
|
function getProcessSessionRemoteEventFileSource(): string {
|
|
return `import { spawn } from "node:child_process";
|
|
import { promises as fs, constants as fsConstants } from "node:fs";
|
|
import path from "node:path";
|
|
|
|
const sessionDir = process.env.PAPERCLIP_PROCESS_SESSION_DIR;
|
|
const commandPayload = process.env.PAPERCLIP_PROCESS_SESSION_COMMAND_B64;
|
|
if (!sessionDir || !commandPayload) throw new Error("Missing process session bridge env.");
|
|
|
|
const stdinDir = path.posix.join(sessionDir, "stdin");
|
|
const eventsDir = path.posix.join(sessionDir, "events");
|
|
let seq = 0;
|
|
let stdinClosed = false;
|
|
let shuttingDown = false;
|
|
let terminated = false;
|
|
let killTimer = null;
|
|
|
|
const config = JSON.parse(Buffer.from(commandPayload, "base64").toString("utf8"));
|
|
await fs.mkdir(stdinDir, { recursive: true });
|
|
await fs.mkdir(eventsDir, { recursive: true });
|
|
|
|
let writeChain = Promise.resolve();
|
|
|
|
function writeEvent(event) {
|
|
seq += 1;
|
|
const file = path.posix.join(eventsDir, String(seq).padStart(12, "0") + ".json");
|
|
const write = writeChain.then(async () => {
|
|
await fs.writeFile(file + ".tmp", JSON.stringify(event) + "\\n", "utf8");
|
|
await fs.rename(file + ".tmp", file);
|
|
});
|
|
writeChain = write.catch(() => undefined);
|
|
return write;
|
|
}
|
|
|
|
// Hardening (I3): refuse a symbolic link on a control path before this
|
|
// wrapper reads or writes through it. A symbolic link here could let another
|
|
// sandbox process redirect the wrapper's file I/O outside the session tree.
|
|
async function isSymbolicLink(candidatePath) {
|
|
try {
|
|
const stats = await fs.lstat(candidatePath);
|
|
return stats.isSymbolicLink();
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
if ((await isSymbolicLink(sessionDir)) || (await isSymbolicLink(stdinDir))) {
|
|
await writeEvent({ type: "error", message: "Refusing a symbolic link on a process session control path." });
|
|
process.exitCode = 1;
|
|
process.exit(1);
|
|
}
|
|
|
|
// Hardening (I3, not containment): the wrapper's own launch env carries the
|
|
// session dir and the command payload. Scrub both keys before they reach the
|
|
// spawned child, so the child never inherits a path to its own control files.
|
|
const childEnv = { ...process.env, ...(config.env || {}) };
|
|
delete childEnv.PAPERCLIP_PROCESS_SESSION_DIR;
|
|
delete childEnv.PAPERCLIP_PROCESS_SESSION_COMMAND_B64;
|
|
|
|
// I1: exactly one child process per emitted wrapper. Do not add a second
|
|
// tracked child handle.
|
|
const child = spawn(config.command, Array.isArray(config.args) ? config.args : [], {
|
|
cwd: config.cwd || process.cwd(),
|
|
env: childEnv,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
});
|
|
|
|
child.stdout.on("data", (chunk) => void writeEvent({ type: "data", stream: "stdout", data: Buffer.from(chunk).toString("base64") }));
|
|
child.stderr.on("data", (chunk) => void writeEvent({ type: "data", stream: "stderr", data: Buffer.from(chunk).toString("base64") }));
|
|
child.on("error", (error) => void writeEvent({ type: "error", message: error.message }));
|
|
// "close" (not "exit") so stdout/stderr fully drain before the exit event;
|
|
// the write chain then guarantees the exit file lands after every data file.
|
|
// Queue the exit event first, then run terminate(), so the poll loop ends
|
|
// even when the child closes on its own, with no stdinEnd and no shutdown
|
|
// message ever received. writeEvent() only queues an asynchronous write.
|
|
// terminate()'s own synchronous work (ending the child's stdin and sending
|
|
// SIGTERM) already runs in this same handler by the time the exit file
|
|
// becomes readable on disk. terminate() is idempotent and its child.kill()
|
|
// call here is always a no-op (I2): the child's process handle is already
|
|
// gone by the time "close" fires. An error event carries no such guarantee:
|
|
// child.on("error", ...) below does not call terminate(), and neither does
|
|
// the poll loop's own error writes, so those fire while the wrapper and its
|
|
// child are still fully alive.
|
|
child.on("close", (code, signal) => {
|
|
void writeEvent({ type: "exit", code, signal });
|
|
process.exitCode = typeof code === "number" ? code : 1;
|
|
void terminate();
|
|
});
|
|
|
|
${PROCESS_SESSION_STDIN_POLL_TAIL}`;
|
|
}
|
|
|
|
/** The default deadline for the duplex readiness handshake, in milliseconds. */
|
|
const DEFAULT_DUPLEX_READINESS_TIMEOUT_MS = 10_000;
|
|
|
|
/** The default bounded budget to close a partial duplex channel, in milliseconds. */
|
|
const DEFAULT_DUPLEX_CLEANUP_BUDGET_MS = 2_000;
|
|
|
|
/**
|
|
* Reserve a loopback port the host assigns to the duplex gateway. The host binds
|
|
* an ephemeral listener on `127.0.0.1`, reads the port the operating system
|
|
* chose, then closes the listener. The number is very likely free when the
|
|
* gateway binds it a moment later. The gateway binds exactly this port or exits
|
|
* nonzero, so a taken port fails closed to the file bridge and never steers the
|
|
* endpoint.
|
|
*/
|
|
async function reserveHostAssignedLoopbackPort(): Promise<number> {
|
|
return new Promise<number>((resolve, reject) => {
|
|
const probe = net.createServer();
|
|
probe.once("error", reject);
|
|
probe.listen(0, "127.0.0.1", () => {
|
|
const address = probe.address();
|
|
if (!address || typeof address === "string") {
|
|
probe.close(() => reject(new Error("Could not reserve a loopback port for the duplex gateway.")));
|
|
return;
|
|
}
|
|
const reserved = address.port;
|
|
probe.close(() => resolve(reserved));
|
|
});
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Build the argument vector that launches the duplex gateway in the sandbox. The
|
|
* host passes the assigned port and the per-open nonce only through the launch
|
|
* environment, so the argument vector sets them as environment assignments in
|
|
* front of the node command. No addressing data comes from the channel.
|
|
*
|
|
* The script uses `exec env NAME=value ... command`. A POSIX shell accepts an
|
|
* environment-assignment prefix only on a plain command, never on `exec`. The
|
|
* form `exec NAME=value command` exits with status 127. The `env` utility carries
|
|
* the assignments, and `exec` still replaces the shell with the gateway process,
|
|
* so the gateway keeps the process slot and the assigned environment.
|
|
*/
|
|
export function buildDuplexGatewayLaunchArgv(input: {
|
|
shellCommand: "bash" | "sh";
|
|
remoteEntrypoint: string;
|
|
nodeCommand?: string | null;
|
|
env: Record<string, string>;
|
|
}): string[] {
|
|
const assignments = Object.entries(input.env)
|
|
.map(([key, value]) => `${key}=${shellQuote(value)}`)
|
|
.join(" ");
|
|
const nodeCommand = input.nodeCommand?.trim() || "node";
|
|
const script = `exec env ${assignments} ${shellQuote(nodeCommand)} ${shellQuote(input.remoteEntrypoint)}`;
|
|
return [input.shellCommand, ...shellCommandArgs(script)];
|
|
}
|
|
|
|
/** The reason the duplex readiness handshake did not pass. */
|
|
type DuplexReadinessFailure = "protocol_contamination" | "nonce_mismatch" | "channel_exit" | "timeout";
|
|
|
|
/** The outcome of the duplex readiness handshake. */
|
|
type DuplexReadinessResult =
|
|
| { ok: true }
|
|
| { ok: false; reason: DuplexReadinessFailure };
|
|
|
|
/**
|
|
* Map a readiness failure to the fixed fallback reason. Each readiness failure
|
|
* maps to exactly one reason from the closed telemetry set, so the fallback
|
|
* counter and the transport event carry only an approved value.
|
|
*/
|
|
function duplexReadinessFallbackReason(reason: DuplexReadinessFailure): DuplexFallbackReason {
|
|
switch (reason) {
|
|
case "protocol_contamination":
|
|
return "contaminated";
|
|
case "nonce_mismatch":
|
|
return "ready_nonce_mismatch";
|
|
case "timeout":
|
|
return "ready_timeout";
|
|
case "channel_exit":
|
|
return "ready_invalid";
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The fixed marker the worker manager puts in a route-busy rejection. The manager
|
|
* defines the constant; this module matches the fixed string, because the two
|
|
* layers ship in separate packages and share no import. The marker names the
|
|
* process-scoped route ceiling, so it carries no route, query, body, or token.
|
|
*/
|
|
const DUPLEX_ROUTE_BUSY_ERROR_MARKER = "DUPLEX_CHANNEL_ROUTE_BUSY";
|
|
|
|
/**
|
|
* Report whether the caught open error is a route-busy rejection. The host maps it
|
|
* to the `route_busy` fallback stage, so a full process-scoped route ceiling never
|
|
* folds into a generic open failure.
|
|
*/
|
|
function isDuplexRouteBusyError(error: unknown): boolean {
|
|
return error instanceof Error && error.message.includes(DUPLEX_ROUTE_BUSY_ERROR_MARKER);
|
|
}
|
|
|
|
/**
|
|
* The cap on the pre-READY readiness buffer, in bytes. The gate reads untrusted
|
|
* bytes before the READY line arrives, so it bounds the buffer. The cap is the
|
|
* codec frame-size bound plus one line of margin, so a legitimate maximum-size
|
|
* READY frame still fits. Past the cap with no newline, the stream cannot be a
|
|
* valid READY frame, so the gate finishes with protocol contamination.
|
|
*/
|
|
const DUPLEX_READINESS_BUFFER_CAP_BYTES = DEFAULT_MAX_DUPLEX_FRAME_BYTES + 4_096;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// http2_v1: the client connection preface scan and the run disposition latch.
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* The HTTP/2 client connection preface: 24 octets, `PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n`
|
|
* (RFC 9113, Section 3.4). A Node `http2.performServerHandshake` call needs a
|
|
* `Duplex` whose readable side starts at this exact sequence; one extra
|
|
* leading byte makes the preface invalid, and the server reports a
|
|
* `PROTOCOL_ERROR`. The server does not skip a leading byte and does not
|
|
* search for the sequence, so the host finds the offset itself before it
|
|
* hands the channel to the server.
|
|
*/
|
|
const HTTP2_CLIENT_CONNECTION_PREFACE = Buffer.from(
|
|
"505249202a20485454502f322e300d0a0d0a534d0d0a0d0a",
|
|
"hex",
|
|
);
|
|
|
|
/** The one shared empty buffer. The preface scan starts and resets its two
|
|
* retained buffers to it. */
|
|
const HTTP2_PREFACE_EMPTY_BUFFER = Buffer.alloc(0);
|
|
|
|
// The count of bytes the preface scan's substring search examines, in bytes.
|
|
// A search that always starts from the beginning of the retained buffer
|
|
// re-examines the whole buffer on every fragment, so this count grows
|
|
// quadratically in the number of fragments. `findPrefaceFrom` instead
|
|
// resumes from where the prior search left off, so this count stays linear
|
|
// in the bytes received. A test reads this count to prove the search work
|
|
// does not regress to the quadratic shape. Production code never reads this
|
|
// count.
|
|
let http2PrefaceScanSearchUnits = 0;
|
|
|
|
// The count of bytes the pre-preface scan buffer copies while it grows its
|
|
// backing storage. A one-copy-per-fragment append copies the whole retained
|
|
// buffer on every fragment, so this count grows quadratically in the number
|
|
// of fragments. The doubling-growth approach copies only on a reallocation,
|
|
// so this count stays linear in the bytes received. A test reads this count
|
|
// to prove the growth work does not regress to the quadratic shape.
|
|
// Production code never reads this count.
|
|
let http2PrefaceScanBufferGrowthCopyUnits = 0;
|
|
|
|
// The same count as `http2PrefaceScanBufferGrowthCopyUnits`, for the
|
|
// post-preface replay buffer instead of the pre-preface scan buffer.
|
|
let http2PrefaceReplayBufferGrowthCopyUnits = 0;
|
|
|
|
/**
|
|
* A byte buffer that grows its backing storage by doubling its capacity,
|
|
* instead of copying the whole retained buffer on every appended fragment. A
|
|
* sender that trickles input in many small fragments would otherwise force
|
|
* one full copy of the whole retained buffer per fragment: with the buffer
|
|
* growing toward its cap, that is quadratic work in the number of fragments.
|
|
* Doubling the backing storage's capacity only when the current capacity
|
|
* runs out reallocates and copies a logarithmic number of times, so the
|
|
* total copy work stays linear in the bytes received. `countGrowthCopy`
|
|
* receives the number of bytes each reallocation copies, so a test can add
|
|
* these up and prove the growth work stays linear.
|
|
*/
|
|
function createGrowableByteBuffer(countGrowthCopy: (copiedBytes: number) => void): {
|
|
view: () => Buffer;
|
|
length: () => number;
|
|
append: (chunk: Uint8Array) => void;
|
|
reset: () => void;
|
|
} {
|
|
let used: Buffer = HTTP2_PREFACE_EMPTY_BUFFER;
|
|
let storage: Buffer = HTTP2_PREFACE_EMPTY_BUFFER;
|
|
return {
|
|
view: () => used,
|
|
length: () => used.length,
|
|
append: (chunk: Uint8Array): void => {
|
|
const usedLength = used.length;
|
|
const neededLength = usedLength + chunk.byteLength;
|
|
if (neededLength > storage.length) {
|
|
let nextCapacity = storage.length === 0 ? chunk.byteLength : storage.length * 2;
|
|
while (nextCapacity < neededLength) {
|
|
nextCapacity *= 2;
|
|
}
|
|
const grown = Buffer.allocUnsafe(nextCapacity);
|
|
storage.copy(grown, 0, 0, usedLength);
|
|
countGrowthCopy(usedLength);
|
|
storage = grown;
|
|
}
|
|
storage.set(chunk, usedLength);
|
|
used = storage.subarray(0, neededLength);
|
|
},
|
|
reset: (): void => {
|
|
used = HTTP2_PREFACE_EMPTY_BUFFER;
|
|
storage = HTTP2_PREFACE_EMPTY_BUFFER;
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Find the client connection preface in `buffer` at or after index `from`
|
|
* and return its offset, or -1. This counts the real scan distance for a
|
|
* test: from `from` up to the found preface's end, or to the end of the
|
|
* buffer when it finds none. The count stays linear in the bytes received
|
|
* when the caller advances `from` to just short of the buffer's end on every
|
|
* miss, instead of always searching from the start of the buffer.
|
|
*/
|
|
function findPrefaceFrom(buffer: Buffer, from: number): number {
|
|
const offset = buffer.indexOf(HTTP2_CLIENT_CONNECTION_PREFACE, from);
|
|
const scannedTo =
|
|
offset === -1 ? buffer.length : offset + HTTP2_CLIENT_CONNECTION_PREFACE.length;
|
|
http2PrefaceScanSearchUnits += Math.max(scannedTo - from, 0);
|
|
return offset;
|
|
}
|
|
|
|
/**
|
|
* Wrap the readiness gate's broker channel so its `onData` delivers no byte
|
|
* until the client connection preface appears, then delivers every byte from
|
|
* the preface onward.
|
|
*
|
|
* The wrapped channel opens its scan window only on the bytes the readiness
|
|
* gate already retained after it accepted the READY line (constraint: the
|
|
* scan window opens only after the gate accepts the nonce). Nothing before
|
|
* that line ever reaches this scan, because the gate itself discards the
|
|
* whole pre-READY buffer on acceptance. The scan buffers at most
|
|
* {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}; past that bound with no preface
|
|
* found, it calls `onMissing` exactly one time and stops buffering, so the
|
|
* caller can abort the open and fall back to `queue_v1`. The function holds
|
|
* no prologue byte count: it always scans for the fixed 24-octet sequence,
|
|
* never a length.
|
|
*
|
|
* The bytes that follow the found preface, before the HTTP/2 server binds a
|
|
* downstream listener, land in `pendingAfterPreface`. This buffer holds
|
|
* untrusted bytes on the same footing as the scan buffer, so it carries the
|
|
* same {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap. A chunk that would pass
|
|
* the cap fails closed: the function drops the buffer and stops the channel.
|
|
* The caller reads {@link replayOverflowed} after the preface settles and, on
|
|
* `true`, treats the open the same as a missing preface.
|
|
*/
|
|
function createHttp2PrefaceScanningChannel(
|
|
channel: CommandManagedDuplexChannel,
|
|
options: {
|
|
capBytes: number;
|
|
onFound: () => void;
|
|
onMissing: () => void;
|
|
},
|
|
): {
|
|
channel: CommandManagedDuplexChannel;
|
|
replayOverflowed: () => boolean;
|
|
disposeScanBuffer: () => void;
|
|
} {
|
|
// The pre-preface scan buffer. `scanBuf.append` grows its backing storage
|
|
// by doubling, instead of copying the whole retained buffer on every
|
|
// fragment — see {@link createGrowableByteBuffer}. `scanSearchFrom` is the
|
|
// first index the next search must examine: `findPrefaceFrom` advances it
|
|
// to just short of the buffer's end on every miss, so a fragmented preface
|
|
// is found without a full rescan of the retained buffer on every fragment.
|
|
const scanBuf = createGrowableByteBuffer((copiedBytes) => {
|
|
http2PrefaceScanBufferGrowthCopyUnits += copiedBytes;
|
|
});
|
|
let scanSearchFrom = 0;
|
|
let sawPreface = false;
|
|
let failed = false;
|
|
let downstream: ((chunk: Uint8Array) => void) | null = null;
|
|
// Bytes found after the preface before a downstream listener attaches. The
|
|
// wrapped channel replays them on attach, the same pattern the readiness
|
|
// gate itself uses for its own post-READY replay buffer. This also grows
|
|
// by doubling, for the same reason as `scanBuf`: fragmented post-preface
|
|
// input must not force a full copy of the retained buffer per fragment.
|
|
const pendingAfterPreface = createGrowableByteBuffer((copiedBytes) => {
|
|
http2PrefaceReplayBufferGrowthCopyUnits += copiedBytes;
|
|
});
|
|
let replayOverflow = false;
|
|
|
|
// Drop the pending buffer and stop the channel. The caller reads
|
|
// `replayOverflowed()` after the preface settles and falls back the same
|
|
// way it does for a missing preface.
|
|
function overflowAndStop(): void {
|
|
replayOverflow = true;
|
|
pendingAfterPreface.reset();
|
|
channel.stop();
|
|
}
|
|
|
|
function deliver(chunk: Buffer): void {
|
|
if (downstream) {
|
|
downstream(chunk);
|
|
return;
|
|
}
|
|
if (replayOverflow) return;
|
|
if (pendingAfterPreface.length() + chunk.byteLength > options.capBytes) {
|
|
overflowAndStop();
|
|
return;
|
|
}
|
|
pendingAfterPreface.append(chunk);
|
|
}
|
|
|
|
channel.onData((chunk) => {
|
|
if (failed || replayOverflow) return;
|
|
if (sawPreface) {
|
|
deliver(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
|
|
return;
|
|
}
|
|
const rawChunk = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
|
|
// Reject the chunk on its prospective length before it grows the scan
|
|
// buffer, the same way `deliver` bounds `pendingAfterPreface`. A single
|
|
// oversized chunk — or a chunk that tips an already-large buffer past
|
|
// the cap — must fail closed here, before `scanBuf.append` performs the
|
|
// allocation. Checking the cap only after the append still bounds the
|
|
// retained buffer, but it lets one untrusted chunk force an allocation
|
|
// as large as the chunk itself, unbounded by `capBytes`.
|
|
if (scanBuf.length() + rawChunk.byteLength > options.capBytes) {
|
|
failed = true;
|
|
scanBuf.reset();
|
|
scanSearchFrom = 0;
|
|
options.onMissing();
|
|
return;
|
|
}
|
|
scanBuf.append(rawChunk);
|
|
const scanBuffer = scanBuf.view();
|
|
const offset = findPrefaceFrom(scanBuffer, scanSearchFrom);
|
|
if (offset === -1) {
|
|
// No match yet. Resume the next search just short of the buffer's
|
|
// end, keeping back an overlap of one octet less than the preface
|
|
// length, so a preface split across this fragment and the next one is
|
|
// still found. Each byte enters that overlap window a bounded number
|
|
// of times, so the total search work stays linear in the bytes
|
|
// received, not quadratic in the number of fragments.
|
|
scanSearchFrom = Math.max(
|
|
0,
|
|
scanBuffer.length - (HTTP2_CLIENT_CONNECTION_PREFACE.length - 1),
|
|
);
|
|
return;
|
|
}
|
|
sawPreface = true;
|
|
options.onFound();
|
|
const fromPreface = Buffer.from(scanBuffer.subarray(offset));
|
|
scanBuf.reset();
|
|
scanSearchFrom = 0;
|
|
deliver(fromPreface);
|
|
});
|
|
|
|
return {
|
|
channel: {
|
|
write: (data: Uint8Array) => channel.write(data),
|
|
onData: (listener: (chunk: Uint8Array) => void) => {
|
|
downstream = listener;
|
|
if (pendingAfterPreface.length() > 0) {
|
|
// Copy the exact retained bytes instead of handing the listener
|
|
// the growable buffer's backing view. That backing storage can
|
|
// run ahead of the bytes in use (the doubling growth in
|
|
// `pendingAfterPreface.append` over-provisions it), so a raw view
|
|
// would keep the whole over-provisioned allocation alive for as
|
|
// long as the listener holds its reference.
|
|
const replay = Buffer.from(pendingAfterPreface.view());
|
|
pendingAfterPreface.reset();
|
|
listener(replay);
|
|
}
|
|
},
|
|
onExit: (listener: (exit: { exitCode: number | null }) => void) => channel.onExit(listener),
|
|
stop: () => channel.stop(),
|
|
close: () => channel.close(),
|
|
},
|
|
replayOverflowed: () => replayOverflow,
|
|
/**
|
|
* Drop the scan buffer, for a caller-side terminal path this function
|
|
* itself never reaches — the bound readiness timeout elapsing while the
|
|
* scan is still searching, with no preface found and no cap refusal of
|
|
* its own. A call after the preface already matched, or after the cap
|
|
* already failed the scan closed, is a no-op: both paths already reset
|
|
* the scan buffer themselves.
|
|
*/
|
|
disposeScanBuffer: (): void => {
|
|
if (sawPreface || failed) return;
|
|
failed = true;
|
|
scanBuf.reset();
|
|
scanSearchFrom = 0;
|
|
},
|
|
};
|
|
}
|
|
|
|
/** The terminal outcome of the preface scan: either the client preface
|
|
* appeared inside the bounded readiness buffer, or it did not. */
|
|
type Http2PrefaceScanResult = "found" | "missing";
|
|
|
|
/**
|
|
* Wait for {@link createHttp2PrefaceScanningChannel} to settle: either the
|
|
* preface appears, or the scan passes the bound with no match, or the bound
|
|
* readiness timeout elapses first. Reapplying the readiness timeout here
|
|
* keeps one configured value for both the READY-line wait and this
|
|
* immediately-following preface wait; the task adds no new timeout setting.
|
|
* Returns the scanning channel alongside the settled result, so the caller
|
|
* binds the HTTP/2 server to it only on a `found` result. On a `found`
|
|
* result, the caller must still read `replayOverflowed()`: the post-preface
|
|
* buffer can overflow its cap after the preface settles as `found` and
|
|
* before the caller binds a downstream listener.
|
|
*/
|
|
function scanForHttp2ClientPreface(
|
|
channel: CommandManagedDuplexChannel,
|
|
options: { capBytes: number; timeoutMs: number },
|
|
): {
|
|
scanned: CommandManagedDuplexChannel;
|
|
settled: Promise<Http2PrefaceScanResult>;
|
|
replayOverflowed: () => boolean;
|
|
} {
|
|
let resolveSettled!: (result: Http2PrefaceScanResult) => void;
|
|
let settledOnce = false;
|
|
const settled = new Promise<Http2PrefaceScanResult>((resolve) => {
|
|
resolveSettled = resolve;
|
|
});
|
|
// Reassigned to the real function once `createHttp2PrefaceScanningChannel`
|
|
// returns, below. `settle` can only actually run after that point (the
|
|
// timer fires later, and `onMissing`/`onFound` fire from inside the
|
|
// channel's own `onData`, which registers after this call), so the
|
|
// placeholder never runs for real.
|
|
let disposeScanBuffer: () => void = () => {};
|
|
const settle = (result: Http2PrefaceScanResult): void => {
|
|
if (settledOnce) return;
|
|
settledOnce = true;
|
|
clearTimeout(timer);
|
|
// The bound readiness timeout can elapse while the scan still searches,
|
|
// with no preface found and no cap refusal of its own. That path holds
|
|
// no other cleanup, so drop the scan buffer here. A `found` result, or a
|
|
// `missing` result the scan itself already failed closed, is a no-op
|
|
// inside `disposeScanBuffer`.
|
|
if (result === "missing") disposeScanBuffer();
|
|
resolveSettled(result);
|
|
};
|
|
const timer = setTimeout(() => settle("missing"), options.timeoutMs);
|
|
timer.unref?.();
|
|
const scan = createHttp2PrefaceScanningChannel(channel, {
|
|
capBytes: options.capBytes,
|
|
onFound: () => settle("found"),
|
|
onMissing: () => settle("missing"),
|
|
});
|
|
disposeScanBuffer = scan.disposeScanBuffer;
|
|
return { scanned: scan.channel, settled, replayOverflowed: scan.replayOverflowed };
|
|
}
|
|
|
|
/**
|
|
* Test-only surface for {@link scanForHttp2ClientPreface}. A test drives the
|
|
* post-preface replay cap across every terminal path without the whole
|
|
* bridge. Production code never reads this export.
|
|
*/
|
|
export const __http2PrefaceScanTesting = {
|
|
scanForHttp2ClientPreface: (
|
|
channel: CommandManagedDuplexChannel,
|
|
options: { capBytes: number; timeoutMs: number },
|
|
) => scanForHttp2ClientPreface(channel, options),
|
|
readScanSearchUnits: (): number => http2PrefaceScanSearchUnits,
|
|
resetScanSearchUnits: (): void => {
|
|
http2PrefaceScanSearchUnits = 0;
|
|
},
|
|
readScanBufferGrowthCopyUnits: (): number => http2PrefaceScanBufferGrowthCopyUnits,
|
|
resetScanBufferGrowthCopyUnits: (): void => {
|
|
http2PrefaceScanBufferGrowthCopyUnits = 0;
|
|
},
|
|
readReplayBufferGrowthCopyUnits: (): number => http2PrefaceReplayBufferGrowthCopyUnits,
|
|
resetReplayBufferGrowthCopyUnits: (): void => {
|
|
http2PrefaceReplayBufferGrowthCopyUnits = 0;
|
|
},
|
|
};
|
|
|
|
/**
|
|
* The terminal run disposition for the `http2_v1` path, in the same shape as
|
|
* {@link DuplexBrokerRunDisposition}. A `failed` disposition means a terminal
|
|
* loss ordered before an orderly completion, so the run must not report
|
|
* success.
|
|
*/
|
|
interface Http2RunDispositionLatch {
|
|
readonly disposition: DuplexBrokerRunDisposition;
|
|
/**
|
|
* Record a terminal loss. Returns `true` when the call flipped the
|
|
* disposition to failed; returns `false` when a loss or an orderly
|
|
* completion already latched, so the run already has its terminal result.
|
|
* The first recorded loss latches — a later call never overrides it.
|
|
*/
|
|
recordLoss(reason: DuplexLossReason): boolean;
|
|
/** Mark the host-observed orderly completion of the agent turn. A loss that
|
|
* already latched keeps the failure. */
|
|
markOrderlyCompletion(): void;
|
|
/** Atomically mark the orderly completion and read the disposition. */
|
|
settleRunDisposition(): DuplexBrokerRunDisposition;
|
|
/**
|
|
* Register a listener that fires once, only on the call to `recordLoss`
|
|
* that actually latches a new terminal loss. Returns a function that
|
|
* unregisters the listener.
|
|
*/
|
|
onLoss(listener: (reason: DuplexLossReason) => void): () => void;
|
|
}
|
|
|
|
function createHttp2RunDispositionLatch(): Http2RunDispositionLatch {
|
|
let lossOrdered = false;
|
|
let lossReason: DuplexLossReason | null = null;
|
|
let completionOrdered = false;
|
|
let lossListener: ((reason: DuplexLossReason) => void) | null = null;
|
|
const markOrderlyCompletion = (): void => {
|
|
if (completionOrdered || lossOrdered) return;
|
|
completionOrdered = true;
|
|
};
|
|
return {
|
|
get disposition(): DuplexBrokerRunDisposition {
|
|
return { failed: lossOrdered, lossReason };
|
|
},
|
|
recordLoss(reason: DuplexLossReason): boolean {
|
|
if (lossOrdered || completionOrdered) return false;
|
|
lossOrdered = true;
|
|
lossReason = reason;
|
|
lossListener?.(reason);
|
|
return true;
|
|
},
|
|
markOrderlyCompletion,
|
|
settleRunDisposition(): DuplexBrokerRunDisposition {
|
|
markOrderlyCompletion();
|
|
return { failed: lossOrdered, lossReason };
|
|
},
|
|
onLoss(listener: (reason: DuplexLossReason) => void): () => void {
|
|
lossListener = listener;
|
|
return () => {
|
|
if (lossListener === listener) lossListener = null;
|
|
};
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Create the run-log directory on the sandbox before the tail starts. The file
|
|
* bridge worker creates this directory on the file path. The duplex path starts
|
|
* no worker, so the host creates the directory here. The tail then reads a real
|
|
* directory from its first tick.
|
|
*
|
|
* This step is best effort. The broker already serves the duplex transport when
|
|
* the host reaches it, and the tail wrap command runs its own `mkdir -p` as a
|
|
* backstop. So a create failure must not tear down a working duplex transport;
|
|
* the host swallows it and still builds the tail. The log line names no raw
|
|
* error, so no raw error rides a log line here.
|
|
*/
|
|
async function ensureSandboxRunLogDirectory(input: {
|
|
runner: CommandManagedRuntimeRunner;
|
|
remoteCwd: string;
|
|
logsDir: string;
|
|
shellCommand: "bash" | "sh";
|
|
timeoutMs: number | null | undefined;
|
|
}): Promise<void> {
|
|
try {
|
|
await input.runner.execute({
|
|
command: input.shellCommand,
|
|
args: shellCommandArgs(`mkdir -p ${shellQuote(input.logsDir)}`),
|
|
cwd: input.remoteCwd,
|
|
timeoutMs: input.timeoutMs ?? undefined,
|
|
});
|
|
} catch {
|
|
// Best effort: the tail wrap command creates the directory before it writes.
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The duplex readiness gate. The gate owns the single data listener and the
|
|
* single exit listener of the channel while the host waits for a valid READY
|
|
* frame. It resolves the handshake, then hands the channel to the broker.
|
|
*
|
|
* The gate authenticates readiness with the nonce and the strict READY schema,
|
|
* not with the line position. A PTY channel echoes the launch wrapper line before
|
|
* it sets raw mode, so the first line is often not the READY frame. The gate skips
|
|
* each pre-READY line that does not decode as a READY frame, then accepts the first
|
|
* line that decodes as a READY frame with the matching nonce. A line that decodes
|
|
* as a READY frame with a wrong nonce fails the handshake with a nonce mismatch. An
|
|
* early exit or a timeout also fails the handshake. The gate never dispatches a
|
|
* request; the broker does that after readiness passes.
|
|
*
|
|
* The skipped bytes stay in the capped buffer, so the O(1) cap and the readiness
|
|
* timeout still bound the wait. The gate enforces the cap on every pre-READY path
|
|
* and before it decodes a candidate line, so an over-cap prefix never reaches READY
|
|
* acceptance. The cap check has priority over READY acceptance on every path.
|
|
*/
|
|
// The count of the pre-READY newline-scan work, in bytes. Each search adds the
|
|
// number of bytes it can read. A test reads this count to prove the scan work
|
|
// stays linear in the bytes received. Production code never reads this count.
|
|
let duplexReadinessNewlineScanUnits = 0;
|
|
|
|
// The count of bytes `appendReadinessBytes` copies while it grows the pre-READY
|
|
// buffer's backing storage. A one-copy-per-fragment approach copies the whole
|
|
// retained buffer on every fragment, so this count grows quadratically in the
|
|
// number of fragments. The doubling-growth approach copies only on a
|
|
// reallocation, so this count stays linear in the bytes received. A test reads
|
|
// this count to prove the growth work does not regress to the quadratic shape.
|
|
// Production code never reads this count.
|
|
let duplexReadinessBufferGrowthCopyUnits = 0;
|
|
|
|
/** The newline byte. The readiness buffer is a byte buffer, not a string. */
|
|
const READINESS_NEWLINE_BYTE = 0x0a;
|
|
/** The opening-brace byte. The bracketed-paste retry scans for it, not a string. */
|
|
const READINESS_OPEN_BRACE_BYTE = 0x7b;
|
|
/** The one shared empty buffer. The gate starts and resets `buffer` to it. */
|
|
const READINESS_EMPTY_BUFFER = Buffer.alloc(0);
|
|
|
|
/**
|
|
* Find the first newline in `buffer` at or after index `from` and return its
|
|
* index, or -1. `Buffer#indexOf` reads the bytes from `from` up to the newline
|
|
* it finds, or to the end of the buffer when it finds none. This helper counts
|
|
* that real scan distance for a test. The count stays linear in the bytes
|
|
* received when the caller advances `from` past each newline it consumes.
|
|
*/
|
|
function findNewlineFrom(buffer: Buffer, from: number): number {
|
|
const newlineIndex = buffer.indexOf(READINESS_NEWLINE_BYTE, from);
|
|
const scanned = newlineIndex === -1 ? buffer.length - from : newlineIndex - from + 1;
|
|
duplexReadinessNewlineScanUnits += scanned;
|
|
return newlineIndex;
|
|
}
|
|
|
|
/**
|
|
* Test-only surface for the pre-READY readiness gate. A test reads the scan
|
|
* count to prove the newline search work stays linear in the bytes received.
|
|
* Production code does not use this object.
|
|
*/
|
|
export const __duplexReadinessTesting = {
|
|
readNewlineScanUnits: (): number => duplexReadinessNewlineScanUnits,
|
|
resetNewlineScanUnits: (): void => {
|
|
duplexReadinessNewlineScanUnits = 0;
|
|
},
|
|
readBufferGrowthCopyUnits: (): number => duplexReadinessBufferGrowthCopyUnits,
|
|
resetBufferGrowthCopyUnits: (): void => {
|
|
duplexReadinessBufferGrowthCopyUnits = 0;
|
|
},
|
|
// Build one readiness gate over a supplied channel, so a test can drive the
|
|
// readiness-replay cap lifecycle across every terminal path without the whole
|
|
// bridge. Production code never reads this factory.
|
|
createReadinessGate: (channel: CommandManagedDuplexChannel, options: { nonce: string; timeoutMs: number }) =>
|
|
createDuplexReadinessGate(channel, options),
|
|
};
|
|
|
|
interface DuplexReadinessGate {
|
|
/** Resolves with the handshake outcome. It never rejects. */
|
|
readonly ready: Promise<DuplexReadinessResult>;
|
|
/**
|
|
* The channel view the broker consumes after readiness passes. It replays the
|
|
* bytes that followed the READY frame, then forwards each later chunk and the
|
|
* exit. The gate keeps one real data listener, so the broker never double-binds
|
|
* the channel.
|
|
*/
|
|
readonly brokerChannel: CommandManagedDuplexChannel;
|
|
/**
|
|
* Report whether a post-READY pre-bind chunk tipped the pending replay buffer
|
|
* past {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}. On such an overflow the gate
|
|
* drops the pending replay buffer and stops the channel. The caller reads this
|
|
* after `ready` resolves `ok`, and before it binds the broker.
|
|
*/
|
|
replayOverflowed(): boolean;
|
|
/**
|
|
* Drop the pending replay buffer. The caller runs this on a terminal path that
|
|
* abandons the pending replay without a broker handoff: a readiness failure, a
|
|
* replay overflow, or a broker-construction failure. The normal handoff already
|
|
* drops the buffer inside `brokerChannel.onData`, so a later call here is a
|
|
* no-op.
|
|
*/
|
|
disposePendingReplay(): void;
|
|
/**
|
|
* Test-only. Report the length of the retained pre-READY buffer, in bytes. A
|
|
* test reads this to prove the gate drops the pre-READY buffer on READY
|
|
* acceptance, so the process does not retain the sandbox-controlled prefix.
|
|
* Production code does not read this.
|
|
*/
|
|
retainedReadinessBufferLength(): number;
|
|
}
|
|
|
|
function createDuplexReadinessGate(
|
|
channel: CommandManagedDuplexChannel,
|
|
options: {
|
|
nonce: string;
|
|
timeoutMs: number;
|
|
},
|
|
): DuplexReadinessGate {
|
|
let settled = false;
|
|
let readyOk = false;
|
|
// The gate sets this when a post-READY pre-bind chunk tips the pending replay
|
|
// buffer past the cap. On that overflow the gate drops the pending buffer and
|
|
// stops the channel. The caller reads it through `replayOverflowed` and selects
|
|
// the file bridge.
|
|
let replayOverflow = false;
|
|
// The raw bytes the host reads before the READY frame completes. `buffer` is
|
|
// append-only and always a zero-copy view over the used prefix of `storage`,
|
|
// so the O(1) cap check on `buffer.length` stays valid.
|
|
let buffer: Buffer = READINESS_EMPTY_BUFFER;
|
|
// The backing storage for `buffer`. `appendReadinessBytes` grows this by
|
|
// doubling its capacity, instead of copying the whole retained buffer on
|
|
// every fragment. See `appendReadinessBytes` for why this bounds the total
|
|
// copy work.
|
|
let storage: Buffer = READINESS_EMPTY_BUFFER;
|
|
// The start index of the current line in `buffer`. A leading blank line
|
|
// advances this cursor past its newline without a buffer copy.
|
|
let lineStart = 0;
|
|
// The next index to search for a newline. The gate scans from here, so each
|
|
// byte is read at most one time for the newline search.
|
|
let scanFrom = 0;
|
|
|
|
/**
|
|
* Append `chunk` to the pre-READY buffer without copying the bytes already
|
|
* retained. A sender that trickles the handshake in many small fragments
|
|
* (a slow socket, a byte-at-a-time PTY echo) would otherwise force one full
|
|
* copy of the whole retained buffer per fragment: with `buffer` growing
|
|
* toward the {@link DUPLEX_READINESS_BUFFER_CAP_BYTES} cap, that is
|
|
* quadratic work in the number of fragments. This instead grows `storage`
|
|
* by doubling its capacity only when the current capacity runs out, so the
|
|
* backing store reallocates and copies a logarithmic number of times, and
|
|
* each append copies only the incoming chunk. The used length still reads
|
|
* in O(1) through `buffer.length`, so every cap check and slice below stays
|
|
* unchanged.
|
|
*/
|
|
function appendReadinessBytes(chunk: Uint8Array): void {
|
|
const usedLength = buffer.length;
|
|
const neededLength = usedLength + chunk.byteLength;
|
|
if (neededLength > storage.length) {
|
|
let nextCapacity = storage.length === 0 ? chunk.byteLength : storage.length * 2;
|
|
while (nextCapacity < neededLength) {
|
|
nextCapacity *= 2;
|
|
}
|
|
const grown = Buffer.allocUnsafe(nextCapacity);
|
|
storage.copy(grown, 0, 0, usedLength);
|
|
duplexReadinessBufferGrowthCopyUnits += usedLength;
|
|
storage = grown;
|
|
}
|
|
storage.set(chunk, usedLength);
|
|
buffer = storage.subarray(0, neededLength);
|
|
}
|
|
// The bytes that followed the READY frame, held until the broker binds.
|
|
let pending: Uint8Array = READINESS_EMPTY_BUFFER;
|
|
// The exit that arrived after READY but before the broker bound, if any.
|
|
let pendingExit: { exitCode: number | null } | null = null;
|
|
let dataSink: ((chunk: Uint8Array) => void) | null = null;
|
|
let exitSink: ((exit: { exitCode: number | null }) => void) | null = null;
|
|
let resolveReady!: (result: DuplexReadinessResult) => void;
|
|
const ready = new Promise<DuplexReadinessResult>((resolve) => {
|
|
resolveReady = resolve;
|
|
});
|
|
|
|
const timer = setTimeout(() => finish({ ok: false, reason: "timeout" }), options.timeoutMs);
|
|
timer.unref?.();
|
|
|
|
function finish(result: DuplexReadinessResult): void {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
if (result.ok) readyOk = true;
|
|
resolveReady(result);
|
|
}
|
|
|
|
channel.onData((chunk) => {
|
|
if (dataSink) {
|
|
dataSink(chunk);
|
|
return;
|
|
}
|
|
if (readyOk) {
|
|
// READY already passed; hold the bytes until the broker binds. Bound this
|
|
// buffer directly against {@link DUPLEX_READINESS_BUFFER_CAP_BYTES}, the
|
|
// same way the preface-scan gate's own post-preface replay buffer bounds
|
|
// itself: a worker that keeps sending bytes after READY, faster than the
|
|
// broker can bind, cannot grow this buffer past the cap. A chunk that
|
|
// would pass the cap fails closed: the gate drops the pending buffer,
|
|
// stops the channel, and sets the overflow flag. The caller reads the
|
|
// flag and selects the file bridge, because `ready` already resolved
|
|
// before this synchronous post-READY chunk arrived.
|
|
if (pending.length + chunk.byteLength > DUPLEX_READINESS_BUFFER_CAP_BYTES) {
|
|
replayOverflow = true;
|
|
pending = READINESS_EMPTY_BUFFER;
|
|
channel.stop();
|
|
return;
|
|
}
|
|
// Copy a first chunk instead of aliasing the caller's `Uint8Array`, so a
|
|
// channel that reuses its delivered buffer across calls cannot corrupt the
|
|
// bytes this gate holds for the broker replay.
|
|
pending = pending.length === 0 ? Buffer.from(chunk) : Buffer.concat([pending, chunk]);
|
|
return;
|
|
}
|
|
if (settled) {
|
|
// The handshake already failed. The channel is untrusted and can keep
|
|
// sending bytes until the host closes it. Drop them, so a failed handshake
|
|
// never grows the buffer after the gate settles.
|
|
return;
|
|
}
|
|
// Append the new bytes and continue the newline search from `scanFrom`, the
|
|
// first index not yet examined. Each byte is read at most one time for the
|
|
// search, and `appendReadinessBytes` copies at most the incoming chunk, so
|
|
// the total work stays linear in the bytes received, not in the number of
|
|
// fragments they arrive in.
|
|
appendReadinessBytes(chunk);
|
|
for (;;) {
|
|
const newlineIndex = findNewlineFrom(buffer, scanFrom);
|
|
if (newlineIndex === -1) {
|
|
// No complete line yet. The whole buffer up to the end is now scanned.
|
|
scanFrom = buffer.length;
|
|
// The gate reads untrusted bytes, so bound the pre-READY buffer. Past
|
|
// the cap with no complete READY line, the stream cannot be a valid
|
|
// READY frame, so finish with protocol contamination. The retained
|
|
// skipped lines count against the cap; that is acceptable and fail-closed.
|
|
//
|
|
// `buffer` is a byte buffer, so `buffer.length` is the exact byte count,
|
|
// read in O(1).
|
|
if (buffer.length > DUPLEX_READINESS_BUFFER_CAP_BYTES) {
|
|
finish({ ok: false, reason: "protocol_contamination" });
|
|
}
|
|
return;
|
|
}
|
|
if (newlineIndex === lineStart) {
|
|
// Skip a blank line, the same as the frame decoder. Advance the line-start
|
|
// cursor past the newline without a buffer copy, then search the next line
|
|
// from the position after the newline. Enforce the cap after the advance,
|
|
// so a blank-line flood past the cap fails closed before READY acceptance.
|
|
lineStart = newlineIndex + 1;
|
|
scanFrom = newlineIndex + 1;
|
|
if (lineStart > DUPLEX_READINESS_BUFFER_CAP_BYTES) {
|
|
finish({ ok: false, reason: "protocol_contamination" });
|
|
return;
|
|
}
|
|
continue;
|
|
}
|
|
// A complete non-blank candidate line spans `[lineStart, newlineIndex)`.
|
|
// Enforce the cap on the line's end offset before the decode, so an over-cap
|
|
// prefix never reaches READY acceptance on a completed line.
|
|
if (newlineIndex + 1 > DUPLEX_READINESS_BUFFER_CAP_BYTES) {
|
|
finish({ ok: false, reason: "protocol_contamination" });
|
|
return;
|
|
}
|
|
// Slice only the single candidate line for the decode.
|
|
const line = buffer.slice(lineStart, newlineIndex);
|
|
let decoded = decodeDuplexLine(line);
|
|
if (!decoded.ok) {
|
|
// The whole line did not decode. A terminal can put bytes in front of the
|
|
// gateway's first frame on the same line, with no newline between them: a
|
|
// shell with bracketed paste enabled writes its disable sequence and a
|
|
// bare carriage return (`ESC [ ? 2 0 0 4 l CR`) immediately before the
|
|
// child's first output. So retry the decode from the first `{` in the
|
|
// line, which is where a frame can start.
|
|
//
|
|
// This does not weaken the handshake. The retry still runs the same
|
|
// strict decode over the remainder of the line, and readiness still
|
|
// authenticates on the nonce below, so a prefix cannot forge a frame or
|
|
// smuggle a second one — it can only be discarded.
|
|
const braceIndex = line.indexOf(READINESS_OPEN_BRACE_BYTE);
|
|
if (braceIndex > 0) {
|
|
decoded = decodeDuplexLine(line.slice(braceIndex));
|
|
}
|
|
}
|
|
if (decoded.ok && decoded.frame.type === "ready") {
|
|
// A line that decodes as a READY frame authenticates by the nonce. A wrong
|
|
// nonce fails the handshake; the matching nonce passes it.
|
|
if (decoded.frame.nonce !== options.nonce) {
|
|
finish({ ok: false, reason: "nonce_mismatch" });
|
|
return;
|
|
}
|
|
// The bytes that follow the READY line become the replay buffer for the
|
|
// broker.
|
|
//
|
|
// Copy the suffix instead of slicing it off `buffer`. `buffer` is a view
|
|
// over `storage`, and `storage`'s capacity can run ahead of the bytes in
|
|
// use (the doubling growth in `appendReadinessBytes` over-provisions it).
|
|
// A slice would keep that whole over-provisioned allocation alive for as
|
|
// long as the broker replay holds its reference. The copy is exactly
|
|
// `suffix.length` bytes, one time, not a per-fragment cost.
|
|
const suffix = Buffer.from(buffer.subarray(newlineIndex + 1));
|
|
// Drop the original pre-READY buffer and its backing storage now. The
|
|
// gate keeps only the retained suffix as `pending`. This clear also
|
|
// covers the broker handoff and the replay disposal. Both run later and
|
|
// read no buffer bytes.
|
|
buffer = READINESS_EMPTY_BUFFER;
|
|
storage = READINESS_EMPTY_BUFFER;
|
|
pending = suffix;
|
|
finish({ ok: true });
|
|
return;
|
|
}
|
|
// The line does not decode as a READY frame. A PTY echo line or any other
|
|
// pre-READY noise reaches here. Skip it and keep scanning; the nonce and the
|
|
// strict schema, not the line position, authenticate readiness. Enforce the
|
|
// cap after the advance, so a noise-line flood past the cap fails closed.
|
|
lineStart = newlineIndex + 1;
|
|
scanFrom = newlineIndex + 1;
|
|
if (lineStart > DUPLEX_READINESS_BUFFER_CAP_BYTES) {
|
|
finish({ ok: false, reason: "protocol_contamination" });
|
|
return;
|
|
}
|
|
}
|
|
});
|
|
|
|
channel.onExit((exit) => {
|
|
if (exitSink) {
|
|
exitSink(exit);
|
|
return;
|
|
}
|
|
if (readyOk) {
|
|
// The channel exited after READY but before the broker bound. Hold the
|
|
// exit so the broker still learns of the loss.
|
|
pendingExit = exit;
|
|
return;
|
|
}
|
|
finish({ ok: false, reason: "channel_exit" });
|
|
});
|
|
|
|
const brokerChannel: CommandManagedDuplexChannel = {
|
|
write: (data: Uint8Array) => channel.write(data),
|
|
onData: (listener: (chunk: Uint8Array) => void) => {
|
|
dataSink = listener;
|
|
if (pending.length > 0) {
|
|
const replay = pending;
|
|
pending = READINESS_EMPTY_BUFFER;
|
|
listener(replay);
|
|
return;
|
|
}
|
|
},
|
|
onExit: (listener: (exit: { exitCode: number | null }) => void) => {
|
|
exitSink = listener;
|
|
if (pendingExit) {
|
|
const exit = pendingExit;
|
|
pendingExit = null;
|
|
listener(exit);
|
|
}
|
|
},
|
|
stop: () => channel.stop(),
|
|
close: () => channel.close(),
|
|
};
|
|
|
|
return {
|
|
ready,
|
|
brokerChannel,
|
|
replayOverflowed: () => replayOverflow,
|
|
disposePendingReplay: () => {
|
|
pending = READINESS_EMPTY_BUFFER;
|
|
},
|
|
retainedReadinessBufferLength: () => buffer.length,
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Close a partial duplex channel inside a bounded budget, then stop the child.
|
|
* The host runs this on any readiness failure, so a failed duplex attempt leaves
|
|
* no live provider session before the fallback to the file bridge.
|
|
*/
|
|
async function closeDuplexChannelWithinBudget(
|
|
channel: CommandManagedDuplexChannel,
|
|
budgetMs: number,
|
|
): Promise<void> {
|
|
try {
|
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
const budget = new Promise<void>((resolve) => {
|
|
timer = setTimeout(resolve, budgetMs);
|
|
timer.unref?.();
|
|
});
|
|
await Promise.race([channel.close().catch(() => undefined), budget]);
|
|
if (timer !== undefined) clearTimeout(timer);
|
|
} catch {
|
|
// Best effort: the stop below still removes the child.
|
|
} finally {
|
|
try {
|
|
channel.stop();
|
|
} catch {
|
|
// The channel is already gone; nothing more to do.
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function startAdapterExecutionTargetPaperclipBridge(input: {
|
|
runId: string;
|
|
target: AdapterExecutionTarget | null | undefined;
|
|
runtimeRootDir: string | null | undefined;
|
|
adapterKey: string;
|
|
timeoutSec?: number | null;
|
|
hostApiToken: string | null | undefined;
|
|
hostApiUrl?: string | null;
|
|
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
|
maxBodyBytes?: number | null;
|
|
// The deadline for one forward call, in milliseconds. This is the inner budget
|
|
// of the duplex broker's nested budget set. The default is the forward budget
|
|
// in `DEFAULT_DUPLEX_BROKER_BUDGETS` (30 s), so the current behavior does not
|
|
// change when the caller sets no option.
|
|
forwardTimeoutMs?: number | null;
|
|
// The first strict gate for the duplex transport. The host selects duplex only
|
|
// when this is exactly `true` and the resolved capability
|
|
// `duplexCommandStream` is exactly `true`. Any other value of either gate
|
|
// selects the file bridge. The caller reads this from the experimental instance
|
|
// setting `enableSandboxDuplexBridge`. The default is the file bridge.
|
|
// HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback.
|
|
enableSandboxDuplexBridge?: boolean | null;
|
|
// The deadline for the duplex readiness handshake, in milliseconds. On a
|
|
// timeout the host closes the partial channel and selects the file bridge. The
|
|
// default is `DEFAULT_DUPLEX_READINESS_TIMEOUT_MS`.
|
|
duplexReadinessTimeoutMs?: number | null;
|
|
// Return the current-run parent-context token. The factory threads it into the
|
|
// callback bridge worker, which reads it per request so each request
|
|
// `sandbox.exec` span parents to the live run span. When it is absent, the
|
|
// request work runs with an empty store, exactly like the earlier behavior.
|
|
getRuntimeParentContext?: () => StartupSpanContext | undefined;
|
|
// Wrap each callback request in a `sandbox.callbackBridge.relayRequest` span.
|
|
// The factory threads it into the worker, which uses it per request so each
|
|
// request's execs group under one wrapper span. When it is absent, the request
|
|
// work runs under the run parent with no wrapper span.
|
|
runtimeSpan?: RuntimeSpanRunner;
|
|
// The injected recorder for the fixed duplex observability surface. The factory
|
|
// binds it to a provider-scoped telemetry facade, which records the channel-open
|
|
// span, the request span, the guarded counters, and the transport event. The
|
|
// default is a no-op recorder, so the surface stays inert until the host injects
|
|
// a real recorder.
|
|
duplexObservabilityRecorder?: DuplexObservabilityRecorder | null;
|
|
}): Promise<AdapterExecutionTargetPaperclipBridgeHandle | null> {
|
|
if (!adapterExecutionTargetUsesPaperclipBridge(input.target)) {
|
|
return null;
|
|
}
|
|
if (!input.target || input.target.kind !== "remote") {
|
|
return null;
|
|
}
|
|
|
|
const target = input.target;
|
|
const onLog = input.onLog ?? (async () => {});
|
|
const hostApiToken = input.hostApiToken?.trim() ?? "";
|
|
if (hostApiToken.length === 0) {
|
|
throw new Error("Sandbox bridge mode requires a host-side Paperclip API token.");
|
|
}
|
|
// The forward budget for one relayed request. It stays at the broker's default
|
|
// forward budget (30 s) when the caller sets no option, so current behavior
|
|
// does not change.
|
|
const forwardTimeoutMs = input.forwardTimeoutMs ?? DEFAULT_DUPLEX_BROKER_BUDGETS.forwardTimeoutMs;
|
|
|
|
const runtimeRootDir =
|
|
input.runtimeRootDir?.trim().length
|
|
? input.runtimeRootDir.trim()
|
|
: path.posix.join(target.remoteCwd, ".paperclip-runtime", input.adapterKey);
|
|
const bridgeRuntimeDir = path.posix.join(runtimeRootDir, "paperclip-bridge");
|
|
const queueDir = path.posix.join(bridgeRuntimeDir, "queue");
|
|
const assetRemoteDir = path.posix.join(bridgeRuntimeDir, "server");
|
|
const bridgeToken = createSandboxCallbackBridgeToken();
|
|
const maxBodyBytes =
|
|
typeof input.maxBodyBytes === "number" && Number.isFinite(input.maxBodyBytes) && input.maxBodyBytes > 0
|
|
? Math.trunc(input.maxBodyBytes)
|
|
: DEFAULT_SANDBOX_CALLBACK_BRIDGE_MAX_BODY_BYTES;
|
|
// The bridge worker runs inside the same process that serves the Paperclip
|
|
// API, so forwarded sandbox calls must target the LOCAL listen origin. The
|
|
// PAPERCLIP_RUNTIME_API_URL / PAPERCLIP_API_URL exports now prefer a
|
|
// configured public base URL, which is the origin browsers and external
|
|
// agents use; routing this in-process loopback hop through the network edge
|
|
// breaks deployments whose public origin sits behind a session-gated proxy
|
|
// (every forwarded agent API call is rejected at the edge). Server boot
|
|
// exports PAPERCLIP_LISTEN_HOST / PAPERCLIP_LISTEN_PORT before any run
|
|
// executes, and resolveDefaultPaperclipApiUrl() maps wildcard listen hosts
|
|
// to the loopback address of the same family (0.0.0.0 -> 127.0.0.1,
|
|
// :: -> [::1]), so the fallback is always loopback-reachable.
|
|
// input.hostApiUrl stays available as an explicit override seam.
|
|
const hostApiUrl = input.hostApiUrl?.trim() || resolveDefaultPaperclipApiUrl();
|
|
const shellCommand = adapterExecutionTargetShellCommand(target);
|
|
const runner = adapterExecutionTargetCommandRunner(target);
|
|
const bridgeTimeoutMs =
|
|
typeof input.timeoutSec === "number" && Number.isFinite(input.timeoutSec) && input.timeoutSec > 0
|
|
? Math.trunc(input.timeoutSec * 1000)
|
|
: adapterExecutionTargetTimeoutMs(target);
|
|
|
|
await onLog(
|
|
"stdout",
|
|
`[paperclip] Starting sandbox callback bridge for ${input.adapterKey} in ${bridgeRuntimeDir}.\n`,
|
|
);
|
|
|
|
const bridgeAsset = await createSandboxCallbackBridgeAsset();
|
|
|
|
// The provider-scoped telemetry facade for the fixed duplex observability
|
|
// surface. It maps the raw provider key through the allowlist one time, so no
|
|
// raw plugin key reaches a span, a counter, or the event. The default recorder
|
|
// is a no-op, so the facade is inert until the host injects a real recorder.
|
|
const duplexProviderKey =
|
|
"providerKey" in target ? target.providerKey ?? undefined : undefined;
|
|
const duplexObservability = createDuplexObservability({
|
|
recorder: input.duplexObservabilityRecorder ?? undefined,
|
|
providerKey: duplexProviderKey,
|
|
// http2_v1 is the one active non-file transport now; every non-file
|
|
// record this facade produces stamps the `http2` transport value.
|
|
transport: "http2",
|
|
});
|
|
|
|
// PAPERCLIP_BRIDGE_DEBUG opts into verbose stdout logs of every bridge proxy
|
|
// request/response. The query string is logged verbatim, so callers who pass
|
|
// auth tokens or other sensitive values as query parameters should be aware
|
|
// those values appear in the host process's stdout when this flag is enabled.
|
|
// Only intended for active debugging in trusted environments.
|
|
const bridgeDebugEnabled = isBridgeDebugEnabled(process.env);
|
|
|
|
// One forward of a relayed sandbox request onto the existing Paperclip API
|
|
// path. The forward applies the real host token and the signed run id, so the
|
|
// token replacement and the run attribution stay in one place for both the
|
|
// file bridge and the duplex broker. The sandbox request carries only the
|
|
// bridge token; the real agent token never leaves the host.
|
|
const forwardBridgeRequest = async (
|
|
request: {
|
|
method: string;
|
|
path: string;
|
|
query: string;
|
|
headers: Record<string, string>;
|
|
/** The file bridge passes the whole request body here as one string.
|
|
* The HTTP/2 bridge passes it as the raw `Buffer` it read off the wire. */
|
|
body?: string | Buffer;
|
|
},
|
|
signal?: AbortSignal,
|
|
options?: {
|
|
suppressDebugLog?: boolean;
|
|
/**
|
|
* The caller's stream reservation owner, if it has one. The HTTP/2
|
|
* bridge passes the stream's own owner here, so the response body copy
|
|
* reserves against the same ceiling the request body copy already
|
|
* reserved against. The queue transport passes no owner, so its
|
|
* response-body read enforces only the per-request size ceiling, exactly
|
|
* as it did before this option existed.
|
|
*/
|
|
reservation?: BridgeBodyReservation;
|
|
},
|
|
): Promise<{ status: number; headers: Record<string, string>; body: Buffer }> => {
|
|
const method = request.method.trim().toUpperCase() || "GET";
|
|
// The per-request debug log prints the method, the path, and the query. The
|
|
// duplex path suppresses it, so no route or query rides a log line there. The
|
|
// file path keeps the existing behavior.
|
|
const emitDebugLog = bridgeDebugEnabled && options?.suppressDebugLog !== true;
|
|
if (emitDebugLog) {
|
|
await onLog(
|
|
"stdout",
|
|
`[paperclip] Bridge proxy ${method} ${request.path}${request.query ? `?${request.query}` : ""}\n`,
|
|
);
|
|
}
|
|
const headers = new Headers();
|
|
for (const [key, value] of Object.entries(request.headers)) {
|
|
if (value.trim().length === 0) continue;
|
|
headers.set(key, value);
|
|
}
|
|
headers.set("authorization", `Bearer ${hostApiToken}`);
|
|
headers.set("x-paperclip-run-id", input.runId);
|
|
// Abort the forward when the caller aborts the request (its per-iteration
|
|
// timeout or watchdog fired, or the broker's forward budget ended), or after
|
|
// the forward budget here, whichever comes first.
|
|
const timeoutSignal = AbortSignal.timeout(forwardTimeoutMs);
|
|
const forwardSignal = signal ? AbortSignal.any([signal, timeoutSignal]) : timeoutSignal;
|
|
// Build the request-body init. A GET or a HEAD carries no body. The file
|
|
// bridge passes the whole body as one string; the HTTP/2 bridge passes it
|
|
// as a raw `Buffer`. Undici accepts a `Buffer` request body directly (a
|
|
// `Buffer` is an `ArrayBufferView`), so neither shape needs a conversion.
|
|
// The cast below only bridges a `BodyInit` typing gap: the DOM library
|
|
// type this project's ambient `RequestInit` resolves to excludes a
|
|
// `Buffer`, though Undici accepts one at runtime.
|
|
const forwardInit: RequestInit = {
|
|
method,
|
|
headers,
|
|
signal: forwardSignal,
|
|
};
|
|
if (method !== "GET" && method !== "HEAD" && request.body !== undefined) {
|
|
forwardInit.body = request.body as BodyInit;
|
|
}
|
|
const response = await fetch(buildBridgeForwardUrl(hostApiUrl, request), forwardInit);
|
|
if (emitDebugLog) {
|
|
await onLog(
|
|
"stdout",
|
|
`[paperclip] Bridge proxy response ${response.status} for ${method} ${request.path}${request.query ? `?${request.query}` : ""}\n`,
|
|
);
|
|
}
|
|
// The host delivered response headers, so the response-body read starts after
|
|
// the host processed the request. A later response-body read failure (a body
|
|
// over the size limit, or a stream read error) needs a classification by
|
|
// method safety. A safe method (GET, HEAD, OPTIONS, TRACE) never changes host
|
|
// state, so a retry cannot double-apply a mutation and the failure stays
|
|
// retryable. A mutating or otherwise unsafe method may have committed on the
|
|
// host, so a retryable status is unsafe: it makes the caller repeat the
|
|
// request with a new request id outside the broker deduplication set, and the
|
|
// host applies the mutation twice. For an unsafe method the code returns a
|
|
// non-retryable 504 and marks the outcome indeterminate, exactly like an
|
|
// aborted in-flight forward. The in-sandbox server maps the indeterminate 504
|
|
// to a non-retryable 409 for both the file bridge and the duplex broker.
|
|
let responseBody: Buffer;
|
|
try {
|
|
responseBody = await readBridgeForwardResponseBody(response, maxBodyBytes, options?.reservation);
|
|
} catch (error) {
|
|
// A denied reservation is retryable capacity pressure for a safe
|
|
// method, not a body-read fault: rethrow it before the method-safety
|
|
// classification below runs, so it reaches the HTTP/2 bridge's own
|
|
// capacity-denial catch (which answers the retryable 503 and settles
|
|
// the stream) instead of this function turning it into a 502. For an
|
|
// unsafe (mutating) method, the host has already delivered response
|
|
// headers by this point, so it may already have committed the
|
|
// mutation. Rethrowing there too would let the retryable 503 reach a
|
|
// caller that repeats the request, applying the mutation twice. An
|
|
// unsafe method's capacity denial falls through to the same
|
|
// non-retryable indeterminate 504 any other response-body read fault
|
|
// gets below.
|
|
if (error instanceof BridgeProcessCapacityError && isSafeBridgeMethod(method)) throw error;
|
|
if (isSafeBridgeMethod(method)) {
|
|
// The method is safe, so a retry cannot double-apply a mutation. Return a
|
|
// retryable 502 with no indeterminate marker, so the gateway passes it
|
|
// through as a retryable status.
|
|
return {
|
|
status: 502,
|
|
headers: { "content-type": "application/json" },
|
|
body: Buffer.from(
|
|
JSON.stringify({
|
|
error: error instanceof Error ? error.message : String(error),
|
|
}),
|
|
"utf8",
|
|
),
|
|
};
|
|
}
|
|
return {
|
|
status: 504,
|
|
headers: {
|
|
"content-type": "application/json",
|
|
"x-paperclip-bridge-outcome": "indeterminate",
|
|
},
|
|
body: Buffer.from(
|
|
JSON.stringify({
|
|
error: error instanceof Error ? error.message : String(error),
|
|
outcome: "indeterminate",
|
|
retryable: false,
|
|
}),
|
|
"utf8",
|
|
),
|
|
};
|
|
}
|
|
const commentMarker = postedIssueCommentLogMarker(method, request.path, response.status, responseBody);
|
|
if (commentMarker) await onLog("stdout", commentMarker);
|
|
return {
|
|
status: response.status,
|
|
headers: buildBridgeResponseHeaders(response),
|
|
body: responseBody,
|
|
};
|
|
};
|
|
|
|
// Two strict gates guard the duplex transport. Select duplex only when the
|
|
// experimental setting is exactly `true`, the resolved capability
|
|
// `duplexCommandStream` is exactly `true`, and the runner exposes the duplex
|
|
// channel. Any other value of either gate selects the file bridge below.
|
|
// HTTP/2 is the preferred transport. `queue_v1` is the soft-deprecated fallback.
|
|
const duplexRequested = input.enableSandboxDuplexBridge === true;
|
|
const capabilityGranted =
|
|
"effectiveCapabilities" in target &&
|
|
target.effectiveCapabilities?.duplexCommandStream === true;
|
|
const openDuplexChannel = runner.openDuplexChannel?.bind(runner);
|
|
// Record the pre-attempt fallback for a file-bridge selection that opens no
|
|
// channel. `gate_off` marks the kill switch off; `capability_absent` marks the
|
|
// capability or the runner method absent. A later channel-open failure records
|
|
// its own fallback through the channel-open attempt below.
|
|
if (!duplexRequested) {
|
|
duplexObservability.recordFallback("gate_off");
|
|
} else if (!capabilityGranted || typeof openDuplexChannel !== "function") {
|
|
duplexObservability.recordFallback("capability_absent");
|
|
}
|
|
if (duplexRequested && capabilityGranted && typeof openDuplexChannel === "function") {
|
|
// Begin the channel-open attempt. The block reports exactly one terminal:
|
|
// `ready` on success, or `fallback(reason)` on an open or a readiness failure.
|
|
const duplexChannelOpen = duplexObservability.startChannelOpen();
|
|
const readinessTimeoutMs =
|
|
typeof input.duplexReadinessTimeoutMs === "number" &&
|
|
Number.isFinite(input.duplexReadinessTimeoutMs) &&
|
|
input.duplexReadinessTimeoutMs > 0
|
|
? Math.trunc(input.duplexReadinessTimeoutMs)
|
|
: DEFAULT_DUPLEX_READINESS_TIMEOUT_MS;
|
|
|
|
// The host assigns the loopback port before it opens the channel. It passes
|
|
// the port and one random per-open nonce to the gateway only through the
|
|
// launch environment. It builds the sandbox-facing origin from its own
|
|
// stored port. No field of any channel frame contributes to the endpoint.
|
|
//
|
|
// Provider-boundary rationale (a security-review condition): the Daytona SDK
|
|
// surface the plugin uses (`sandbox.process` sessions, PTY, and exec;
|
|
// `sandbox.fs`) exposes no listener identity that the provider control plane
|
|
// binds to a launched process. No channel-supplied address can be attested,
|
|
// so this design uses zero channel-supplied addressing data. The only actor
|
|
// that can pre-bind the host-assigned loopback port before the agent starts
|
|
// is the provider itself, which already delivers the agent launch
|
|
// environment. The gateway bind-or-exit contract fails closed. This is an
|
|
// accepted provider-untrusted residual, not endpoint authentication: the
|
|
// nonce is a liveness signal, because a compromised channel can read the
|
|
// launch environment; the endpoint stays safe because it never derives from
|
|
// the channel.
|
|
const assignedPort = await reserveHostAssignedLoopbackPort();
|
|
const nonce = randomBytes(16).toString("hex");
|
|
const sandboxOrigin = `http://127.0.0.1:${assignedPort}`;
|
|
|
|
let channel: CommandManagedDuplexChannel | null = null;
|
|
// The open runs two stages under one try: the entrypoint sync, then the
|
|
// channel open. The stage names the exact open-failure outcome in the catch.
|
|
let openStage: "entrypoint_sync" | "channel_open" = "entrypoint_sync";
|
|
try {
|
|
const assetSync = await syncSandboxCallbackBridgeEntrypoint({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
assetRemoteDir,
|
|
bridgeAsset,
|
|
timeoutMs: bridgeTimeoutMs,
|
|
shellCommand,
|
|
});
|
|
const gatewayEnv: Record<string, string> = {
|
|
PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE,
|
|
PAPERCLIP_BRIDGE_TOKEN: bridgeToken,
|
|
PAPERCLIP_BRIDGE_HOST: "127.0.0.1",
|
|
PAPERCLIP_BRIDGE_PORT: String(assignedPort),
|
|
PAPERCLIP_BRIDGE_NONCE: nonce,
|
|
PAPERCLIP_BRIDGE_MAX_BODY_BYTES: String(maxBodyBytes),
|
|
};
|
|
const command = buildDuplexGatewayLaunchArgv({
|
|
shellCommand,
|
|
remoteEntrypoint: assetSync.remoteEntrypoint,
|
|
env: gatewayEnv,
|
|
});
|
|
openStage = "channel_open";
|
|
channel = await openDuplexChannel({ command });
|
|
} catch (error) {
|
|
// The channel never opened, so no request could carry the bridge token.
|
|
// The open call is the last statement of the try, so `channel` is still
|
|
// null here; there is nothing to close. Fall through to the file bridge
|
|
// below. Bind and keep the caught error, so the fallback names the exact
|
|
// open-failure stage: a full process-scoped route ceiling is `route_busy`;
|
|
// otherwise the stage is the entrypoint sync or the channel open. The log
|
|
// line names only the fixed stage enum, so no raw provider error rides a log
|
|
// line on the duplex path.
|
|
const reason: DuplexFallbackReason = isDuplexRouteBusyError(error)
|
|
? "route_busy"
|
|
: openStage === "entrypoint_sync"
|
|
? "entrypoint_sync_failed"
|
|
: "channel_open_failed";
|
|
duplexChannelOpen.fallback(reason);
|
|
await onLog(
|
|
"stderr",
|
|
`[paperclip] Could not open the sandbox duplex channel (${reason}). Using the file bridge.\n`,
|
|
);
|
|
channel = null;
|
|
}
|
|
|
|
if (channel) {
|
|
const gate = createDuplexReadinessGate(channel, {
|
|
nonce,
|
|
timeoutMs: readinessTimeoutMs,
|
|
});
|
|
const readiness = await gate.ready;
|
|
if (!readiness.ok) {
|
|
// Fail closed. Close the partial channel inside a bounded budget, then
|
|
// select the file bridge. The broker never started, so no request that
|
|
// carries the bridge token reached the channel or any endpoint. The reason
|
|
// is a fixed enum, so it rides the log line and the fallback telemetry
|
|
// with no raw value.
|
|
gate.disposePendingReplay();
|
|
await closeDuplexChannelWithinBudget(channel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
|
|
duplexChannelOpen.fallback(duplexReadinessFallbackReason(readiness.reason));
|
|
await onLog(
|
|
"stderr",
|
|
`[paperclip] Sandbox duplex readiness failed (${readiness.reason}). Using the file bridge.\n`,
|
|
);
|
|
} else {
|
|
// Readiness passed. The gate retained every byte that followed the
|
|
// accepted READY line. Scan those retained bytes for the HTTP/2
|
|
// client connection preface — the scan window opens only now, after
|
|
// the gate accepted the nonce — and start the HTTP/2 session at that
|
|
// offset, inclusive. A missing preface inside the bounded readiness
|
|
// buffer aborts the open; the run falls back to `queue_v1` exactly
|
|
// one time (accepted security fix 6).
|
|
const openedChannel = channel;
|
|
const prefaceScan = scanForHttp2ClientPreface(gate.brokerChannel, {
|
|
capBytes: DUPLEX_READINESS_BUFFER_CAP_BYTES,
|
|
timeoutMs: readinessTimeoutMs,
|
|
});
|
|
const prefaceResult = await prefaceScan.settled;
|
|
if (prefaceResult === "missing") {
|
|
// Fail closed, the same shape as a readiness failure: close the
|
|
// partial channel inside the cleanup budget, then select the file
|
|
// bridge. No HTTP/2 server ever bound to this channel, so no
|
|
// request reached it or any endpoint.
|
|
gate.disposePendingReplay();
|
|
await closeDuplexChannelWithinBudget(openedChannel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
|
|
duplexChannelOpen.fallback("preface_missing");
|
|
await onLog(
|
|
"stderr",
|
|
"[paperclip] Sandbox HTTP/2 client preface did not appear inside the bounded readiness buffer (preface_missing). Using the file bridge.\n",
|
|
);
|
|
} else {
|
|
// The run disposition latch for the http2_v1 path, in the same
|
|
// shape the retired duplex_v1 broker exposed. A loss ordered before
|
|
// an orderly completion reports a failure with the typed loss
|
|
// reason; every other state reports a success.
|
|
const dispositionLatch = createHttp2RunDispositionLatch();
|
|
// Set before the first await inside the forward handler, so a loss
|
|
// that lands mid-request still classifies as `post_dispatch`.
|
|
let anyStreamDispatched = false;
|
|
|
|
const recordHttp2Loss = (event: Http2TelemetryEventName): void => {
|
|
const reason = mapHttp2EventToDuplexLossReason(event);
|
|
const disposedNow = dispositionLatch.recordLoss(reason);
|
|
if (!disposedNow && (reason === "provider_exit" || reason === "transport_closed")) {
|
|
// A clean channel end that orders after a host-observed orderly
|
|
// completion is a normal teardown, not a loss: the run already
|
|
// completed. Emit no loss telemetry and no log line for it, the
|
|
// same policy the retired duplex_v1 broker applied.
|
|
return;
|
|
}
|
|
const lossClass = anyStreamDispatched ? "post_dispatch" : "pre_dispatch";
|
|
duplexObservability.recordLoss(lossClass, reason);
|
|
void onLog("stderr", `[paperclip] Sandbox HTTP/2 channel lost (${reason}). The run fails.\n`);
|
|
};
|
|
|
|
// The forward handler applies the real host token and the run id
|
|
// through the existing `forwardBridgeRequest` — the same function
|
|
// the file bridge uses. The route allowlist, the header allowlist,
|
|
// and the per-request debug-log suppression already ran inside
|
|
// `createHttp2BridgeServer`'s own stream handler before this call.
|
|
// It records the request span with the same latency-and-outcome
|
|
// shape the retired duplex_v1 broker recorded: `ok` for any
|
|
// delivered host response (any status), `error` only when the
|
|
// forward call itself throws.
|
|
const http2ForwardRequest: Http2BridgeForwardHandler = async (request) => {
|
|
anyStreamDispatched = true;
|
|
const dispatchStartMs = Date.now();
|
|
try {
|
|
const result = await forwardBridgeRequest(
|
|
{
|
|
method: request.method,
|
|
path: request.pathname,
|
|
query: request.query,
|
|
headers: request.headers,
|
|
body: request.body,
|
|
},
|
|
request.signal,
|
|
{ suppressDebugLog: true, reservation: request.reservation },
|
|
);
|
|
duplexObservability.recordRequest({ latencyMs: Date.now() - dispatchStartMs, outcome: "ok" });
|
|
return { status: result.status, headers: result.headers, body: result.body };
|
|
} catch (error) {
|
|
duplexObservability.recordRequest({ latencyMs: Date.now() - dispatchStartMs, outcome: "error" });
|
|
throw error;
|
|
}
|
|
};
|
|
|
|
const http2Server = createHttp2BridgeServer({
|
|
bridgeToken,
|
|
forwardRequest: http2ForwardRequest,
|
|
routes: HTTP2_SANDBOX_CALLBACK_BRIDGE_ROUTE_ALLOWLIST,
|
|
// The same resolved limit the launch environment hands the
|
|
// sandbox-side gateway (`PAPERCLIP_BRIDGE_MAX_BODY_BYTES`,
|
|
// below), so the host check and the gateway check enforce one
|
|
// value instead of the host silently falling back to the
|
|
// package default.
|
|
maxBodyBytes,
|
|
onGoaway: () => recordHttp2Loss("session_goaway"),
|
|
onSessionError: () => recordHttp2Loss("session_error"),
|
|
});
|
|
// Combine the loss hook with the forward-to-Duplex handoff inside
|
|
// one `onExit` registration: the channel primitive holds exactly
|
|
// one listener slot, and `bindChannel` below registers the one that
|
|
// ends the wrapped `Duplex`.
|
|
//
|
|
// This object's `stop()` is also the real sandbox-side effect of
|
|
// the post-bind read backpressure bound `bindChannel` applies
|
|
// (`wrapDuplexChannelAsNodeDuplex` in `http2-bridge-server.ts`):
|
|
// this raw provider channel exposes no pause, so once the bounded
|
|
// read queue there overflows, it calls `stop()` through this exact
|
|
// chain, down to `prefaceScan.scanned.stop()` and on to the real
|
|
// channel, instead of letting sandbox-controlled bytes grow host
|
|
// memory with no bound.
|
|
const channelForHttp2Server: CommandManagedDuplexChannel = {
|
|
write: (data: Uint8Array) => prefaceScan.scanned.write(data),
|
|
onData: (listener: (chunk: Uint8Array) => void) => prefaceScan.scanned.onData(listener),
|
|
onExit: (listener: (exit: { exitCode: number | null }) => void) => {
|
|
prefaceScan.scanned.onExit((exit) => {
|
|
recordHttp2Loss("channel_exit");
|
|
listener(exit);
|
|
});
|
|
},
|
|
stop: () => prefaceScan.scanned.stop(),
|
|
close: () => prefaceScan.scanned.close(),
|
|
};
|
|
const boundDuplex = http2Server.bindChannel(channelForHttp2Server);
|
|
// Also catches the `Duplex` this wrapper destroys when the bounded
|
|
// read backpressure queue overflows post-bind, so that loss still
|
|
// reaches `recordHttp2Loss` the same way any other write fault does.
|
|
boundDuplex.on("error", () => recordHttp2Loss("write_error"));
|
|
|
|
duplexChannelOpen.ready();
|
|
await onLog(
|
|
"stdout",
|
|
"[paperclip] Sandbox HTTP/2 transport ready; serving the host-assigned origin.\n",
|
|
);
|
|
// Stream run logs on the http2 path with the same gate and the same
|
|
// log line as the file path. The http2 path starts no file-bridge
|
|
// worker, so create the log directory before the tail starts.
|
|
let duplexRunLogTail: SandboxRunLogTailFactory | null = null;
|
|
if (target.transport === "sandbox" && target.streamRunLogs !== false) {
|
|
const duplexLogsDir = sandboxCallbackBridgeDirectories(queueDir).logsDir;
|
|
await ensureSandboxRunLogDirectory({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
logsDir: duplexLogsDir,
|
|
shellCommand,
|
|
timeoutMs: bridgeTimeoutMs,
|
|
});
|
|
duplexRunLogTail = createSandboxRunLogTailFactory({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
logsDir: duplexLogsDir,
|
|
shellCommand,
|
|
});
|
|
await onLog("stdout", "[paperclip] Sandbox run log streaming enabled for this run.\n");
|
|
}
|
|
return {
|
|
env: {
|
|
PAPERCLIP_API_URL: sandboxOrigin,
|
|
PAPERCLIP_API_KEY: bridgeToken,
|
|
PAPERCLIP_API_BRIDGE_MODE: SANDBOX_CALLBACK_BRIDGE_HTTP2_MODE,
|
|
},
|
|
runLogTail: duplexRunLogTail,
|
|
readRunDisposition: (): DuplexBrokerRunDisposition => dispositionLatch.disposition,
|
|
// Atomically read the latch and mark the orderly completion for the
|
|
// ACP success-eligible terminal, so no await separates the read from
|
|
// the mark and a teardown loss cannot slip in between.
|
|
settleRunDisposition: (): DuplexBrokerRunDisposition => dispositionLatch.settleRunDisposition(),
|
|
markOrderlyCompletion: (): void => dispositionLatch.markOrderlyCompletion(),
|
|
onLoss: (listener: (reason: DuplexLossReason) => void): (() => void) =>
|
|
dispositionLatch.onLoss(listener),
|
|
stop: async () => {
|
|
// Close the HTTP/2 server's sessions, then the channel, before
|
|
// lease release, so no live provider session remains when the
|
|
// caller releases the lease.
|
|
await http2Server.close();
|
|
await closeDuplexChannelWithinBudget(openedChannel, DEFAULT_DUPLEX_CLEANUP_BUDGET_MS);
|
|
await bridgeAsset.cleanup();
|
|
},
|
|
};
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
let server: Awaited<ReturnType<typeof startSandboxCallbackBridgeServer>> | null = null;
|
|
let worker: Awaited<ReturnType<typeof startSandboxCallbackBridgeWorker>> | null = null;
|
|
try {
|
|
const client = createCommandManagedSandboxCallbackBridgeQueueClient({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
timeoutMs: bridgeTimeoutMs,
|
|
shellCommand,
|
|
});
|
|
// `startSandboxCallbackBridgeWorker` keeps its awaited queue-directory
|
|
// setup on the active `bridge.paperclip` step, and runs each request under
|
|
// the run parent context (see `runWithRuntimeParent` inside that function).
|
|
// So the startup `mkdir` execs stay parented to the step, and every later
|
|
// request `sandbox.exec` span parents to the live run span.
|
|
worker = await startSandboxCallbackBridgeWorker({
|
|
client,
|
|
queueDir,
|
|
maxBodyBytes,
|
|
getRuntimeParentContext: input.getRuntimeParentContext,
|
|
runtimeSpan: input.runtimeSpan,
|
|
// The queue transport writes the response body to a text file, so this
|
|
// is the one place the forward path decodes the response `Buffer` to a
|
|
// UTF-8 string. The queue's own on-wire behavior does not change.
|
|
handleRequest: async (request, options) => {
|
|
const result = await forwardBridgeRequest(request, options?.signal);
|
|
return { status: result.status, headers: result.headers, body: result.body.toString("utf8") };
|
|
},
|
|
});
|
|
server = await startSandboxCallbackBridgeServer({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
assetRemoteDir,
|
|
queueDir,
|
|
bridgeToken,
|
|
bridgeAsset,
|
|
timeoutMs: bridgeTimeoutMs,
|
|
maxBodyBytes,
|
|
shellCommand,
|
|
});
|
|
} catch (error) {
|
|
await Promise.allSettled([
|
|
server?.stop(),
|
|
worker?.stop(),
|
|
bridgeAsset.cleanup(),
|
|
]);
|
|
throw error;
|
|
}
|
|
|
|
let runLogTail: SandboxRunLogTailFactory | null = null;
|
|
if (target.transport === "sandbox" && target.streamRunLogs !== false) {
|
|
runLogTail = createSandboxRunLogTailFactory({
|
|
runner,
|
|
remoteCwd: target.remoteCwd,
|
|
logsDir: sandboxCallbackBridgeDirectories(queueDir).logsDir,
|
|
shellCommand,
|
|
});
|
|
await onLog("stdout", "[paperclip] Sandbox run log streaming enabled for this run.\n");
|
|
}
|
|
|
|
return {
|
|
env: {
|
|
PAPERCLIP_API_URL: server.baseUrl,
|
|
PAPERCLIP_API_KEY: bridgeToken,
|
|
PAPERCLIP_API_BRIDGE_MODE: "queue_v1",
|
|
PAPERCLIP_BRIDGE_QUEUE_DIR: queueDir,
|
|
},
|
|
runLogTail,
|
|
stop: async () => {
|
|
await Promise.allSettled([
|
|
server?.stop(),
|
|
]);
|
|
await Promise.allSettled([
|
|
worker?.stop(),
|
|
bridgeAsset.cleanup(),
|
|
]);
|
|
},
|
|
};
|
|
}
|