/** * PluginWorkerManager — spawns and manages out-of-process plugin worker child * processes, routes JSON-RPC 2.0 calls over stdio, and handles lifecycle * management including crash recovery with exponential backoff. * * Each installed plugin gets one dedicated worker process. The host sends * JSON-RPC requests over the child's stdin and reads responses from stdout. * Worker stderr is captured and forwarded to the host logger. * * Process Model (from PLUGIN_SPEC.md §12): * - One worker process per installed plugin * - Failure isolation: plugin crashes do not affect the host * - Graceful shutdown: 10-second drain, then SIGTERM, then SIGKILL * - Automatic restart with exponential backoff on unexpected exits * * @see PLUGIN_SPEC.md §12 — Process Model * @see PLUGIN_SPEC.md §12.5 — Graceful Shutdown Policy * @see PLUGIN_SPEC.md §13 — Host-Worker Protocol */ import { fork, type ChildProcess } from "node:child_process"; import { randomUUID } from "node:crypto"; import { EventEmitter } from "node:events"; import { createInterface, type Interface as ReadlineInterface } from "node:readline"; import type { PaperclipPluginManifestV1 } from "@paperclipai/shared"; import { JSONRPC_VERSION, JSONRPC_ERROR_CODES, PLUGIN_RPC_ERROR_CODES, createRequest, createErrorResponse, parseMessage, serializeMessage, isJsonRpcResponse, isJsonRpcRequest, isJsonRpcNotification, isJsonRpcSuccessResponse, JsonRpcParseError, JsonRpcCallError, LOGIN_PTY_OUTPUT_NOTIFICATION, LOGIN_PTY_EXIT_NOTIFICATION, DUPLEX_CHANNEL_DATA_NOTIFICATION, DUPLEX_CHANNEL_EXIT_NOTIFICATION, encodeChannelBytes, decodeChannelBytes, } from "@paperclipai/plugin-sdk"; import type { JsonRpcId, PluginInvocationContext, PluginInvocationScope, JsonRpcResponse, JsonRpcRequest, JsonRpcNotification, WorkerHostCallContext, HostToWorkerMethodName, HostToWorkerMethods, WorkerToHostMethodName, WorkerToHostMethods, InitializeParams, } from "@paperclipai/plugin-sdk"; import { getActiveStepContext } from "@paperclipai/adapter-utils/acpx-engine/startup-timing"; import { isLoginCommandKey, validateLoginSessionHome, type LoginCommandKey, } from "./login-command.js"; import { logger } from "../middleware/logger.js"; import { traceparentFromContextToken } from "../instrumentation.js"; // --------------------------------------------------------------------------- // Constants // --------------------------------------------------------------------------- /** Default timeout for RPC calls in milliseconds. */ const DEFAULT_RPC_TIMEOUT_MS = 30_000; /** * Upper bound for the *default* RPC timeout path (15 minutes). Explicit * caller-supplied timeouts are not subject to this cap: execute-class RPCs such * as `environmentExecute` run entire sandboxed agent sessions in one call and * their callers deliberately request multi-hour budgets (see * `resolvePluginExecuteRpcTimeoutMs` in plugin-environment-driver.ts). * Clamping those explicit budgets here killed long sandboxed runs mid-work. */ const MAX_RPC_TIMEOUT_MS = 15 * 60 * 1_000; /** * Maximum delay accepted by Node timers before Node clamps the timeout to 1ms. * Keep accepted explicit RPC budgets inside this range before calling * setTimeout, otherwise a huge timeout can expire almost immediately. */ const MAX_NODE_TIMER_TIMEOUT_MS = 2_147_483_647; /** Timeout for the initialize RPC call. */ const INITIALIZE_TIMEOUT_MS = 15_000; /** Timeout for the shutdown RPC call before escalating to SIGTERM. */ const SHUTDOWN_DRAIN_MS = 10_000; /** Time to wait after SIGTERM before sending SIGKILL. */ const SIGTERM_GRACE_MS = 5_000; /** Minimum backoff delay for crash recovery (1 second). */ const MIN_BACKOFF_MS = 1_000; /** Maximum backoff delay for crash recovery (5 minutes). */ const MAX_BACKOFF_MS = 5 * 60 * 1_000; /** Backoff multiplier on each consecutive crash. */ const BACKOFF_MULTIPLIER = 2; /** Maximum number of consecutive crashes before giving up on auto-restart. */ const MAX_CONSECUTIVE_CRASHES = 10; /** Time window in which crashes are considered consecutive (10 minutes). */ const CRASH_WINDOW_MS = 10 * 60 * 1_000; /** Maximum number of stderr characters retained for worker failure context. */ const MAX_STDERR_EXCERPT_CHARS = 8_000; /** Maximum characters accepted for one `execute.log` chunk. A larger chunk is * dropped, so a faulty or hostile worker cannot flood the host with one * unbounded notification. */ const MAX_EXECUTE_LOG_CHUNK_CHARS = 1_000_000; /** * Maximum characters accepted for one incoming worker stdout line before the * host parses it as JSON. The host drops a longer line without a parse, so a * faulty or hostile worker cannot force the host to parse an unbounded document * and exhaust memory. The bound sits far above the largest legitimate framed * message, so a real large command result still passes. A worker can override * it through `WorkerStartOptions.executeLogLimits`. */ const MAX_WORKER_MESSAGE_CHARS = 128 * 1024 * 1024; /** * Default ceiling for the total characters one execute call may stream through * `execute.log`. The host counts the delivered characters for each active * execute route and drops further chunks past this bound, so one runaway or * hostile execution cannot flood the host and the run-log sink without limit. * The final command result still delivers the complete output through its own * capture path. A worker can override it through * `WorkerStartOptions.executeLogLimits`. */ const MAX_EXECUTE_LOG_TOTAL_CHARS = 128 * 1024 * 1024; /** Maximum characters for one live login pseudo-terminal output notification. */ const MAX_LOGIN_PTY_CHUNK_CHARS = 1_000_000; /** Maximum cumulative output characters for one login pseudo-terminal route. */ const MAX_LOGIN_PTY_TOTAL_CHARS = 8 * 1024 * 1024; /** Default maximum output and exit records the host queues before the bind. */ const MAX_LOGIN_PTY_PRE_BIND_FRAMES = 10_000; /** Default maximum cumulative output characters the host queues before the bind. */ const MAX_LOGIN_PTY_PRE_BIND_CHARS = 8 * 1024 * 1024; /** The default open timeout for one login pseudo-terminal route, in milliseconds. */ const LOGIN_PTY_OPEN_TIMEOUT_MS = 30_000; /** The default close timeout for one login pseudo-terminal route, in milliseconds. */ const LOGIN_PTY_CLOSE_TIMEOUT_MS = 10_000; /** * The fixed non-secret error a disallowed login command key returns. The manager * forwards only a closed login command key to the worker pseudo-terminal. It * rejects a key outside the closed set before the worker call, so a caller cannot * select an arbitrary command in the sandbox. */ const LOGIN_PTY_COMMAND_NOT_ALLOWED = "LOGIN_PTY_COMMAND_NOT_ALLOWED"; /** The fixed non-secret error a rejected second credential open returns. */ const LOGIN_PTY_ROUTE_BUSY = "LOGIN_PTY_ROUTE_BUSY"; /** The fixed non-secret error a failed open returns. */ const LOGIN_PTY_OPEN_FAILED = "LOGIN_PTY_OPEN_FAILED"; // Bounds and timeouts for the generic duplex channel route. The route mirrors the // login pseudo-terminal route, but it carries no command allowlist and adds seven // explicit bounds the pseudo-terminal route lacks. Each bound ends the route when // it passes the limit, so a faulty or hostile worker cannot flood the host. /** The default maximum characters for one duplex channel data notification. This * matches the host duplex frame bound ({@link DEFAULT_MAX_DUPLEX_FRAME_BYTES} in * `duplex-frame-codec.ts`), so one chunk always fits inside one frame. */ const MAX_DUPLEX_CHANNEL_CHUNK_CHARS = 262_144; /** * The default maximum cumulative bytes the host retains, on one duplex channel * route, for input the worker has not yet delivered: the pre-bind hold (before * the worker session id binds), the post-bind buffered queue (bound, before a * data listener attaches), and the terminal buffer (a route that ended before * a listener attached). All three representations share this one route-local * bound, so a worker cannot grow any of them past it. This bound is a fixed * share of the fixed per-route byte budget the host now bounds every duplex * retention site against; it holds no measurement of real traffic. */ export const DUPLEX_ROUTE_INPUT_MAX_BYTES = 393_216; /** * The default maximum number of data frames the host buffers for one duplex * channel route before a data listener attaches. */ const MAX_DUPLEX_CHANNEL_PRE_BIND_FRAMES = 10_000; /** * The margin the pre-bind hold ceiling keeps above the pre-bind buffered-frame * bound (`maxDuplexChannelPreBindFrames`). A worker can batch data and exit * frames with the open reply, so the host reads them before the route binds and * before it can apply the per-frame bounds. The host holds these frames and * replays them after the bind. The hold ceiling bounds that hold, so a worker * that floods frames before the open reply cannot make the host hold an * unbounded number of frames. * * The hold ceiling derives from the buffered bound, not a fixed constant. A * fixed ceiling at or below a caller-configured buffered bound would drop the * frame that must instead trip the buffered bound during replay, so the route * would never end. One frame of margin is enough. It lets the frame that * exceeds the buffered bound reach the hold, so the replay's buffered-bound * check, not the hold, ends the route. */ const DUPLEX_CHANNEL_PRE_BIND_HOLD_MARGIN_FRAMES = 1; /** * The default maximum number of in-flight host→worker requests for one duplex * channel route. A worker that never replies cannot make the host hold an * unbounded number of pending requests. */ const MAX_DUPLEX_CHANNEL_PENDING_REQUESTS = 256; /** The default maximum characters for one host→worker duplex channel write. This * matches the host duplex frame bound, so one write always fits inside one * frame. */ const MAX_DUPLEX_CHANNEL_WRITE_CHARS = 262_144; /** * The default maximum cumulative bytes the host retains, on one duplex channel * route, for a host→worker write the worker has not yet acknowledged: the raw * write payload and the serialized child-stdin frame the host built from it. * This bound is route-local and independent of the pending-request count * bound above; a worker that never replies still cannot make one route hold * an unbounded number of write bytes, even under a raised pending-request * count. This bound is a fixed share of the fixed per-route byte budget; it * holds no measurement of real traffic. */ export const DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES = 393_216; /** * The default maximum number of protocol errors for one duplex channel route. * A protocol error is one malformed or mismatched data frame. The route ends * when the count passes this budget, so a flood of bad frames bounds the route. */ const MAX_DUPLEX_CHANNEL_PROTOCOL_ERRORS = 100; /** * The default maximum cumulative bytes the host forwards for one duplex channel * route over its whole life. The host counts the bytes of every inbound chunk, * before and after a data listener attaches. The route ends when the count * passes this cap, so an active route with a bound listener cannot stream an * unbounded number of bytes. */ const MAX_DUPLEX_CHANNEL_TOTAL_DATA_BYTES = 256 * 1024 * 1024; /** * The default maximum lifetime for one duplex channel route, in milliseconds. * The host starts a timer when the route opens and ends the route when the * timer expires, so a route cannot live without limit. */ const MAX_DUPLEX_CHANNEL_DURATION_MS = 60 * 60 * 1000; /** The default open timeout for one duplex channel route, in milliseconds. */ const DUPLEX_CHANNEL_OPEN_TIMEOUT_MS = 30_000; /** The default close timeout for one duplex channel route, in milliseconds. */ const DUPLEX_CHANNEL_CLOSE_TIMEOUT_MS = 10_000; /** The fixed non-secret error a rejected second duplex channel open returns. */ const DUPLEX_CHANNEL_ROUTE_BUSY = "DUPLEX_CHANNEL_ROUTE_BUSY"; /** The fixed non-secret error a failed duplex channel open returns. */ const DUPLEX_CHANNEL_OPEN_FAILED = "DUPLEX_CHANNEL_OPEN_FAILED"; // The process-scoped monotonic route-generation source. The host mints one // strictly increasing, non-reusable `hostRouteId` on each duplex channel open, // across every worker in the process. A retired generation never returns, so a // late frame for a closed pair never collides with a new open. The host owns the // value; no worker field sets it. let duplexHostRouteIdSequence = 0; /** Mint the next monotonic, non-reusable host route identifier for a duplex channel open. */ function nextDuplexHostRouteId(): string { duplexHostRouteIdSequence += 1; return `duplex-route-${duplexHostRouteIdSequence}`; } // The separator between the two identifiers of a duplex route pair key. It is a // null byte, which appears in neither a host route id nor a worker session id, so // two distinct pairs never collapse to one key. const DUPLEX_PAIR_KEY_SEPARATOR = "\u0000"; /** Build the exact-pair routing key from the host route id and the worker session id. */ function duplexPairKey(hostRouteId: string, workerSessionId: string): string { return `${hostRouteId}${DUPLEX_PAIR_KEY_SEPARATOR}${workerSessionId}`; } // The maximum number of tombstoned duplex pairs one worker retains. The worker // keeps every tombstone until it retires, so a closed pair never returns. Before // the set would exceed this bound, the host retires the worker, which drops every // route and every tombstone at once. A tombstone overflow fails closed; the host // never evicts a tombstone and lets a pair return. const MAX_DUPLEX_ROUTE_TOMBSTONES = 4096; /** Minimum time between two dropped-`execute.log` debug records. The router * rate-limits the record so a flood of dropped chunks writes at most one line * per window with a running count. */ const EXECUTE_LOG_DROP_LOG_INTERVAL_MS = 1_000; // --------------------------------------------------------------------------- // Types // --------------------------------------------------------------------------- /** * Status of a managed worker process. */ export type WorkerStatus = | "stopped" | "starting" | "running" | "stopping" | "crashed" | "backoff"; /** * Worker-to-host method handler. The host registers these to service calls * that the plugin worker makes back to the host (e.g. state.get, events.emit). */ export type WorkerToHostHandler = ( params: WorkerToHostMethods[M][0], context?: WorkerHostCallContext, ) => Promise; /** * A map of all worker-to-host method handlers provided by the host. */ export type WorkerToHostHandlers = { [M in WorkerToHostMethodName]?: WorkerToHostHandler; }; /** * Events emitted by a PluginWorkerHandle. */ export interface WorkerHandleEvents { /** Worker process started and is ready (initialize succeeded). */ "ready": { pluginId: string }; /** Worker process exited. */ "exit": { pluginId: string; code: number | null; signal: NodeJS.Signals | null }; /** Worker process crashed unexpectedly. */ "crash": { pluginId: string; code: number | null; signal: NodeJS.Signals | null; willRestart: boolean }; /** Worker process errored (e.g. spawn failure). */ "error": { pluginId: string; error: Error }; /** Worker status changed. */ "status": { pluginId: string; status: WorkerStatus; previousStatus: WorkerStatus }; } type WorkerHandleEventName = keyof WorkerHandleEvents; export function appendStderrExcerpt(current: string, chunk: string): string { const next = current ? `${current}\n${chunk}` : chunk; return next.length <= MAX_STDERR_EXCERPT_CHARS ? next : next.slice(-MAX_STDERR_EXCERPT_CHARS); } export function formatWorkerFailureMessage(message: string, stderrExcerpt: string): string { const excerpt = stderrExcerpt.trim(); if (!excerpt) return message; if (message.includes(excerpt)) return message; return `${message}\n\nWorker stderr:\n${excerpt}`; } /** * Resolve the effective timeout for an RPC call. * * An explicit, positive, finite caller-supplied timeout bypasses the 15-minute * RPC cap after normalization to Node's timer-safe integer range. Callers that * pass one (e.g. the environment driver for `environmentExecute`) own their * budget, and independent inactivity/safety guards bound hung runs. Only the * default path (no usable explicit timeout) is clamped to MAX_RPC_TIMEOUT_MS so * ordinary plugin calls stay bounded. */ export function resolveRpcCallTimeoutMs( explicitTimeoutMs: number | undefined, defaultTimeoutMs: number, ): number { if ( explicitTimeoutMs !== undefined && Number.isFinite(explicitTimeoutMs) && explicitTimeoutMs > 0 ) { return Math.min(Math.max(Math.trunc(explicitTimeoutMs), 1), MAX_NODE_TIMER_TIMEOUT_MS); } return Math.min(defaultTimeoutMs, MAX_RPC_TIMEOUT_MS); } /** * Options for starting a worker process. */ /** * The process-scoped aggregate route-slot controller. The manager injects one * shared instance into every worker handle, so the ceiling counts every * concurrent duplex route across the process, not one agent's setting. */ export interface DuplexRouteSlotController { /** Reserve one route slot. Return true when a slot was free and is now held. */ tryAcquire(): boolean; /** Release one held route slot. */ release(): void; } export interface WorkerStartOptions { /** Absolute path to the plugin worker entrypoint (CJS bundle). */ entrypointPath: string; /** Plugin manifest. */ manifest: PaperclipPluginManifestV1; /** Resolved plugin configuration. */ config: Record; /** Host instance information for the initialize call. */ instanceInfo: { instanceId: string; hostVersion: string; }; /** Host API version. */ apiVersion: number; /** Host-derived plugin database namespace, when declared. */ databaseNamespace?: string | null; /** Handlers for worker→host RPC calls. */ hostHandlers: WorkerToHostHandlers; /** Default timeout for RPC calls (ms). Defaults to 30s. */ rpcTimeoutMs?: number; /** Whether to auto-restart on crash. Defaults to true. */ autoRestart?: boolean; /** Node.js execArgv passed to the child process. */ execArgv?: string[]; /** Environment variables passed to the child process. */ env?: Record; /** * The process-scoped aggregate route-slot controller for the duplex channel * ceiling. The manager injects one shared instance into every worker handle. * When it is absent, the worker admits every duplex open unbounded (a unit test * constructs a handle this way). */ duplexRouteSlots?: DuplexRouteSlotController | null; /** * Companies this worker may act on from proactive (no-invocation) worker→host * calls — the plugin's configured companies. Seeded onto the handle at * creation, BEFORE the child process spawns, so a proactive plugin that * issues host calls during setup() (e.g. the chat gateway's one-shot * `events.subscribe`, which runs while `startWorker` is still awaiting the * initialize response) is already authorized when those calls arrive. The set * can still be replaced at runtime via `setProactiveCompanyScopes` (e.g. on a * config change). Never widens access beyond the listed companies (LOOA-695). */ proactiveCompanyScopes?: readonly string[]; /** * Callback for stream notifications from the worker (streams.open/emit/close). * The host wires this to the PluginStreamBus to fan out events to SSE clients. */ onStreamNotification?: (method: string, params: Record) => void; /** * Framing and flood limits for the `execute.log` route. The defaults bound * one incoming line before the JSON parse and the total streamed output for * one execute call. A test overrides them to exercise the drop paths without * huge inputs. */ executeLogLimits?: { /** Max characters for one incoming worker line before the JSON parse. */ maxIncomingMessageChars?: number; /** Max total characters one execute call may stream through `execute.log`. */ maxTotalCharsPerExecute?: number; }; /** * Bounds and timeouts for the login pseudo-terminal route. The * defaults bound one output notification, the cumulative output per route, and * the open and the close timeouts. A test overrides them to exercise the * terminalize paths without huge inputs or long waits. */ loginPtyLimits?: { /** Max characters for one login pseudo-terminal output notification. */ maxChunkChars?: number; /** Max cumulative output characters for one login pseudo-terminal route. */ maxTotalChars?: number; /** Max number of pre-bind output and exit records the host queues before the bind. */ maxPreBindFrames?: number; /** Max cumulative output characters the host queues before the bind. */ maxPreBindChars?: number; /** The open timeout for one login pseudo-terminal route, in milliseconds. */ openTimeoutMs?: number; /** The close timeout for one login pseudo-terminal route, in milliseconds. */ closeTimeoutMs?: number; }; /** * Bounds and timeouts for the generic duplex channel route. The defaults bound * one data notification, the pre-bind buffer, the in-flight request count, one * host→worker write, the protocol-error budget, and the open and close * timeouts. A test overrides them to exercise each bound without huge inputs or * long waits. */ duplexChannelLimits?: { /** Max characters for one duplex channel data notification. */ maxChunkChars?: number; /** Max cumulative characters the host buffers before a data listener attaches. */ maxPreBindBufferedChars?: number; /** Max number of data frames the host buffers before a data listener attaches. */ maxPreBindBufferedFrames?: number; /** Max number of in-flight host→worker requests for one route. */ maxPendingRequests?: number; /** Max characters for one host→worker duplex channel write. */ maxWriteChars?: number; /** Max cumulative bytes the host retains, for one route, for a * host→worker write pending the worker's acknowledgement. */ maxPendingWriteBytes?: number; /** Max number of protocol errors for one route before the route ends. */ maxProtocolErrors?: number; /** Max cumulative bytes the host forwards for one route over its whole life. */ maxTotalDataBytes?: number; /** The maximum lifetime for one route, in milliseconds. */ maxDurationMs?: number; /** The open timeout for one duplex channel route, in milliseconds. */ openTimeoutMs?: number; /** The close timeout for one duplex channel route, in milliseconds. */ closeTimeoutMs?: number; }; } /** * A pending RPC call waiting for a response from the worker. */ interface PendingRequest { /** The request ID. */ id: JsonRpcId; /** Method name (for logging). */ method: string; /** Resolve the promise with the response. */ resolve: (response: JsonRpcResponse) => void; /** Timeout timer handle. */ timer: ReturnType; /** Timestamp when the request was sent. */ sentAt: number; /** Active host-owned invocation id attached to this host→worker call. */ invocationId?: string; } interface ActiveInvocation { scope: PluginInvocationScope; timer?: ReturnType; // The host-minted W3C `traceparent` for the active startup span, or undefined // when no startup span is active. The span host handler reads it to mint the // parentage, so a worker never supplies the parent itself. traceparent?: string; } /** * Sink for one incremental output chunk of an active `environmentExecute` call. * The host runner passes it to `call` for the execute method, and the manager * delivers each `execute.log` chunk to it. The sink may return a promise; the * caller owns the ordering. */ export type ExecuteLogSink = ( stream: "stdout" | "stderr", chunk: string, ) => void | Promise; /** * The input the manager needs to open one live login pseudo-terminal route. The * manager mints the host route identifier; the caller supplies the sandbox scope, * the provider lease id, and the host-resolved launch descriptor. The launch * descriptor carries a closed command key and a validated session home. It * carries no command string, so a caller cannot select the command. */ export interface LoginPtyOpenInput { /** The environment driver key. It routes the worker; it confers no command authority. */ driverKey: string; companyId: string; environmentId: string; providerLeaseId: string; /** The host-resolved fixed command identity. The worker maps it to a compile-time command. */ loginCommandKey: LoginCommandKey; /** * The server-controlled, validated session home. The shape is exact: * `/tmp/paperclip-adapter-login/`. */ sessionHome: string; } /** * One live login pseudo-terminal session the manager hands to the login * transport. The shape matches the sandbox provider login * pseudo-terminal session, so the transport consumes it with no adapter. */ export interface LoginPtyHostSession { /** Registers the one output listener. The session streams each raw chunk in order. */ onData(listener: (chunk: string) => void): void; /** Writes raw input bytes to the pseudo-terminal. */ write(data: string): void; /** Resolves with the child exit code when the command ends or the route terminalizes. */ wait(): Promise<{ exitCode: number | null }>; /** Stops the child process. Safe to call more than one time. */ kill(): void; /** Closes the route and releases the terminal. Safe to call more than one time. */ close(): Promise; } /** * The input the manager needs to open one generic duplex channel. The manager * mints the host route identifier. The caller supplies the sandbox scope, the * provider lease id, and the command. The duplex channel carries no command * allowlist, so the caller owns the command. */ export interface DuplexChannelOpenInput { driverKey: string; companyId: string; environmentId: string; providerLeaseId: string; /** * The command argument vector the worker runs on the channel. Element 0 is the * program. The worker runs the vector with no shell, so a shell metacharacter * in an element cannot inject a command. */ command: readonly string[]; } /** * One live duplex channel the manager hands to a caller. The shape matches the * login pseudo-terminal session, so a caller consumes one live bidirectional * stream with the same methods. */ export interface DuplexChannelHostSession { /** Registers the one data listener. The session streams each raw byte chunk in order. */ onData(listener: (chunk: Uint8Array) => void): void; /** Writes raw input bytes to the channel. */ write(data: Uint8Array): void; /** * Resolves when the command ends or the route ends. A numeric `exitCode` is a * real process exit. `transportClosed` is true when the provider transport * closed with no exit data, so the broker can tell a real process exit from a * reason-less transport close. */ wait(): Promise<{ exitCode: number | null; transportClosed?: boolean }>; /** Stops the child process. Safe to call more than one time. */ kill(): void; /** Closes the route and releases the channel. Safe to call more than one time. */ close(): Promise; } /** * Host-owned route for one active execute call. The host mints the invocation * id and stores the exact company id and log sink here. A worker never selects * this record; the host looks it up by the host-issued invocation id on the * message envelope. The company id is the single authority for the delivery * target, so an `execute.log` notification never carries a company id. */ interface ExecuteLogRoute { companyId: string; onLog: ExecuteLogSink; /** * The count of characters delivered through this route. The router bounds the * per-execute total and drops chunks past the configured ceiling. */ deliveredChars: number; /** * Latched when the router cannot bind the shared worker pipe to a single * company, because a second company's execute overlapped this one. After the * latch the router drops every further chunk for this route and lets the final * command result deliver the complete output. The latch keeps the delivered * prefix contiguous, so the run log never shows a gap. */ crossCompanyBlocked: boolean; } // --------------------------------------------------------------------------- // PluginWorkerHandle — manages a single worker process // --------------------------------------------------------------------------- /** * Handle for a single plugin worker process. * * Callers use `start()` to spawn the worker, `call()` to send RPC requests, * and `stop()` to gracefully shut down. The handle manages crash recovery * with exponential backoff automatically when `autoRestart` is enabled. */ export interface PluginWorkerHandle { /** The plugin ID this worker serves. */ readonly pluginId: string; /** Current worker status. */ readonly status: WorkerStatus; /** Start the worker process. Resolves when initialize completes. */ start(): Promise; /** * Stop the worker process gracefully. * * Sends a `shutdown` RPC call, waits up to 10 seconds for the worker to * exit, then escalates to SIGTERM, and finally SIGKILL if needed. */ stop(): Promise; /** * Restart the worker process (stop + start). */ restart(): Promise; /** * Send a typed host→worker RPC call. * * @param method - The RPC method name * @param params - Method parameters * @param timeoutMs - Optional per-call timeout override * @returns The method result * @throws {JsonRpcCallError} if the worker returns an error response * @throws {Error} if the worker is not running or the call times out */ call( method: M, params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, ): Promise; /** * Send a fire-and-forget notification to the worker (no response expected). */ notify(method: string, params: unknown): void; /** * Open one live login pseudo-terminal route on this worker. The * manager mints the host route identifier, reserves the route, drives the open, * binds the worker session identifier one time, and returns a session the login * transport drives. It permits one active credential pseudo-terminal per worker. */ openLoginPtySession( input: LoginPtyOpenInput, ): Promise; /** * Open one generic duplex channel on this worker. The manager mints the host * route identifier, reserves the route, drives the open, binds the worker * session identifier one time, and returns a session a caller drives. It * permits one active duplex channel per worker. It enforces five explicit * bounds and ends the route when a bound passes its limit. */ openDuplexChannel( input: DuplexChannelOpenInput, ): Promise; /** * Authorize the set of companies this worker may act on from proactive * (non-invocation) context. Replaces any previously-authorized set. See the * proactive-company-scope note in `createPluginWorkerHandle` for rationale. */ setProactiveCompanyScopes(companyIds: readonly string[]): void; /** Subscribe to worker events. */ on( event: K, listener: (payload: WorkerHandleEvents[K]) => void, ): void; /** Unsubscribe from worker events. */ off( event: K, listener: (payload: WorkerHandleEvents[K]) => void, ): void; /** Optional methods the worker reported during initialization. */ readonly supportedMethods: string[]; /** Get diagnostic info about the worker. */ diagnostics(): WorkerDiagnostics; } /** * Diagnostic information about a worker process. */ export interface WorkerDiagnostics { pluginId: string; status: WorkerStatus; pid: number | null; uptime: number | null; consecutiveCrashes: number; totalCrashes: number; pendingRequests: number; lastCrashAt: number | null; nextRestartAt: number | null; } // --------------------------------------------------------------------------- // PluginWorkerManager — manages all plugin workers // --------------------------------------------------------------------------- /** * The top-level manager that holds all plugin worker handles. * * Provides a registry of workers keyed by plugin ID, with convenience methods * for starting/stopping all workers and routing RPC calls. */ export interface PluginWorkerManager { /** * Register and start a worker for a plugin. * * @returns The worker handle * @throws if a worker is already registered for this plugin */ startWorker(pluginId: string, options: WorkerStartOptions): Promise; /** * Stop and unregister a specific plugin worker. */ stopWorker(pluginId: string): Promise; /** * Get the worker handle for a plugin. */ getWorker(pluginId: string): PluginWorkerHandle | undefined; /** * Check if a worker is registered and running for a plugin. */ isRunning(pluginId: string): boolean; /** * Authorize the companies a plugin's worker may act on from proactive * (non-invocation) context. No-op if the worker is not registered. */ setProactiveCompanyScopes(pluginId: string, companyIds: readonly string[]): void; /** * Stop all managed workers. Called during server shutdown. */ stopAll(): Promise; /** * Get diagnostic info for all workers. */ diagnostics(): WorkerDiagnostics[]; /** * Send an RPC call to a specific plugin worker. * * @throws if the worker is not running */ call( pluginId: string, method: M, params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, ): Promise; /** * Open one live login pseudo-terminal route on a specific plugin worker * See {@link PluginWorkerHandle.openLoginPtySession}. * * @throws if the worker is not registered. */ openLoginPtySession( pluginId: string, input: LoginPtyOpenInput, ): Promise; } // --------------------------------------------------------------------------- // Implementation: createPluginWorkerHandle // --------------------------------------------------------------------------- /** * Create a handle for a single plugin worker process. * * @internal Exported for testing; consumers should use `createPluginWorkerManager`. */ export function createPluginWorkerHandle( pluginId: string, options: WorkerStartOptions, ): PluginWorkerHandle { const log = logger.child({ service: "plugin-worker", pluginId }); const emitter = new EventEmitter(); /** * Higher than default (10) to accommodate multiple subscribers to * crash/ready/exit events during integration tests and runtime monitoring. */ emitter.setMaxListeners(50); // Worker process state let childProcess: ChildProcess | null = null; let readline: ReadlineInterface | null = null; let stderrReadline: ReadlineInterface | null = null; let status: WorkerStatus = "stopped"; let startedAt: number | null = null; let stderrExcerpt = ""; // Pending RPC requests awaiting a response const pendingRequests = new Map(); let nextRequestId = 1; const activeInvocations = new Map(); // Host-owned execute routes, keyed by the host-issued invocation id. Only an // `environmentExecute` call with a log sink registers a route here. The // `execute.log` router delivers only through this map — never through the // generic `activeInvocations` record — so a non-execute call can never become // a log target. const activeExecuteRoutes = new Map(); // Rate-limit state for dropped `execute.log` notifications. The debug record // never carries chunk bytes. let executeLogDropCount = 0; let executeLogDropLoggedAtMs = 0; // Rate-limit state for dropped oversized worker lines. The warn record carries // only the length, never the line bytes. let oversizedLineDropCount = 0; let oversizedLineLoggedAtMs = 0; // Framing and flood limits for the `execute.log` route. The defaults bound one // incoming line before the JSON parse and the total streamed output for one // execute call. A caller (a test) can lower them. const maxIncomingMessageChars = options.executeLogLimits?.maxIncomingMessageChars ?? MAX_WORKER_MESSAGE_CHARS; const maxExecuteLogTotalChars = options.executeLogLimits?.maxTotalCharsPerExecute ?? MAX_EXECUTE_LOG_TOTAL_CHARS; // Bounds and timeouts for the login pseudo-terminal route. A caller // (a test) can lower them to exercise the terminalize paths. const maxLoginPtyChunkChars = options.loginPtyLimits?.maxChunkChars ?? MAX_LOGIN_PTY_CHUNK_CHARS; const maxLoginPtyTotalChars = options.loginPtyLimits?.maxTotalChars ?? MAX_LOGIN_PTY_TOTAL_CHARS; const maxLoginPtyPreBindFrames = options.loginPtyLimits?.maxPreBindFrames ?? MAX_LOGIN_PTY_PRE_BIND_FRAMES; const maxLoginPtyPreBindChars = options.loginPtyLimits?.maxPreBindChars ?? MAX_LOGIN_PTY_PRE_BIND_CHARS; const loginPtyOpenTimeoutMs = options.loginPtyLimits?.openTimeoutMs ?? LOGIN_PTY_OPEN_TIMEOUT_MS; const loginPtyCloseTimeoutMs = options.loginPtyLimits?.closeTimeoutMs ?? LOGIN_PTY_CLOSE_TIMEOUT_MS; // Bounds and timeouts for the generic duplex channel route. A caller (a test) // can lower them to exercise each bound and the terminalize paths. Each // "Chars" name is a historical holdover: the channel now carries `Uint8Array` // chunks, so every one of these bounds counts raw bytes (`.length` on a // `Uint8Array` is its byte count), not UTF-16 code units. const maxDuplexChannelChunkChars = options.duplexChannelLimits?.maxChunkChars ?? MAX_DUPLEX_CHANNEL_CHUNK_CHARS; const maxDuplexChannelPreBindChars = options.duplexChannelLimits?.maxPreBindBufferedChars ?? DUPLEX_ROUTE_INPUT_MAX_BYTES; const maxDuplexChannelPreBindFrames = options.duplexChannelLimits?.maxPreBindBufferedFrames ?? MAX_DUPLEX_CHANNEL_PRE_BIND_FRAMES; // The pre-bind hold ceiling stays one frame above the buffered bound, so the // replay, not the hold, ends the route on the buffered bound. See // DUPLEX_CHANNEL_PRE_BIND_HOLD_MARGIN_FRAMES for why the margin must hold. const maxDuplexChannelPreBindHoldFrames = maxDuplexChannelPreBindFrames + DUPLEX_CHANNEL_PRE_BIND_HOLD_MARGIN_FRAMES; const maxDuplexChannelPendingRequests = options.duplexChannelLimits?.maxPendingRequests ?? MAX_DUPLEX_CHANNEL_PENDING_REQUESTS; const maxDuplexChannelWriteChars = options.duplexChannelLimits?.maxWriteChars ?? MAX_DUPLEX_CHANNEL_WRITE_CHARS; const maxDuplexRoutePendingWriteBytes = options.duplexChannelLimits?.maxPendingWriteBytes ?? DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES; const maxDuplexChannelProtocolErrors = options.duplexChannelLimits?.maxProtocolErrors ?? MAX_DUPLEX_CHANNEL_PROTOCOL_ERRORS; const maxDuplexChannelTotalDataBytes = options.duplexChannelLimits?.maxTotalDataBytes ?? MAX_DUPLEX_CHANNEL_TOTAL_DATA_BYTES; const maxDuplexChannelDurationMs = options.duplexChannelLimits?.maxDurationMs ?? MAX_DUPLEX_CHANNEL_DURATION_MS; const duplexChannelOpenTimeoutMs = options.duplexChannelLimits?.openTimeoutMs ?? DUPLEX_CHANNEL_OPEN_TIMEOUT_MS; const duplexChannelCloseTimeoutMs = options.duplexChannelLimits?.closeTimeoutMs ?? DUPLEX_CHANNEL_CLOSE_TIMEOUT_MS; // ------------------------------------------------------------------ // Proactive company scopes (LOOA-629) // ------------------------------------------------------------------ // A proactive plugin (e.g. the chat gateway) does company-scoped work from // its own timers/loops — not inside a host-issued top-level invocation // (onEvent/performAction/executeTool/configChanged). Those worker→host calls // carry no `paperclipInvocationId`, so the governed-access gate // (host-client-factory.ts) rejects any company-scoped request with // "company context is required" (regression class from #9557). The host // authorizes a bounded set of companies — the plugin's configured companies, // set by the loader after startup config delivery — for such proactive work. // A no-invocation call that references one of these companies resolves to // that company's scope; a call referencing any other company stays denied, // and in-invocation calls keep their strict single-company match. // // Seeded from options at handle creation — before the child process is // spawned — so a proactive plugin's setup()-time host calls (which land while // `startWorker` is still awaiting initialize) are authorized in time. The // loader used to call setProactiveCompanyScopes only AFTER startWorker // resolved, which was too late for the gateway's one-shot events.subscribe // and left outbound push permanently dead (LOOA-695). const proactiveCompanyScopes = new Set(); for (const id of options.proactiveCompanyScopes ?? []) { const trimmed = readNonEmptyString(id); if (trimmed) proactiveCompanyScopes.add(trimmed); } // Optional methods reported by the worker during initialization let supportedMethods: string[] = []; // Crash tracking for exponential backoff let consecutiveCrashes = 0; let totalCrashes = 0; let lastCrashAt: number | null = null; let backoffTimer: ReturnType | null = null; let nextRestartAt: number | null = null; // Track open stream channels so we can emit synthetic close on crash. // Maps channel → companyId. const openStreamChannels = new Map(); // Shutdown coordination let intentionalStop = false; const rpcTimeoutMs = options.rpcTimeoutMs ?? DEFAULT_RPC_TIMEOUT_MS; const autoRestart = options.autoRestart ?? true; // ----------------------------------------------------------------------- // Status management // ----------------------------------------------------------------------- function setStatus(newStatus: WorkerStatus): void { const prev = status; if (prev === newStatus) return; status = newStatus; log.debug({ from: prev, to: newStatus }, "worker status change"); emitter.emit("status", { pluginId, status: newStatus, previousStatus: prev }); } // ----------------------------------------------------------------------- // JSON-RPC message sending // ----------------------------------------------------------------------- function sendMessage(message: unknown): void { if (!childProcess?.stdin?.writable) { throw new Error(`Worker process for plugin "${pluginId}" is not writable`); } const serialized = serializeMessage(message as any); childProcess.stdin.write(serialized); } function errorCodeForWorkerHostError(err: unknown): number { const code = (err as { code?: unknown } | null)?.code; const pluginErrorCodes: readonly number[] = Object.values(PLUGIN_RPC_ERROR_CODES); return typeof code === "number" && pluginErrorCodes.includes(code) ? code : JSONRPC_ERROR_CODES.INTERNAL_ERROR; } // ----------------------------------------------------------------------- // Incoming message handling // ----------------------------------------------------------------------- function handleLine(line: string): void { if (!line.trim()) return; // Enforce the framing bound BEFORE the JSON parse. A line longer than the // limit is dropped without a parse, so a faulty or hostile worker cannot // force the host to parse an unbounded document and exhaust memory. if (line.length > maxIncomingMessageChars) { dropOversizedLine(line.length); return; } let message: unknown; try { message = parseMessage(line); } catch (err) { if (err instanceof JsonRpcParseError) { log.warn({ rawLine: line.slice(0, 200) }, "unparseable message from worker"); } else { log.warn({ err }, "error parsing worker message"); } return; } if (isJsonRpcResponse(message)) { handleResponse(message); } else if (isJsonRpcRequest(message)) { handleWorkerRequest(message as JsonRpcRequest); } else if (isJsonRpcNotification(message)) { handleWorkerNotification(message as JsonRpcNotification); } else { log.warn("unknown message type from worker"); } } /** * Handle a JSON-RPC response from the worker (matching a pending request). */ function handleResponse(response: JsonRpcResponse): void { const id = response.id; if (id === null || id === undefined) { log.warn("received response with null/undefined id"); return; } const pending = pendingRequests.get(id); if (!pending) { log.warn({ id }, "received response for unknown request id"); return; } clearTimeout(pending.timer); pendingRequests.delete(id); pending.resolve(response); } function readNonEmptyString(value: unknown): string | null { return typeof value === "string" && value.trim().length > 0 ? value.trim() : null; } function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } function deriveInvocationScope( method: HostToWorkerMethodName | string, params: unknown, ): PluginInvocationScope | null { if (!isRecord(params)) return null; const directCompanyId = readNonEmptyString(params.companyId); if (directCompanyId) return { companyId: directCompanyId }; if (method === "performAction" && isRecord(params.actorContext)) { const companyId = readNonEmptyString(params.actorContext.companyId); return companyId ? { companyId } : null; } if (method === "executeTool" && isRecord(params.runContext)) { const companyId = readNonEmptyString(params.runContext.companyId); return companyId ? { companyId } : null; } if (method === "onEvent" && isRecord(params.event)) { const companyId = readNonEmptyString(params.event.companyId); return companyId ? { companyId } : null; } return null; } function registerInvocation(scope: PluginInvocationScope, ttlMs?: number): PluginInvocationContext { // Mint a W3C `traceparent` from the active startup span, so the worker's // provider span can parent to it. The host keeps the value on its own record // (below) and never trusts the worker to supply the parent. Outside a // measured startup step there is no active span, so this is undefined. const activeStep = getActiveStepContext(); const traceparent = activeStep ? traceparentFromContextToken(activeStep.parentContext) : undefined; const invocation: PluginInvocationContext = { id: randomUUID(), scope, ...(traceparent ? { traceparent } : {}), }; const entry: ActiveInvocation = { scope, traceparent }; if (ttlMs !== undefined) { entry.timer = setTimeout(() => { activeInvocations.delete(invocation.id); }, ttlMs); if (entry.timer.unref) entry.timer.unref(); } activeInvocations.set(invocation.id, entry); return invocation; } function clearInvocation(invocation: PluginInvocationContext | null): void { if (!invocation) return; const entry = activeInvocations.get(invocation.id); if (entry?.timer) clearTimeout(entry.timer); activeInvocations.delete(invocation.id); } // Store the host-owned execute route for one active execute call. The host // holds the exact company id and log sink; the worker never supplies them. function registerExecuteRoute( invocationId: string, companyId: string, onLog: ExecuteLogSink, ): void { activeExecuteRoutes.set(invocationId, { companyId, onLog, deliveredChars: 0, crossCompanyBlocked: false, }); } function clearExecuteRoute(invocationId: string | undefined): void { if (invocationId) activeExecuteRoutes.delete(invocationId); } // Drop an oversized incoming worker line before the JSON parse. Write a // rate-limited warn record with the length and a running drop count. The // record never carries the line bytes. function dropOversizedLine(lineLength: number): void { oversizedLineDropCount += 1; const nowMs = Date.now(); if (nowMs - oversizedLineLoggedAtMs >= EXECUTE_LOG_DROP_LOG_INTERVAL_MS) { log.warn( { lineLength, maxIncomingMessageChars, droppedSinceLastLog: oversizedLineDropCount }, "dropping oversized worker line before JSON parse", ); oversizedLineLoggedAtMs = nowMs; oversizedLineDropCount = 0; } } // Drop an `execute.log` notification. Write a rate-limited debug record with // the reason and a running drop count. The record never carries the chunk // bytes, the company id, or command data. function dropExecuteLogNotification(reason: string): void { executeLogDropCount += 1; const nowMs = Date.now(); if (nowMs - executeLogDropLoggedAtMs >= EXECUTE_LOG_DROP_LOG_INTERVAL_MS) { log.debug( { reason, droppedSinceLastLog: executeLogDropCount }, "dropping execute.log notification", ); executeLogDropLoggedAtMs = nowMs; executeLogDropCount = 0; } } // Route one `execute.log` notification to its host-owned execute route. The // route is the single authority for the delivery target and the company // binding. This never reads a company id from the notification and never // routes through the generic active-invocation record. // // Complete mediation: the host and the worker share one stdio pipe, and the // worker process sees every active invocation id. So the host cannot prove // which concurrent invocation produced a notification, and it must NOT treat // the worker-supplied `paperclipInvocationId` alone as proof of origin. The // host validates the exact company scope instead: it delivers only while every // active execute route on this worker belongs to ONE company. When a second // company's execute overlaps, the host fails closed — it latches the active // routes and drops the chunk — so a worker that runs company A can never forge // company B's active id and inject output into B's route. The final command // result still delivers the complete output, so no byte is lost; only the live // stream pauses while two companies overlap. function routeExecuteLogNotification(notification: JsonRpcNotification): void { const invocationId = readNonEmptyString( (notification as { paperclipInvocationId?: unknown }).paperclipInvocationId, ); const params = isRecord(notification.params) ? notification.params : {}; const stream = params.stream; const chunk = params.chunk; // Runtime-validate the payload. Drop invalid input without a throw. if (stream !== "stdout" && stream !== "stderr") { dropExecuteLogNotification("invalid-stream"); return; } if ( typeof chunk !== "string" || chunk.length === 0 || chunk.length > MAX_EXECUTE_LOG_CHUNK_CHARS ) { dropExecuteLogNotification("invalid-chunk"); return; } if (!invocationId) { dropExecuteLogNotification("missing-invocation"); return; } const route = activeExecuteRoutes.get(invocationId); if (!route) { // No active execute route for this id: a late chunk after settlement or // timeout, a non-execute invocation, or an unknown id. Drop it. dropExecuteLogNotification("no-active-route"); return; } // The route already lost single-company attribution earlier in its life, so // it stays closed for the rest of the call. if (route.crossCompanyBlocked) { dropExecuteLogNotification("cross-company-scope"); return; } // Validate the exact company scope. Deliver only while every active execute // route on this worker belongs to one company. A second company's active // route makes the shared pipe ambiguous, so the host fails closed: it // latches every active route and drops the chunk. let onlyCompanyId: string | null = null; let crossCompany = false; for (const active of activeExecuteRoutes.values()) { if (onlyCompanyId === null) { onlyCompanyId = active.companyId; } else if (onlyCompanyId !== active.companyId) { crossCompany = true; break; } } if (crossCompany) { for (const active of activeExecuteRoutes.values()) { active.crossCompanyBlocked = true; } dropExecuteLogNotification("cross-company-scope"); return; } // Bound the total characters one execute call may stream. Past the ceiling // the host drops further chunks, so one runaway or hostile execution cannot // flood the host and the run-log sink without limit. if (route.deliveredChars + chunk.length > maxExecuteLogTotalChars) { dropExecuteLogNotification("execute-output-cap"); return; } route.deliveredChars += chunk.length; try { const delivery = route.onLog(stream, chunk); if (delivery && typeof (delivery as Promise).then === "function") { void (delivery as Promise).catch((err) => { log.error( { err: err instanceof Error ? err.message : String(err) }, "execute.log delivery failed", ); }); } } catch (err) { log.error( { err: err instanceof Error ? err.message : String(err) }, "execute.log delivery threw", ); } } // ----------------------------------------------------------------------- // Host-owned login pseudo-terminal route gate // ----------------------------------------------------------------------- // The manager owns one live login pseudo-terminal route per worker. It mints a // host-owned opaque route identifier, carries it in the open call, and keys the // close on it, so it closes a worker-created terminal even when the open reply // was lost and no worker session identifier arrived. It binds the worker // session identifier one time while the route is `opening`, for output only. It // never trusts a worker-supplied identifier as proof of origin: it delivers // output only while the route is `open` and the notification carries the exact // bound identifier and valid bounded bytes, and it never logs the raw bytes. It // terminalizes the route exactly once on every open failure path, closes the // terminal by the host route identifier, and admits a new open only after it // verifies a close acknowledgement bound to that identifier; it retires the // worker on an unconfirmed close. // A single-consumer route state. The login pseudo-terminal route and the // generic duplex channel route share it. type RouteState = "reserved" | "opening" | "open" | "closed"; // Shared route-binding helpers. The login pseudo-terminal route and the duplex // channel route both use them, so the two routes bind and settle one way. // Settle the route wait exactly once. Replace the settler with a no-op, so a // later exit or terminalize never settles the wait a second time. function settleRouteWait( route: { settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void }, value: { exitCode: number | null; transportClosed?: boolean }, ): void { const settle = route.settleWait; route.settleWait = () => {}; settle(value); } // Read the worker session identifier from an open reply, but only when the // route can still bind. Return null for a malformed reply, or for a route that // already left `opening` or terminalized. A late or a duplicate reply never // binds, revives, or reopens a route. function readBindableWorkerSessionId( route: { state: RouteState; terminalized: boolean }, openResult: unknown, ): string | null { const workerSessionId = readNonEmptyString( isRecord(openResult) ? openResult.workerSessionId : null, ); if (!workerSessionId || route.state !== "opening" || route.terminalized) { return null; } return workerSessionId; } type LoginPtyRouteState = RouteState; // One pre-bind login pseudo-terminal record: an output chunk or an exit, // normalized to a narrow scalar shape (never the raw notification object). interface LoginPtyPreBindOutputRecord { kind: "output"; workerSessionId: string; chunk: string; } interface LoginPtyPreBindExitRecord { kind: "exit"; workerSessionId: string; exitCode: number | null; } type LoginPtyPreBindRecord = LoginPtyPreBindOutputRecord | LoginPtyPreBindExitRecord; interface LoginPtyRoute { hostRouteId: string; state: LoginPtyRouteState; workerSessionId: string | null; listener: ((chunk: string) => void) | null; buffered: string[]; deliveredChars: number; terminalized: boolean; settleWait: (value: { exitCode: number | null }) => void; // Every output and exit record that arrived before the bind, in arrival order. preBind: LoginPtyPreBindRecord[]; // The cumulative characters `preBind` holds. Each record charges its // `workerSessionId` characters, plus the `chunk` characters for an output // record. preBindChars: number; } // At most one active credential pseudo-terminal per worker. A non-null route // blocks a second open until the manager confirms the first route's close. let loginPtyRoute: LoginPtyRoute | null = null; // Close the worker terminal by the host route identifier and verify the bound // acknowledgement. Return true only when the worker returns an acknowledgement // that carries the exact host route identifier. An absent, malformed, // mismatched, or timed-out acknowledgement returns false, so the caller fails // closed. async function closeLoginPtyTerminal(hostRouteId: string): Promise { try { const ack = await callInternal( "loginPtyClose", { hostRouteId }, loginPtyCloseTimeoutMs, ); return isRecord(ack) && readNonEmptyString(ack.hostRouteId) === hostRouteId; } catch { return false; } } // Terminalize the route exactly once. Resolve the login wait, close the worker // terminal by the host route identifier, and free the per-worker slot only // after the close resolves. Retire the worker when the close is unconfirmed. async function terminalizeLoginPtyRoute(route: LoginPtyRoute): Promise { if (route.terminalized) return; route.terminalized = true; route.state = "closed"; route.listener = null; route.buffered = []; // A terminalized route never replays a queued pre-bind record. route.preBind = []; route.preBindChars = 0; // A terminalized route reports a null exit code, which the runner treats as a // failure. settleRouteWait(route, { exitCode: null }); const confirmed = await closeLoginPtyTerminal(route.hostRouteId); if (loginPtyRoute === route) loginPtyRoute = null; if (!confirmed) { // The worker did not acknowledge the close, so the host cannot prove the // terminal is gone. Fail closed: retire the worker before any reuse. log.error( { pluginId }, "login pseudo-terminal close not acknowledged; retiring worker", ); void killProcess(); } } // Route one login pseudo-terminal output notification to the per-session // listener. Deliver only while the route is `open` and the notification carries // the exact bound worker session identifier and valid bounded bytes. Queue the // notification while the route is still `opening`. Drop an unknown, late, // malformed, or mismatched notification. Never log the raw bytes. function routeLoginPtyOutput(notification: JsonRpcNotification): void { const route = loginPtyRoute; if (!route) return; if (route.state === "opening") { queuePreBindLoginPtyOutput(route, notification); return; } if (route.state !== "open") return; const params = isRecord(notification.params) ? notification.params : {}; const workerSessionId = readNonEmptyString(params.workerSessionId); if (!workerSessionId || workerSessionId !== route.workerSessionId) return; const chunk = params.chunk; if ( typeof chunk !== "string" || chunk.length === 0 || chunk.length > maxLoginPtyChunkChars ) { return; } if (route.deliveredChars + chunk.length > maxLoginPtyTotalChars) { // The cumulative output passed the per-route bound. Terminalize the route. void terminalizeLoginPtyRoute(route); return; } route.deliveredChars += chunk.length; if (route.listener) route.listener(chunk); else route.buffered.push(chunk); } // Route one login pseudo-terminal exit notification to the login wait. Resolve // only while the route is `open` and the notification carries the exact bound // worker session identifier. Queue the notification while the route is still // `opening`. A resolved exit moves the state off `open`, so a later record — // live or replayed — finds a closed route and drops there. function routeLoginPtyExit(notification: JsonRpcNotification): void { const route = loginPtyRoute; if (!route) return; if (route.state === "opening") { queuePreBindLoginPtyExit(route, notification); return; } if (route.state !== "open") return; const params = isRecord(notification.params) ? notification.params : {}; const workerSessionId = readNonEmptyString(params.workerSessionId); if (!workerSessionId || workerSessionId !== route.workerSessionId) return; const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; route.state = "closed"; settleRouteWait(route, { exitCode }); } // Queue one login pseudo-terminal output notification that arrived before the // bind, in arrival order. Terminalize the route fail-closed on a frame-count // or character-bound breach. Never log the raw chunk. function queuePreBindLoginPtyOutput( route: LoginPtyRoute, notification: JsonRpcNotification, ): void { const params = isRecord(notification.params) ? notification.params : {}; const workerSessionId = readNonEmptyString(params.workerSessionId); const chunk = params.chunk; if ( !workerSessionId || typeof chunk !== "string" || chunk.length === 0 || chunk.length > maxLoginPtyChunkChars ) { return; } // Charge the record's full retained size: the worker session identifier // plus the chunk. A record that retains only the identifier still counts. const recordChars = workerSessionId.length + chunk.length; if ( route.preBind.length + 1 > maxLoginPtyPreBindFrames || route.preBindChars + recordChars > maxLoginPtyPreBindChars ) { log.warn( { pluginId }, "login pseudo-terminal pre-bind queue exceeded a bound; terminalizing route", ); void terminalizeLoginPtyRoute(route); return; } route.preBind.push({ kind: "output", workerSessionId, chunk }); route.preBindChars += recordChars; } // Queue one login pseudo-terminal exit notification that arrived before the // bind, in the same queue the output records use. An exit record carries no // chunk, but it still retains a worker session identifier, so it charges // against the character bound too. function queuePreBindLoginPtyExit( route: LoginPtyRoute, notification: JsonRpcNotification, ): void { const params = isRecord(notification.params) ? notification.params : {}; const workerSessionId = readNonEmptyString(params.workerSessionId); if (!workerSessionId) return; const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; const recordChars = workerSessionId.length; if ( route.preBind.length + 1 > maxLoginPtyPreBindFrames || route.preBindChars + recordChars > maxLoginPtyPreBindChars ) { log.warn( { pluginId }, "login pseudo-terminal pre-bind queue exceeded a bound; terminalizing route", ); void terminalizeLoginPtyRoute(route); return; } route.preBind.push({ kind: "exit", workerSessionId, exitCode }); route.preBindChars += recordChars; } // Replay the records a route held before it bound, in arrival order, // through the same live router the open route uses (`routeLoginPtyOutput`, // `routeLoginPtyExit`), so a forged worker session identifier still fails // the exact-match gate and a replayed exit still closes the route and // drops every record behind it. function replayPreBindLoginPtyRecords(route: LoginPtyRoute): void { const held = route.preBind; route.preBind = []; route.preBindChars = 0; for (const record of held) { if (route.terminalized) return; if (record.kind === "output") { routeLoginPtyOutput({ jsonrpc: "2.0", method: LOGIN_PTY_OUTPUT_NOTIFICATION, params: { workerSessionId: record.workerSessionId, chunk: record.chunk }, }); } else { routeLoginPtyExit({ jsonrpc: "2.0", method: LOGIN_PTY_EXIT_NOTIFICATION, params: { workerSessionId: record.workerSessionId, exitCode: record.exitCode }, }); } } } // Close the one route on a worker exit. The worker is gone, so the manager // resolves the login wait with the fixed non-secret exit and clears the route // one time. The pending pseudo-terminal calls reject through `rejectAllPending`. function closeLoginPtyRouteOnWorkerExit(): void { const route = loginPtyRoute; if (!route) return; loginPtyRoute = null; route.terminalized = true; route.state = "closed"; route.listener = null; route.buffered = []; route.preBind = []; route.preBindChars = 0; settleRouteWait(route, { exitCode: null }); } // Open one live login pseudo-terminal route. Reserve the route // before the open call, bind the worker session identifier one time on the // first successful open reply, and return a session the login transport drives. // Terminalize the route on every open failure path. async function openLoginPtySession( input: LoginPtyOpenInput, ): Promise { if (!isLoginCommandKey(input.loginCommandKey)) { // Allowlist the login command key. Only a key in the closed set may open a // sandbox pseudo-terminal. Reject a key outside the set with one fixed // non-secret error before the worker call, so a caller cannot select an // arbitrary command in the sandbox pseudo-terminal. throw new Error(LOGIN_PTY_COMMAND_NOT_ALLOWED); } // Revalidate the server-controlled session home shape before the worker RPC. // The binding validated it at the service boundary; this is the last host gate // before the worker call, so a malformed home fails closed here too. validateLoginSessionHome(input.sessionHome); if (loginPtyRoute) { // A route for this worker is not yet closed and confirmed. Reject the // second open with one fixed non-secret error before it reaches the worker. throw new Error(LOGIN_PTY_ROUTE_BUSY); } const hostRouteId = randomUUID(); let settleWait: (value: { exitCode: number | null }) => void = () => {}; const waitPromise = new Promise<{ exitCode: number | null }>((resolve) => { settleWait = resolve; }); const route: LoginPtyRoute = { hostRouteId, state: "reserved", workerSessionId: null, listener: null, buffered: [], deliveredChars: 0, terminalized: false, settleWait, preBind: [], preBindChars: 0, }; loginPtyRoute = route; route.state = "opening"; let openResult: HostToWorkerMethods["loginPtyOpen"][1]; try { openResult = await callInternal( "loginPtyOpen", { hostRouteId, driverKey: input.driverKey, companyId: input.companyId, environmentId: input.environmentId, providerLeaseId: input.providerLeaseId, loginCommandKey: input.loginCommandKey, sessionHome: input.sessionHome, }, loginPtyOpenTimeoutMs, ); } catch (err) { // A send failure, an RPC rejection, or an open timeout. Terminalize the // route exactly once and fail closed. await terminalizeLoginPtyRoute(route); throw err instanceof Error ? err : new Error(LOGIN_PTY_OPEN_FAILED); } const workerSessionId = readBindableWorkerSessionId(route, openResult); if (!workerSessionId) { // A malformed reply, or a route that already left `opening`. A late or a // duplicate reply never binds, revives, or reopens a route. await terminalizeLoginPtyRoute(route); throw new Error(LOGIN_PTY_OPEN_FAILED); } // Bind the worker session identifier one time and move the route to `open`. route.workerSessionId = workerSessionId; route.state = "open"; // Replay every record the route queued before the bind, in arrival order. replayPreBindLoginPtyRecords(route); return { onData(listener: (chunk: string) => void): void { route.listener = listener; if (route.buffered.length > 0) { const pending = route.buffered; route.buffered = []; for (const chunk of pending) listener(chunk); } }, write(data: string): void { const sid = route.workerSessionId; if (route.state !== "open" || !sid) return; void callInternal( "loginPtyInput", { workerSessionId: sid, data }, loginPtyOpenTimeoutMs, ).catch(() => {}); }, wait(): Promise<{ exitCode: number | null }> { return waitPromise; }, kill(): void { const sid = route.workerSessionId; if (!sid) return; void callInternal( "loginPtyStop", { workerSessionId: sid }, loginPtyOpenTimeoutMs, ).catch(() => {}); }, async close(): Promise { await terminalizeLoginPtyRoute(route); }, }; } // ----------------------------------------------------------------------- // Host-owned generic duplex channel route // ----------------------------------------------------------------------- // The duplex channel route mirrors the login pseudo-terminal route model. The // host owns the route identifier, binds the worker session identifier one time // on a valid open reply, and keys the close on the host route identifier. It // rejects a late or a duplicate open reply, and it retires the worker on an // unconfirmed close. The duplex channel carries no command allowlist, so the // caller owns the command. // // The route adds seven explicit bounds the pseudo-terminal route lacks. Each // bound ends the route when it passes its limit: // 1. pre-bind buffered bytes — the cumulative characters the host buffers // before a data listener attaches; // 2. pre-bind buffered frame count — the number of data frames the host // buffers before a data listener attaches; // 3. pending request count — the number of in-flight host→worker requests; // 4. host→worker write size — the characters for one write; // 5. protocol error rate — the count of malformed or mismatched data frames; // 6. total data bytes — the cumulative inbound bytes over the whole life, // counted before and after a data listener attaches; // 7. route lifetime — the milliseconds from the open to the terminal end. // One buffered data chunk retained for a late listener drain. It carries the raw // chunk bytes. interface BufferedDuplexChunk { chunk: Uint8Array; } // One pre-bind data event, normalized to the narrow duplex-event schema. The host // retains only these bounded scalar fields, never the original arbitrary // notification graph. The bind re-resolves the pair from `workerSessionId`, so a // frame whose pair does not match the bound pair still fails closed. interface HeldDuplexEvent { workerSessionId: string; chunk: Uint8Array; } // One pre-bind exit event, normalized to the narrow duplex-event schema. interface HeldDuplexExitEvent { workerSessionId: string; exitCode: number | null; transportClosed?: boolean; } interface DuplexChannelRoute { hostRouteId: string; state: RouteState; workerSessionId: string | null; listener: ((chunk: Uint8Array) => void) | null; buffered: BufferedDuplexChunk[]; bufferedChars: number; pendingRequests: number; protocolErrors: number; totalDataBytes: number; lifetimeTimer: ReturnType | null; terminalized: boolean; settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void; // The bounded raw data events that arrived before the bind, held in order. The // bind replays them through the exact-pair routing, so an early frame is never // lost and a frame whose pair does not match the bound pair still fails closed. // The hold ceiling is `maxDuplexChannelPreBindHoldFrames`, one frame above the // buffered bound, so the replay's buffered-bound check ends the route, not the // hold. The host never retains the original arbitrary notification graph. preBind: HeldDuplexEvent[]; // The cumulative raw bytes `preBind` holds. `DUPLEX_ROUTE_INPUT_MAX_BYTES` // bounds this route-local counter, so a worker cannot flood the pre-bind // hold with bytes the frame-count ceiling alone does not catch (a handful // of large frames, each well under the frame-count limit). This counter // covers only the hold; the post-bind buffered queue counts its own bytes // in `bufferedChars`, against the same named bound. preBindBytes: number; // The single bounded exit event that arrived before the bind. An exit never // consumes a data hold slot, so a worker that batches an exit among enough data // frames to fill the hold cannot crowd out a data frame. The bind replays the // held data events first, then this exit last. preBindExit: HeldDuplexExitEvent | null; // The cumulative raw payload bytes of every host→worker write this route // has sent but the worker has not yet acknowledged. The window this // counter tracks spans both retained representations of a pending write: // the raw payload the pending request object holds, and the serialized // child-stdin frame the host builds from it and writes before the request // settles. `DUPLEX_ROUTE_PENDING_WRITE_MAX_BYTES` bounds this route-local // counter, independent of `pendingRequests`: a worker that never replies // cannot make one route hold an unbounded number of write bytes, even // under a raised pending-request count. `sendBoundedRequest` increments it // before the send and decrements it once the request settles, on every // path: a reply, a rejection, a timeout, a worker exit, or a shutdown. pendingWriteBytes: number; } // The live duplex routes on this worker, keyed by the exact // `{ hostRouteId, workerSessionId }` pair. The host binds one pair once, at // open, and routes each data, exit, and close frame only to the route that owns // its exact pair. The host never routes by the worker session id alone. const liveDuplexRoutes = new Map(); // The reserved-or-opening routes, keyed by the host route id. A route lives here // from the open call until the worker session id binds. The host then moves it // to `liveDuplexRoutes` under the exact pair key. const openingDuplexRoutes = new Map(); // The tombstoned pairs on this worker, keyed by the exact pair key. The host // installs a tombstone atomically when it removes a live binding, and retains it // until the worker retires. A late frame for a tombstoned pair reaches no // listener. A closed pair never returns. const duplexPairTombstones = new Set(); // The process-scoped aggregate route-slot controller. The manager injects it, so // the ceiling counts every duplex route across every worker in the process, not // one agent's setting. When it is absent, the worker admits every open, so the // handle runs unbounded in isolation (a unit test constructs it this way). const duplexRouteSlots = options.duplexRouteSlots ?? null; // The routes that currently hold one aggregate slot. The host releases a slot one // time per route, so a double terminalize never releases two slots. const duplexRouteSlotHolders = new Set(); // The terminalized routes that still hold buffered bytes for a late listener // drain. A terminalized route leaves the opening and live maps, so this registry // keeps the worker-exit sweep able to clear its still-buffered representations. const terminalDuplexRoutes = new Set(); // Try to reserve one aggregate route slot for a route. Return true when the // route holds a slot after the call. When no controller is present, the route // always holds a slot. function acquireDuplexRouteSlot(route: DuplexChannelRoute): boolean { if (!duplexRouteSlots) return true; if (!duplexRouteSlots.tryAcquire()) return false; duplexRouteSlotHolders.add(route); return true; } // Release the aggregate route slot a route holds, one time. A route that never // held a slot releases nothing. function releaseDuplexRouteSlot(route: DuplexChannelRoute): void { if (!duplexRouteSlotHolders.delete(route)) return; duplexRouteSlots?.release(); } // Retire the worker at once. The host kills the process, so every live route, // every reserved route, and every tombstone drops together. The host uses this // as the fail-closed response to an ownership violation and to a tombstone-set // overflow. The host never logs the raw frame content on a violation. function retireDuplexWorkerOnViolation(reason: string): void { log.error({ pluginId }, `duplex channel ownership violation (${reason}); retiring worker`); void killProcess(); } // Install a tombstone for a bound pair atomically with the removal of its live // binding. This runs in one synchronous step, before any slot release or reuse, // so a late frame for the pair reaches no listener. Before the set would exceed // its hard bound, retire the worker, which drops every route and every tombstone // at once. A tombstone overflow fails closed; the host never evicts a tombstone // and lets a pair return. function tombstoneBoundDuplexPair(route: DuplexChannelRoute): void { if (route.workerSessionId === null) return; const pairKey = duplexPairKey(route.hostRouteId, route.workerSessionId); liveDuplexRoutes.delete(pairKey); if (!duplexPairTombstones.has(pairKey)) { if (duplexPairTombstones.size >= MAX_DUPLEX_ROUTE_TOMBSTONES) { // The set is full. Do not evict a tombstone. Retire the worker, which // drops every route and every tombstone together, so no closed pair ever // returns. retireDuplexWorkerOnViolation("tombstone set overflow"); return; } duplexPairTombstones.add(pairKey); } } // Close the worker channel by the host route identifier and verify the bound // acknowledgement. Return true only when the worker returns an acknowledgement // for the exact pair. Before a session binds, the acknowledgement carries the // host route identifier only, so a lost open reply still permits a route-only // close. After a session binds, the acknowledgement must echo both the host // route identifier and the bound worker session identifier. An absent, // malformed, mismatched, or timed-out acknowledgement returns false, so the // caller fails closed. async function closeDuplexChannelTerminal( hostRouteId: string, boundWorkerSessionId: string | null, ): Promise { try { const ack = await callInternal( "duplexChannelClose", { hostRouteId }, duplexChannelCloseTimeoutMs, ); if (!isRecord(ack) || readNonEmptyString(ack.hostRouteId) !== hostRouteId) { return false; } if (boundWorkerSessionId !== null) { // A bound close: the acknowledgement must echo the exact worker session // identifier. A missing or a mismatched identifier is invalid. return readNonEmptyString(ack.workerSessionId) === boundWorkerSessionId; } return true; } catch { return false; } } // Terminalize the route exactly once. Resolve the wait, close the worker // channel by the host route identifier, and free the per-worker slot only // after the close resolves. Retire the worker when the close is unconfirmed. // Clear the route lifetime timer one time. Every terminal path and the // worker-exit path calls this, so a timer never fires after the route ends. function clearDuplexChannelLifetimeTimer(route: DuplexChannelRoute): void { if (route.lifetimeTimer) { clearTimeout(route.lifetimeTimer); route.lifetimeTimer = null; } } async function terminalizeDuplexChannelRoute(route: DuplexChannelRoute): Promise { if (route.terminalized) return; route.terminalized = true; route.state = "closed"; route.listener = null; // Stop admission first. Keep the buffered chunks the host accepted before the // route ended, so a listener that attaches after the end still drains them. A // frame can end the route during the pre-bind replay, before a listener // attaches, and the chunks the host accepted before that frame are valid data // the listener must still receive. The buffered bytes stay bounded by the // pre-bind buffered bound, and `onData` clears them once it drains them. // Register the route in the terminal registry, so the worker-exit sweep can // clear the still-buffered chunks later. route.preBind = []; route.preBindBytes = 0; route.preBindExit = null; if (route.buffered.length > 0) { terminalDuplexRoutes.add(route); } clearDuplexChannelLifetimeTimer(route); // Remove the live binding and install the tombstone in one synchronous step, // before the worker close and before any reuse. A reserved route that never // bound leaves the opening map only; it has no pair to tombstone, and the // monotonic host route id never returns. openingDuplexRoutes.delete(route.hostRouteId); tombstoneBoundDuplexPair(route); releaseDuplexRouteSlot(route); // A terminalized route reports a null exit code, which the caller treats as a // failure. settleRouteWait(route, { exitCode: null }); const confirmed = await closeDuplexChannelTerminal(route.hostRouteId, route.workerSessionId); if (!confirmed) { // The worker did not acknowledge the close, so the host cannot prove the // channel is gone. Fail closed: retire the worker before any reuse. log.error( { pluginId }, "duplex channel close not acknowledged; retiring worker", ); void killProcess(); } } // Count one protocol error for the route. End the route when the count passes // the per-route budget, so a flood of malformed or mismatched frames bounds the // route. function recordDuplexChannelProtocolError(route: DuplexChannelRoute): void { route.protocolErrors += 1; if (route.protocolErrors > maxDuplexChannelProtocolErrors) { void terminalizeDuplexChannelRoute(route); } } // Deliver one duplex channel chunk to the bound listener in isolation. A // listener that throws must not escape the worker stdout notification handler // or the buffered replay, so a throw here breaks neither the notification // dispatch loop nor the pre-bind drain. The manager catches the error and logs // it without the raw bytes. This mirrors the `execute.log` delivery isolation. function deliverDuplexChannelChunk( listener: (chunk: Uint8Array) => void, chunk: Uint8Array, ): void { try { listener(chunk); } catch (err) { log.error( { err: err instanceof Error ? err.message : String(err) }, "duplex channel data delivery threw", ); } } // Route one duplex channel data notification to the per-session listener. // Deliver only while the route is `open` and the notification carries the exact // bound worker session identifier and a valid chunk. Count a mismatched or // malformed frame as a protocol error. End the route at once when one chunk is // larger than the per-chunk limit or when the cumulative bytes pass the total // cap. Buffer a valid frame under the pre-bind bounds when no listener has // attached yet. Never log the raw bytes. // Resolve one inbound duplex frame to its owning route by the exact pair. Return // the live route on an exact live-pair match. Return `"tombstoned"` for a late // frame whose exact pair the host already closed; that frame reaches no listener // and changes no state. Return `"violation"` for any unknown, foreign, // duplicate, ambiguous, or malformed pair; the caller fails closed and retires // the worker. The lookup never routes by the worker session id alone. function resolveDuplexRouteByPair( hostRouteId: string | null, workerSessionId: string | null, ): DuplexChannelRoute | "tombstoned" | "opening" | "violation" { if (!hostRouteId || !workerSessionId) return "violation"; const pairKey = duplexPairKey(hostRouteId, workerSessionId); const live = liveDuplexRoutes.get(pairKey); if (live && live.state === "open") return live; if (duplexPairTombstones.has(pairKey)) return "tombstoned"; // A frame whose host route id names a route that is still opening arrived // before the bind. It is the host's own reserved route, not a foreign frame. // The caller defers it, so the bind replays it through the exact-pair routing. if (openingDuplexRoutes.has(hostRouteId)) return "opening"; return "violation"; } // Route one duplex channel data frame. On a live route with a listener, deliver // the chunk transiently. On a live route with no listener, buffer the chunk. function routeDuplexChannelData(notification: JsonRpcNotification): void { const params = isRecord(notification.params) ? notification.params : {}; const hostRouteId = readNonEmptyString(params.hostRouteId); const workerSessionId = readNonEmptyString(params.workerSessionId); const resolved = resolveDuplexRouteByPair(hostRouteId, workerSessionId); if (resolved === "tombstoned") { // A late frame for a closed pair. It reaches no listener and changes no // state. A bind replay never lands here, because the replayed route is live; // a carried token stays on its route and the terminal cleanup releases it. // Never log the raw frame content. return; } if (resolved === "violation") { // An unknown, foreign, duplicate, ambiguous, or malformed pair. Fail closed: // retire the worker. Never log the raw frame content. retireDuplexWorkerOnViolation("data frame pair"); return; } if (resolved === "opening") { // The frame arrived before the bind. A replay never lands here, because the // route is live during replay. Hold the live frame; the bind replays it. bufferPreBindDuplexFrame(hostRouteId, notification); return; } const route = resolved; // The wire carries the chunk as a base64 string (JSON has no binary type; see // `ChannelBytesWireValue` in protocol.ts). Decode it back to raw bytes before // any bound check, so every bound below counts real bytes, not base64 text. // A bind replay (`replayPreBindDuplexFrames`) reuses this function with a // held event whose chunk the host already decoded once; it carries the // already-decoded `Uint8Array` straight through with no second decode. const rawChunk = params.chunk; const chunk = rawChunk instanceof Uint8Array ? rawChunk : decodeChannelBytes(rawChunk); if (chunk === null || chunk.byteLength === 0) { // The exact pair matches, but the chunk is malformed or empty. Count one // per-route protocol error. recordDuplexChannelProtocolError(route); return; } if (chunk.byteLength > maxDuplexChannelChunkChars) { // One inbound chunk is larger than the per-chunk limit. End the route at // once. Do not count the chunk as a protocol error. void terminalizeDuplexChannelRoute(route); return; } // Enforce the cumulative total-byte cap before and after a listener attaches. // End the route when the cap is exceeded, so a bound listener cannot receive // data past the cap. const chunkBytes = chunk.byteLength; if (route.totalDataBytes + chunkBytes > maxDuplexChannelTotalDataBytes) { void terminalizeDuplexChannelRoute(route); return; } route.totalDataBytes += chunkBytes; if (route.listener) { // A listener is attached. Deliver the chunk transiently. deliverDuplexChannelChunk(route.listener, chunk); return; } // No listener attached yet. Buffer the frame under the pre-bind bounds. End // the route when the cumulative bytes or the frame count passes the bound. if ( route.buffered.length + 1 > maxDuplexChannelPreBindFrames || route.bufferedChars + chunk.length > maxDuplexChannelPreBindChars ) { void terminalizeDuplexChannelRoute(route); return; } route.buffered.push({ chunk }); route.bufferedChars += chunk.length; } // Route one duplex channel exit notification to the wait. Resolve only while // the route is `open` and the notification carries the exact bound worker // session identifier. function routeDuplexChannelExit(notification: JsonRpcNotification): void { const params = isRecord(notification.params) ? notification.params : {}; const hostRouteId = readNonEmptyString(params.hostRouteId); const workerSessionId = readNonEmptyString(params.workerSessionId); const resolved = resolveDuplexRouteByPair(hostRouteId, workerSessionId); if (resolved === "tombstoned") { // A late exit for a closed pair. It reaches no wait and changes no state. return; } if (resolved === "violation") { // An unknown, foreign, duplicate, ambiguous, or malformed pair. Fail closed: // retire the worker. retireDuplexWorkerOnViolation("exit frame pair"); return; } if (resolved === "opening") { // The exit arrived before the bind. Hold it; the bind replays it through the // exact-pair routing. bufferPreBindDuplexFrame(hostRouteId, notification); return; } const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; // A reason-less transport close carries `transportClosed`; a real process exit // does not. Carry the discriminator to the wait only when it is a transport // close, so a real exit keeps the plain `{ exitCode }` result and the broker // still keeps the two apart in the loss taxonomy. if (params.transportClosed === true) { settleRouteWait(resolved, { exitCode, transportClosed: true }); return; } settleRouteWait(resolved, { exitCode }); } // Hold one frame that arrived before its route bound, in order. The bind replays // the held frames through the exact-pair routing. An exit frame never consumes a // data hold slot; the host holds it in the single `preBindExit` slot instead, so // a worker that batches an exit among enough data frames to fill the hold cannot // crowd out a data frame. The data hold ceiling stays one frame above the // buffered bound, so the replay's buffered-bound check, not the hold, ends the // route. The host counts one protocol error for each data frame past the ceiling, // so an early flood bounds the hold instead of growing without limit. function bufferPreBindDuplexFrame( hostRouteId: string | null, notification: JsonRpcNotification, ): void { if (!hostRouteId) return; const route = openingDuplexRoutes.get(hostRouteId); if (!route) return; const params = isRecord(notification.params) ? notification.params : {}; const workerSessionId = readNonEmptyString(params.workerSessionId); if (!workerSessionId) { // A malformed pair. Count one per-route protocol error and hold nothing. recordDuplexChannelProtocolError(route); return; } if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) { // Normalize the exit to the narrow duplex-event schema. A replaced exit // simply overwrites the earlier held exit. const exitCode = typeof params.exitCode === "number" ? params.exitCode : null; route.preBindExit = { workerSessionId, exitCode, ...(params.transportClosed === true ? { transportClosed: true } : {}), }; return; } // A data event. Validate and normalize it to the narrow duplex-event schema // before any retention. The wire carries the chunk as base64; decode it back // to raw bytes before any bound check or retention. const chunk = decodeChannelBytes(params.chunk); if (chunk === null || chunk.byteLength === 0) { recordDuplexChannelProtocolError(route); return; } if (route.preBind.length >= maxDuplexChannelPreBindHoldFrames) { recordDuplexChannelProtocolError(route); return; } // The route-local input bound covers the pre-bind hold directly. A handful // of large frames can pass the frame-count ceiling above while still // flooding the host with bytes, so this route ends at once on the byte // bound, the same way the post-bind buffered queue already does. if (route.preBindBytes + chunk.byteLength > maxDuplexChannelPreBindChars) { void terminalizeDuplexChannelRoute(route); return; } route.preBind.push({ workerSessionId, chunk }); route.preBindBytes += chunk.byteLength; } // Replay the frames a route held before it bound. The route is live now, so the // exact-pair routing delivers each held frame or fails closed on a mismatch. // Replay the held data frames first, in order, then the held exit last, so the // data delivers before the exit resolves the wait, which matches a real worker's // order. A frame that ends the route terminalizes it, and every later frame in // the replay is a no-op, because the routing functions drop a frame for a route // that is not `open`. function replayPreBindDuplexFrames(route: DuplexChannelRoute): void { const held = route.preBind; route.preBind = []; route.preBindBytes = 0; for (const event of held) { if (route.terminalized) { // The route ended mid-replay. The remaining held events are dropped. continue; } // Reconstruct a transient data notification for the exact-pair routing. The // host holds only the bounded event, so it builds this notification for the // routing step and discards it at once. routeDuplexChannelData({ jsonrpc: "2.0", method: DUPLEX_CHANNEL_DATA_NOTIFICATION, params: { hostRouteId: route.hostRouteId, workerSessionId: event.workerSessionId, chunk: event.chunk, }, }); } const heldExit = route.preBindExit; route.preBindExit = null; if (heldExit) { // Resolve the wait through the exact-pair routing when the route still lives. if (!route.terminalized) { routeDuplexChannelExit({ jsonrpc: "2.0", method: DUPLEX_CHANNEL_EXIT_NOTIFICATION, params: { hostRouteId: route.hostRouteId, workerSessionId: heldExit.workerSessionId, exitCode: heldExit.exitCode, ...(heldExit.transportClosed === true ? { transportClosed: true } : {}), }, }); } } } // Close every route on a worker exit. The worker is gone, so the manager // resolves each wait with the fixed non-secret exit, releases each aggregate // slot, and clears the live, opening, and tombstone state. A worker exit is a // worker retirement, so the host drops every tombstone; the monotonic host route // id never returns, so no closed pair can revive on a restart. The pending // channel calls reject through `rejectAllPending`. function closeDuplexChannelRouteOnWorkerExit(): void { // Enumerate the opening index, the live index, and the terminal registry. A // terminalized route left the opening and live maps but may still hold buffered // bytes for a late listener; the worker is gone, so the host releases them now. const routes = [ ...openingDuplexRoutes.values(), ...liveDuplexRoutes.values(), ...terminalDuplexRoutes, ]; openingDuplexRoutes.clear(); liveDuplexRoutes.clear(); duplexPairTombstones.clear(); for (const route of routes) { if (!route.terminalized) { route.terminalized = true; route.state = "closed"; route.listener = null; route.bufferedChars = 0; clearDuplexChannelLifetimeTimer(route); releaseDuplexRouteSlot(route); settleRouteWait(route, { exitCode: null }); } // Clear the route's retained buffers, exactly once. A route already drained // by a late listener holds nothing, so this is harmless. route.preBind = []; route.preBindBytes = 0; route.preBindExit = null; route.buffered = []; route.bufferedChars = 0; terminalDuplexRoutes.delete(route); } // Clear the terminal registry. Every route in it was just discarded above. terminalDuplexRoutes.clear(); } // Open one live generic duplex channel route. Reserve the route before the open // call, bind the worker session identifier one time on the first successful // open reply, and return a session a caller drives. Terminalize the route on // every open failure path. async function openDuplexChannel( input: DuplexChannelOpenInput, ): Promise { const hostRouteId = nextDuplexHostRouteId(); let settleWait: (value: { exitCode: number | null; transportClosed?: boolean }) => void = () => {}; const waitPromise = new Promise<{ exitCode: number | null; transportClosed?: boolean }>( (resolve) => { settleWait = resolve; }, ); const route: DuplexChannelRoute = { hostRouteId, state: "reserved", workerSessionId: null, listener: null, buffered: [], bufferedChars: 0, pendingRequests: 0, protocolErrors: 0, totalDataBytes: 0, lifetimeTimer: null, terminalized: false, settleWait, preBind: [], preBindBytes: 0, preBindExit: null, pendingWriteBytes: 0, }; // Reserve one aggregate route slot before any work. When the process-scoped // ceiling is full, reject with the fixed route-busy error and open nothing, so // an active channel never downgrades and the ceiling never overcommits. if (!acquireDuplexRouteSlot(route)) { throw new Error(DUPLEX_CHANNEL_ROUTE_BUSY); } openingDuplexRoutes.set(hostRouteId, route); route.state = "opening"; let openResult: HostToWorkerMethods["duplexChannelOpen"][1]; try { openResult = await callInternal( "duplexChannelOpen", { hostRouteId, driverKey: input.driverKey, companyId: input.companyId, environmentId: input.environmentId, providerLeaseId: input.providerLeaseId, command: input.command, }, duplexChannelOpenTimeoutMs, ); } catch (err) { // A send failure, an RPC rejection, or an open timeout. Terminalize the // route exactly once and fail closed. await terminalizeDuplexChannelRoute(route); throw err instanceof Error ? err : new Error(DUPLEX_CHANNEL_OPEN_FAILED); } const workerSessionId = readBindableWorkerSessionId(route, openResult); // Verify the worker echoed the exact host route id the open request carried. // A reply with a missing or a mismatched host route id never binds, so the // host binds only a reply that proves the worker holds the exact pair. const echoedHostRouteId = isRecord(openResult) ? readNonEmptyString(openResult.hostRouteId) : null; if (!workerSessionId || echoedHostRouteId !== hostRouteId) { // A malformed reply, a mismatched host route id, or a route that already // left `opening`. A late or a duplicate reply never binds, revives, or // reopens a route. await terminalizeDuplexChannelRoute(route); throw new Error(DUPLEX_CHANNEL_OPEN_FAILED); } const pairKey = duplexPairKey(hostRouteId, workerSessionId); if (liveDuplexRoutes.has(pairKey) || duplexPairTombstones.has(pairKey)) { // The worker returned a pair that is already live or already tombstoned. Fail // closed: terminalize this route and retire the worker before any reuse. await terminalizeDuplexChannelRoute(route); retireDuplexWorkerOnViolation("duplicate bound pair"); throw new Error(DUPLEX_CHANNEL_OPEN_FAILED); } // Bind the worker session identifier one time and move the route from the // opening map to the live map under the exact pair key. route.workerSessionId = workerSessionId; route.state = "open"; openingDuplexRoutes.delete(hostRouteId); liveDuplexRoutes.set(pairKey, route); // Start the route lifetime timer now the route is open. The route ends when // the timer expires. Every terminal path and the worker-exit path clears the // timer. Unreference the timer so it never blocks the host process shutdown. route.lifetimeTimer = setTimeout(() => { void terminalizeDuplexChannelRoute(route); }, maxDuplexChannelDurationMs); route.lifetimeTimer.unref?.(); // Replay any frame that arrived before the bind. The route is live now, so the // exact-pair routing delivers each held frame or fails closed on a mismatch. replayPreBindDuplexFrames(route); // Send one host→worker request under the pending-request bound. End the route // when too many requests are in-flight, so a worker that never replies cannot // make the host hold an unbounded number of pending requests. const sendBoundedRequest = < M extends "duplexChannelWrite" | "duplexChannelStop", >( method: M, params: HostToWorkerMethods[M][0], // The exact raw byte count of a `duplexChannelWrite` payload, before base64 // encoding. The caller supplies it: `params.data` on the wire is the base64 // form (`ChannelBytesWireValue`), so its string length no longer equals the // real payload bytes. A stop request carries no payload and omits this. writeByteCount?: number, ): void => { if (route.state !== "open") return; if (route.pendingRequests >= maxDuplexChannelPendingRequests) { void terminalizeDuplexChannelRoute(route); return; } // The route-local pending-write byte bound: a worker that never replies // cannot make one route hold an unbounded number of write bytes, even // under a raised pending-request count. A stop request carries no // payload, so it never counts against this bound. const pendingWriteBytesToCharge = method === "duplexChannelWrite" ? writeByteCount ?? 0 : 0; if (route.pendingWriteBytes + pendingWriteBytesToCharge > maxDuplexRoutePendingWriteBytes) { void terminalizeDuplexChannelRoute(route); return; } route.pendingRequests += 1; route.pendingWriteBytes += pendingWriteBytesToCharge; void callInternal(method, params, duplexChannelOpenTimeoutMs, undefined) // A send failure, an RPC rejection, a timeout, or a worker exit ends up // here. The route already handles a lost write through its own state // (a worker exit or a stop rejects every pending request through the // normal call machinery), so this fire-and-forget call swallows the // rejection instead of leaving it unhandled. .catch(() => {}) .finally(() => { route.pendingRequests -= 1; // Release the route-local pending-write bytes one time, after the RPC // settles on any path: success, error, timeout, worker exit, or // shutdown. route.pendingWriteBytes -= pendingWriteBytesToCharge; }); }; return { onData(listener: (chunk: Uint8Array) => void): void { route.listener = listener; if (route.buffered.length > 0) { const pending = route.buffered; route.buffered = []; route.bufferedChars = 0; for (const record of pending) { // Deliver each buffered record before the drain proceeds to the next // record. A terminal route drains through this same code. deliverDuplexChannelChunk(listener, record.chunk); } // The buffered records are gone, so the route no longer holds terminal // bytes for a late listener. terminalDuplexRoutes.delete(route); } }, write(data: Uint8Array): void { const sid = route.workerSessionId; if (route.state !== "open" || !sid) return; if (data.byteLength > maxDuplexChannelWriteChars) { // The write is larger than the size bound. End the route before the // write reaches the worker. void terminalizeDuplexChannelRoute(route); return; } // Encode the raw bytes to the wire-safe base64 form (JSON carries no // binary type) and pass the exact raw byte count separately, so the // pending-write byte bound charges the real payload bytes, not the // inflated base64 string length. sendBoundedRequest( "duplexChannelWrite", { hostRouteId: route.hostRouteId, workerSessionId: sid, data: encodeChannelBytes(data), }, data.byteLength, ); }, wait(): Promise<{ exitCode: number | null }> { return waitPromise; }, kill(): void { const sid = route.workerSessionId; if (!sid) return; sendBoundedRequest("duplexChannelStop", { hostRouteId: route.hostRouteId, workerSessionId: sid }); }, async close(): Promise { await terminalizeDuplexChannelRoute(route); }, }; } /** * Extract the single company a worker→host call references, mirroring the SDK * governed-access gate's own derivation (host-client-factory.ts * `requestedCompanyScope`) so a proactive call resolves to exactly the company * the gate would require: * - explicit `params.companyId`; * - a company-scoped state key (`scopeKind: "company"` + `scopeId`); * - `events.subscribe`'s `params.filter.companyId` (how the SDK's * `ctx.events.on(name, { companyId }, fn)` issues its subscribe). * * Returns null whenever the gate treats the call as a wildcard (`companies.list`, * a `scopeKind: "company"` key with no `scopeId`) or as referencing no company * (instance-scoped state, an unfiltered subscribe). A wildcard is deliberately * NOT granted proactively: proactive resolution only ever admits a single, * explicit company, never "all". This keeps the resolver and the gate in * lockstep in the functional direction (LOOA-693 AC#4 / LOOA-695). */ function referencedCompanyId(method: string, params: unknown): string | null { // Gate returns { kind: "all" } for companies.list regardless of params — // never a single company — so proactive access declines it here. if (method === "companies.list") return null; if (!isRecord(params)) return null; const direct = readNonEmptyString(params.companyId); if (direct) return direct; if (params.scopeKind === "company") { // scopeId present → that company; absent → wildcard ("all") in the gate, // which we never grant proactively → null. return readNonEmptyString(params.scopeId); } if (method === "events.subscribe" && isRecord(params.filter)) { return readNonEmptyString(params.filter.companyId); } return null; } function contextForWorkerMessage(message: JsonRpcRequest | JsonRpcNotification): WorkerHostCallContext { const invocationId = readNonEmptyString( (message as { paperclipInvocationId?: unknown }).paperclipInvocationId, ); if (!invocationId) { // No host-issued invocation is being echoed. This is a genuinely // proactive worker→host call (timer/loop). If it references a company the // plugin is authorized to act on proactively, resolve it to that // company's scope so the governed-access gate admits it. This never // widens access beyond the plugin's configured companies, and only // applies when the worker is NOT inside a host-issued invocation (which // would carry an id and keep its strict single-company match below). const proactiveCompanyId = referencedCompanyId( message.method, (message as { params?: unknown }).params, ); if (proactiveCompanyId && proactiveCompanyScopes.has(proactiveCompanyId)) { return { invocationScope: { companyId: proactiveCompanyId } }; } const hasActiveInvocation = activeInvocations.size > 0 || Array.from(pendingRequests.values()).some((pending) => pending.invocationId); return hasActiveInvocation ? { invalidInvocationScope: true } : {}; } const entry = activeInvocations.get(invocationId); if (!entry) return { invalidInvocationScope: true }; return { invocationScope: entry.scope, traceparent: entry.traceparent }; } /** * Handle a JSON-RPC request from the worker (worker→host call). */ async function handleWorkerRequest(request: JsonRpcRequest): Promise { const method = request.method as WorkerToHostMethodName; const handler = options.hostHandlers[method] as | ((params: unknown, context?: WorkerHostCallContext) => Promise) | undefined; if (!handler) { log.warn({ method }, "worker called unregistered host method"); try { sendMessage( createErrorResponse( request.id, JSONRPC_ERROR_CODES.METHOD_NOT_FOUND, `Host does not handle method "${method}"`, ), ); } catch { // Worker may have exited, ignore send error } return; } try { const result = await handler(request.params, contextForWorkerMessage(request)); sendMessage({ jsonrpc: JSONRPC_VERSION, id: request.id, result: result ?? null, }); } catch (err) { const errorMessage = err instanceof Error ? err.message : String(err); log.error({ method, err: errorMessage }, "host handler error"); try { sendMessage( createErrorResponse( request.id, errorCodeForWorkerHostError(err), errorMessage, ), ); } catch { // Worker may have exited, ignore send error } } } /** * Handle a JSON-RPC notification from the worker (fire-and-forget). * * The `log` notification is the primary case — worker `ctx.logger` calls * arrive here. We append structured plugin context (pluginId, timestamp, * level) so that every log entry is queryable per the spec (§26.1). */ function handleWorkerNotification(notification: JsonRpcNotification): void { if (notification.method === "log") { const params = notification.params as { level?: string; message?: string; meta?: Record; } | null; const level = params?.level ?? "info"; const msg = params?.message ?? ""; const meta = params?.meta; // Build a structured log object that includes the plugin context fields // required by §26.1: pluginId, timestamp, level, message, and metadata. // The child logger already carries `pluginId` in its bindings, but we // add explicit `pluginLogLevel` and `pluginTimestamp` so downstream // consumers (log storage, UI queries) can filter without parsing. const logFields: Record = { ...meta, pluginLogLevel: level, pluginTimestamp: new Date().toISOString(), }; if (level === "error") { log.error(logFields, `[plugin] ${msg}`); } else if (level === "warn") { log.warn(logFields, `[plugin] ${msg}`); } else if (level === "debug") { log.debug(logFields, `[plugin] ${msg}`); } else { log.info(logFields, `[plugin] ${msg}`); } return; } // Execute-log notifications: deliver one incremental output chunk to the // host-owned execute route for the active execute call. if (notification.method === "execute.log") { routeExecuteLogNotification(notification); return; } // Login pseudo-terminal notifications: deliver output // and the exit to the one host-owned login route, bound by the worker session // identifier while the route is open. if (notification.method === LOGIN_PTY_OUTPUT_NOTIFICATION) { routeLoginPtyOutput(notification); return; } if (notification.method === LOGIN_PTY_EXIT_NOTIFICATION) { routeLoginPtyExit(notification); return; } // Duplex channel notifications: deliver data and the exit to the one // host-owned duplex route, bound by the worker session identifier while the // route is open. if (notification.method === DUPLEX_CHANNEL_DATA_NOTIFICATION) { routeDuplexChannelData(notification); return; } if (notification.method === DUPLEX_CHANNEL_EXIT_NOTIFICATION) { routeDuplexChannelExit(notification); return; } // Stream notifications: forward to the stream bus via callback if ( notification.method === "streams.open" || notification.method === "streams.emit" || notification.method === "streams.close" ) { const params = (notification.params ?? {}) as Record; const companyId = String(params.companyId ?? ""); const context = contextForWorkerMessage(notification); if (context.invalidInvocationScope) { log.warn( { method: notification.method, companyId }, "dropping plugin stream notification with invalid invocation scope", ); return; } const allowedCompanyId = context.invocationScope?.companyId; if (allowedCompanyId && companyId !== allowedCompanyId) { log.warn( { method: notification.method, companyId, allowedCompanyId }, "dropping plugin stream notification outside invocation company scope", ); return; } // Track open channels so we can emit synthetic close on crash if (notification.method === "streams.open") { const ch = String(params.channel ?? ""); if (ch) openStreamChannels.set(ch, companyId); } else if (notification.method === "streams.close") { openStreamChannels.delete(String(params.channel ?? "")); } if (options.onStreamNotification) { try { options.onStreamNotification(notification.method, params); } catch (err) { log.error( { method: notification.method, err: err instanceof Error ? err.message : String(err), }, "stream notification handler failed", ); } } return; } log.debug({ method: notification.method }, "received notification from worker"); } // ----------------------------------------------------------------------- // Process lifecycle // ----------------------------------------------------------------------- function spawnProcess(): ChildProcess { // Security: Do NOT spread process.env into the worker. Plugins should only // receive a minimal, controlled environment to prevent leaking host // secrets (like DATABASE_URL, internal API keys, etc.). const workerEnv: Record = { ...options.env, PATH: process.env.PATH ?? "", NODE_PATH: process.env.NODE_PATH ?? "", PAPERCLIP_PLUGIN_ID: pluginId, NODE_ENV: process.env.NODE_ENV ?? "production", TZ: process.env.TZ ?? "UTC", }; const child = fork(options.entrypointPath, [], { stdio: ["pipe", "pipe", "pipe", "ipc"], execArgv: options.execArgv ?? [], env: workerEnv, // Don't let the child keep the parent alive detached: false, }); return child; } function attachStdioHandlers(child: ChildProcess): void { // Read NDJSON from stdout if (child.stdout) { readline = createInterface({ input: child.stdout }); readline.on("line", handleLine); } // The `error` listener stops an unhandled EPIPE from a child that closed its // stdin. if (child.stdin) { child.stdin.on("error", () => {}); child.stdin.on("close", () => {}); } // Capture stderr for logging if (child.stderr) { stderrReadline = createInterface({ input: child.stderr }); stderrReadline.on("line", (line: string) => { stderrExcerpt = appendStderrExcerpt(stderrExcerpt, line); log.warn({ stream: "stderr" }, `[plugin stderr] ${line}`); }); } // Handle process exit child.on("exit", (code, signal) => { handleProcessExit(code, signal); }); // Handle process errors (e.g. spawn failure) child.on("error", (err) => { log.error({ err: err.message }, "worker process error"); if (emitter.listenerCount("error") > 0) { emitter.emit("error", { pluginId, error: err }); } if (status === "starting") { setStatus("crashed"); rejectAllPending( new Error(formatWorkerFailureMessage( `Worker process failed to start: ${err.message}`, stderrExcerpt, )), ); } }); } function handleProcessExit( code: number | null, signal: NodeJS.Signals | null, ): void { const wasIntentional = intentionalStop; // Clean up readline interfaces if (readline) { readline.close(); readline = null; } if (stderrReadline) { stderrReadline.close(); stderrReadline = null; } childProcess = null; startedAt = null; // Reject all pending requests rejectAllPending( new Error(formatWorkerFailureMessage( `Worker process exited (code=${code}, signal=${signal})`, stderrExcerpt, )), ); // Close the one login pseudo-terminal route with a fixed non-secret exit and // clear the route one time. The pending pseudo-terminal calls // already rejected through `rejectAllPending`. closeLoginPtyRouteOnWorkerExit(); // Close the one duplex channel route the same way. The pending channel calls // already rejected through `rejectAllPending`. closeDuplexChannelRouteOnWorkerExit(); // Emit synthetic close for any orphaned stream channels so SSE clients // are notified instead of hanging indefinitely. if (openStreamChannels.size > 0 && options.onStreamNotification) { for (const [channel, companyId] of openStreamChannels) { try { options.onStreamNotification("streams.close", { channel, companyId }); } catch { // Best-effort cleanup — don't let it interfere with exit handling } } openStreamChannels.clear(); } emitter.emit("exit", { pluginId, code, signal }); if (wasIntentional) { // Graceful stop — status is already "stopping" or will be set to "stopped" setStatus("stopped"); log.info({ code, signal }, "worker process stopped"); return; } // Unexpected exit — crash recovery totalCrashes++; const now = Date.now(); // Reset consecutive crash counter if enough time passed if (lastCrashAt !== null && now - lastCrashAt > CRASH_WINDOW_MS) { consecutiveCrashes = 0; } consecutiveCrashes++; lastCrashAt = now; log.error( { code, signal, consecutiveCrashes, totalCrashes }, "worker process crashed", ); const willRestart = autoRestart && consecutiveCrashes <= MAX_CONSECUTIVE_CRASHES; setStatus("crashed"); emitter.emit("crash", { pluginId, code, signal, willRestart }); if (willRestart) { scheduleRestart(); } else { log.error( { consecutiveCrashes, maxCrashes: MAX_CONSECUTIVE_CRASHES }, "max consecutive crashes reached, not restarting", ); } } function rejectAllPending(error: Error): void { for (const [id, pending] of pendingRequests) { clearTimeout(pending.timer); pending.resolve( createErrorResponse( pending.id, PLUGIN_RPC_ERROR_CODES.WORKER_UNAVAILABLE, error.message, ) as JsonRpcResponse, ); } pendingRequests.clear(); for (const invocation of activeInvocations.values()) { if (invocation.timer) clearTimeout(invocation.timer); } activeInvocations.clear(); } // ----------------------------------------------------------------------- // Crash recovery with exponential backoff // ----------------------------------------------------------------------- function computeBackoffMs(): number { // Exponential backoff: MIN_BACKOFF * MULTIPLIER^(consecutiveCrashes - 1) const delay = MIN_BACKOFF_MS * Math.pow(BACKOFF_MULTIPLIER, consecutiveCrashes - 1); // Add jitter: ±25% const jitter = delay * 0.25 * (Math.random() * 2 - 1); return Math.min(Math.round(delay + jitter), MAX_BACKOFF_MS); } function scheduleRestart(): void { const delay = computeBackoffMs(); nextRestartAt = Date.now() + delay; setStatus("backoff"); log.info( { delayMs: delay, consecutiveCrashes }, "scheduling restart with backoff", ); backoffTimer = setTimeout(async () => { backoffTimer = null; nextRestartAt = null; try { await startInternal(); } catch (err) { log.error( { err: err instanceof Error ? err.message : String(err) }, "restart after backoff failed", ); } }, delay); } function cancelPendingRestart(): void { if (backoffTimer !== null) { clearTimeout(backoffTimer); backoffTimer = null; nextRestartAt = null; } } // ----------------------------------------------------------------------- // Start / Stop // ----------------------------------------------------------------------- async function startInternal(): Promise { if (status === "running" || status === "starting") { throw new Error(`Worker for plugin "${pluginId}" is already ${status}`); } intentionalStop = false; setStatus("starting"); stderrExcerpt = ""; const child = spawnProcess(); childProcess = child; attachStdioHandlers(child); startedAt = Date.now(); // Send the initialize RPC call const initParams: InitializeParams = { manifest: options.manifest, config: options.config, instanceInfo: options.instanceInfo, apiVersion: options.apiVersion, databaseNamespace: options.databaseNamespace ?? null, }; try { const result = await callInternal( "initialize", initParams, INITIALIZE_TIMEOUT_MS, ) as { ok?: boolean; supportedMethods?: string[] } | undefined; if (!result || !result.ok) { throw new Error("Worker initialize returned ok=false"); } supportedMethods = result.supportedMethods ?? []; } catch (err) { // Initialize failed — kill the process and propagate const msg = err instanceof Error ? err.message : String(err); log.error({ err: msg }, "worker initialize failed"); await killProcess(); setStatus("crashed"); throw new Error(`Worker initialize failed for "${pluginId}": ${msg}`); } // Reset crash counter on successful start consecutiveCrashes = 0; setStatus("running"); emitter.emit("ready", { pluginId }); log.info({ pid: child.pid }, "worker process started and initialized"); } async function stopInternal(): Promise { cancelPendingRestart(); if (status === "stopped" || status === "stopping") { return; } intentionalStop = true; setStatus("stopping"); if (!childProcess) { setStatus("stopped"); return; } // Step 1: Send shutdown RPC and wait for the worker to exit gracefully. // We race the shutdown call against a timeout. The worker should process // the shutdown and exit on its own within the drain period. try { await Promise.race([ callInternal("shutdown", {} as Record, SHUTDOWN_DRAIN_MS), waitForExit(SHUTDOWN_DRAIN_MS), ]); } catch { // Shutdown call failed or timed out — proceed to kill log.warn("shutdown RPC failed or timed out, escalating to SIGTERM"); } // Give the process a brief moment to exit after the shutdown response if (childProcess) { await waitForExit(500); } // Check if process already exited if (!childProcess) { setStatus("stopped"); return; } // Step 2: Send SIGTERM and wait log.info("worker did not exit after shutdown RPC, sending SIGTERM"); await killWithSignal("SIGTERM", SIGTERM_GRACE_MS); if (!childProcess) { setStatus("stopped"); return; } // Step 3: Forcefully kill with SIGKILL log.warn("worker did not exit after SIGTERM, sending SIGKILL"); await killWithSignal("SIGKILL", 2_000); if (childProcess) { log.error("worker process still alive after SIGKILL — this should not happen"); } setStatus("stopped"); } /** * Wait for the child process to exit, up to `timeoutMs`. * Resolves immediately if the process is already gone. */ function waitForExit(timeoutMs: number): Promise { return new Promise((resolve) => { if (!childProcess) { resolve(); return; } let settled = false; const timer = setTimeout(() => { if (settled) return; settled = true; resolve(); }, timeoutMs); childProcess.once("exit", () => { if (settled) return; settled = true; clearTimeout(timer); resolve(); }); }); } function killWithSignal( signal: NodeJS.Signals, waitMs: number, ): Promise { return new Promise((resolve) => { if (!childProcess) { resolve(); return; } const timer = setTimeout(() => { resolve(); }, waitMs); childProcess.once("exit", () => { clearTimeout(timer); resolve(); }); try { childProcess.kill(signal); } catch { clearTimeout(timer); resolve(); } }); } async function killProcess(): Promise { if (!childProcess) return; intentionalStop = true; try { childProcess.kill("SIGKILL"); } catch { // Process may already be dead } // Wait briefly for exit event await new Promise((resolve) => { if (!childProcess) { resolve(); return; } const timer = setTimeout(() => { resolve(); }, 1_000); childProcess.once("exit", () => { clearTimeout(timer); resolve(); }); }); } // ----------------------------------------------------------------------- // RPC call implementation // ----------------------------------------------------------------------- function callInternal( method: M, params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, ): Promise { const rpcPromise = new Promise((resolve, reject) => { if (!childProcess?.stdin?.writable) { reject( new Error( `Cannot call "${method}" — worker for "${pluginId}" is not running`, ), ); return; } const id = nextRequestId++; const timeout = resolveRpcCallTimeoutMs(timeoutMs, rpcTimeoutMs); const invocationScope = deriveInvocationScope(method, params); const invocation = invocationScope ? registerInvocation(invocationScope) : null; // Register the host-owned execute route only for an execute call that // carries a log sink. The company id comes from the host-derived // invocation scope, never from the worker. This binds the sink to the // exact company for the life of the call. if (invocation && invocationScope && executeLogSink && method === "environmentExecute") { registerExecuteRoute(invocation.id, invocationScope.companyId, executeLogSink); } // Guard against double-settlement. When a process exits all pending // requests are rejected via rejectAllPending(), but the timeout timer // may still be running. Without this guard the timer's reject fires on // an already-settled promise, producing an unhandled rejection. let settled = false; const settle = (fn: (value: T) => void, value: T): void => { if (settled) return; settled = true; clearTimeout(timer); pendingRequests.delete(id); clearInvocation(invocation); clearExecuteRoute(invocation?.id); fn(value); }; const timer = setTimeout(() => { settle( reject, new JsonRpcCallError({ code: PLUGIN_RPC_ERROR_CODES.TIMEOUT, message: `RPC call "${method}" timed out after ${timeout}ms`, }), ); }, timeout); const pending: PendingRequest = { id, method, resolve: (response: JsonRpcResponse) => { if (isJsonRpcSuccessResponse(response)) { settle(resolve, response.result as HostToWorkerMethods[M][1]); } else if ("error" in response && response.error) { settle(reject, new JsonRpcCallError(response.error)); } else { settle(reject, new Error(`Unexpected response format for "${method}"`)); } }, timer, sentAt: Date.now(), invocationId: invocation?.id, }; pendingRequests.set(id, pending); try { const request = { ...createRequest(method, params, id), ...(invocation ? { paperclipInvocation: invocation } : {}), }; sendMessage(request); } catch (err) { clearTimeout(timer); pendingRequests.delete(id); clearInvocation(invocation); clearExecuteRoute(invocation?.id); reject( new Error( `Failed to send "${method}" to worker: ${ err instanceof Error ? err.message : String(err) }`, ), ); } }); // Some call sites hand these promises across async boundaries before // attaching their own handlers. Mark the promise as handled here so a // worker-side JSON-RPC error can fail the caller without killing the host // process via an unhandled rejection. void rpcPromise.catch(() => undefined); return rpcPromise; } // ----------------------------------------------------------------------- // Public API // ----------------------------------------------------------------------- const handle: PluginWorkerHandle = { get pluginId() { return pluginId; }, get status() { return status; }, get supportedMethods() { return supportedMethods; }, async start() { await startInternal(); }, async stop() { await stopInternal(); }, async restart() { await stopInternal(); await startInternal(); }, call( method: M, params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, ): Promise { if (status !== "running" && status !== "starting") { return Promise.reject( new Error( `Cannot call "${method}" — worker for "${pluginId}" is ${status}`, ), ); } return callInternal(method, params, timeoutMs, executeLogSink); }, openLoginPtySession(input: LoginPtyOpenInput) { if (status !== "running" && status !== "starting") { return Promise.reject( new Error( `Cannot open a login pseudo-terminal — worker for "${pluginId}" is ${status}`, ), ); } return openLoginPtySession(input); }, openDuplexChannel(input: DuplexChannelOpenInput) { if (status !== "running" && status !== "starting") { return Promise.reject( new Error( `Cannot open a duplex channel — worker for "${pluginId}" is ${status}`, ), ); } return openDuplexChannel(input); }, notify(method: string, params: unknown) { if (status !== "running") return; const invocationScope = deriveInvocationScope(method, params); // Notifications have no response to settle on, so the invocation scope // is GC'd by TTL. Call-path invocations are registered without a TTL and // cleared on settlement, so they survive arbitrarily long call timeouts. const invocation = invocationScope ? registerInvocation(invocationScope, MAX_RPC_TIMEOUT_MS) : null; try { sendMessage({ jsonrpc: JSONRPC_VERSION, method, params, ...(invocation ? { paperclipInvocation: invocation } : {}), }); } catch { clearInvocation(invocation); log.warn({ method }, "failed to send notification to worker"); } }, on( event: K, listener: (payload: WorkerHandleEvents[K]) => void, ) { emitter.on(event, listener); }, off( event: K, listener: (payload: WorkerHandleEvents[K]) => void, ) { emitter.off(event, listener); }, setProactiveCompanyScopes(companyIds: readonly string[]): void { proactiveCompanyScopes.clear(); for (const id of companyIds) { const trimmed = readNonEmptyString(id); if (trimmed) proactiveCompanyScopes.add(trimmed); } }, diagnostics(): WorkerDiagnostics { return { pluginId, status, pid: childProcess?.pid ?? null, uptime: startedAt !== null && status === "running" ? Date.now() - startedAt : null, consecutiveCrashes, totalCrashes, pendingRequests: pendingRequests.size, lastCrashAt, nextRestartAt, }; }, }; return handle; } // --------------------------------------------------------------------------- // Implementation: createPluginWorkerManager // --------------------------------------------------------------------------- /** * Options for creating a PluginWorkerManager. */ export interface PluginWorkerManagerOptions { /** * Optional callback invoked when a worker emits a lifecycle event * (crash, restart). Used by the server to publish global live events. */ onWorkerEvent?: (event: { type: "plugin.worker.crashed" | "plugin.worker.restarted"; pluginId: string; code?: number | null; signal?: string | null; willRestart?: boolean; }) => void; /** * The process-scoped aggregate ceiling for concurrent duplex channel routes, * across every worker in the process. The manager builds one shared slot * controller from it and injects it into every worker handle, so one tenant can * never exhaust the manager-wide resource. The manager validates it and falls * back to {@link DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES} for an absent or an * invalid value. It is not the per-agent `heartbeat.maxConcurrentRuns`, which * stays upstream admission only. */ maxConcurrentDuplexRoutes?: number | null; } /** * The default process-scoped aggregate ceiling for concurrent duplex channel * routes. It caps the manager-wide resource, not one agent's run budget. The host * reports an explicit route-busy outcome when the ceiling is full. * * Known aggregate behavior: this ceiling bounds route count only, not * retained bytes. Each HTTP/2 bridge route bounds its own retained body * bytes to 8,388,608 bytes (see `HTTP2_BRIDGE_MAX_CONCURRENT_STREAMS` in * `http2-bridge-server.ts`), so this route ceiling caps the process's * aggregate retained body bytes at 128 * 8,388,608 = 1,073,741,824 bytes * (1 GiB). This is accepted, known behavior, not a defect: the process * tracks no aggregate byte ledger across routes. */ export const DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES = 128; /** * Build one process-scoped aggregate route-slot controller. The controller holds a * strictly positive integer ceiling and a live count. `tryAcquire` reserves one * slot only when a slot is free, so the count never passes the ceiling. */ export function createDuplexRouteSlotController(maxRoutes?: number | null): DuplexRouteSlotController { const ceiling = typeof maxRoutes === "number" && Number.isInteger(maxRoutes) && maxRoutes > 0 ? maxRoutes : DEFAULT_MAX_CONCURRENT_DUPLEX_ROUTES; let active = 0; return { tryAcquire(): boolean { if (active >= ceiling) return false; active += 1; return true; }, release(): void { if (active > 0) active -= 1; }, }; } /** * Create a new PluginWorkerManager. * * The manager holds all plugin worker handles and provides a unified API for * starting, stopping, and communicating with plugin workers. * * @example * ```ts * const manager = createPluginWorkerManager(); * * const handle = await manager.startWorker("acme.linear", { * entrypointPath: "/path/to/worker.cjs", * manifest, * config: resolvedConfig, * instanceInfo: { instanceId: "inst-1", hostVersion: "1.0.0" }, * apiVersion: 1, * hostHandlers: { "config.get": async () => resolvedConfig, ... }, * }); * * // Send RPC call to the worker * const health = await manager.call("acme.linear", "health", {}); * * // Shutdown all workers on server exit * await manager.stopAll(); * ``` */ export function createPluginWorkerManager( managerOptions?: PluginWorkerManagerOptions, ): PluginWorkerManager { const log = logger.child({ service: "plugin-worker-manager" }); const workers = new Map(); /** Per-plugin startup locks to prevent concurrent spawn races. */ const startupLocks = new Map>(); // The one shared, process-scoped aggregate route-slot controller. The manager // injects it into every worker handle, so the duplex route ceiling counts every // concurrent route across the process, not one agent's setting. const duplexRouteSlots = createDuplexRouteSlotController( managerOptions?.maxConcurrentDuplexRoutes, ); return { async startWorker( pluginId: string, options: WorkerStartOptions, ): Promise { // Mutex: if a start is already in-flight for this plugin, wait for it const inFlight = startupLocks.get(pluginId); if (inFlight) { log.warn({ pluginId }, "concurrent startWorker call — waiting for in-flight start"); return inFlight; } const existing = workers.get(pluginId); if (existing && existing.status !== "stopped") { throw new Error( `Worker already registered for plugin "${pluginId}" (status: ${existing.status})`, ); } const handle = createPluginWorkerHandle(pluginId, { // Inject the shared process-scoped route-slot controller, unless the // caller already supplied its own (a test may inject its own). duplexRouteSlots, ...options, }); workers.set(pluginId, handle); // Subscribe to crash/ready events for live event forwarding if (managerOptions?.onWorkerEvent) { const notify = managerOptions.onWorkerEvent; handle.on("crash", (payload) => { notify({ type: "plugin.worker.crashed", pluginId: payload.pluginId, code: payload.code, signal: payload.signal, willRestart: payload.willRestart, }); }); handle.on("ready", (payload) => { // Only emit restarted if this was a crash recovery (totalCrashes > 0) const diag = handle.diagnostics(); if (diag.totalCrashes > 0) { notify({ type: "plugin.worker.restarted", pluginId: payload.pluginId, }); } }); } log.info({ pluginId }, "starting plugin worker"); // Set the lock before awaiting start() to prevent concurrent spawns const startPromise = handle.start().then(() => handle).finally(() => { startupLocks.delete(pluginId); }); startupLocks.set(pluginId, startPromise); return startPromise; }, async stopWorker(pluginId: string): Promise { const handle = workers.get(pluginId); if (!handle) { log.warn({ pluginId }, "no worker registered for plugin, nothing to stop"); return; } log.info({ pluginId }, "stopping plugin worker"); await handle.stop(); workers.delete(pluginId); }, getWorker(pluginId: string): PluginWorkerHandle | undefined { return workers.get(pluginId); }, isRunning(pluginId: string): boolean { const handle = workers.get(pluginId); return handle?.status === "running"; }, setProactiveCompanyScopes(pluginId: string, companyIds: readonly string[]): void { workers.get(pluginId)?.setProactiveCompanyScopes(companyIds); }, async stopAll(): Promise { log.info({ count: workers.size }, "stopping all plugin workers"); const promises = Array.from(workers.values()).map(async (handle) => { try { await handle.stop(); } catch (err) { log.error( { pluginId: handle.pluginId, err: err instanceof Error ? err.message : String(err), }, "error stopping worker during shutdown", ); } }); await Promise.all(promises); workers.clear(); }, diagnostics(): WorkerDiagnostics[] { return Array.from(workers.values()).map((h) => h.diagnostics()); }, call( pluginId: string, method: M, params: HostToWorkerMethods[M][0], timeoutMs?: number, executeLogSink?: ExecuteLogSink, ): Promise { const handle = workers.get(pluginId); if (!handle) { return Promise.reject( new Error(`No worker registered for plugin "${pluginId}"`), ); } return handle.call(method, params, timeoutMs, executeLogSink); }, openLoginPtySession(pluginId: string, input: LoginPtyOpenInput) { const handle = workers.get(pluginId); if (!handle) { return Promise.reject( new Error(`No worker registered for plugin "${pluginId}"`), ); } return handle.openLoginPtySession(input); }, }; }