diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index bb233059f4..a6bd70d1d1 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -49,6 +49,12 @@ an installed server does not depend on a separate system Rust installation or a manually copied binary. `pnpm-lock.yaml` remains under the repository's existing lockfile process. +The package also builds `paperclip-runner-acpx-sidecar`. This bounded v2 +stdin/stdout bridge admits the qualified Codex ACPX profile only. It validates +the exact model, session identity, tool catalog, structured input, and terminal +settlement at the process boundary. Runnerd and the server do not select this +sidecar in this slice. Other ACPX agents remain unavailable. + Run the complete contract gate with: ```sh diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index c7859a4871..6dfc54abee 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -7,6 +7,9 @@ "node": ">=24.11.0" }, "type": "module", + "bin": { + "paperclip-runner-acpx-sidecar": "./dist/cli/acpx-runtime-sidecar.js" + }, "exports": { ".": { "types": "./dist/index.d.ts", @@ -59,6 +62,7 @@ }, "devDependencies": { "@types/node": "^24.0.0", + "tsx": "^4.23.12", "typescript": "^7.0.2", "vitest": "^4.1.10" } diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts new file mode 100644 index 0000000000..a0a9a0aefa --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts @@ -0,0 +1,563 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { ACPX_SIDECAR_PROTOCOL_VERSION } from "../drivers/acpx/sidecar-protocol.js"; +import { + awaitSidecarCleanupWithin, + closeActiveSidecarHostWithin, + closeSidecarHostForCommand, + combineSidecarAdmissionCleanups, + combineSidecarHostCleanups, + hasSidecarSessionOwnership, + observeSidecarCleanupWithin, + parseAcpxRunAttachment, + readSidecarHostStatusWithin, + recoverAndCombineSidecarHostCleanup, + recoverSidecarHostCleanup, + reportAuthoritativeSidecarHostCleanupFailure, + requireSidecarCommandHost, + verifyOpenedAcpxSidecarHost, +} from "./acpx-sidecar-lifecycle.js"; + +const children = new Set(); + +afterEach(async () => { + await Promise.all([...children].map((child) => child.close())); + children.clear(); +}); + +describe("Codex ACPX runtime sidecar", () => { + it("keeps session admission closed while any cleanup owner remains", () => { + const cleanup = Promise.resolve(); + + expect(hasSidecarSessionOwnership(null, null, null)).toBe(false); + expect(hasSidecarSessionOwnership({}, null, null)).toBe(true); + expect(hasSidecarSessionOwnership(null, cleanup, null)).toBe(true); + expect(hasSidecarSessionOwnership(null, null, cleanup)).toBe(true); + }); + + it("allows only an explicit cleanup retry to reach a retained host", () => { + const host = { identity: () => ({ kind: "acpx" }) }; + const cleanup = new Promise(() => undefined); + + expect(() => requireSidecarCommandHost(host, cleanup)).toThrow( + "cleanup is in progress", + ); + expect( + requireSidecarCommandHost(host, cleanup, { allowCleanupRetry: true }), + ).toBe(host); + expect(() => + requireSidecarCommandHost(null, cleanup, { allowCleanupRetry: true }), + ).toThrow("session is not open"); + }); + + it("closes an opened host when post-open verification fails", async () => { + const close = vi.fn().mockResolvedValue(undefined); + const host = { + identity: () => ({ kind: "acpx" }), + status: vi.fn().mockRejectedValue(new Error("status failed")), + close, + }; + + await expect(verifyOpenedAcpxSidecarHost(host, () => ({}))).rejects.toThrow( + "status failed", + ); + expect(close).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledWith({ + reason: "ACPX session open verification failed", + }); + }); + + it("bounds failed-admission cleanup when the host does not settle", async () => { + let finishCleanup!: () => void; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const close = vi.fn(() => cleanup); + const retainCleanup = vi.fn(); + const host = { + identity: () => ({ kind: "acpx" }), + status: vi.fn().mockRejectedValue(new Error("status failed")), + close, + }; + + await expect( + verifyOpenedAcpxSidecarHost(host, () => ({}), 1, retainCleanup), + ).rejects.toThrow("verification and provider cleanup failed"); + expect(close).toHaveBeenCalledOnce(); + expect(retainCleanup).toHaveBeenCalledWith(cleanup); + finishCleanup(); + await cleanup; + }); + + it("bounds shutdown waiting without releasing retained cleanup ownership", async () => { + let finishCleanup!: () => void; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + + await expect(awaitSidecarCleanupWithin(cleanup, 1)).resolves.toBe( + "deferred", + ); + let settled = false; + void cleanup.then(() => { + settled = true; + }); + expect(settled).toBe(false); + finishCleanup(); + await cleanup; + expect(settled).toBe(true); + await expect(awaitSidecarCleanupWithin(cleanup, 1)).resolves.toBe( + "settled", + ); + }); + + it("preserves retained cleanup failure for shutdown accounting", async () => { + const failure = new Error("provider cleanup failed"); + await expect( + observeSidecarCleanupWithin(Promise.reject(failure), 1), + ).resolves.toEqual({ status: "failed", error: failure }); + await expect( + observeSidecarCleanupWithin(new Promise(() => undefined), 1), + ).resolves.toEqual({ status: "deferred" }); + }); + + it("preserves failed-admission rejection until every cleanup settles", async () => { + let finishCleanup!: () => void; + const pending = new Promise((resolve) => { + finishCleanup = resolve; + }); + const retained = combineSidecarAdmissionCleanups([ + Promise.reject(new Error("provider survived termination")), + pending, + ]); + let settled = false; + void retained.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + + await Promise.resolve(); + expect(settled).toBe(false); + finishCleanup(); + await expect(retained).rejects.toThrow( + "did not release provider ownership", + ); + }); + + it("bounds active-host cleanup during sidecar shutdown", async () => { + const cleanup = new Promise(() => undefined); + const close = vi.fn(() => cleanup); + const retainCleanup = vi.fn(); + + await expect( + closeActiveSidecarHostWithin({ close }, "SIGTERM", 1, retainCleanup), + ).resolves.toBe("deferred"); + expect(close).toHaveBeenCalledWith({ reason: "SIGTERM" }); + expect(retainCleanup).toHaveBeenCalledWith(cleanup); + }); + + it("bounds command cleanup without replacing its exact owner", async () => { + let finishCleanup!: () => void; + const cleanup = new Promise((resolve) => { + finishCleanup = resolve; + }); + const close = vi.fn(() => cleanup); + const retainCleanup = vi.fn(); + + await expect( + closeSidecarHostForCommand({ close }, "session close", 1, retainCleanup), + ).rejects.toThrow("cleanup exceeded its command timeout"); + expect(close).toHaveBeenCalledOnce(); + expect(retainCleanup).toHaveBeenCalledWith(cleanup); + + finishCleanup(); + await cleanup; + }); + + it("preserves a settled command cleanup failure", async () => { + const cleanup = Promise.reject(new Error("runtime close failed")); + await expect( + closeSidecarHostForCommand({ close: () => cleanup }, "session close", 10), + ).rejects.toThrow("runtime close failed"); + }); + + it("recovers a rejected active-host cleanup sequentially", async () => { + const close = vi + .fn<() => Promise>() + .mockRejectedValueOnce(new Error("first close failed")) + .mockResolvedValue(undefined); + const host = { close }; + const initialCleanup = host.close(); + + await expect( + recoverSidecarHostCleanup(host, initialCleanup), + ).resolves.toBeUndefined(); + expect(close).toHaveBeenCalledTimes(2); + }); + + it("bounds repeated active-host cleanup failures", async () => { + const close = vi + .fn<() => Promise>() + .mockRejectedValue(new Error("close failed")); + const host = { close }; + const initialCleanup = host.close(); + + await expect( + recoverSidecarHostCleanup(host, initialCleanup), + ).rejects.toThrow("close failed"); + expect(close).toHaveBeenCalledTimes(4); + }); + + it("retains a pending cleanup owner after a command retry succeeds", async () => { + let finishPending!: () => void; + const pending = new Promise((resolve) => { + finishPending = resolve; + }); + const successfulRetry = Promise.resolve(); + const owner = combineSidecarHostCleanups([pending, successfulRetry]); + let settled = false; + void owner.then(() => { + settled = true; + }); + + await Promise.resolve(); + expect(settled).toBe(false); + finishPending(); + await expect(owner).resolves.toBeUndefined(); + }); + + it("accepts a coalesced rejection after sequential recovery succeeds", async () => { + let rejectCoalesced!: (error: unknown) => void; + const coalesced = new Promise((_resolve, reject) => { + rejectCoalesced = reject; + }); + const close = vi + .fn<() => Promise>() + .mockReturnValueOnce(coalesced) + .mockReturnValueOnce(coalesced) + .mockResolvedValue(undefined); + const host = { close }; + const recoveredPrior = recoverSidecarHostCleanup(host, host.close()); + const owner = recoverAndCombineSidecarHostCleanup( + host, + host.close(), + recoveredPrior, + ); + let settled = false; + void owner + .finally(() => { + settled = true; + }) + .catch(() => undefined); + + await Promise.resolve(); + expect(settled).toBe(false); + rejectCoalesced(new Error("coalesced close failed before recovery")); + await expect(owner).resolves.toBeUndefined(); + expect(close).toHaveBeenCalledTimes(4); + }); + + it("rejects when every active-host cleanup owner fails", async () => { + const owner = combineSidecarHostCleanups([ + Promise.reject(new Error("recovery exhausted")), + Promise.reject(new Error("retry failed")), + ]); + + await expect(owner).rejects.toThrow("did not release provider ownership"); + }); + + it("accepts a later recovery after an older owner exhausts", async () => { + await expect( + combineSidecarHostCleanups([ + Promise.reject(new Error("older recovery exhausted")), + Promise.resolve(), + ]), + ).resolves.toBeUndefined(); + }); + + it("does not escalate a superseded cleanup owner failure", async () => { + let rejectOlder!: (error: unknown) => void; + const older = new Promise((_resolve, reject) => { + rejectOlder = reject; + }); + const replacement = combineSidecarHostCleanups([ + older, + Promise.resolve(), + ]); + const reportFailure = vi.fn(); + void older.catch((error: unknown) => { + reportAuthoritativeSidecarHostCleanupFailure( + false, + replacement, + older, + error, + reportFailure, + ); + }); + + rejectOlder(new Error("older recovery exhausted")); + await expect(replacement).resolves.toBeUndefined(); + expect(reportFailure).not.toHaveBeenCalled(); + }); + + it("escalates only an authoritative cleanup owner's terminal failure", async () => { + const owner = combineSidecarHostCleanups([ + Promise.reject(new Error("older recovery exhausted")), + Promise.reject(new Error("replacement recovery exhausted")), + ]); + const reportFailure = vi.fn(); + + await owner.catch((error: unknown) => { + reportAuthoritativeSidecarHostCleanupFailure( + false, + owner, + owner, + error, + reportFailure, + ); + }); + expect(reportFailure).toHaveBeenCalledOnce(); + expect(reportFailure.mock.calls[0]?.[0]).toBeInstanceOf(AggregateError); + + reportAuthoritativeSidecarHostCleanupFailure( + true, + owner, + owner, + new Error("shutdown cleanup failed"), + reportFailure, + ); + expect(reportFailure).toHaveBeenCalledOnce(); + }); + + it("bounds status verification before cleaning up the opened host", async () => { + const close = vi.fn().mockResolvedValue(undefined); + const host = { + identity: () => ({ kind: "acpx" }), + status: vi.fn(() => new Promise(() => undefined)), + close, + }; + + await expect( + verifyOpenedAcpxSidecarHost(host, () => ({}), 1), + ).rejects.toThrow("status read exceeded its timeout"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("bounds ordinary status reads so serialized shutdown can proceed", async () => { + const host = { + status: vi.fn(() => new Promise(() => undefined)), + }; + + await expect(readSidecarHostStatusWithin(host, 1)).rejects.toThrow( + "status read exceeded its timeout", + ); + }); + + it("validates a complete run attachment before it can be committed", () => { + let attachedRunId: string | null = null; + const attach = (params: Record) => { + const attachment = parseAcpxRunAttachment(params); + attachedRunId = attachment.runId; + return attachment; + }; + + expect(() => attach({ runId: "run-1", catalogRevision: 0 })).toThrow( + "catalogRevision must be a positive integer", + ); + expect(attachedRunId).toBeNull(); + expect(attach({ runId: "run-1", catalogRevision: 2 })).toEqual({ + runId: "run-1", + catalogRevision: 2, + }); + expect(attachedRunId).toBe("run-1"); + }); + + it("recovers after malformed input and reports its qualified Codex profile", async () => { + const sidecar = startSidecar(); + sidecar.write({ + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + id: 1, + command: "initialize", + params: {}, + unexpected: true, + }); + await expect( + sidecar.next((frame) => frame.eventType === "runtime.diagnostic"), + ).resolves.toMatchObject({ + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + eventType: "runtime.diagnostic", + payload: { code: "malformed_frame" }, + }); + + sidecar.write(initializeRequest(2, "codex")); + + await expect( + sidecar.next((frame) => frame.id === 2), + ).resolves.toMatchObject({ + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + id: 2, + ok: true, + result: { + profile: { + agent: "codex", + qualificationModel: "gpt-5.6-sol", + }, + capabilities: { + persistentSessions: true, + exactModelVerification: true, + structuredInput: "paperclip.question_set.v1", + }, + }, + }); + expect(sidecar.stderr()).toContain("malformed_frame"); + + sidecar.write(initializeRequest(3, "codex")); + await expect( + sidecar.next((frame) => frame.id === 3), + ).resolves.toMatchObject({ + id: 3, + ok: false, + error: { message: "ACPX sidecar is already initialized" }, + }); + }); + + it("fails closed after an unsupported provider bootstrap", async () => { + const sidecar = startSidecar(); + sidecar.write(initializeRequest(1, "pi")); + + await expect( + sidecar.next((frame) => frame.id === 1), + ).resolves.toMatchObject({ + id: 1, + ok: false, + error: { + code: "acpx_sidecar_command_failed", + message: "This production ACPX sidecar supports Codex only", + retryable: false, + }, + }); + + sidecar.write(initializeRequest(2, "codex")); + + await expect( + sidecar.next((frame) => frame.id === 2), + ).resolves.toMatchObject({ + id: 2, + ok: false, + error: { + message: expect.stringContaining( + "ACPX provider bootstrap failed before initialize", + ), + retryable: false, + }, + }); + }); +}); + +function initializeRequest(id: number, agent: string): Record { + return { + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + id, + command: "initialize", + params: { agent, model: "gpt-5.6-sol" }, + }; +} + +function startSidecar(): SidecarProcess { + const sidecar = new SidecarProcess(); + children.add(sidecar); + return sidecar; +} + +class SidecarProcess { + readonly #child: ChildProcessWithoutNullStreams; + readonly #frames: Array> = []; + readonly #signals: Array<() => void> = []; + #stderr = ""; + #closed = false; + + constructor() { + this.#child = spawn( + fileURLToPath(new URL("../../node_modules/.bin/tsx", import.meta.url)), + [fileURLToPath(new URL("./acpx-runtime-sidecar.ts", import.meta.url))], + { stdio: ["pipe", "pipe", "pipe"] }, + ); + let stdout = ""; + this.#child.stdout.setEncoding("utf8"); + this.#child.stdout.on("data", (chunk: string) => { + stdout += chunk; + for (;;) { + const newline = stdout.indexOf("\n"); + if (newline < 0) break; + const line = stdout.slice(0, newline); + stdout = stdout.slice(newline + 1); + if (!line.trim()) continue; + this.#frames.push(JSON.parse(line) as Record); + for (const signal of this.#signals.splice(0)) signal(); + } + }); + this.#child.stderr.setEncoding("utf8"); + this.#child.stderr.on("data", (chunk: string) => { + this.#stderr += chunk; + }); + } + + write(value: Record): void { + this.#child.stdin.write(`${JSON.stringify(value)}\n`); + } + + stderr(): string { + return this.#stderr; + } + + async next( + predicate: (frame: Record) => boolean, + ): Promise> { + const deadline = Date.now() + 5_000; + for (;;) { + const index = this.#frames.findIndex(predicate); + if (index >= 0) return this.#frames.splice(index, 1)[0]!; + const remaining = deadline - Date.now(); + if (remaining <= 0) { + throw new Error( + `Timed out waiting for sidecar frame. stderr=${JSON.stringify(this.#stderr)}`, + ); + } + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + const index = this.#signals.indexOf(signal); + if (index >= 0) this.#signals.splice(index, 1); + reject(new Error("Timed out waiting for sidecar output")); + }, remaining); + const signal = () => { + clearTimeout(timer); + resolve(); + }; + this.#signals.push(signal); + }); + } + } + + async close(): Promise { + if (this.#closed) return; + this.#closed = true; + this.#child.stdin.end(); + const exit = new Promise((resolve) => { + this.#child.once("exit", () => resolve()); + }); + const timeout = new Promise((resolve) => { + setTimeout(() => { + if (this.#child.exitCode === null) this.#child.kill("SIGKILL"); + resolve(); + }, 2_000).unref(); + }); + await Promise.race([exit, timeout]); + } +} diff --git a/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts new file mode 100644 index 0000000000..74653c8ee6 --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-runtime-sidecar.ts @@ -0,0 +1,1168 @@ +#!/usr/bin/env node +import { createHash } from "node:crypto"; +import { createInterface } from "node:readline"; + +import type { + AcpElicitationContext, + AcpElicitationRequest, + AcpElicitationResponse, + AcpRuntimeEvent, +} from "acpx/runtime"; + +import { + PRP_BLOCK_TOOL_NAME, + PRP_COMPLETION_TOOL_NAME, +} from "../contracts/completion-result.js"; +import { + parseHarnessRuntimeRequestResolution, + type HarnessRuntimeRequestResolution, +} from "../contracts/harness-driver.js"; +import { + normalizeAcpFormElicitation, + type NormalizedAcpForm, +} from "../drivers/acpx/acp-question-adapter.js"; +import { openCodexAcpxRuntime } from "../drivers/acpx/codex-runtime-adapter.js"; +import { resolveQualifiedAcpxProfile } from "../drivers/acpx/qualified-profiles.js"; +import { + AcpxRuntimeHost, + type AcpxRetainedCleanupFailure, + type AcpxRuntimeTurn, +} from "../drivers/acpx/runtime-host.js"; +import { + ACPX_SIDECAR_MAX_FRAME_BYTES, + ACPX_SIDECAR_PROTOCOL_VERSION, + boundedSidecarValue, + parseAcpxSidecarRequest, + record, + sanitizeAcpxPlanEntries, + text, + type AcpxExpectedSessionIdentity, + type AcpxSidecarEvent, + type AcpxSidecarOpenParams, + type AcpxSidecarRequest, + type AcpxSidecarResponse, +} from "../drivers/acpx/sidecar-protocol.js"; +import { safeAcpxLocations } from "./acpx-sidecar-locations.js"; +import { validatePrpStructuredRunResult } from "../protocol/replay-contract.js"; +import type { RunnerToolCall } from "../drivers/runner-tool-bridge.js"; +import { + acpxBootstrapBlockedError, + enqueueAcpxSidecarInput, + recordAcpxBootstrapFailure, +} from "./acpx-sidecar-input.js"; +import { + boundedIdentity, + closeSidecarHostForCommand, + combineSidecarAdmissionCleanups, + hasSidecarSessionOwnership, + observeSidecarCleanupWithin, + parseAcpxRunAttachment, + readSidecarHostStatusWithin, + recoverAndCombineSidecarHostCleanup, + reportAuthoritativeSidecarHostCleanupFailure, + requireSidecarCommandHost, + verifyOpenedAcpxSidecarHost, +} from "./acpx-sidecar-lifecycle.js"; + +const MAX_PENDING_TOOLS = 512; +const MAX_PENDING_INPUTS = 16; + +function reportRetainedAcpxCleanupFailure( + input: AcpxRetainedCleanupFailure, +): void { + const errorName = input.error instanceof Error ? input.error.name : "Error"; + process.emitWarning( + JSON.stringify({ + schema: "paperclip.runner.retained_cleanup_failure.v1", + resource: input.resource, + attempt: input.attempt, + errorName, + }), + { + code: "PAPERCLIP_ACPX_RETAINED_CLEANUP_FAILURE", + type: "PaperclipRunnerCleanupWarning", + }, + ); +} + +interface PendingTool { + turnId: string; + settle(value: unknown): void; + reject(error: Error): void; + cleanup(): void; +} + +interface PendingInput { + turnId: string; + normalized: NormalizedAcpForm; + settle(response: AcpElicitationResponse): void; + cleanup(): void; +} + +let host: AcpxRuntimeHost | null = null; +let activeHostCleanup: Promise | null = null; +let failedAdmissionCleanup: Promise | null = null; +let openParams: AcpxSidecarOpenParams | null = null; +let runId: string | null = null; +let turnId: string | null = null; +let sequence = 0; +let requestSequence = 0; +let closing = false; +let shutdownRequested = false; +let pendingInput = Promise.resolve(); +let bootstrapFailure: Error | null = null; +let initializedModel: string | null = null; +const tools = new Map(); +const inputs = new Map(); + +const lines = createInterface({ + input: process.stdin, + crlfDelay: Infinity, + terminal: false, +}); +lines.on("line", (line) => { + pendingInput = enqueueAcpxSidecarInput( + pendingInput, + () => receiveLine(line), + (error) => diagnostic("sidecar_input_failed", safeMessage(error)), + ); +}); +lines.on("close", () => { + requestShutdown("sidecar stdin closed"); +}); +process.once("SIGTERM", () => { + requestShutdown("sidecar received SIGTERM"); +}); +process.once("SIGINT", () => { + requestShutdown("sidecar received SIGINT"); +}); + +function requestShutdown(reason: string): void { + if (shutdownRequested) return; + shutdownRequested = true; + lines.pause(); + pendingInput = enqueueAcpxSidecarInput( + pendingInput, + () => shutdown(reason), + (error) => diagnostic("sidecar_shutdown_failed", safeMessage(error)), + ); +} + +async function receiveLine(line: string): Promise { + if (shutdownRequested) return; + if (!line.trim()) return; + if (Buffer.byteLength(line) > ACPX_SIDECAR_MAX_FRAME_BYTES) { + diagnostic("oversized_frame", "Rejected an oversized sidecar request."); + return; + } + let request: AcpxSidecarRequest; + try { + request = parseAcpxSidecarRequest(JSON.parse(line)); + } catch (error) { + diagnostic("malformed_frame", safeMessage(error)); + return; + } + try { + const blocked = acpxBootstrapBlockedError( + bootstrapFailure, + request.command, + ); + if (blocked) throw blocked; + response(request.id, true, await dispatch(request)); + } catch (error) { + const normalized = + error instanceof Error ? error : new Error(String(error)); + bootstrapFailure = recordAcpxBootstrapFailure( + bootstrapFailure, + request.command, + normalized, + ); + response(request.id, false, undefined, { + code: safeCode(record(normalized).code, "acpx_sidecar_command_failed"), + message: safeMessage(normalized), + retryable: record(normalized).retryable === true, + }); + } +} + +async function dispatch( + request: AcpxSidecarRequest, +): Promise> { + if (request.command === "initialize") { + if (initializedModel) + throw new Error("ACPX sidecar is already initialized"); + requireCodexAgent(request.params.agent); + const model = requiredText(request.params.model, "model"); + const profile = resolveQualifiedAcpxProfile("codex", model); + initializedModel = model; + return { + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + sidecarPid: process.pid, + profile, + capabilities: { + persistentSessions: true, + exactModelVerification: true, + permissions: "runner_policy", + semanticTools: "runner_bridge", + structuredInput: "paperclip.question_set.v1", + }, + }; + } + if (request.command === "session.open") { + if ( + hasSidecarSessionOwnership( + host, + activeHostCleanup, + failedAdmissionCleanup, + ) + ) { + throw new Error("ACPX sidecar already owns a session or its cleanup"); + } + if (!initializedModel) throw new Error("initialize the ACPX sidecar first"); + const params = parseOpenParams(request.params); + if (params.model !== initializedModel) { + throw new Error("ACPX session model differs from its initialization"); + } + const openedHost = await AcpxRuntimeHost.open( + { + runtimeDirectory: params.runtimeDirectory, + normalizedSessionId: params.normalizedSessionId, + workingDirectory: params.workingDirectory, + agent: "codex", + model: params.model, + permissionMode: params.permissionMode, + systemInstructions: params.systemInstructions, + environment: process.env, + expectedIdentity: params.expectedIdentity, + semanticTools: { + tools: params.tools, + handler: waitForTool, + }, + }, + { + retainAdmissionCleanup: retainFailedAdmissionCleanup, + reportRetainedCleanupFailure: reportRetainedAcpxCleanupFailure, + openRuntime: (options) => + openCodexAcpxRuntime(options, { + retainCleanup: retainFailedAdmissionCleanup, + }), + }, + ); + const opened = await verifyOpenedAcpxSidecarHost( + openedHost, + sanitizeRuntimeStatus, + undefined, + retainFailedAdmissionCleanup, + ); + host = openedHost; + openParams = params; + emit("runtime.process", { + role: "sidecar", + pid: process.pid, + processGroupId: null, + startedAt: new Date().toISOString(), + }); + return { + identity: opened.identity, + sidecarPid: process.pid, + status: opened.status, + }; + } + if (request.command === "run.attach") { + requireHost(); + if (turnId) throw new Error("cannot attach a run during an active turn"); + const attachedTools = parseTools(request.params.tools); + if ( + canonicalJson(attachedTools) !== canonicalJson(openParams?.tools ?? []) + ) { + throw new Error("ACPX run tool catalog differs from the opened session"); + } + const attachment = parseAcpxRunAttachment(request.params); + runId = attachment.runId; + return { + runId: attachment.runId, + catalogRevision: attachment.catalogRevision, + }; + } + if (request.command === "turn.start") { + const activeHost = requireHost(); + if (!runId) throw new Error("attach a run before starting an ACPX turn"); + if (turnId) throw new Error("ACPX sidecar already has an active turn"); + const currentTurnId = boundedIdentity(request.params.turnId, "turnId"); + turnId = currentTurnId; + let runtimeTurn: AcpxRuntimeTurn; + try { + runtimeTurn = activeHost.startTurn({ + requestId: `${runId}:${currentTurnId}`, + text: boundedText(request.params.message, "message", 1024 * 1024), + onElicitation: (providerRequest, context) => + waitForInput(currentTurnId, providerRequest, context), + }); + } catch (error) { + turnId = null; + throw error; + } + void pumpTurn(currentTurnId, runtimeTurn); + return { turnId: currentTurnId }; + } + if (request.command === "turn.cancel") { + const expected = boundedIdentity(request.params.turnId, "turnId"); + if (expected !== turnId) throw new Error("cannot cancel a stale ACPX turn"); + await requireHost().interruptActiveTurn( + boundedOptionalText( + request.params.reason, + "Paperclip cancellation", + 4_000, + ), + ); + return { cancelled: true }; + } + if (request.command === "permission.resolve") { + throw new Error( + "Codex ACPX permissions are resolved by the admitted runner policy", + ); + } + if (request.command === "input.resolve") { + const requestId = boundedIdentity(request.params.requestId, "requestId"); + const expectedTurnId = boundedIdentity(request.params.turnId, "turnId"); + const pending = inputs.get(requestId); + if ( + !pending || + pending.turnId !== expectedTurnId || + turnId !== expectedTurnId + ) { + throw new Error("input request is stale or unknown"); + } + const resolution = parseHarnessRuntimeRequestResolution( + "elicitation", + request.params.resolution, + pending.normalized.questionSet, + ); + const providerResponse = elicitationResponse( + pending.normalized, + resolution, + ); + if (!inputs.delete(requestId)) + throw new Error("input request lost its settlement race"); + pending.cleanup(); + pending.settle(providerResponse); + return { resolved: true }; + } + if (request.command === "tool.resolve") { + const callId = boundedIdentity(request.params.callId, "callId"); + const expectedTurnId = boundedIdentity(request.params.turnId, "turnId"); + const pending = tools.get(callId); + if ( + !pending || + pending.turnId !== expectedTurnId || + turnId !== expectedTurnId + ) { + throw new Error("tool call is stale or unknown"); + } + if (!tools.delete(callId)) + throw new Error("tool call lost its settlement race"); + pending.cleanup(); + if (request.params.error) { + pending.reject( + new Error( + safeText( + text(record(request.params.error).message, "Paperclip tool failed"), + ), + ), + ); + } else { + pending.settle(structuredClone(request.params.result)); + } + return { resolved: true }; + } + if (request.command === "session.read") { + const activeHost = requireHost(); + return { + identity: activeHost.identity(), + status: sanitizeRuntimeStatus( + await readSidecarHostStatusWithin(activeHost), + ), + }; + } + if (request.command === "session.snapshot") { + const activeHost = requireHost(); + return { + identity: activeHost.identity(), + status: sanitizeRuntimeStatus( + await readSidecarHostStatusWithin(activeHost), + ), + runId, + turnId, + sequence, + pendingToolCount: tools.size, + pendingInputCount: inputs.size, + }; + } + if (request.command === "session.suspend") { + if (turnId || tools.size > 0 || inputs.size > 0) { + throw new Error("ACPX session is not at a safe suspension point"); + } + // Cleanup retries must be able to reach the retained host. The command is + // still serialized, and retainActiveHostCleanup keeps admission closed + // until one sequential close proves ownership was released. + const activeHost = requireHost({ allowCleanupRetry: true }); + const identity = activeHost.identity(); + await closeSidecarHostForCommand( + activeHost, + boundedOptionalText(request.params.reason, "Paperclip suspension", 4_000), + undefined, + (cleanup) => retainActiveHostCleanup(activeHost, cleanup), + ); + host = null; + openParams = null; + runId = null; + return { suspended: true, identity }; + } + if (request.command === "session.close") { + if (request.params.discardPersistentState === true) { + throw new Error( + "Codex ACPX persistent state cannot be discarded by this sidecar", + ); + } + const closingTurnId = turnId; + if (closingTurnId) rejectTurnWaiters(closingTurnId, "ACPX session closed"); + if (host) { + const activeHost = host; + await closeSidecarHostForCommand( + activeHost, + boundedOptionalText(request.params.reason, "Paperclip close", 4_000), + undefined, + (cleanup) => retainActiveHostCleanup(activeHost, cleanup), + ); + } + host = null; + openParams = null; + runId = null; + turnId = null; + return { closed: true, discarded: false }; + } + throw new Error("unreachable ACPX sidecar command"); +} + +async function pumpTurn( + currentTurnId: string, + runtimeTurn: AcpxRuntimeTurn, +): Promise { + try { + for await (const event of runtimeTurn.events) { + emit("runtime.event", sanitizeRuntimeEvent(event), currentTurnId); + } + const result = await runtimeTurn.result; + emit("runtime.turn_terminal", boundedSidecarValue(result), currentTurnId); + } catch (error) { + emit( + "runtime.turn_terminal", + { + status: "failed", + error: { message: safeMessage(error), retryable: false }, + }, + currentTurnId, + ); + } finally { + rejectTurnWaiters(currentTurnId, "ACPX turn became terminal"); + if (turnId === currentTurnId) turnId = null; + } +} + +async function waitForTool(call: RunnerToolCall): Promise { + const activeTurnId = turnId; + if (!activeTurnId || call.signal.aborted) { + throw new Error("ACPX tool call is not bound to an active turn"); + } + const callId = boundedIdentity(call.callId, "callId"); + if (tools.has(callId)) throw new Error("ACPX tool call is duplicated"); + const operationId = boundedIdentity(call.tool, "operationId"); + if ( + operationId === PRP_COMPLETION_TOOL_NAME || + operationId === PRP_BLOCK_TOOL_NAME + ) { + const validation = validatePrpStructuredRunResult(call.arguments); + if (!validation.ok) { + throw new Error("ACPX semantic result failed PRP schema validation"); + } + const blocked = validation.result.reportedWorkDisposition === "blocked"; + if ( + (operationId === PRP_BLOCK_TOOL_NAME && !blocked) || + (operationId === PRP_COMPLETION_TOOL_NAME && blocked) + ) { + throw new Error( + "ACPX semantic result disposition does not match its terminal operation", + ); + } + emit("runtime.event", { + type: "semantic_result", + callId, + operationId, + result: validation.result, + }); + return { accepted: true }; + } + if (tools.size >= MAX_PENDING_TOOLS) { + throw new Error("ACPX pending tool limit reached"); + } + emit("runtime.tool_called", { + callId, + operationId, + input: boundedSidecarValue(record(call.arguments)), + }); + return await new Promise((settle, reject) => { + const abort = () => { + const pending = tools.get(callId); + if (!pending || !tools.delete(callId)) return; + pending.cleanup(); + reject(new Error("ACPX tool call was cancelled")); + }; + call.signal.addEventListener("abort", abort, { once: true }); + tools.set(callId, { + turnId: activeTurnId, + settle, + reject, + cleanup: () => call.signal.removeEventListener("abort", abort), + }); + if (call.signal.aborted) abort(); + }); +} + +async function waitForInput( + activeTurnId: string, + request: AcpElicitationRequest, + context: AcpElicitationContext, +): Promise { + if (turnId !== activeTurnId || context.signal.aborted) { + return { action: "cancel" }; + } + if (inputs.size >= MAX_PENDING_INPUTS) { + diagnostic( + "runtime_input_limit_reached", + "The active ACPX turn has too many pending input requests.", + ); + return { action: "cancel" }; + } + let normalized: NormalizedAcpForm | null; + try { + normalized = normalizeAcpFormElicitation(request); + } catch (error) { + diagnostic("runtime_input_rejected", safeMessage(error)); + return { action: "cancel" }; + } + if (!normalized) { + diagnostic( + "runtime_input_unsupported", + "The Codex ACPX provider requested an unsupported input mode.", + ); + return { action: "cancel" }; + } + if ( + normalized.questionSet.questions.some( + (question) => question.textValidation?.pattern !== undefined, + ) + ) { + diagnostic( + "runtime_input_pattern_unsupported", + "ACPX form patterns require a bounded regular expression dialect.", + ); + return { action: "cancel" }; + } + const requestId = stableRequestId( + activeTurnId, + ++requestSequence, + context.requestId, + ); + emit( + "runtime.input_requested", + { + requestId, + questionSet: normalized.questionSet, + origin: { + adapter: "acpx-runtime-sidecar", + provider: "codex", + method: "elicitation/create", + }, + }, + activeTurnId, + ); + return await new Promise((settle) => { + const abort = () => { + const pending = inputs.get(requestId); + if (!pending || !inputs.delete(requestId)) return; + pending.cleanup(); + settle({ action: "cancel" }); + }; + context.signal.addEventListener("abort", abort, { once: true }); + inputs.set(requestId, { + turnId: activeTurnId, + normalized, + settle, + cleanup: () => context.signal.removeEventListener("abort", abort), + }); + if (context.signal.aborted) abort(); + }); +} + +function elicitationResponse( + normalized: NormalizedAcpForm, + resolution: HarnessRuntimeRequestResolution, +): AcpElicitationResponse { + if (resolution.action === "submit") { + if (!("response" in resolution)) { + throw new Error("ACPX form submission requires a canonical response"); + } + return normalized.accept(resolution.response); + } + if (resolution.action === "decline" || resolution.action === "cancel") { + return { action: resolution.action }; + } + throw new Error("unsupported ACPX input resolution action"); +} + +function rejectTurnWaiters(terminalTurnId: string, message: string): void { + for (const [callId, pending] of tools) { + if (pending.turnId !== terminalTurnId || !tools.delete(callId)) continue; + pending.cleanup(); + pending.reject(new Error(message)); + } + for (const [requestId, pending] of inputs) { + if (pending.turnId !== terminalTurnId || !inputs.delete(requestId)) + continue; + pending.cleanup(); + pending.settle({ action: "cancel" }); + } +} + +function sanitizeRuntimeEvent(event: AcpRuntimeEvent): Record { + const runtimeType = text(record(event).type); + if (runtimeType === "plan") { + return { + type: "plan", + entries: sanitizeAcpxPlanEntries(record(event).entries), + }; + } + if (event.type === "text_delta") { + return { + type: event.stream === "thought" ? "thinking" : "text_delta", + text: boundedOptionalText(event.text, "", 64 * 1024), + stream: event.stream, + tag: event.tag ?? null, + messageId: event.messageId?.slice(0, 240) ?? null, + }; + } + if (event.type === "status") { + return boundedSidecarValue({ + type: "status", + text: boundedOptionalText(event.text, "", 4_000), + tag: event.tag ?? null, + used: safeNonNegativeNumber(event.used), + size: safeNonNegativeNumber(event.size), + ...safeUsage(event.cost, event.breakdown), + }); + } + if (event.type === "tool_call") { + return boundedSidecarValue( + { + type: "tool_call", + toolCallId: event.toolCallId?.slice(0, 240) ?? null, + status: event.status?.slice(0, 100) ?? null, + title: event.title?.slice(0, 4_000) ?? null, + kind: event.kind ?? null, + locations: safeAcpxLocations( + event.locations, + openParams?.workingDirectory, + ), + ...safeOutput(event.rawOutput), + }, + 128 * 1024, + ); + } + if (event.type === "error") { + return { + type: "error", + code: event.code?.slice(0, 160) ?? null, + message: safeText(event.message), + retryable: event.retryable ?? false, + }; + } + if (event.type === "done") { + return { + type: "done", + stopReason: event.stopReason?.slice(0, 160) ?? null, + }; + } + return { + type: "provider_notice", + category: `unclassified_acp_${safeCode(record(event).type, "unknown")}`, + summary: "The qualified ACP agent emitted an unclassified runtime update.", + }; +} + +function sanitizeRuntimeStatus(value: unknown): Record { + const status = record(value); + const models = record(status.models); + return boundedSidecarValue( + { + summary: safeText(text(status.summary)).slice(0, 4_000) || null, + agentSessionId: + safeText(text(status.agentSessionId)).slice(0, 240) || null, + models: { + currentModelId: + safeText(text(models.currentModelId)).slice(0, 240) || null, + availableModelCount: Array.isArray(models.availableModelIds) + ? Math.min(models.availableModelIds.length, 100_000) + : 0, + }, + }, + 32 * 1024, + ); +} + +function safeUsage(cost: unknown, breakdown: unknown): Record { + const nativeCost = record(cost); + const nativeBreakdown = record(breakdown); + return { + cost: + cost === undefined || cost === null + ? null + : { + amount: safeNonNegativeNumber(nativeCost.amount), + currency: safeText(text(nativeCost.currency)).slice(0, 16) || null, + }, + breakdown: + breakdown === undefined || breakdown === null + ? null + : { + inputTokens: safeNonNegativeNumber(nativeBreakdown.inputTokens), + outputTokens: safeNonNegativeNumber(nativeBreakdown.outputTokens), + cachedReadTokens: safeNonNegativeNumber( + nativeBreakdown.cachedReadTokens, + ), + cachedWriteTokens: safeNonNegativeNumber( + nativeBreakdown.cachedWriteTokens, + ), + thoughtTokens: safeNonNegativeNumber(nativeBreakdown.thoughtTokens), + totalTokens: safeNonNegativeNumber(nativeBreakdown.totalTokens), + }, + }; +} + +function safeNonNegativeNumber(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? value + : null; +} + +function safeOutput(value: unknown): Record { + if (value === undefined || value === null) { + return { + output: null, + outputBytes: 0, + outputTruncated: false, + outputDigest: null, + }; + } + let raw: string; + try { + raw = typeof value === "string" ? value : JSON.stringify(value); + } catch { + return { + output: null, + outputBytes: 0, + outputTruncated: false, + outputDigest: null, + }; + } + if (!raw) { + return { + output: null, + outputBytes: 0, + outputTruncated: false, + outputDigest: null, + }; + } + const redacted = redactSecrets(raw); + const bytes = Buffer.from(redacted); + return { + output: bytes + .subarray(Math.max(0, bytes.length - 64 * 1024)) + .toString("utf8"), + outputBytes: bytes.length, + outputTruncated: bytes.length > 64 * 1024, + outputDigest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + }; +} + +function parseOpenParams( + value: Record, +): AcpxSidecarOpenParams { + requireCodexAgent(value.agent); + const model = requiredText(value.model, "model"); + resolveQualifiedAcpxProfile("codex", model); + if (value.runtimeContext !== undefined && value.runtimeContext !== null) { + throw new Error( + "Codex ACPX sidecar runtime context must be pre-materialized", + ); + } + if ( + value.providerSessionKey !== undefined && + value.providerSessionKey !== null + ) { + throw new Error( + "Codex ACPX replacement provider sessions are not available in this release", + ); + } + return { + runtimeDirectory: requiredText(value.runtimeDirectory, "runtimeDirectory"), + normalizedSessionId: boundedIdentity( + value.normalizedSessionId, + "normalizedSessionId", + ), + workingDirectory: requiredText(value.workingDirectory, "workingDirectory"), + agent: "codex", + model, + permissionMode: requiredPermissionMode(value.permissionMode), + permissionModePinned: value.permissionModePinned === true, + systemInstructions: boundedText( + value.systemInstructions, + "systemInstructions", + 1024 * 1024, + ), + runtimeContext: null, + tools: parseTools(value.tools), + ...(value.expectedIdentity === undefined || value.expectedIdentity === null + ? {} + : { expectedIdentity: parseExpectedIdentity(value.expectedIdentity) }), + }; +} + +function parseTools(value: unknown): Readonly>[] { + if (!Array.isArray(value) || value.length > 512) { + throw new Error("tools must be an array with at most 512 entries"); + } + return value.map((tool, index) => { + if (typeof tool !== "object" || tool === null || Array.isArray(tool)) { + throw new Error(`tool ${index + 1} must be an object`); + } + return structuredClone(tool as Record); + }); +} + +function parseExpectedIdentity(value: unknown): AcpxExpectedSessionIdentity { + const input = record(value); + if (input.kind !== "acpx") + throw new Error("ACPX recovery identity kind is invalid"); + return { + kind: "acpx", + normalizedSessionId: boundedIdentity( + input.normalizedSessionId, + "expected normalizedSessionId", + ), + acpxRecordId: boundedIdentity(input.acpxRecordId, "expected acpxRecordId"), + backendSessionId: boundedIdentity( + input.backendSessionId, + "expected backendSessionId", + ), + agentSessionId: boundedIdentity( + input.agentSessionId, + "expected agentSessionId", + ), + profileDigest: digest(input.profileDigest, "expected profileDigest"), + workspaceDigest: digest(input.workspaceDigest, "expected workspaceDigest"), + requestedModel: boundedIdentity( + input.requestedModel, + "expected requestedModel", + ), + effectiveModel: boundedIdentity( + input.effectiveModel, + "expected effectiveModel", + ), + ...(input.permissionMode === undefined + ? {} + : { permissionMode: requiredPermissionMode(input.permissionMode) }), + }; +} + +function requiredPermissionMode( + value: unknown, +): AcpxSidecarOpenParams["permissionMode"] { + if ( + value === "approve-all" || + value === "approve-reads" || + value === "deny-all" + ) { + return value; + } + throw new Error( + "permissionMode must be approve-all, approve-reads, or deny-all", + ); +} + +function emit( + eventType: AcpxSidecarEvent["eventType"], + payload: Record, + eventTurnId: string | null = turnId, +): void { + if (sequence >= Number.MAX_SAFE_INTEGER) { + throw new Error("ACPX sidecar event sequence exhausted"); + } + writeFrame({ + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + sequence: ++sequence, + eventType, + runId, + turnId: eventTurnId, + payload, + }); +} + +function diagnostic(code: string, message: string): void { + const safe = safeText(message); + process.stderr.write(`[paperclip-acpx-sidecar] ${code}: ${safe}\n`); + emit("runtime.diagnostic", { code: code.slice(0, 160), message: safe }); +} + +function response( + id: number, + ok: boolean, + result?: Record, + error?: AcpxSidecarResponse["error"], +): void { + writeFrame({ + protocolVersion: ACPX_SIDECAR_PROTOCOL_VERSION, + id, + ok, + ...(result ? { result } : {}), + ...(error ? { error } : {}), + }); +} + +function writeFrame(value: AcpxSidecarEvent | AcpxSidecarResponse): void { + const line = JSON.stringify(value); + if (Buffer.byteLength(line) > ACPX_SIDECAR_MAX_FRAME_BYTES) { + process.stderr.write("[paperclip-acpx-sidecar] output_frame_too_large\n"); + return; + } + process.stdout.write(`${line}\n`); +} + +function requireHost( + options: { allowCleanupRetry?: boolean } = {}, +): AcpxRuntimeHost { + return requireSidecarCommandHost(host, activeHostCleanup, options); +} + +function requireCodexAgent(value: unknown): void { + if (value !== "codex") { + throw new Error("This production ACPX sidecar supports Codex only"); + } +} + +function requiredText(value: unknown, field: string): string { + const result = text(value).trim(); + if (!result) throw new Error(`${field} is required`); + return result; +} + +function boundedText(value: unknown, field: string, maxBytes: number): string { + const result = requiredText(value, field); + if (Buffer.byteLength(result) > maxBytes) { + throw new Error(`${field} exceeds its byte limit`); + } + return result; +} + +function boundedOptionalText( + value: unknown, + fallback: string, + maxBytes: number, +): string { + const result = text(value, fallback); + const bytes = Buffer.from(result); + return bytes.length <= maxBytes + ? result + : bytes.subarray(0, maxBytes).toString("utf8"); +} + +function safeCode(value: unknown, fallback: string): string { + const code = text(value, fallback) + .replace(/[^A-Za-z0-9._:-]+/g, "_") + .slice(0, 160); + return code || fallback; +} + +function digest(value: unknown, field: string): string { + const result = requiredText(value, field); + if (!/^sha256:[a-f0-9]{64}$/.test(result)) { + throw new Error(`${field} is invalid`); + } + return result; +} + +function stableRequestId( + activeTurnId: string, + index: number, + nativeRequestId: string | number | null, +): string { + const digest = createHash("sha256") + .update( + `${activeTurnId}:${index}:${typeof nativeRequestId}:${String(nativeRequestId)}`, + ) + .digest("hex") + .slice(0, 24); + return `acpx-input-${digest}`; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const object = value as Record; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +function safeText(value: unknown, maxBytes = 8_192): string { + const message = redactSecrets(text(value, String(value))); + const bytes = Buffer.from(message); + return bytes.length <= maxBytes + ? message + : bytes.subarray(0, maxBytes).toString("utf8"); +} + +function redactSecrets(value: string): string { + return value.replace( + /(key|token|secret|password|authorization)\s*[:=]\s*[^\s,}\]]+/gi, + "$1=[REDACTED]", + ); +} + +function safeMessage(error: unknown): string { + return safeText(error instanceof Error ? error.message : error); +} + +async function shutdown(reason: string): Promise { + if (closing) return; + closing = true; + if (turnId) rejectTurnWaiters(turnId, reason); + const cleanupOwners: Array<{ + kind: "active_host" | "failed_admission"; + cleanup: Promise; + }> = []; + if (activeHostCleanup) { + // A timed-out close or suspension already owns sequential provider + // cleanup. Join that exact owner; starting another host.close here would + // overlap it and still lose its eventual failure. + cleanupOwners.push({ kind: "active_host", cleanup: activeHostCleanup }); + } else if (host) { + const activeHost = host; + const cleanup = activeHost.close({ reason }); + retainActiveHostCleanup(activeHost, cleanup); + cleanupOwners.push({ kind: "active_host", cleanup }); + } + if (failedAdmissionCleanup) { + cleanupOwners.push({ + kind: "failed_admission", + cleanup: failedAdmissionCleanup, + }); + } + let cleanupIncomplete = false; + const outcomes = await Promise.all( + cleanupOwners.map(async (owner) => ({ + ...owner, + outcome: await observeSidecarCleanupWithin(owner.cleanup), + })), + ); + for (const { kind, outcome } of outcomes) { + if (outcome.status === "deferred") { + cleanupIncomplete = true; + diagnostic( + kind === "active_host" + ? "active_host_cleanup_deferred" + : "failed_admission_cleanup_deferred", + kind === "active_host" + ? "ACPX active-host cleanup exceeded the bounded shutdown wait." + : "ACPX failed-admission cleanup remains owned after the bounded shutdown wait.", + ); + } else if (outcome.status === "failed") { + cleanupIncomplete = true; + diagnostic( + kind === "active_host" + ? "active_host_cleanup_failed" + : "failed_admission_cleanup_failed", + safeMessage(outcome.error), + ); + } + } + if (!cleanupIncomplete) host = null; + openParams = null; + runId = null; + turnId = null; + lines.close(); + process.stdin.pause(); + process.exitCode = cleanupIncomplete ? 1 : 0; +} + +function retainActiveHostCleanup( + activeHost: AcpxRuntimeHost, + cleanup: Promise, +): void { + const prior = activeHostCleanup; + const retained = closing + ? cleanup + : recoverAndCombineSidecarHostCleanup(activeHost, cleanup, prior); + activeHostCleanup = retained; + void retained + .then( + () => { + if (host === activeHost) { + host = null; + openParams = null; + runId = null; + turnId = null; + } + }, + (error: unknown) => { + reportAuthoritativeSidecarHostCleanupFailure( + closing, + activeHostCleanup, + retained, + error, + (authoritativeError) => { + diagnostic( + "active_host_cleanup_failed", + safeMessage(authoritativeError), + ); + requestShutdown("ACPX active-host cleanup could not recover"); + }, + ); + }, + ) + .finally(() => { + if (activeHostCleanup === retained) activeHostCleanup = null; + }); +} + +function retainFailedAdmissionCleanup(cleanup: Promise): void { + const prior = failedAdmissionCleanup; + const retained = combineSidecarAdmissionCleanups( + prior ? [prior, cleanup] : [cleanup], + ); + failedAdmissionCleanup = retained; + void retained.then( + () => { + if (failedAdmissionCleanup === retained) failedAdmissionCleanup = null; + }, + (error: unknown) => { + // A rejected cleanup can mean the provider survived termination. Keep + // the admission guard owned and retire this sidecar; never turn rejection + // into a successful settlement that permits a second provider. + diagnostic("failed_admission_cleanup_failed", safeMessage(error)); + requestShutdown("ACPX failed-admission cleanup could not recover"); + }, + ); +} diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts new file mode 100644 index 0000000000..902798507b --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-lifecycle.ts @@ -0,0 +1,286 @@ +export interface OpenedAcpxSidecarHost { + identity(): unknown; + status(): Promise; + close(options: { reason: string }): Promise; +} + +const FAILED_ADMISSION_CLOSE_TIMEOUT_MS = 8_000; +const ACTIVE_HOST_CLEANUP_ATTEMPTS = 4; + +export function hasSidecarSessionOwnership( + host: unknown, + activeHostCleanup: Promise | null, + failedAdmissionCleanup: Promise | null, +): boolean { + return Boolean(host || activeHostCleanup || failedAdmissionCleanup); +} + +/** + * Ordinary commands cannot observe a host while cleanup owns it. An explicit + * cleanup retry may reuse that same host so its close can supersede a stale + * owner; admission remains guarded separately by hasSidecarSessionOwnership. + */ +export function requireSidecarCommandHost( + host: T | null, + activeHostCleanup: Promise | null, + options: { allowCleanupRetry?: boolean } = {}, +): T { + if (!host) throw new Error("ACPX session is not open"); + if (activeHostCleanup && options.allowCleanupRetry !== true) { + throw new Error("ACPX session cleanup is in progress"); + } + return host; +} + +export async function readSidecarHostStatusWithin( + host: Pick, + timeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + host.status(), + new Promise((_resolve, reject) => { + timer = setTimeout( + () => + reject(new Error("ACPX session status read exceeded its timeout")), + timeoutMs, + ); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +export async function awaitSidecarCleanupWithin( + cleanup: Promise, + timeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, +): Promise<"settled" | "deferred"> { + const outcome = await observeSidecarCleanupWithin(cleanup, timeoutMs); + return outcome.status === "deferred" ? "deferred" : "settled"; +} + +export type SidecarCleanupOutcome = + | { status: "settled" } + | { status: "deferred" } + | { status: "failed"; error: unknown }; + +export async function observeSidecarCleanupWithin( + cleanup: Promise, + timeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + cleanup.then( + () => ({ status: "settled" as const }), + (error: unknown) => ({ status: "failed" as const, error }), + ), + new Promise((resolve) => { + timer = setTimeout(() => resolve({ status: "deferred" }), timeoutMs); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +export async function combineSidecarAdmissionCleanups( + cleanups: readonly Promise[], +): Promise { + const outcomes = await Promise.allSettled(cleanups); + const errors = outcomes.flatMap((outcome) => + outcome.status === "rejected" ? [outcome.reason as unknown] : [], + ); + if (errors.length > 0) { + throw new AggregateError( + errors, + "ACPX failed-admission cleanup did not release provider ownership", + ); + } +} + +/** + * Keep ownership until every cleanup started for the same host settles. Once + * all observers are terminal, one successful close proves that host released + * its provider resources; an intermediate coalesced rejection must not erase + * that proof. Reject only when every cleanup owner failed. + */ +export async function combineSidecarHostCleanups( + cleanups: readonly [Promise, Promise], +): Promise { + const outcomes = await Promise.allSettled(cleanups); + if (outcomes.some((outcome) => outcome.status === "fulfilled")) return; + const errors = outcomes.flatMap((outcome) => + outcome.status === "rejected" ? [outcome.reason as unknown] : [], + ); + throw new AggregateError( + errors, + "ACPX active-host cleanup did not release provider ownership", + ); +} + +export function recoverAndCombineSidecarHostCleanup( + host: Pick, + cleanup: Promise, + prior: Promise | null, +): Promise { + const recovered = recoverSidecarHostCleanup(host, cleanup); + return prior + ? combineSidecarHostCleanups([prior, recovered]) + : recovered; +} + +export function reportAuthoritativeSidecarHostCleanupFailure( + closing: boolean, + activeCleanup: Promise | null, + failedCleanup: Promise, + error: unknown, + reportFailure: (error: unknown) => void, +): void { + if (!closing && activeCleanup === failedCleanup) reportFailure(error); +} + +export async function closeActiveSidecarHostWithin( + host: Pick, + reason: string, + timeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, + retainCleanup: (cleanup: Promise) => void = () => undefined, +): Promise<"settled" | "deferred"> { + const cleanup = host.close({ reason }); + retainCleanup(cleanup); + return await awaitSidecarCleanupWithin(cleanup, timeoutMs); +} + +export async function closeSidecarHostForCommand( + host: Pick, + reason: string, + timeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, + retainCleanup: (cleanup: Promise) => void = () => undefined, +): Promise { + const cleanup = host.close({ reason }); + retainCleanup(cleanup); + const disposition = await awaitSidecarCleanupWithin(cleanup, timeoutMs); + if (disposition === "deferred") { + throw new Error("ACPX session cleanup exceeded its command timeout"); + } + // The bounded wait only reports settlement; preserve the exact close error + // for the command response and keep the host available for a later retry. + await cleanup; +} + +export async function recoverSidecarHostCleanup( + host: Pick, + initialCleanup: Promise, + maxAttempts = ACTIVE_HOST_CLEANUP_ATTEMPTS, +): Promise { + let cleanup = initialCleanup; + for (let attempt = 1; ; attempt += 1) { + try { + await cleanup; + return; + } catch (error) { + if (attempt >= maxAttempts) throw error; + // AcpxRuntimeHost releases its failed close promise before propagating + // the rejection, so this starts a new sequential cleanup attempt rather + // than reusing or overlapping the rejected operation. + cleanup = host.close({ reason: "Paperclip cleanup recovery" }); + } + } +} + +export async function verifyOpenedAcpxSidecarHost( + host: OpenedAcpxSidecarHost, + sanitizeStatus: (value: unknown) => Record, + closeTimeoutMs = FAILED_ADMISSION_CLOSE_TIMEOUT_MS, + retainCleanup: (cleanup: Promise) => void = () => undefined, +): Promise<{ identity: unknown; status: Record }> { + try { + const identity = host.identity(); + const status = sanitizeStatus( + await readSidecarHostStatusWithin(host, closeTimeoutMs), + ); + return { identity, status }; + } catch (error) { + const cleanup = host.close({ + reason: "ACPX session open verification failed", + }); + // The admission timeout bounds the command response, not ownership. The + // sidecar retains this exact close operation so shutdown can still await + // provider termination after the bounded verification path returns. + retainCleanup(cleanup); + const cleanupError = await boundedFailedAdmissionClose( + cleanup, + closeTimeoutMs, + ); + if (cleanupError) { + throw new AggregateError( + [error, cleanupError], + "ACPX session verification and provider cleanup failed", + ); + } + throw error; + } +} + +async function boundedFailedAdmissionClose( + close: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + close.then( + () => null, + (error: unknown) => error, + ), + new Promise((resolve) => { + timer = setTimeout( + () => + resolve( + new Error( + "ACPX failed-admission cleanup exceeded its shutdown timeout", + ), + ), + timeoutMs, + ); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +export interface AcpxRunAttachment { + runId: string; + catalogRevision: number; +} + +export function parseAcpxRunAttachment( + params: Record, +): AcpxRunAttachment { + return { + runId: boundedIdentity(params.runId, "runId"), + catalogRevision: positiveInteger(params.catalogRevision, "catalogRevision"), + }; +} + +export function boundedIdentity(value: unknown, field: string): string { + const result = typeof value === "string" ? value.trim() : ""; + if (!result) throw new Error(`${field} is required`); + if (result.length > 240 || /[\u0000-\u001f\u007f]/.test(result)) { + throw new Error(`${field} is invalid`); + } + return result; +} + +function positiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error(`${field} must be a positive integer`); + } + return Number(value); +} diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-locations.test.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-locations.test.ts new file mode 100644 index 0000000000..122cbb6873 --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-locations.test.ts @@ -0,0 +1,59 @@ +import { describe, expect, it } from "vitest"; + +import { safeAcpxLocations } from "./acpx-sidecar-locations.js"; + +describe("ACPX sidecar locations", () => { + it("preserves valid host-relative display names without admitting escape", () => { + expect( + safeAcpxLocations( + [ + { path: "src/main.ts", line: 4 }, + { path: "reports/100%/summary.txt" }, + { path: "../outside.txt" }, + { path: "/etc/passwd" }, + { uri: "https://example.test/private" }, + { path: "bad\0name" }, + ], + "/workspace/project", + ), + ).toEqual([ + { + path: "src/main.ts", + line: 4, + pathBoundary: "paperclip.workspace_relative_display.v1", + }, + { + path: "reports/100%/summary.txt", + line: null, + pathBoundary: "paperclip.workspace_relative_display.v1", + }, + ]); + }); + + it.runIf(process.platform !== "win32")( + "preserves POSIX literal colon and backslash filename characters", + () => { + expect( + safeAcpxLocations( + [{ path: "src:main.ts" }, { path: String.raw`folder\literal` }], + "/workspace/project", + ), + ).toEqual([ + { + path: "src:main.ts", + line: null, + pathBoundary: "paperclip.workspace_relative_display.v1", + }, + { + path: String.raw`folder\literal`, + line: null, + pathBoundary: "paperclip.workspace_relative_display.v1", + }, + ]); + }, + ); + + it("omits every location until the session working directory is bound", () => { + expect(safeAcpxLocations([{ path: "src/main.ts" }], undefined)).toEqual([]); + }); +}); diff --git a/packages/paperclip-runner/src/cli/acpx-sidecar-locations.ts b/packages/paperclip-runner/src/cli/acpx-sidecar-locations.ts new file mode 100644 index 0000000000..3a4cc3011f --- /dev/null +++ b/packages/paperclip-runner/src/cli/acpx-sidecar-locations.ts @@ -0,0 +1,49 @@ +import { isAbsolute, relative, resolve, sep } from "node:path"; + +export const ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY = + "paperclip.workspace_relative_display.v1"; + +function record(value: unknown): Record { + return value !== null && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : {}; +} + +/** + * Converts provider paths to workspace-relative display targets using the + * sidecar host's path semantics. A URI is not a path. Windows separators are + * canonicalized for PRP; POSIX backslashes and colons remain literal filename + * characters. Consumers must treat the result as display data, never as an + * authorization to access a file. + */ +export function safeAcpxLocations( + locations: readonly unknown[] | null | undefined, + workingDirectory: string | null | undefined, +): Array> { + if (!workingDirectory) return []; + const cwd = resolve(workingDirectory); + return (locations ?? []).slice(0, 2_000).flatMap((location) => { + const candidate = record(location); + const rawPath = typeof candidate.path === "string" ? candidate.path : ""; + if (!rawPath || rawPath.includes("\0")) return []; + const absolute = isAbsolute(rawPath) + ? resolve(rawPath) + : resolve(cwd, rawPath); + const local = relative(cwd, absolute); + if (!local || isAbsolute(local)) return []; + const portable = sep === "\\" ? local.replaceAll("\\", "/") : local; + if ( + portable.startsWith("/") || + portable.split("/").some((segment) => segment === "..") + ) { + return []; + } + return [ + { + path: [...portable].slice(0, 4_000).join(""), + line: candidate.line ?? null, + pathBoundary: ACPX_WORKSPACE_RELATIVE_DISPLAY_BOUNDARY, + }, + ]; + }); +} diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts index 218b702664..163184d1a9 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts @@ -511,9 +511,9 @@ describe("Codex ACPX harness driver", () => { fixture.host.close.mockImplementation(({ reason }) => reason.includes("scheduled quarantined cleanup recovery") - ? // Exercise the complete production host bound: two seconds for - // active-turn cancellation plus six seconds for protocol/TERM/KILL. - new Promise((resolve) => setTimeout(resolve, 8_500)) + ? // Exercise the complete production host bound: active-turn + // cancellation plus bounded protocol, TERM, and guardian-group KILL. + new Promise((resolve) => setTimeout(resolve, 9_500)) : Promise.resolve(), ); await vi.advanceTimersToNextTimerAsync(); @@ -536,7 +536,7 @@ describe("Codex ACPX harness driver", () => { reason: "runtime close persistently failed (scheduled quarantined cleanup recovery)", }); - await vi.advanceTimersByTimeAsync(8_499); + await vi.advanceTimersByTimeAsync(9_499); expect(admissionSettled).toBe(false); await vi.advanceTimersByTimeAsync(1); await expect(admission).resolves.toBeDefined(); @@ -638,7 +638,7 @@ describe("Codex ACPX harness driver", () => { setTimeout(() => { if (attempt < 3) reject(new Error("transient quarantine failure")); else resolve(); - }, 8_000); + }, 9_000); }); }); const session = await fixture.driver.openSession({ @@ -669,7 +669,7 @@ describe("Codex ACPX harness driver", () => { admissionSettled = true; }, ); - await vi.advanceTimersByTimeAsync(23_000); + await vi.advanceTimersByTimeAsync(26_000); expect(admissionSettled).toBe(false); expect(fixture.host.close).toHaveBeenCalledTimes(7); await vi.advanceTimersByTimeAsync(2_000); @@ -976,7 +976,7 @@ describe("Codex ACPX harness driver", () => { admissionSettled = true; }) .catch(() => undefined); - await vi.advanceTimersByTimeAsync(34_999); + await vi.advanceTimersByTimeAsync(38_999); expect(admissionSettled).toBe(false); await vi.advanceTimersByTimeAsync(2); await expect(admission).rejects.toThrow("exceeded the admission grace"); @@ -1702,9 +1702,11 @@ describe("Codex ACPX harness driver", () => { const cancellation = new Error("recovery cancelled before start"); controller.abort(cancellation); - await expect(fixture.driver.recoverSession!(snapshot, { - signal: controller.signal, - })).resolves.toEqual({ + await expect( + fixture.driver.recoverSession!(snapshot, { + signal: controller.signal, + }), + ).resolves.toEqual({ recovered: false, reason: cancellation.message, }); @@ -1850,7 +1852,9 @@ describe("Codex ACPX harness driver", () => { semanticResult: { turnId: semanticTurn.turnId }, }); expect(snapshot.terminalTurns?.at(-1)?.turnId).toBe(laterTurn.turnId); - await session.close({ reason: "simulate unsuccessful follow-up recovery" }); + await session.close({ + reason: "simulate unsuccessful follow-up recovery", + }); await expect(fixture.driver.recoverSession!(snapshot)).resolves.toEqual({ recovered: false, @@ -1882,7 +1886,11 @@ describe("Codex ACPX harness driver", () => { }); fixture.finishTurn({ status: "failed", - error: { code: "provider_retry", message: "Retry the turn", retryable: true }, + error: { + code: "provider_retry", + message: "Retry the turn", + retryable: true, + }, }); await firstTerminal; @@ -1906,10 +1914,12 @@ describe("Codex ACPX harness driver", () => { }); expect(snapshot.semanticResult?.turnId).not.toBe(first.turnId); expect(snapshot.terminalTurns?.at(-1)?.turnId).toBe(second.turnId); - expect(snapshot.terminalTurns).toEqual(expect.arrayContaining([ - expect.objectContaining({ turnId: first.turnId }), - expect.objectContaining({ turnId: second.turnId }), - ])); + expect(snapshot.terminalTurns).toEqual( + expect.arrayContaining([ + expect.objectContaining({ turnId: first.turnId }), + expect.objectContaining({ turnId: second.turnId }), + ]), + ); const successfulTerminal = snapshot.terminalTurns?.find( (terminal) => terminal.turnId === second.turnId, ); @@ -1919,15 +1929,19 @@ describe("Codex ACPX harness driver", () => { }); await session.close({ reason: "simulate successful retry recovery" }); - await expect(fixture.driver.recoverSession!({ - ...snapshot, - activeTurnId: first.turnId, - })).resolves.toEqual({ + await expect( + fixture.driver.recoverSession!({ + ...snapshot, + activeTurnId: first.turnId, + }), + ).resolves.toEqual({ recovered: false, reason: "persisted Codex ACPX active turn is not the completed semantic settlement", }); - await expect(fixture.driver.recoverSession!(snapshot)).resolves.toMatchObject({ + await expect( + fixture.driver.recoverSession!(snapshot), + ).resolves.toMatchObject({ recovered: true, }); }); @@ -1984,7 +1998,9 @@ describe("Codex ACPX harness driver", () => { }); await session.close({ reason: "simulate reaffirmed result recovery" }); - await expect(fixture.driver.recoverSession!(snapshot)).resolves.toMatchObject({ + await expect( + fixture.driver.recoverSession!(snapshot), + ).resolves.toMatchObject({ recovered: true, }); }); @@ -2031,9 +2047,7 @@ describe("Codex ACPX harness driver", () => { }); const failedEvents = await failedTerminal; expect( - failedEvents.filter( - (event) => event.eventType === "run.result.proposed", - ), + failedEvents.filter((event) => event.eventType === "run.result.proposed"), ).toHaveLength(1); const snapshot = await session.snapshot(); @@ -2296,10 +2310,12 @@ describe("Codex ACPX harness driver", () => { })); for (const activeTurnId of [turnId, null]) { - await expect(fixture.driver.recoverSession!({ - ...snapshot, - activeTurnId, - })).resolves.toEqual({ + await expect( + fixture.driver.recoverSession!({ + ...snapshot, + activeTurnId, + }), + ).resolves.toEqual({ recovered: false, reason: "persisted Codex ACPX resultless recovery requires a completed terminal turn", @@ -2403,7 +2419,9 @@ function driverFixture( finishTurn(result: Awaited): void; } { let turnCount = 0; - let activeResult: ReturnType>> | null = null; + let activeResult: ReturnType< + typeof deferred> + > | null = null; const createTurn = (): AcpxRuntimeTurn => { activeResult = deferred>(); return { @@ -2442,7 +2460,10 @@ function driverFixture( }; }; const host = fakeHost(createTurn, () => - activeResult?.resolve({ status: "cancelled", stopReason: "session_closed" }), + activeResult?.resolve({ + status: "cancelled", + stopReason: "session_closed", + }), ); let hostOptions: OpenAcpxRuntimeHostOptions | null = null; const openHost = vi.fn( diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts index bd5f96248b..7231259638 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts @@ -62,6 +62,7 @@ import { import { ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS, AcpxRuntimeHost, + type AcpxRetainedCleanupFailure, type AcpxRuntimeTurn, type OpenAcpxRuntimeHostOptions, } from "./runtime-host.js"; @@ -197,8 +198,6 @@ export class CodexAcpxDriver implements HarnessDriver { AcpxRuntimeHost.open(hostOptions, { openRuntime: openCodexAcpxRuntime, reportRetainedCleanupFailure: reportRetainedAcpxCleanupFailure, - } as Parameters[1] & { - reportRetainedCleanupFailure: typeof reportRetainedAcpxCleanupFailure; })); this.#closeSettlementTimeoutMs = dependencies.closeSettlementTimeoutMs ?? CLOSE_TURN_SETTLEMENT_TIMEOUT_MS; @@ -1999,11 +1998,9 @@ function safeMessage(error: unknown): string { .slice(0, 4_000); } -function reportRetainedAcpxCleanupFailure(input: { - resource: "credential" | "command" | "runtime"; - attempt: number; - error: unknown; -}): void { +function reportRetainedAcpxCleanupFailure( + input: AcpxRetainedCleanupFailure, +): void { const errorName = input.error instanceof Error ? input.error.name : "Error"; process.emitWarning( JSON.stringify({ diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts index 9b2a5d4eb4..6e31392e06 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts @@ -70,6 +70,15 @@ describe("managed Codex credentials", () => { }); expect(lease.mode).toBe("inline_json"); + expect(lease.lifetimeFenceFds).toHaveLength(2); + expect(lease.lifetimeFenceFds.every(Number.isSafeInteger)).toBe(true); + expect(lease.lifetimeFenceFds[0]).not.toBe(lease.lifetimeFenceFds[1]); + await expect( + lease.activateLifetimeOwner(process.pid), + ).resolves.toBeUndefined(); + await expect(lease.activateLifetimeOwner(0)).rejects.toThrow( + "lifetime owner is invalid", + ); const cleanupIntent = join( fixture.home, ".paperclip-auth-cleanup-required", diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts index 5abec9e887..4f343dc077 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-credentials.ts @@ -46,6 +46,8 @@ try { interface CredentialHomeLock { assertHeld(): void; + inheritanceFds(): readonly [number, number]; + activateLifetimeOwner(pid: number): Promise; release(): Promise; } @@ -112,6 +114,10 @@ export type ManagedCodexCredentialMode = export interface ManagedCodexCredentialLease { readonly path: string; readonly mode: ManagedCodexCredentialMode; + /** Duplicate both quorum listeners into the provider lifetime sentinel. */ + readonly lifetimeFenceFds: readonly [number, number]; + /** Validate the guardian while the credential quorum is still held. */ + activateLifetimeOwner(pid: number): Promise; close(): Promise; } @@ -340,6 +346,24 @@ async function acquireCredentialHomeLock( await Promise.allSettled(servers.map(closeCredentialLeaseServer)); throw error; } + let inheritanceFds: readonly [number, number]; + try { + const first = credentialLeaseServerFd(servers[0]!); + const second = credentialLeaseServerFd(servers[1]!); + if (first === second) { + throw new Error( + "Managed Codex credential ownership listeners are not distinct", + ); + } + inheritanceFds = Object.freeze([first, second]) as readonly [ + number, + number, + ]; + } catch (error) { + released = true; + await Promise.allSettled(servers.map(closeCredentialLeaseServer)); + throw error; + } return Object.freeze({ assertHeld(): void { @@ -352,6 +376,16 @@ async function acquireCredentialHomeLock( throw new Error("Managed Codex credential ownership was lost"); } }, + inheritanceFds(): readonly [number, number] { + this.assertHeld(); + return inheritanceFds; + }, + async activateLifetimeOwner(pid: number): Promise { + this.assertHeld(); + if (!Number.isSafeInteger(pid) || pid < 1) { + throw new Error("Managed Codex credential lifetime owner is invalid"); + } + }, async release(): Promise { if (released) return; const outcomes = await Promise.allSettled( @@ -396,6 +430,15 @@ function credentialLeasePorts(home: string): readonly number[] { ); } +function credentialLeaseServerFd(server: Server): number { + const fd = (server as Server & { _handle?: { fd?: unknown } })._handle?.fd; + if (!Number.isSafeInteger(fd) || (fd as number) < 0) { + throw new Error( + "Managed Codex credential ownership listener cannot be inherited", + ); + } + return fd as number; +} async function listenForCredentialLease( server: Server, port: number, @@ -551,13 +594,29 @@ function credentialLease( lock.assertHeld(); let closed = false; let closeAttempt: Promise | null = null; + let lifetimeOwnerAttempt: Promise | null = null; return Object.freeze({ path, mode, + lifetimeFenceFds: lock.inheritanceFds(), + async activateLifetimeOwner(pid: number): Promise { + if (closed || closeAttempt !== null) { + throw new Error("Managed Codex credential lease is closing"); + } + if (lifetimeOwnerAttempt !== null) return await lifetimeOwnerAttempt; + const attempt = lock.activateLifetimeOwner(pid); + lifetimeOwnerAttempt = attempt; + try { + await attempt; + } finally { + if (lifetimeOwnerAttempt === attempt) lifetimeOwnerAttempt = null; + } + }, async close(): Promise { if (closed) return; if (closeAttempt !== null) return await closeAttempt; const attempt = (async () => { + await lifetimeOwnerAttempt?.catch(() => undefined); const activeGeneration = activeCredentialLeaseGenerations.get(home); if (activeGeneration !== ownerGeneration) { // A failed close releases its generation only after publishing a diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 6c900f2951..a29bbb1a98 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -53,34 +53,6 @@ describe("Codex ACPX runtime adapter", () => { expect(command.spawn).not.toHaveBeenCalled(); }); - it("rejects Windows before constructing or spawning ACPX", async () => { - const command = fakeCommand(); - const createRuntime = vi.fn(); - const platformDescriptor = Object.getOwnPropertyDescriptor( - process, - "platform", - ); - if (platformDescriptor === undefined) { - throw new Error("Node process.platform descriptor is unavailable"); - } - Object.defineProperty(process, "platform", { - ...platformDescriptor, - value: "win32", - }); - try { - await expect( - openCodexAcpxRuntime(openOptions(command), { createRuntime }), - ).rejects.toThrow( - "provider process-tree containment unavailable on Windows", - ); - } finally { - Object.defineProperty(process, "platform", platformDescriptor); - } - - expect(createRuntime).not.toHaveBeenCalled(); - expect(command.spawn).not.toHaveBeenCalled(); - }); - it("opens a persistent Codex session without persisting launch secrets", async () => { const runtime = fakeRuntime(); let runtimeOptions: AcpRuntimeOptions | undefined; @@ -130,100 +102,35 @@ describe("Codex ACPX runtime adapter", () => { it("launches only through the verified command lease", async () => { const runtime = fakeRuntime(); - let runtimeOptions: AcpRuntimeOptions | undefined; const command = fakeCommand(); - await openCodexAcpxRuntime(openOptions(command), { - createRegistry: () => registry(), - createStore: () => store(), - createRuntime: (options) => { - runtimeOptions = options; - return runtime; - }, - }); const child = fakeChild(); vi.mocked(command.spawn).mockReturnValue(child); const spawnOptions = { cwd: "/runtime/spawn" }; - - expect( - runtimeOptions?.spawnAgent?.({ - command: "/attacker/replacement", - args: ["--stdio"], - options: spawnOptions, - }), - ).toBe(child); - expect(command.spawn).toHaveBeenCalledWith(["--stdio"], { - ...spawnOptions, - detached: true, + await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + vi.mocked(runtime.ensureSession).mockImplementation(async () => { + expect( + options.spawnAgent?.({ + command: "/attacker/replacement", + args: ["--stdio"], + options: spawnOptions, + }), + ).toBe(child); + return HANDLE; + }); + return runtime; + }, + }); + expect(command.spawn).toHaveBeenCalledWith(["--stdio"], spawnOptions, { + credentialFenceFds: [42, 43], + activateCredentialFenceOwner: expect.any(Function), }); }); - it.runIf(process.platform !== "win32")( - "terminates the provider process group after its leader exits", - async () => { - const child = fakeProcessGroupChild(54_321); - const command = fakeCommand(); - vi.mocked(command.spawn).mockReturnValue(child); - const handshakeFailure = new Error("ACP handshake rejected"); - const runtime = fakeRuntime(); - let groupRunning = true; - const processKill = vi - .spyOn(process, "kill") - .mockImplementation((pid, signal) => { - expect(pid).toBe(-54_321); - if (signal === 0) { - if (!groupRunning) { - throw Object.assign(new Error("process group exited"), { - code: "ESRCH", - }); - } - return true; - } - if (signal === "SIGTERM") { - child.signalCode = "SIGTERM"; - queueMicrotask(() => child.emit("exit", null, "SIGTERM")); - return true; - } - if (signal === "SIGKILL") { - groupRunning = false; - return true; - } - return true; - }); - vi.useFakeTimers(); - try { - const openingError = openCodexAcpxRuntime(openOptions(command), { - createRegistry: () => registry(), - createStore: () => store(), - createRuntime: (runtimeOptions) => { - vi.mocked(runtime.ensureSession).mockImplementation(async () => { - runtimeOptions.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); - throw handshakeFailure; - }); - return runtime; - }, - }).then( - () => undefined, - (error: unknown) => error, - ); - for (let turn = 0; turn < 5; turn += 1) await Promise.resolve(); - expect(command.spawn).toHaveBeenCalledOnce(); - - await vi.advanceTimersByTimeAsync(2_001); - await expect(openingError).resolves.toBe(handshakeFailure); - expect(processKill).toHaveBeenCalledWith(-54_321, "SIGTERM"); - expect(processKill).toHaveBeenCalledWith(-54_321, "SIGKILL"); - expect(child.kill).not.toHaveBeenCalled(); - } finally { - processKill.mockRestore(); - vi.useRealTimers(); - } - }, - ); - it("revalidates a recovered workspace immediately before provider spawn", async () => { const runtime = fakeRuntime(); let runtimeOptions: AcpRuntimeOptions | undefined; @@ -254,7 +161,6 @@ describe("Codex ACPX runtime adapter", () => { expect(assertWorkspaceHeld).toHaveBeenCalledOnce(); expect(command.spawn).not.toHaveBeenCalled(); }); - it("maps status, model selection, and state-preserving close", async () => { const runtime = fakeRuntime(); vi.mocked(runtime.getStatus!).mockResolvedValue({ @@ -300,20 +206,15 @@ describe("Codex ACPX runtime adapter", () => { const child = fakeChild(); const command = fakeCommand(); vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const firstClose = expect( port.close({ reason: "runtime close stalled" }), @@ -352,20 +253,15 @@ describe("Codex ACPX runtime adapter", () => { const child = fakeChild(); const command = fakeCommand(); vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const firstClose = expect( port.close({ reason: "runtime close stalled" }), @@ -409,20 +305,15 @@ describe("Codex ACPX runtime adapter", () => { const child = fakeChild(); const command = fakeCommand(); vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const firstClose = expect( port.close({ reason: "first protocol close stalls" }), @@ -473,20 +364,15 @@ describe("Codex ACPX runtime adapter", () => { const child = fakeChild(); const command = fakeCommand(); vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const firstClose = expect( port.close({ reason: "first protocol close stalls" }), @@ -813,23 +699,40 @@ describe("Codex ACPX runtime adapter", () => { const finalReconciliation = new Promise((resolve) => { resolveFinalReconciliation = resolve; }); + let resolveFinalReconciliationStarted!: () => void; + const finalReconciliationStarted = new Promise((resolve) => { + resolveFinalReconciliationStarted = resolve; + }); + let resolveRenewedReconciliation!: () => void; + const renewedReconciliation = new Promise((resolve) => { + resolveRenewedReconciliation = resolve; + }); + let resolveRenewedReconciliationStarted!: () => void; + const renewedReconciliationStarted = new Promise((resolve) => { + resolveRenewedReconciliationStarted = resolve; + }); vi.mocked(runtime.close) .mockReturnValueOnce(initialClose) .mockRejectedValueOnce(new Error("reconciliation 1 failed")) .mockRejectedValueOnce(new Error("reconciliation 2 failed")) - .mockReturnValueOnce(finalReconciliation) - .mockResolvedValueOnce(undefined); - const child = fakeChild(); - child.kill = vi.fn(() => true); + .mockImplementationOnce(() => { + resolveFinalReconciliationStarted(); + return finalReconciliation; + }) + .mockImplementationOnce(() => { + resolveRenewedReconciliationStarted(); + return renewedReconciliation; + }); + const child = failingSignalChild(); const command = fakeCommand(); - vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; + vi.mocked(command.spawn).mockReturnValue(child.child); const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, runtimeCloseTimeoutMs: 1, }); @@ -841,37 +744,26 @@ describe("Codex ACPX runtime adapter", () => { await vi.advanceTimersByTimeAsync(1); await initialObserver; rejectInitialClose(new Error("external close failed late")); - for ( - let turn = 0; - turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4; - turn += 1 - ) { - await Promise.resolve(); - } + await finalReconciliationStarted; expect(runtime.close).toHaveBeenCalledTimes(4); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const finalObserver = expect( port.close({ reason: "external observer of failed process cleanup" }), ).rejects.toThrow("ACPX runtime and provider cleanup failed"); resolveFinalReconciliation(); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(4_000); await finalObserver; - - await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); + await renewedReconciliationStarted; + expect(runtime.close).toHaveBeenCalledTimes(5); expect(runtime.close).toHaveBeenLastCalledWith({ handle: HANDLE, reason: "ACPX late protocol cleanup reconciliation 1", discardPersistentState: false, }); - child.signalCode = "SIGKILL"; - child.emit("exit", null, "SIGKILL"); + child.child.signalCode = "SIGKILL"; + child.child.emit("exit", null, "SIGKILL"); + resolveRenewedReconciliation(); await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); expect(runtime.close).toHaveBeenCalledTimes(5); } finally { vi.useRealTimers(); @@ -890,23 +782,36 @@ describe("Codex ACPX runtime adapter", () => { const finalReconciliation = new Promise((resolve) => { resolveFinalReconciliation = resolve; }); + let resolveFinalReconciliationStarted!: () => void; + const finalReconciliationStarted = new Promise((resolve) => { + resolveFinalReconciliationStarted = resolve; + }); + let resolveRenewedReconciliationStarted!: () => void; + const renewedReconciliationStarted = new Promise((resolve) => { + resolveRenewedReconciliationStarted = resolve; + }); vi.mocked(runtime.close) .mockReturnValueOnce(initialClose) .mockRejectedValueOnce(new Error("reconciliation 1 failed")) .mockRejectedValueOnce(new Error("reconciliation 2 failed")) - .mockReturnValueOnce(finalReconciliation) - .mockResolvedValueOnce(undefined); - const child = fakeChild(); - child.kill = vi.fn(() => true); + .mockImplementationOnce(() => { + resolveFinalReconciliationStarted(); + return finalReconciliation; + }) + .mockImplementationOnce(() => { + resolveRenewedReconciliationStarted(); + return Promise.resolve(); + }); + const child = failingSignalChild(); const command = fakeCommand(); - vi.mocked(command.spawn).mockReturnValue(child); - let runtimeOptions: AcpRuntimeOptions | undefined; + vi.mocked(command.spawn).mockReturnValue(child.child); const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, runtimeCloseTimeoutMs: 1, }); @@ -918,20 +823,9 @@ describe("Codex ACPX runtime adapter", () => { await vi.advanceTimersByTimeAsync(1); await initialObserver; rejectInitialClose(new Error("external close failed late")); - for ( - let turn = 0; - turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4; - turn += 1 - ) { - await Promise.resolve(); - } + await finalReconciliationStarted; expect(runtime.close).toHaveBeenCalledTimes(4); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); const finalObserver = expect( port.close({ reason: "external observer of failed process cleanup" }), ).rejects.toThrow("ACPX runtime and provider cleanup failed"); @@ -941,10 +835,11 @@ describe("Codex ACPX runtime adapter", () => { await finalObserver; expect(runtime.close).toHaveBeenCalledTimes(4); - child.signalCode = "SIGKILL"; - child.emit("exit", null, "SIGKILL"); + child.child.signalCode = "SIGKILL"; + child.child.emit("exit", null, "SIGKILL"); resolveFinalReconciliation(); - await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); + await renewedReconciliationStarted; + expect(runtime.close).toHaveBeenCalledTimes(5); expect(runtime.close).toHaveBeenLastCalledWith({ handle: HANDLE, reason: "ACPX late protocol cleanup reconciliation 1", @@ -1090,6 +985,241 @@ describe("Codex ACPX runtime adapter", () => { } }); + it("retains cleanup after guardian exit until provider exit is proven", async () => { + const runtime = fakeRuntime(); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let proveProviderExit!: () => void; + const providerExit = new Promise((resolve) => { + proveProviderExit = resolve; + }); + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: async () => await providerExit, + createRuntime: (options) => runtimeWithProvider(runtime, options), + }); + + let settled = false; + const closing = port.close({ reason: "guardian exited first" }); + void closing.then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.waitFor(() => + expect(child.kill).toHaveBeenCalledWith("SIGTERM"), + ); + await Promise.resolve(); + + expect(child.signalCode).toBe("SIGTERM"); + expect(settled).toBe(false); + + proveProviderExit(); + await expect(closing).resolves.toBeUndefined(); + expect(settled).toBe(true); + }); + + it("kills a TERM-resistant provider through its live guardian", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + const child = childThatExitsOnKill(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + return runtimeWithProvider(runtime, options); + }, + }); + + const closing = expect( + port.close({ reason: "provider ignored shutdown" }), + ).rejects.toMatchObject({ + errors: [ + expect.objectContaining({ + message: "ACPX provider did not exit after SIGTERM", + }), + ], + }); + await vi.advanceTimersByTimeAsync(4_000); + await closing; + + expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(child.kill).toHaveBeenCalledTimes(2); + expect(child.signalCode).toBe("SIGKILL"); + // Cleanup never transfers a copied numeric PGID or releases local + // ownership before the verified guardian's exit is observed. + expect(child.unref).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("retains local ownership when guardian group exit cannot be confirmed", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + const child = stubbornChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + return runtimeWithProvider(runtime, options); + }, + }); + + const closing = expect( + port.close({ reason: "provider group exit unconfirmed" }), + ).rejects.toMatchObject({ + errors: expect.arrayContaining([ + expect.objectContaining({ + message: "ACPX provider did not exit after SIGKILL", + }), + ]), + }); + await vi.advanceTimersByTimeAsync(4_000); + await closing; + + expect(child.kill).toHaveBeenNthCalledWith(1, "SIGTERM"); + expect(child.kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(child.unref).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("rejects Windows before runtime construction or provider spawn", async () => { + const command = fakeCommand(); + const createRuntime = vi.fn(() => fakeRuntime()); + + await expect( + openCodexAcpxRuntime(openOptions(command), { + platform: "win32", + createRuntime, + }), + ).rejects.toThrow( + "The production ACPX runtime is unavailable on Windows because verified provider launch requires atomic no-follow file opening", + ); + + expect(createRuntime).not.toHaveBeenCalled(); + expect(command.spawn).not.toHaveBeenCalled(); + }); + + it.each([ + ["one fence", [42]], + ["duplicate fences", [42, 42]], + ["an invalid fence", [42, -1]], + ])("rejects %s before runtime construction", async (_label, fences) => { + const command = fakeCommand(); + const createRuntime = vi.fn(() => fakeRuntime()); + + await expect( + openCodexAcpxRuntime( + { + ...openOptions(command), + credentialFenceFds: fences as unknown as readonly [number, number], + }, + { createRuntime }, + ), + ).rejects.toThrow( + "The production ACPX runtime requires an inherited credential-home fence", + ); + + expect(createRuntime).not.toHaveBeenCalled(); + expect(command.spawn).not.toHaveBeenCalled(); + }); + + it("rejects a missing credential owner activation callback", async () => { + const command = fakeCommand(); + const createRuntime = vi.fn(() => fakeRuntime()); + + await expect( + openCodexAcpxRuntime( + { + ...openOptions(command), + activateCredentialFenceOwner: null, + }, + { createRuntime }, + ), + ).rejects.toThrow( + "The production ACPX runtime requires an inherited credential-home fence", + ); + + expect(createRuntime).not.toHaveBeenCalled(); + expect(command.spawn).not.toHaveBeenCalled(); + }); + + it("rejects and independently cleans providers spawned during termination", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + const firstChild = childThatExitsOnKill(); + const lateChild = childThatExitsOnKill(); + const command = fakeCommand(); + vi.mocked(command.spawn) + .mockReturnValueOnce(firstChild) + .mockReturnValueOnce(lateChild); + const retainedCleanups: Promise[] = []; + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, + retainCleanup: (cleanup) => retainedCleanups.push(cleanup), + createRuntime: (options) => { + runtimeOptions = options; + return runtimeWithProvider(runtime, options); + }, + }); + + let settled = false; + const closing = port.close({ reason: "join late provider" }).then( + () => { + settled = true; + }, + () => { + settled = true; + }, + ); + await vi.advanceTimersByTimeAsync(0); + expect(firstChild.kill).toHaveBeenCalledWith("SIGTERM"); + await vi.advanceTimersByTimeAsync(1_500); + expect(() => + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }), + ).toThrow("provider spawned after cleanup was sealed"); + expect(lateChild.kill).toHaveBeenCalledWith("SIGKILL"); + expect(retainedCleanups).toHaveLength(1); + + await vi.advanceTimersByTimeAsync(500); + expect(firstChild.kill).toHaveBeenCalledWith("SIGKILL"); + await closing; + await retainedCleanups[0]; + expect(settled).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + it("maps prompt turns to the admitted ACPX handle", async () => { const runtime = fakeRuntime(); const turn = { @@ -1228,33 +1358,42 @@ describe("Codex ACPX runtime adapter", () => { }); }); - it("retains failed ordinary admission cleanup until a retry succeeds", async () => { - const runtime = fakeRuntime({ ...HANDLE, agentSessionId: undefined }); - const firstCloseFailure = new Error("runtime close failed"); - vi.mocked(runtime.close) - .mockRejectedValueOnce(firstCloseFailure) - .mockResolvedValueOnce(undefined); - const retainedCleanups: Promise[] = []; - - await expect( - openCodexAcpxRuntime(openOptions(fakeCommand()), { + it("bounds invalid-identity cleanup before terminating the provider", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime({ ...HANDLE, agentSessionId: undefined }); + vi.mocked(runtime.close).mockImplementation( + () => new Promise(() => undefined), + ); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + const opening = openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), - createRuntime: () => runtime, - retainCleanup: (cleanup) => retainedCleanups.push(cleanup), - }), - ).rejects.toMatchObject({ - errors: [ - expect.objectContaining({ - message: "ACPX runtime omitted agentSessionId", + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => ({ + ...runtime, + ensureSession: vi.fn(async () => { + options.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + return { ...HANDLE, agentSessionId: undefined }; + }), }), - firstCloseFailure, - ], - }); + }); + const rejected = expect(opening).rejects.toThrow( + "identity validation and cleanup failed", + ); - expect(retainedCleanups).toHaveLength(1); - await expect(retainedCleanups[0]).resolves.toBeUndefined(); - expect(runtime.close).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(2_000); + await rejected; + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + } finally { + vi.useRealTimers(); + } }); it("terminates a provider spawned before the session handshake rejects", async () => { @@ -1268,6 +1407,7 @@ describe("Codex ACPX runtime adapter", () => { openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { vi.mocked(runtime.ensureSession).mockImplementation(async () => { options.spawnAgent?.({ @@ -1302,6 +1442,7 @@ describe("Codex ACPX runtime adapter", () => { { createRegistry: () => registry(), createStore: () => store(), + awaitProviderExit: providerOwnershipEstablished, createRuntime: (runtimeOptions) => { vi.mocked(runtime.ensureSession).mockImplementation(async () => { runtimeOptions.spawnAgent?.({ @@ -1447,19 +1588,15 @@ describe("Codex ACPX runtime adapter", () => { ); }); - it("retains ownership of a late close until its exact attempt settles", async () => { + it("terminalizes a never-settling late close without overlapping it", async () => { let resolveHandshake: ((handle: AcpRuntimeHandle) => void) | undefined; const blockedHandshake = new Promise((resolve) => { resolveHandshake = resolve; }); const runtime = fakeRuntime(); vi.mocked(runtime.ensureSession).mockReturnValue(blockedHandshake); - let resolveClose: (() => void) | undefined; vi.mocked(runtime.close).mockImplementation( - () => - new Promise((resolve) => { - resolveClose = resolve; - }), + () => new Promise(() => undefined), ); const controller = new AbortController(); const cancellation = new Error("runtime admission cancelled"); @@ -1486,23 +1623,11 @@ describe("Codex ACPX runtime adapter", () => { expect(retainedCleanups).toHaveLength(2); resolveHandshake?.(HANDLE); - await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce()); - let cleanupSettled = false; - void retainedCleanups[0]!.then( - () => { - cleanupSettled = true; - }, - () => { - cleanupSettled = true; - }, - ); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(cleanupSettled).toBe(false); + await expect(retainedCleanups[0]).rejects.toMatchObject({ + name: "AcpxRuntimeCloseFinalTimeoutError", + }); expect(runtime.close).toHaveBeenCalledOnce(); - - resolveClose?.(); - await expect(Promise.all(retainedCleanups)).resolves.toBeDefined(); - expect(cleanupSettled).toBe(true); + await new Promise((resolve) => setTimeout(resolve, 20)); expect(runtime.close).toHaveBeenCalledOnce(); }); @@ -1694,7 +1819,7 @@ describe("Codex ACPX runtime adapter", () => { rejectHandshake?.(new Error("test handshake stopped")); }); - it("retains cleanup beyond the former admission close retry budget", async () => { + it("exhausts the full retained admission close retry budget", async () => { let rejectHandshake: ((error: Error) => void) | undefined; const blockedHandshake = new Promise( (_resolve, reject) => { @@ -1705,12 +1830,7 @@ describe("Codex ACPX runtime adapter", () => { const closeFailure = new Error("runtime close failed"); const retainedCleanups: Promise[] = []; vi.mocked(runtime.ensureSession).mockReturnValue(blockedHandshake); - vi.mocked(runtime.close) - .mockRejectedValueOnce(closeFailure) - .mockRejectedValueOnce(closeFailure) - .mockRejectedValueOnce(closeFailure) - .mockRejectedValueOnce(closeFailure) - .mockResolvedValueOnce(undefined); + vi.mocked(runtime.close).mockRejectedValue(closeFailure); const controller = new AbortController(); const cancellation = new Error("runtime admission cancelled"); let runtimeOptions: AcpRuntimeOptions | undefined; @@ -1744,22 +1864,25 @@ describe("Codex ACPX runtime adapter", () => { } as never); await expect(opening).rejects.toBeInstanceOf(Error); - // Retain the pending handshake, the discovered runtime handle cleanup, - // and their host-facing aggregate proof until the same obligation settles. expect(retainedCleanups).toHaveLength(3); - const settledCleanups = new Set>(); - for (const cleanup of retainedCleanups) { - void cleanup - .finally(() => settledCleanups.add(cleanup)) - .catch(() => undefined); - } - await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); - await vi.waitFor(() => expect(settledCleanups.size).toBe(1)); - expect(runtime.close).toHaveBeenCalledTimes(5); + const recoveryOutcome = retainedCleanups[0]!.catch( + (error: unknown) => error, + ); + const compositeOutcome = retainedCleanups[2]!.catch( + (error: unknown) => error, + ); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + await expect(recoveryOutcome).resolves.toMatchObject({ + message: "ACPX failed-admission cleanup exhausted 3 retry attempts", + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(runtime.close).toHaveBeenCalledTimes(4); rejectHandshake?.(new Error("test handshake stopped")); - await vi.waitFor(() => expect(settledCleanups.size).toBe(3)); - await expect(Promise.all(retainedCleanups)).resolves.toBeDefined(); + await expect(retainedCleanups[1]).resolves.toBeUndefined(); + await expect(compositeOutcome).resolves.toMatchObject({ + message: "ACPX failed-admission cleanup exhausted 3 retry attempts", + }); }); it("aggregates asynchronous provider signal errors after a failed handshake", async () => { @@ -1768,10 +1891,13 @@ describe("Codex ACPX runtime adapter", () => { vi.mocked(command.spawn).mockReturnValue(child.child); const handshakeError = new Error("ACP handshake rejected"); const runtime = fakeRuntime(); + const retainedCleanups: Promise[] = []; const result = openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderExit: providerOwnershipEstablished, + retainCleanup: (cleanup) => retainedCleanups.push(cleanup), createRuntime: (options) => { vi.mocked(runtime.ensureSession).mockImplementation(async () => { options.spawnAgent?.({ @@ -1788,7 +1914,11 @@ describe("Codex ACPX runtime adapter", () => { await expect(result).rejects.toMatchObject({ errors: [ handshakeError, - ...child.errors, + child.errors[0], + expect.objectContaining({ + message: "ACPX provider did not exit after SIGTERM", + }), + child.errors[1], expect.objectContaining({ message: "ACPX provider did not exit after SIGKILL", }), @@ -1796,31 +1926,14 @@ describe("Codex ACPX runtime adapter", () => { }); expect(child.child.kill).toHaveBeenNthCalledWith(1, "SIGTERM"); expect(child.child.kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(retainedCleanups).toHaveLength(1); child.child.signalCode = "SIGKILL"; child.child.emit("exit", null, "SIGKILL"); + await expect(retainedCleanups[0]).resolves.toBeUndefined(); }); - it("retains provider cleanup beyond the former retry budget", async () => { - const child = new EventEmitter() as ChildProcess; - Object.defineProperties(child, { - exitCode: { value: null, writable: true }, - signalCode: { value: null, writable: true }, - }); - let killAttempts = 0; - child.kill = vi.fn((signal) => { - if (signal === "SIGKILL") { - killAttempts += 1; - if (killAttempts === 5) { - child.signalCode = "SIGKILL"; - queueMicrotask(() => child.emit("exit", null, "SIGKILL")); - return true; - } - } - queueMicrotask(() => - child.emit("error", new Error(`${String(signal)} still pending`)), - ); - return true; - }); + it("retains failed-admission cleanup until the provider exits", async () => { + const child = childThatExitsAfterSignalCalls(12); const command = fakeCommand(); vi.mocked(command.spawn).mockReturnValue(child); const handshakeError = new Error("ACP handshake rejected"); @@ -1831,9 +1944,11 @@ describe("Codex ACPX runtime adapter", () => { openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), - createRuntime: (runtimeOptions) => { + awaitProviderExit: providerOwnershipEstablished, + retainCleanup: (cleanup) => retainedCleanups.push(cleanup), + createRuntime: (options) => { vi.mocked(runtime.ensureSession).mockImplementation(async () => { - runtimeOptions.spawnAgent?.({ + options.spawnAgent?.({ command: "ignored", args: ["--stdio"], options: {}, @@ -1842,13 +1957,13 @@ describe("Codex ACPX runtime adapter", () => { }); return runtime; }, - retainCleanup: (cleanup) => retainedCleanups.push(cleanup), }), - ).rejects.toBeInstanceOf(AggregateError); + ).rejects.toThrow("session handshake and runtime cleanup failed"); expect(retainedCleanups).toHaveLength(1); await expect(retainedCleanups[0]).resolves.toBeUndefined(); - expect(killAttempts).toBe(5); + expect(child.kill).toHaveBeenCalledTimes(12); + expect(child.signalCode).toBe("SIGKILL"); }); it("closes a recovered session when its handshake rejects before another save", async () => { @@ -1961,6 +2076,7 @@ describe("Codex ACPX runtime adapter", () => { openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderExit: providerOwnershipEstablished, createRuntime: (runtimeOptions) => { vi.mocked(runtime.ensureSession).mockImplementation(async () => { await runtimeOptions.sessionStore.save({ @@ -2002,30 +2118,85 @@ describe("Codex ACPX runtime adapter", () => { expect(child.kill).toHaveBeenCalledWith("SIGTERM"); }); + it("waits for a timed-out admission close before retrying it sequentially", async () => { + let rejectFirstClose!: (error: Error) => void; + const firstClose = new Promise((_resolve, reject) => { + rejectFirstClose = reject; + }); + const runtime = fakeRuntime(); + vi.mocked(runtime.close) + .mockImplementationOnce(() => firstClose) + .mockResolvedValueOnce(undefined); + const retainedCleanups: Promise[] = []; + + await expect( + openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + runtimeCloseTimeoutMs: 1, + retainCleanup: (cleanup) => retainedCleanups.push(cleanup), + createRuntime: (runtimeOptions) => { + vi.mocked(runtime.ensureSession).mockImplementation(async () => { + await runtimeOptions.sessionStore.save({ + acpxRecordId: "retry-record", + acpSessionId: "retry-backend-session", + agentSessionId: "retry-agent-session", + name: "retry-runtime-name", + cwd: "/workspace", + } as never); + throw new Error("handshake failed before cleanup retry"); + }); + return runtime; + }, + }), + ).rejects.toThrow("session handshake and runtime cleanup failed"); + + expect(runtime.close).toHaveBeenCalledOnce(); + expect(retainedCleanups).toHaveLength(1); + let recoverySettled = false; + void retainedCleanups[0]!.then(() => { + recoverySettled = true; + }); + await Promise.resolve(); + expect(recoverySettled).toBe(false); + + rejectFirstClose(new Error("late first close failure")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + await expect(retainedCleanups[0]).resolves.toBeUndefined(); + await new Promise((resolve) => setTimeout(resolve, 25)); + expect(runtime.close).toHaveBeenCalledTimes(2); + expect(runtime.close).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + reason: "ACPX session handshake failed", + discardPersistentState: false, + }), + ); + }); + it("rejects close with asynchronous provider signal errors", async () => { const runtime = fakeRuntime(); const command = fakeCommand(); const child = failingSignalChild(); - let runtimeOptions: AcpRuntimeOptions | undefined; vi.mocked(command.spawn).mockReturnValue(child.child); const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); await expect(port.close({ reason: "test complete" })).rejects.toMatchObject( { errors: [ - ...child.errors, + child.errors[0], + expect.objectContaining({ + message: "ACPX provider did not exit after SIGTERM", + }), + child.errors[1], expect.objectContaining({ message: "ACPX provider did not exit after SIGKILL", }), @@ -2034,6 +2205,7 @@ describe("Codex ACPX runtime adapter", () => { ); expect(child.child.kill).toHaveBeenNthCalledWith(1, "SIGTERM"); expect(child.child.kill).toHaveBeenNthCalledWith(2, "SIGKILL"); + expect(child.child.kill).toHaveBeenCalledTimes(2); child.child.signalCode = "SIGKILL"; child.child.emit("exit", null, "SIGKILL"); }); @@ -2043,21 +2215,16 @@ describe("Codex ACPX runtime adapter", () => { const command = fakeCommand(); const child = fakeChild(); const providerError = new Error("provider spawn failed"); - let runtimeOptions: AcpRuntimeOptions | undefined; vi.mocked(command.spawn).mockReturnValue(child); const port = await openCodexAcpxRuntime(openOptions(command), { createRegistry: () => registry(), createStore: () => store(), + awaitProviderOwnership: providerOwnershipEstablished, + awaitProviderExit: providerOwnershipEstablished, createRuntime: (options) => { - runtimeOptions = options; - return runtime; + return runtimeWithProvider(runtime, options); }, }); - runtimeOptions?.spawnAgent?.({ - command: "ignored", - args: ["--stdio"], - options: {}, - }); child.emit("error", providerError); // A real ChildProcess has committed its terminal status before `close`. @@ -2074,6 +2241,162 @@ describe("Codex ACPX runtime adapter", () => { expect(child.kill).not.toHaveBeenCalled(); }); + it("propagates ownership failure from a provider spawned during verification", async () => { + const firstChild = fakeChild(); + const racingChild = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn) + .mockReturnValueOnce(firstChild) + .mockReturnValueOnce(racingChild); + let resolveFirstOwnership!: () => void; + const firstOwnership = new Promise((resolve) => { + resolveFirstOwnership = resolve; + }); + const racingOwnershipFailure = new Error( + "racing guardian ownership failed", + ); + const runtime = fakeRuntime(); + let runtimeOptions: AcpRuntimeOptions | undefined; + const awaitProviderOwnership = vi.fn((child: ChildProcess) => { + if (child !== firstChild) return Promise.reject(racingOwnershipFailure); + return firstOwnership.then(() => { + // This callback runs while verification still awaits its initial + // ownership batch, reproducing the exact append-after-splice race. + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + }); + }); + vi.mocked(runtime.ensureSession).mockImplementation(async () => { + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + return HANDLE; + }); + + const opening = openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + awaitProviderOwnership, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + await vi.waitFor(() => + expect(awaitProviderOwnership).toHaveBeenCalledOnce(), + ); + resolveFirstOwnership(); + + await expect(opening).rejects.toBe(racingOwnershipFailure); + expect(awaitProviderOwnership).toHaveBeenCalledTimes(2); + expect(firstChild.kill).toHaveBeenCalledWith("SIGTERM"); + expect(racingChild.kill).toHaveBeenCalledWith("SIGTERM"); + }); + + it("bounds a stalled session handshake and terminates its provider", async () => { + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + const runtime = fakeRuntime(); + + await expect( + openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + sessionHandshakeTimeoutMs: 1, + awaitProviderExit: providerOwnershipEstablished, + createRuntime: (options) => { + vi.mocked(runtime.ensureSession).mockImplementation(() => { + options.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + return new Promise(() => undefined); + }); + return runtime; + }, + }), + ).rejects.toThrow("session handshake exceeded its admission deadline"); + expect(runtime.close).not.toHaveBeenCalled(); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + }); + + it("rejects provider children created after handshake cleanup is sealed", async () => { + const child = fakeChild(); + const postCleanupChild = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn) + .mockReturnValueOnce(child) + .mockReturnValueOnce(postCleanupChild); + const runtime = fakeRuntime(); + const retainCleanup = vi.fn<(cleanup: Promise) => void>(); + let runtimeOptions: AcpRuntimeOptions | undefined; + let resolveHandshake: ((handle: AcpRuntimeHandle) => void) | undefined; + vi.mocked(runtime.ensureSession).mockImplementation( + () => + new Promise((resolve) => { + resolveHandshake = resolve; + }), + ); + + await expect( + openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + sessionHandshakeTimeoutMs: 1, + awaitProviderExit: providerOwnershipEstablished, + retainCleanup, + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }), + ).rejects.toThrow("session handshake exceeded its admission deadline"); + + expect(retainCleanup).toHaveBeenCalledTimes(2); + const lateHandshakeCleanup = retainCleanup.mock.calls[0]?.[0]; + const cleanupProof = retainCleanup.mock.calls[1]?.[0]; + expect(lateHandshakeCleanup).toBeDefined(); + expect(cleanupProof).toBeDefined(); + + expect(() => + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }), + ).toThrow("provider spawned after cleanup was sealed"); + expect(child.kill).toHaveBeenCalledWith("SIGKILL"); + expect(retainCleanup).toHaveBeenCalledTimes(3); + await retainCleanup.mock.calls[2]?.[0]; + resolveHandshake?.(HANDLE); + + await lateHandshakeCleanup; + await cleanupProof; + expect(runtime.close).toHaveBeenCalledWith({ + handle: HANDLE, + reason: "ACPX session handshake completed after its admission deadline", + discardPersistentState: false, + }); + expect(() => + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }), + ).toThrow("provider spawned after cleanup was sealed"); + expect(postCleanupChild.kill).toHaveBeenCalledWith("SIGKILL"); + expect(retainCleanup).toHaveBeenCalledTimes(4); + await retainCleanup.mock.calls[3]?.[0]; + }); + it("rejects non-Codex profiles before constructing ACPX", async () => { const createRuntime = vi.fn(); await expect( @@ -2126,6 +2449,10 @@ function openOptions( OPENAI_API_KEY: "credential-secret", OMITTED: undefined, }, + // Fake command leases do not inherit these descriptors; production host + // supplies both live credential-quorum listener descriptors. + credentialFenceFds: [42, 43] as const, + activateCredentialFenceOwner: async () => undefined, systemInstructions: "Use Paperclip tools.", mcpServers: [], retainFailedAdmissionCleanup: vi.fn(), @@ -2144,6 +2471,23 @@ function fakeRuntime(handle: AcpRuntimeHandle = HANDLE): AcpRuntime { }; } +function runtimeWithProvider( + runtime: AcpRuntime, + options: AcpRuntimeOptions, +): AcpRuntime { + vi.mocked(runtime.ensureSession).mockImplementationOnce(async () => { + options.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + return HANDLE; + }); + return runtime; +} + +async function providerOwnershipEstablished(): Promise {} + function fakeCommand(): VerifiedAcpxCommandLease { return { spawn: vi.fn(), close: vi.fn() }; } @@ -2162,12 +2506,6 @@ function fakeChild(): ChildProcess { return child; } -function fakeProcessGroupChild(pid: number): ChildProcess { - const child = fakeChild(); - Object.defineProperty(child, "pid", { value: pid }); - return child; -} - function failingSignalChild(): { child: ChildProcess; errors: [Error, Error]; @@ -2186,9 +2524,65 @@ function failingSignalChild(): { queueMicrotask(() => child.emit("error", error)); return true; }); + child.unref = vi.fn(() => child); return { child, errors }; } +function childThatExitsAfterSignalCalls(exitAfter: number): ChildProcess { + const child = new EventEmitter() as ChildProcess; + Object.defineProperties(child, { + exitCode: { value: null, writable: true }, + signalCode: { value: null, writable: true }, + }); + child.kill = vi.fn((signal?: NodeJS.Signals | number) => { + const calls = vi.mocked(child.kill).mock.calls.length; + if (calls >= exitAfter) { + child.signalCode = signal as NodeJS.Signals; + queueMicrotask(() => child.emit("exit", null, signal)); + } else { + queueMicrotask(() => + child.emit("error", new Error(`signal attempt ${calls} failed`)), + ); + } + return true; + }); + child.unref = vi.fn(() => child); + return child; +} + +function stubbornChild(): ChildProcess { + const child = new EventEmitter() as ChildProcess; + Object.defineProperties(child, { + pid: { value: 71_001 }, + exitCode: { value: null, writable: true }, + signalCode: { value: null, writable: true }, + stdin: { value: { destroy: vi.fn() } }, + stdout: { value: { destroy: vi.fn() } }, + stderr: { value: { destroy: vi.fn() } }, + }); + child.kill = vi.fn(() => true); + child.unref = vi.fn(() => child); + return child; +} + +function childThatExitsOnKill(): ChildProcess { + const child = new EventEmitter() as ChildProcess; + Object.defineProperties(child, { + pid: { value: 71_002 }, + exitCode: { value: null, writable: true }, + signalCode: { value: null, writable: true }, + }); + child.kill = vi.fn((signal?: NodeJS.Signals | number) => { + if (signal === "SIGKILL") { + child.signalCode = "SIGKILL"; + queueMicrotask(() => child.emit("exit", null, "SIGKILL")); + } + return true; + }); + child.unref = vi.fn(() => child); + return child; +} + function registry(): AcpAgentRegistry { return { resolve: vi.fn(), list: vi.fn() }; } diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index df2aa93337..3884605a6e 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -18,29 +18,39 @@ import type { AcpxRuntimePortIdentity, AcpxRuntimePortOpenOptions, } from "./runtime-host.js"; +import { + assertVerifiedAcpxProviderPlatform, + awaitVerifiedAcpxProviderExit, + awaitVerifiedAcpxProviderOwnership, +} from "./installation-integrity.js"; import { decideAcpxPermission } from "./permission-policy.js"; const VERIFIED_COMMAND_SENTINEL = "paperclip-verified-acpx-command"; const DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS = 2_000; +const MAX_ADMISSION_CLEANUP_RETRY_ATTEMPTS = 3; +const MAX_ADMISSION_CLEANUP_ATTEMPTS = 1 + MAX_ADMISSION_CLEANUP_RETRY_ATTEMPTS; const RETAINED_ADMISSION_CLEANUP_RETRY_MIN_MS = 10; -const RETAINED_ADMISSION_CLEANUP_RETRY_MAX_MS = 30_000; +const RETAINED_ADMISSION_CLEANUP_RETRY_MAX_MS = 100; const PROVIDER_TERM_EXIT_TIMEOUT_MS = 2_000; const PROVIDER_KILL_EXIT_TIMEOUT_MS = 2_000; +const PROVIDER_SHUTDOWN_SCHEDULING_MARGIN_MS = 1_000; const MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS = 3; // Production shutdown waits for the protocol close bound before beginning the -// sequential TERM/KILL verification windows. Keep this exported package-local -// bound aligned with the implementation so admission can include the complete -// provider cleanup path instead of accounting for only part of it. +// sequential TERM/KILL verification windows plus a finite scheduling margin. +// Keep this exported package-local bound aligned with the complete +// implementation. export const DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS = DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS + PROVIDER_TERM_EXIT_TIMEOUT_MS + - PROVIDER_KILL_EXIT_TIMEOUT_MS; + PROVIDER_KILL_EXIT_TIMEOUT_MS + + PROVIDER_SHUTDOWN_SCHEDULING_MARGIN_MS; // A close may outlive its caller-facing wait bound. Keep every exact attempt // owned until it settles. A handle never starts a second protocol close while // the first remains unresolved; late failure can start bounded reconciliation // only after the exact attempt reaches a terminal outcome. const activeRuntimeCleanupOwners = new Set>(); const activeCodexRuntimeCleanupOwners = new Set>(); +const SESSION_HANDSHAKE_TIMEOUT_MS = 8_000; class AcpxRuntimeCloseTimeoutError extends Error { constructor() { @@ -49,6 +59,20 @@ class AcpxRuntimeCloseTimeoutError extends Error { } } +class AcpxRuntimeCloseFinalTimeoutError extends Error { + constructor() { + super("ACPX runtime close remained pending after its final cleanup watch"); + this.name = "AcpxRuntimeCloseFinalTimeoutError"; + } +} + +class AcpxSessionHandshakeTimeoutError extends Error { + constructor() { + super("ACPX session handshake exceeded its admission deadline"); + this.name = "AcpxSessionHandshakeTimeoutError"; + } +} + export interface CodexAcpxRuntimeDependencies { createRuntime?: (options: AcpRuntimeOptions) => AcpRuntime; createRegistry?: (input: { @@ -56,8 +80,16 @@ export interface CodexAcpxRuntimeDependencies { }) => AcpAgentRegistry; createStore?: (input: { stateDir: string }) => AcpSessionStore; runtimeCloseTimeoutMs?: number; - /** Internal test seam for autonomous failed-admission cleanup ownership. */ + /** Internal test seam for the provider-session admission deadline. */ + sessionHandshakeTimeoutMs?: number; + /** Internal test seam for verified guardian ownership transfer. */ + awaitProviderOwnership?: (child: ChildProcess) => Promise; + /** Internal test seam for independent provider-exit proof. */ + awaitProviderExit?: (child: ChildProcess) => Promise; + /** Retains autonomous cleanup ownership across the sidecar lifecycle. */ retainCleanup?: (cleanup: Promise) => void; + /** Internal test seam for the fail-closed platform admission boundary. */ + platform?: NodeJS.Platform; } /** @@ -77,21 +109,31 @@ export async function openCodexAcpxRuntime( options.retainFailedAdmissionCleanup(Promise.resolve()); throw options.signal.reason; } + // Verified ACPX command admission already fails closed on Windows because + // Node cannot atomically open the provider executable with O_NOFOLLOW there. + // Reject at the adapter boundary too: allowing a fabricated command lease to + // start a provider would create a cleanup state that cannot guarantee both a + // bounded sidecar exit and retained ownership of an unresponsive process + // tree when Node cannot safely signal a verified provider process group. + assertVerifiedAcpxProviderPlatform(dependencies.platform ?? process.platform); if (options.profile.agent !== "codex") { throw new Error( "The production ACPX runtime currently supports Codex only", ); } - // The verified-command boundary already refuses to mint a Windows command - // lease, because Node cannot pin its executable there. Repeat the platform - // gate at this lower boundary so alternate host wiring cannot launch a - // credential-bearing provider without a killable tree. `child.kill()` only - // terminates the direct Windows process, and taskkill cannot reliably find - // descendants after their original parent has exited; Windows support must - // therefore wait for an owned Job Object or equivalent containment. - if (process.platform === "win32") { + options.signal?.throwIfAborted(); + const credentialFenceFds = options.credentialFenceFds; + if ( + !Array.isArray(credentialFenceFds) || + credentialFenceFds.length !== 2 || + credentialFenceFds.some( + (fd) => !Number.isSafeInteger(fd) || (fd as number) < 0, + ) || + credentialFenceFds[0] === credentialFenceFds[1] || + typeof options.activateCredentialFenceOwner !== "function" + ) { throw new Error( - "The production ACPX runtime requires provider process-tree containment unavailable on Windows", + "The production ACPX runtime requires an inherited credential-home fence", ); } @@ -100,11 +142,6 @@ export async function openCodexAcpxRuntime( const createRuntime = dependencies.createRuntime ?? createAcpRuntime; const runtimeCloseTimeoutMs = dependencies.runtimeCloseTimeoutMs ?? DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS; - const children = new SpawnedChildSet(); - const baseStore = createStore({ stateDir: options.stateDirectory }); - let failedHandshakeHandle: AcpRuntimeHandle | null = null; - let admissionCleanup: RuntimeAdmissionCleanup | null = null; - let abortedHandshakeCleanup: Promise | null = null; const retainedCleanupOwners = new WeakSet>(); const retainCleanup = (cleanup: Promise): void => { if (retainedCleanupOwners.has(cleanup)) { @@ -114,6 +151,14 @@ export async function openCodexAcpxRuntime( dependencies.retainCleanup?.(cleanup); retainCodexRuntimeCleanup(cleanup); }; + const children = new SpawnedChildSet( + retainCleanup, + dependencies.awaitProviderOwnership, + dependencies.awaitProviderExit, + ); + const baseStore = createStore({ stateDir: options.stateDirectory }); + let failedHandshakeHandle: AcpRuntimeHandle | null = null; + let admissionCleanup: RuntimeAdmissionCleanup | null = null; const rememberHandshakeHandle = (record: AcpSessionRecord): void => { const runtimeSessionName = record.name?.trim(); if ( @@ -219,16 +264,12 @@ export async function openCodexAcpxRuntime( // been cancelled. Check at the last host-owned boundary so a late // handshake cannot create a provider process after authority is gone. options.signal?.throwIfAborted(); - // A verified provider can create descendants that inherit its launch - // credential. Give the provider a dedicated POSIX process group so - // cleanup authority covers that complete credential-bearing tree. options.assertWorkspaceHeld?.(); return children.add( - options.command.spawn(input.args, { - ...input.options, - detached: true, + options.command.spawn(input.args, input.options, { + credentialFenceFds, + activateCredentialFenceOwner: options.activateCredentialFenceOwner!, }) as ChildProcess, - true, ); }, }); @@ -236,75 +277,66 @@ export async function openCodexAcpxRuntime( runtime, children, runtimeCloseTimeoutMs, - retainCleanup, ); + const handshake = Promise.resolve().then(() => + runtime.ensureSession({ + sessionKey: options.providerSessionKey, + agent: "codex", + mode: "persistent", + cwd: options.cwd, + sessionOptions: { + model: options.profile.qualificationModel, + ...(options.systemInstructions + ? { systemPrompt: { append: options.systemInstructions } } + : {}), + }, + }), + ); let handle: AcpRuntimeHandle | null = null; + let lateCleanup: Promise | null = null; try { - const handshake = Promise.resolve().then(() => - runtime.ensureSession({ - sessionKey: options.providerSessionKey, - agent: "codex", - mode: "persistent", - cwd: options.cwd, - sessionOptions: { - model: options.profile.qualificationModel, - ...(options.systemInstructions - ? { systemPrompt: { append: options.systemInstructions } } - : {}), - }, - }), + const boundedHandshake = boundedSessionHandshake( + handshake, + dependencies.sessionHandshakeTimeoutMs ?? SESSION_HANDSHAKE_TIMEOUT_MS, ); - if (options.signal === undefined) { - handle = await handshake; - } else { - try { - handle = await raceRuntimeHandshakeWithAbort(handshake, options.signal); - } catch (error) { - if (options.signal.aborted) { - abortedHandshakeCleanup = handshake.then( - (lateHandle) => - admissionCleanup!.runRetained( - lateHandle, - "ACPX runtime admission aborted", - ), - () => - admissionCleanup!.runRetained( - failedHandshakeHandle, - "ACPX runtime admission aborted", - ), - ); - retainCleanup(abortedHandshakeCleanup); - } - throw error; - } - // The promise and abort notification can settle in the same turn. Do - // not admit a handle if cancellation won immediately afterward. - options.signal.throwIfAborted(); - } + handle = options.signal + ? await raceRuntimeHandshakeWithAbort(boundedHandshake, options.signal) + : await boundedHandshake; + // A provider can answer only after the verified sentinel is armed, but do + // not admit the session until the owner has observed that exact handoff. + await children.verifyLifetimeOwnership(); + // The handshake or lifetime-ownership observation can settle in the same + // turn as cancellation. Never admit that newly acquired authority. + options.signal?.throwIfAborted(); } catch (error) { + const aborted = options.signal?.aborted === true; + if (aborted || error instanceof AcpxSessionHandshakeTimeoutError) { + lateCleanup = lateHandshakeCleanup( + handshake, + admissionCleanup, + aborted + ? "ACPX runtime admission aborted" + : "ACPX session handshake completed after its admission deadline", + ); + retainCleanup(lateCleanup); + } const cleanupHandle = handle ?? failedHandshakeHandle; + const cleanupReason = aborted + ? "ACPX runtime admission aborted" + : "ACPX session handshake failed"; const cleanupErrors = await admissionCleanup.run( cleanupHandle, - options.signal?.aborted - ? "ACPX runtime admission aborted" - : "ACPX session handshake failed", + cleanupReason, ); const retainedCleanup = cleanupErrors.length === 0 ? Promise.resolve() - : admissionCleanup.runRetained( - cleanupHandle, - options.signal?.aborted - ? "ACPX runtime admission aborted" - : "ACPX session handshake failed", - ); + : admissionCleanup.runRetained(cleanupHandle, cleanupReason); const cleanupProof = - abortedHandshakeCleanup === null + lateCleanup === null ? retainedCleanup - : Promise.all([retainedCleanup, abortedHandshakeCleanup]).then( - () => undefined, - ); + : Promise.all([retainedCleanup, lateCleanup]).then(() => undefined); options.retainFailedAdmissionCleanup(cleanupProof); retainCleanup(cleanupProof); if (cleanupErrors.length > 0) { @@ -330,17 +362,12 @@ export async function openCodexAcpxRuntime( runtimeCloseTimeoutMs, ); } catch (error) { - const cleanupErrors = await admissionCleanup.run( - handle, - "ACPX runtime identity validation failed", - ); + const cleanupReason = "ACPX runtime identity validation failed"; + const cleanupErrors = await admissionCleanup.run(handle, cleanupReason); const cleanupProof = cleanupErrors.length === 0 ? Promise.resolve() - : admissionCleanup.runRetained( - handle, - "ACPX runtime identity validation failed", - ); + : admissionCleanup.runRetained(handle, cleanupReason); options.retainFailedAdmissionCleanup(cleanupProof); retainCleanup(cleanupProof); if (cleanupErrors.length > 0) { @@ -398,6 +425,7 @@ class RuntimeAdmissionCleanup { string, Promise >(); + readonly #handleAttemptCounts = new Map(); readonly #registeredTargets = new Map< string, RuntimeAdmissionCleanupTarget @@ -409,7 +437,6 @@ class RuntimeAdmissionCleanup { private readonly runtime: AcpRuntime, private readonly children: SpawnedChildSet, private readonly runtimeCloseTimeoutMs: number, - private readonly retainCleanup: (cleanup: Promise) => void, ) {} run(handle: AcpRuntimeHandle | null, reason: string): Promise { @@ -417,12 +444,9 @@ class RuntimeAdmissionCleanup { runtimeAdmissionCleanupTargetKey(handle), handle, ); - return this.#runAttempt(targetKey, handle, reason).then(({ errors }) => { - if (errors.length > 0) { - this.retainCleanup(this.runRetained(handle, reason)); - } - return errors; - }); + return this.#runAttempt(targetKey, handle, reason).then( + ({ errors }) => errors, + ); } runRetained(handle: AcpRuntimeHandle | null, reason: string): Promise { @@ -430,12 +454,13 @@ class RuntimeAdmissionCleanup { const targetKey = this.#resolveTargetKey(rawTargetKey, handle); const existing = this.#registeredTargets.get(targetKey); if (existing !== undefined) { - existing.handle = - existing.handle === null - ? handle - : handle === null - ? existing.handle - : preferRuntimeAdmissionCleanupHandle(existing.handle, handle); + if (existing.handle === null) existing.handle = handle; + else if (handle !== null) { + existing.handle = preferRuntimeAdmissionCleanupHandle( + existing.handle, + handle, + ); + } this.#targetAliases.set(rawTargetKey, targetKey); return existing.cleanup!; } @@ -492,24 +517,58 @@ class RuntimeAdmissionCleanup { targetKey: string, target: RuntimeAdmissionCleanupTarget, ): Promise { + let runtimeTerminalError: unknown | null = null; + let processErrors: unknown[] = []; let retryDelayMs = RETAINED_ADMISSION_CLEANUP_RETRY_MIN_MS; - // Retained cleanup is the continuing owner. Keep one runtime-close attempt - // in flight at a time and retry process-tree termination until both are - // confirmed complete; a finite budget would recreate an orphan boundary. for (;;) { const attempt = await this.#runAttempt( targetKey, - target.handle, + runtimeTerminalError === null ? target.handle : null, target.reason, ); - const runtimeNeedsRetry = - target.handle !== null && - attempt.runtimeError !== undefined && - !this.#closedHandles.has(targetKey); - const processNeedsRetry = attempt.processErrors.length > 0; - if (!runtimeNeedsRetry && !processNeedsRetry) { - return; + let runtimeError = attempt.runtimeError; + processErrors = attempt.processErrors; + if (attempt.pendingRuntimeClose !== undefined) { + // The configured timeout bounds the caller-facing pass, not the exact + // close promise. Give that exact promise one final bounded watch. A + // late rejection can then admit the next actual close attempt, while + // a second timeout terminalizes protocol cleanup without overlap. + const lateOutcome = await closeOutcomeWithin( + attempt.pendingRuntimeClose, + this.runtimeCloseTimeoutMs, + ); + if (lateOutcome instanceof AcpxRuntimeCloseTimeoutError) { + runtimeTerminalError = new AcpxRuntimeCloseFinalTimeoutError(); + runtimeError = runtimeTerminalError; + } else { + runtimeError = lateOutcome; + } } + if ( + runtimeTerminalError === null && + runtimeError !== undefined && + (this.#handleAttemptCounts.get(targetKey) ?? 0) >= + MAX_ADMISSION_CLEANUP_ATTEMPTS + ) { + runtimeTerminalError = new AggregateError( + [runtimeError], + `ACPX failed-admission cleanup exhausted ${MAX_ADMISSION_CLEANUP_RETRY_ATTEMPTS} retry attempts`, + ); + } + const runtimeNeedsRetry = + runtimeTerminalError === null && + runtimeError !== undefined && + !this.#closedHandles.has(targetKey); + const processNeedsRetry = processErrors.length > 0; + if (!runtimeNeedsRetry && !processNeedsRetry) { + if (runtimeTerminalError === null) return; + throw runtimeTerminalError; + } + // Runtime close retries are bounded above, but a live provider cannot + // be abandoned merely because its first termination passes failed. + // Keep this retained owner active with bounded backoff until process + // exit is observed. Once the process is gone, any terminal protocol + // cleanup error is still reported to the owner below. await delay(retryDelayMs); retryDelayMs = Math.min( retryDelayMs * 2, @@ -526,13 +585,21 @@ class RuntimeAdmissionCleanup { errors: unknown[]; runtimeError: unknown | undefined; processErrors: unknown[]; + pendingRuntimeClose?: Promise; }> { const cleanup = this.#tail.then(async () => { const errors: unknown[] = []; let runtimeError: unknown | undefined; + let pendingRuntimeClose: Promise | undefined; if (handle !== null && !this.#closedHandles.has(targetKey)) { - runtimeError = await this.#closeHandleWithin(targetKey, handle, reason); + const runtimeOutcome = await this.#closeHandleWithin( + targetKey, + handle, + reason, + ); + runtimeError = runtimeOutcome.error; if (runtimeError !== undefined) errors.push(runtimeError); + pendingRuntimeClose = runtimeOutcome.pendingAttempt; } const processErrors = await this.children.terminate(); errors.push(...processErrors); @@ -540,6 +607,7 @@ class RuntimeAdmissionCleanup { errors, runtimeError, processErrors, + ...(pendingRuntimeClose === undefined ? {} : { pendingRuntimeClose }), }; }); this.#tail = cleanup.then( @@ -553,9 +621,21 @@ class RuntimeAdmissionCleanup { targetKey: string, handle: AcpRuntimeHandle, reason: string, - ): Promise { + ): Promise<{ + error: unknown | undefined; + pendingAttempt?: Promise; + }> { let attempt = this.#activeHandleAttempts.get(targetKey); if (attempt === undefined) { + const attemptCount = this.#handleAttemptCounts.get(targetKey) ?? 0; + if (attemptCount >= MAX_ADMISSION_CLEANUP_ATTEMPTS) { + return { + error: new Error( + `ACPX failed-admission cleanup exhausted ${MAX_ADMISSION_CLEANUP_RETRY_ATTEMPTS} retry attempts`, + ), + }; + } + this.#handleAttemptCounts.set(targetKey, attemptCount + 1); attempt = runtimeCloseOutcome(this.runtime, { handle, reason, @@ -569,7 +649,10 @@ class RuntimeAdmissionCleanup { if (error === undefined) this.#closedHandles.add(targetKey); }); } - return await closeOutcomeWithin(attempt, this.runtimeCloseTimeoutMs); + const error = await closeOutcomeWithin(attempt, this.runtimeCloseTimeoutMs); + return error instanceof AcpxRuntimeCloseTimeoutError + ? { error, pendingAttempt: attempt } + : { error }; } } @@ -644,6 +727,38 @@ function nonEmptyRuntimeIdentity( return typeof value === "string" && value.length > 0 ? value : undefined; } +async function boundedSessionHandshake( + handshake: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + handshake, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new AcpxSessionHandshakeTimeoutError()), + timeoutMs, + ); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +function lateHandshakeCleanup( + handshake: Promise, + cleanup: RuntimeAdmissionCleanup, + reason: string, +): Promise { + return handshake.then( + (lateHandle) => cleanup.runRetained(lateHandle, reason), + () => undefined, + ); +} + function runtimePort( runtime: AcpRuntime, handle: AcpRuntimeHandle, @@ -901,9 +1016,7 @@ function runtimePort( mode: "prompt", requestId: input.requestId, ...(input.signal ? { signal: input.signal } : {}), - ...(input.onElicitation - ? { onElicitation: input.onElicitation } - : {}), + ...(input.onElicitation ? { onElicitation: input.onElicitation } : {}), }); }, close: closeRuntime, @@ -1005,177 +1118,319 @@ async function boundedCloseOutcome( } function delay(timeoutMs: number): Promise { - return new Promise((resolve) => { - const timer = setTimeout(resolve, timeoutMs); - timer.unref?.(); - }); + return new Promise((resolve) => setTimeout(resolve, timeoutMs)); +} + +type ProviderExitOutcome = + | { exited: true } + | { exited: false; error: unknown }; + +class ProviderExitObservation { + #outcome: ProviderExitOutcome | null = null; + readonly #observers = new Set<(outcome: ProviderExitOutcome) => void>(); + + constructor(providerExit: Promise) { + void providerExit.then( + () => this.#settle({ exited: true }), + (error: unknown) => this.#settle({ exited: false, error }), + ); + } + + observe(observer: (outcome: ProviderExitOutcome) => void): void { + if (this.#outcome) observer(this.#outcome); + else this.#observers.add(observer); + } + + async waitWithin( + timeoutMs: number, + ): Promise<{ exited: boolean; error?: unknown }> { + if (this.#outcome) return this.#outcome; + return await new Promise((resolve) => { + const finish = (outcome: ProviderExitOutcome | { exited: false }) => { + clearTimeout(timer); + this.#observers.delete(finish); + resolve(outcome); + }; + const timer = setTimeout(() => finish({ exited: false }), timeoutMs); + timer.unref(); + this.#observers.add(finish); + if (this.#outcome) finish(this.#outcome); + }); + } + + #settle(outcome: ProviderExitOutcome): void { + if (this.#outcome) return; + this.#outcome = outcome; + for (const observer of this.#observers) observer(outcome); + this.#observers.clear(); + } } class SpawnedChildSet { - readonly #children = new Map(); + readonly #children = new Set(); readonly #errors = new Set(); + readonly #providerExits = new Map(); + readonly #terminations = new Map>(); + readonly #lifetimeOwnership: Promise[] = []; + #lifetimeOwnershipSealed = false; + #sealed = false; - add(child: ChildProcess, processGroup: boolean): ChildProcess { + constructor( + private readonly retainCleanup?: (cleanup: Promise) => void, + private readonly awaitProviderOwnership: ( + child: ChildProcess, + ) => Promise = awaitVerifiedAcpxProviderOwnership, + private readonly awaitProviderExit: ( + child: ChildProcess, + ) => Promise = awaitVerifiedAcpxProviderExit, + ) {} + + add(child: ChildProcess): ChildProcess { + let exitProof: Promise; + try { + exitProof = this.awaitProviderExit(child); + } catch (error) { + exitProof = Promise.reject(error); + } + const providerExit = new ProviderExitObservation(exitProof); + this.#providerExits.set(child, providerExit); + this.#track(child, providerExit); + const ownership = this.awaitProviderOwnership(child); + void ownership.catch(() => undefined); + this.#lifetimeOwnership.push(ownership); + if (this.#sealed || this.#lifetimeOwnershipSealed) { + // Once the stable-empty cleanup point is sealed, ACPX no longer has + // authority to create provider work. Retain an immediate-kill attempt + // through exit verification before rejecting the spawn itself. + const termination = this.#startTermination(child, true); + const cleanup = termination.then((errors) => { + if (errors.length > 0) { + throw new AggregateError( + errors, + "ACPX post-seal provider cleanup failed", + ); + } + }); + this.retainCleanup?.(cleanup); + void cleanup.catch(() => undefined); + throw new Error( + this.#sealed + ? "ACPX provider spawned after cleanup was sealed" + : "ACPX provider spawned after ownership admission was sealed", + ); + } + return child; + } + + async verifyLifetimeOwnership(): Promise { + for (;;) { + const ownership = this.#lifetimeOwnership.splice(0); + if (ownership.length === 0) { + // This check and seal are synchronous. Any spawn added while an + // earlier batch was pending is observed by the next loop iteration; + // no later provider can race admission after the stable-empty point. + this.#lifetimeOwnershipSealed = true; + return; + } + await Promise.all(ownership); + } + } + + #track(child: ChildProcess, providerExit: ProviderExitObservation): void { + this.#children.add(child); const onError = (error: unknown) => this.#errors.add(error); - const tracked: SpawnedProviderProcess = { - child, - processGroupId: processGroup ? (child.pid ?? null) : null, - onError, + let guardianExited = !running(child); + let providerExited = false; + const forgetIfReleased = () => { + if (!guardianExited || !providerExited) return; + this.#children.delete(child); + this.#providerExits.delete(child); + child.off("error", onError); + child.off("exit", onGuardianExit); + child.off("close", onGuardianExit); }; - this.#children.set(child, tracked); - const forgetExitedTree = () => { - if (!providerTreeRunning(tracked)) this.#forget(tracked); + const onGuardianExit = () => { + guardianExited = true; + forgetIfReleased(); }; // ChildProcess reports some spawn and signal-delivery failures through an // asynchronous `error` event. Observe those for the child's whole tracked // lifetime so cleanup can report them instead of crashing runnerd. child.on("error", onError); - child.once("exit", forgetExitedTree); - child.once("close", forgetExitedTree); - return child; + child.once("exit", onGuardianExit); + child.once("close", onGuardianExit); + providerExit.observe((outcome) => { + if (outcome.exited) { + providerExited = true; + forgetIfReleased(); + } else { + this.#errors.add(outcome.error); + } + }); } async terminate(): Promise { - const errors: unknown[] = []; - const children = [...this.#children.values()]; - await Promise.all( - children.map(async (tracked) => { - if (providerTreeRunning(tracked)) { - const terminateOutcome = await signalAndWaitForExit( - tracked, - "SIGTERM", - PROVIDER_TERM_EXIT_TIMEOUT_MS, - ); - if (terminateOutcome.error !== undefined) { - pushUnique(errors, terminateOutcome.error); - } - if (!terminateOutcome.exited && providerTreeRunning(tracked)) { - const killOutcome = await signalAndWaitForExit( - tracked, - "SIGKILL", - PROVIDER_KILL_EXIT_TIMEOUT_MS, - ); - if (killOutcome.error !== undefined) { - pushUnique(errors, killOutcome.error); - } - if (!killOutcome.exited && providerTreeRunning(tracked)) { - errors.push( - new Error("ACPX provider did not exit after SIGKILL"), - ); - } - } - } - if (!providerTreeRunning(tracked)) this.#forget(tracked); - }), - ); + // Revoke spawn authority synchronously before the first await. Children + // already owned here receive the normal TERM/KILL sequence; every later + // spawn is rejected and its independently retained post-seal cleanup + // cannot extend this caller-facing shutdown without bound. + this.#sealed = true; + for (const child of this.#children) this.#startTermination(child); + const ownedTerminations = [...this.#terminations.values()]; + const errors = (await Promise.all(ownedTerminations)).flat(); // A failed spawn or signal can emit `error` and then `close` before this // method snapshots the live children. Keep those errors independently of - // child membership, report each object once, and drain them only after all - // in-flight termination attempts have had a chance to emit. + // child membership and report each object once after all owned attempts. for (const error of this.#errors) pushUnique(errors, error); this.#errors.clear(); return errors; } - #forget(tracked: SpawnedProviderProcess): void { - if (this.#children.get(tracked.child) !== tracked) return; - this.#children.delete(tracked.child); - tracked.child.off("error", tracked.onError); + #startTermination( + child: ChildProcess, + immediateKill = false, + ): Promise { + const existing = this.#terminations.get(child); + if (existing) return existing; + const providerExit = + this.#providerExits.get(child) ?? + new ProviderExitObservation( + Promise.reject(new Error("ACPX provider exit proof is unavailable")), + ); + const termination = ( + immediateKill + ? terminatePostSealChild(child, providerExit) + : terminateChild(child, providerExit) + ).catch((error: unknown) => [error]); + this.#terminations.set(child, termination); + termination.then(() => { + if (this.#terminations.get(child) === termination) { + this.#terminations.delete(child); + } + }); + return termination; } } -interface SpawnedProviderProcess { - child: ChildProcess; - processGroupId: number | null; - onError: (error: unknown) => void; +async function terminatePostSealChild( + child: ChildProcess, + providerExit: ProviderExitObservation, +): Promise { + const errors: unknown[] = []; + // Verified production children override ChildProcess.kill so this SIGKILL + // request revokes the owner pipe and wakes the live guardian, which retains + // authority to reap the whole group. Never copy the numeric PGID into a + // later signal owner. + const killOutcome = await signalAndWaitForVerifiedProviderExit( + child, + "SIGKILL", + PROVIDER_KILL_EXIT_TIMEOUT_MS, + providerExit, + ); + for (const error of killOutcome.errors) pushUnique(errors, error); + if (!killOutcome.exited) { + errors.push( + new Error("ACPX post-seal provider did not exit after SIGKILL"), + ); + } + return errors; +} + +async function terminateChild( + child: ChildProcess, + providerExit: ProviderExitObservation, +): Promise { + const errors: unknown[] = []; + const terminateOutcome = await signalAndWaitForVerifiedProviderExit( + child, + "SIGTERM", + PROVIDER_TERM_EXIT_TIMEOUT_MS, + providerExit, + ); + for (const error of terminateOutcome.errors) pushUnique(errors, error); + if (!terminateOutcome.exited) { + errors.push(new Error("ACPX provider did not exit after SIGTERM")); + // A live verified guardian still pins the PGID. Its protected `kill` + // override revokes the owner pipe and wakes it to reap the group. If the + // guardian already exited, do not signal a saved identifier; retain local + // cleanup while waiting for the provider-only descriptor to reach EOF. + const killOutcome = await signalAndWaitForVerifiedProviderExit( + child, + "SIGKILL", + PROVIDER_KILL_EXIT_TIMEOUT_MS, + providerExit, + ); + for (const error of killOutcome.errors) pushUnique(errors, error); + if (!killOutcome.exited) { + errors.push(new Error("ACPX provider did not exit after SIGKILL")); + } + } + // Never unref a child whose guardian exit and provider-only EOF were not + // both observed. Local cleanup retains it instead of transferring a reusable + // PGID or releasing credential ownership early. + return errors; } function running(child: ChildProcess): boolean { return child.exitCode === null && child.signalCode === null; } -function providerTreeRunning(tracked: SpawnedProviderProcess): boolean { - if (tracked.processGroupId === null) return running(tracked.child); - try { - process.kill(-tracked.processGroupId, 0); - return true; - } catch (error) { - return errorCode(error) !== "ESRCH"; - } +async function signalAndWaitForVerifiedProviderExit( + child: ChildProcess, + signal: NodeJS.Signals, + timeoutMs: number, + providerExit: ProviderExitObservation, +): Promise<{ exited: boolean; errors: unknown[] }> { + const [guardian, provider] = await Promise.all([ + signalAndWaitForExit(child, signal, timeoutMs), + providerExit.waitWithin(timeoutMs), + ]); + const errors: unknown[] = []; + if (guardian.error !== undefined) pushUnique(errors, guardian.error); + if (provider.error !== undefined) pushUnique(errors, provider.error); + return { + exited: guardian.exited && provider.exited, + errors, + }; } async function signalAndWaitForExit( - tracked: SpawnedProviderProcess, + child: ChildProcess, signal: NodeJS.Signals, timeoutMs: number, ): Promise<{ exited: boolean; error?: unknown }> { - if (!providerTreeRunning(tracked)) return { exited: true }; - const { child } = tracked; + if (!running(child)) return { exited: true }; return await new Promise<{ exited: boolean; error?: unknown }>((resolve) => { let settled = false; const finish = (outcome: { exited: boolean; error?: unknown }) => { if (settled) return; settled = true; clearTimeout(timer); - if (poll !== undefined) clearInterval(poll); child.off("exit", onExit); child.off("close", onExit); child.off("error", onError); resolve(outcome); }; - const onExit = () => { - if (!providerTreeRunning(tracked)) finish({ exited: true }); - }; + const onExit = () => finish({ exited: true }); const onError = (error: unknown) => finish({ exited: false, error }); - const timer = setTimeout( - () => finish({ exited: !providerTreeRunning(tracked) }), - timeoutMs, - ); + const timer = setTimeout(() => finish({ exited: false }), timeoutMs); timer.unref(); - const poll = - tracked.processGroupId === null - ? undefined - : setInterval(() => { - if (!providerTreeRunning(tracked)) finish({ exited: true }); - }, 25); - poll?.unref(); child.once("exit", onExit); child.once("close", onExit); child.once("error", onError); - if (!providerTreeRunning(tracked)) { + if (!running(child)) { finish({ exited: true }); return; } try { - if (tracked.processGroupId === null) { - if (!child.kill(signal) && providerTreeRunning(tracked)) { - finish({ - exited: false, - error: new Error(`ACPX provider rejected ${signal}`), - }); - return; - } - } else { - process.kill(-tracked.processGroupId, signal); - } - if (!providerTreeRunning(tracked)) finish({ exited: true }); + child.kill(signal); + if (!running(child)) finish({ exited: true }); } catch (error) { - if (errorCode(error) === "ESRCH" && !providerTreeRunning(tracked)) { - finish({ exited: true }); - } else { - finish({ exited: false, error }); - } + finish({ exited: false, error }); } }); } -function errorCode(error: unknown): string | undefined { - if (typeof error !== "object" || error === null || !("code" in error)) { - return undefined; - } - return typeof error.code === "string" ? error.code : undefined; -} - function pushUnique(errors: unknown[], error: unknown): void { if (!errors.includes(error)) errors.push(error); } diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts index e887139139..1dab74e06a 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts @@ -1,11 +1,12 @@ import { createHash } from "node:crypto"; -import type { ChildProcess } from "node:child_process"; +import { fork, type ChildProcess } from "node:child_process"; import { once } from "node:events"; import { chmod, link, mkdir, mkdtemp, + readFile, realpath, rename, rm, @@ -13,6 +14,7 @@ import { stat, writeFile, } from "node:fs/promises"; +import { createServer, type Server } from "node:net"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -20,14 +22,19 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { resolveQualifiedAcpxProfile } from "./qualified-profiles.js"; import { + awaitVerifiedAcpxProviderExit, + awaitVerifiedAcpxProviderOwnership, guardSnapshotModuleLookup, guardSnapshotModuleResolution, + reapCurrentProviderProcessGroup, sanitizedNodeEnvironment, snapshotDescriptorAncestorIndex, snapshotDescriptorResolution, verifiedExecutableOpenFlags, verifyQualifiedAcpxInstallation, + type VerifiedAcpxProviderLifetime, } from "./installation-integrity.js"; +import { stageManagedCodexCredential } from "./codex-credentials.js"; const temporaryDirectories: string[] = []; const descriptorCommandPath = "/proc/self/fd/4/server.js"; @@ -41,6 +48,57 @@ afterEach(async () => { }); describe("ACPX installation integrity", () => { + it("rejects an unregistered provider exit proof", async () => { + await expect( + awaitVerifiedAcpxProviderExit({} as ChildProcess), + ).rejects.toThrow("provider exit proof is unavailable"); + }); + + it("never signals a dead guardian's saved process-group identity", () => { + const signalCurrentGroup = vi.fn( + (_pid: number, _signal: NodeJS.Signals) => true, + ); + reapCurrentProviderProcessGroup( + signalCurrentGroup, + 4_321, + vi.fn((_code: number) => undefined), + ); + expect(signalCurrentGroup).toHaveBeenCalledOnce(); + expect(signalCurrentGroup).toHaveBeenCalledWith(0, "SIGKILL"); + + const signalSelfAfterGroupFailure = vi.fn( + (pid: number, _signal: NodeJS.Signals) => { + if (pid === 0) throw new Error("group signal unavailable"); + }, + ); + const exit = vi.fn((_code: number) => undefined); + reapCurrentProviderProcessGroup(signalSelfAfterGroupFailure, 4_321, exit); + expect(signalSelfAfterGroupFailure.mock.calls).toEqual([ + [0, "SIGKILL"], + [4_321, "SIGKILL"], + ]); + expect(exit).not.toHaveBeenCalled(); + + const failedSignals = vi.fn((_pid: number, _signal: NodeJS.Signals) => { + throw new Error("signal unavailable"); + }); + reapCurrentProviderProcessGroup(failedSignals, 4_321, exit); + expect(failedSignals.mock.calls).toEqual([ + [0, "SIGKILL"], + [4_321, "SIGKILL"], + ]); + expect(exit).toHaveBeenCalledWith(1); + expect( + [ + ...signalCurrentGroup.mock.calls, + ...signalSelfAfterGroupFailure.mock.calls, + ...failedSignals.mock.calls, + ] + .map(([pid]) => pid) + .filter((pid) => pid < 0), + ).toEqual([]); + }); + it("does not delegate non-Linux snapshot filesystem lookups", () => { for (const platform of ["darwin", "freebsd", "win32"] as const) { const nextResolve = vi.fn(() => ({ url: "file:///attacker.js" })); @@ -1115,6 +1173,370 @@ describe("ACPX installation integrity", () => { await expectFailure(child, "requires Linux descriptor-pinned paths"); } }); + it.runIf(process.platform !== "win32")( + "rejects incomplete or duplicate provider credential quorum descriptors", + async () => { + const fixture = await persistentInstallationFixture(); + const installation = await verifyQualifiedAcpxInstallation( + fixture.profile, + fixture.resolve, + ); + const invalidLifetimes = [ + { + credentialFenceFds: [42], + activateCredentialFenceOwner: async () => undefined, + }, + { + credentialFenceFds: [42, 42], + activateCredentialFenceOwner: async () => undefined, + }, + { + credentialFenceFds: [42, -1], + activateCredentialFenceOwner: async () => undefined, + }, + { + credentialFenceFds: [42, 43], + }, + ] as unknown as readonly VerifiedAcpxProviderLifetime[]; + + for (const lifetime of invalidLifetimes) { + const command = await installation.openCommand(); + expect(() => command.spawn([], {}, lifetime)).toThrow( + "ACPX provider credential fence is invalid", + ); + } + }, + ); + + it.runIf(process.platform === "linux")( + "keeps the staged credential fenced through owner SIGKILL and reaps the provider group", + async () => { + const fixture = await persistentInstallationFixture(); + const ownerScript = join(fixture.root, "provider-owner.mjs"); + const pidFile = join(fixture.root, "provider.pid"); + const credentialHome = join(fixture.root, "codex-home"); + await mkdir(credentialHome, { mode: 0o700 }); + const moduleUrl = new URL("./installation-integrity.ts", import.meta.url) + .href; + const credentialModuleUrl = new URL( + "./codex-credentials.ts", + import.meta.url, + ).href; + await writeFile( + ownerScript, + [ + `const module = await import(${JSON.stringify(moduleUrl)});`, + `const credentials = await import(${JSON.stringify(credentialModuleUrl)});`, + `const profile = ${JSON.stringify(fixture.profile)};`, + `const credential = await credentials.stageManagedCodexCredential({ agentHomeDirectory: ${JSON.stringify(credentialHome)}, environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"original"}' } });`, + `const paths = new Map(${JSON.stringify([...fixture.paths])});`, + "const installation = await module.verifyQualifiedAcpxInstallation(profile, (name) => paths.get(name));", + "const lease = await installation.openCommand();", + `const provider = lease.spawn([], { env: { ...process.env, PAPERCLIP_PROVIDER_PID_FILE: ${JSON.stringify(pidFile)} } }, { credentialFenceFds: credential.lifetimeFenceFds, activateCredentialFenceOwner: (pid) => credential.activateLifetimeOwner(pid) });`, + "await module.awaitVerifiedAcpxProviderOwnership(provider);", + 'process.send?.({ type: "ready", guardianPid: provider.pid });', + "process.stdin.resume();", + ].join("\n"), + ); + + const owner = fork(ownerScript, [], { + execArgv: ["--import", "tsx"], + stdio: ["pipe", "ignore", "pipe", "ipc"], + }); + let guardianPid = 0; + let providerPid = 0; + try { + const ready = (await childMessage(owner, "ready")) as { + guardianPid: number; + }; + guardianPid = ready.guardianPid; + providerPid = Number.parseInt(await waitForFile(pidFile), 10); + expect(processAlive(providerPid)).toBe(true); + + process.kill(guardianPid, "SIGSTOP"); + try { + owner.kill("SIGKILL"); + await once(owner, "exit"); + // The stopped sentinel cannot answer any application protocol. Its + // two inherited quorum listeners nevertheless prevent a second owner. + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + expect(processAlive(providerPid)).toBe(true); + } finally { + if (owner.exitCode === null && owner.signalCode === null) { + owner.kill("SIGKILL"); + await once(owner, "exit").catch(() => undefined); + } + // SIGSTOP pins this exact live guardian PID against reuse until the + // matching resume. Owner-pipe EOF then makes it self-reap its group. + process.kill(guardianPid, "SIGCONT"); + await waitUntil(() => !processAlive(providerPid)); + } + let contender: Awaited< + ReturnType + > | null = null; + await waitUntilAsync(async () => { + try { + contender = await stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }); + return true; + } catch { + return false; + } + }); + await contender!.close(); + } finally { + if (owner.exitCode === null && owner.signalCode === null) { + // This direct child handle owns the guardian pipe. Closing it lets + // the live guardian reap only its own still-pinned group; never + // signal a saved guardian PGID from cleanup. + owner.kill("SIGKILL"); + await once(owner, "exit").catch(() => undefined); + } + } + }, + ); + + it.runIf(process.platform === "linux")( + "reaps a fenced provider when its lifetime guardian is SIGKILLed", + async () => { + const fixture = await persistentInstallationFixture(); + const ownerScript = join(fixture.root, "guardian-owner.mjs"); + const pidFile = join(fixture.root, "guardian-provider.pid"); + const credentialHome = join(fixture.root, "guardian-codex-home"); + await mkdir(credentialHome, { mode: 0o700 }); + const moduleUrl = new URL("./installation-integrity.ts", import.meta.url) + .href; + const credentialModuleUrl = new URL( + "./codex-credentials.ts", + import.meta.url, + ).href; + await writeFile( + ownerScript, + [ + `const module = await import(${JSON.stringify(moduleUrl)});`, + `const credentials = await import(${JSON.stringify(credentialModuleUrl)});`, + `const profile = ${JSON.stringify(fixture.profile)};`, + `const credential = await credentials.stageManagedCodexCredential({ agentHomeDirectory: ${JSON.stringify(credentialHome)}, environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"original"}' } });`, + `const paths = new Map(${JSON.stringify([...fixture.paths])});`, + "const installation = await module.verifyQualifiedAcpxInstallation(profile, (name) => paths.get(name));", + "const lease = await installation.openCommand();", + `const provider = lease.spawn([], { env: { ...process.env, PAPERCLIP_PROVIDER_PID_FILE: ${JSON.stringify(pidFile)} } }, { credentialFenceFds: credential.lifetimeFenceFds, activateCredentialFenceOwner: (pid) => credential.activateLifetimeOwner(pid) });`, + "await module.awaitVerifiedAcpxProviderOwnership(provider);", + 'process.send?.({ type: "ready", guardianPid: provider.pid });', + "process.stdin.resume();", + ].join("\n"), + ); + + const owner = fork(ownerScript, [], { + execArgv: ["--import", "tsx"], + stdio: ["pipe", "ignore", "pipe", "ipc"], + }); + let guardianPid = 0; + let providerPid = 0; + try { + const ready = (await childMessage(owner, "ready")) as { + guardianPid: number; + }; + guardianPid = ready.guardianPid; + providerPid = Number.parseInt(await waitForFile(pidFile), 10); + expect(processAlive(providerPid)).toBe(true); + + // Freeze the provider so it cannot process guardian-pipe EOF itself. + // The armed credential-free peer must still reap the current group. + process.kill(providerPid, "SIGSTOP"); + await waitUntilAsync(() => processStopped(providerPid)); + process.kill(guardianPid, "SIGKILL"); + owner.kill("SIGKILL"); + await once(owner, "exit"); + await waitUntil(() => !processAlive(providerPid)); + + const contender = await stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }); + await contender.close(); + } finally { + if (owner.exitCode === null && owner.signalCode === null) { + owner.kill("SIGKILL"); + await once(owner, "exit").catch(() => undefined); + } + if (providerPid > 0 && processAlive(providerPid)) { + // Failure cleanup only: allow the provider's own guardian-loss + // callback to reap its still-pinned group if the watchdog regressed. + process.kill(providerPid, "SIGCONT"); + await waitUntil(() => !processAlive(providerPid)); + } + } + }, + ); + + it.runIf(process.platform === "linux")( + "reaps a stopped provider after an external guardian kill", + async () => { + const fixture = await persistentInstallationFixture(); + const pidFile = join(fixture.root, "provider-exit-proof.pid"); + const fences = await Promise.all([ + listenOnLoopback(), + listenOnLoopback(), + ]); + const fenceFds = fences.map( + (fence) => + (fence as Server & { _handle?: { fd?: number } })._handle?.fd, + ); + expect(fenceFds.every(Number.isSafeInteger)).toBe(true); + const installation = await verifyQualifiedAcpxInstallation( + fixture.profile, + fixture.resolve, + ); + const guardian = (await installation.openCommand()).spawn( + [], + { env: { ...process.env, PAPERCLIP_PROVIDER_PID_FILE: pidFile } }, + { + credentialFenceFds: [fenceFds[0]!, fenceFds[1]!], + activateCredentialFenceOwner: async () => undefined, + }, + ); + await awaitVerifiedAcpxProviderOwnership(guardian); + const providerExit = awaitVerifiedAcpxProviderExit(guardian); + const providerPid = Number.parseInt(await waitForFile(pidFile), 10); + const guardianExit = once(guardian, "exit"); + process.kill(providerPid, "SIGSTOP"); + await waitUntilAsync(() => processStopped(providerPid)); + + try { + // Bypass the protected cleanup method to model SIGKILL/OOM of the + // guardian itself. Its credential-free peer must reap the stopped + // provider without waiting for provider JavaScript to run. + process.kill(guardian.pid!, "SIGKILL"); + await guardianExit; + await providerExit; + await waitUntil(() => !processAlive(providerPid)); + } finally { + if (guardian.exitCode === null && guardian.signalCode === null) { + guardian.kill("SIGKILL"); + await guardianExit.catch(() => undefined); + } + if (processAlive(providerPid)) { + process.kill(providerPid, "SIGCONT"); + await waitUntil(() => !processAlive(providerPid)); + } + await Promise.all(fences.map(closeServer)); + } + }, + ); + + it.runIf(process.platform === "linux")( + "dismisses the lifetime sentinel only after normal provider-group cleanup", + async () => { + const fixture = await persistentInstallationFixture(); + const pidFile = join(fixture.root, "normal-provider.pid"); + const fences = await Promise.all([ + listenOnLoopback(), + listenOnLoopback(), + ]); + const fenceFds = fences.map( + (fence) => + (fence as Server & { _handle?: { fd?: number } })._handle?.fd, + ); + expect(fenceFds.every(Number.isSafeInteger)).toBe(true); + expect(fenceFds[0]).not.toBe(fenceFds[1]); + const installation = await verifyQualifiedAcpxInstallation( + fixture.profile, + fixture.resolve, + ); + const provider = (await installation.openCommand()).spawn( + [], + { env: { ...process.env, PAPERCLIP_PROVIDER_PID_FILE: pidFile } }, + { + credentialFenceFds: [fenceFds[0]!, fenceFds[1]!], + activateCredentialFenceOwner: async () => undefined, + }, + ); + await awaitVerifiedAcpxProviderOwnership(provider); + const providerPid = Number.parseInt(await waitForFile(pidFile), 10); + const ports = fences.map( + (fence) => (fence.address() as { port: number }).port, + ); + provider.kill("SIGTERM"); + await Promise.all(fences.map(closeServer)); + await once(provider, "exit"); + await waitUntil(() => !processAlive(providerPid)); + await Promise.all( + ports.map((port) => + expect(canBindLoopbackPort(port)).resolves.toBe(true), + ), + ); + }, + ); + + it.runIf(process.platform === "linux")( + "reaps a stopped provider across repeated guardian cleanup requests", + async () => { + const fixture = await persistentInstallationFixture(); + const pidFile = join(fixture.root, "emergency-provider.pid"); + const fences = await Promise.all([ + listenOnLoopback(), + listenOnLoopback(), + ]); + const fenceFds = fences.map( + (fence) => + (fence as Server & { _handle?: { fd?: number } })._handle?.fd, + ); + expect(fenceFds.every(Number.isSafeInteger)).toBe(true); + expect(fenceFds[0]).not.toBe(fenceFds[1]); + const installation = await verifyQualifiedAcpxInstallation( + fixture.profile, + fixture.resolve, + ); + const guardian = (await installation.openCommand()).spawn( + [], + { env: { ...process.env, PAPERCLIP_PROVIDER_PID_FILE: pidFile } }, + { + credentialFenceFds: [fenceFds[0]!, fenceFds[1]!], + activateCredentialFenceOwner: async () => undefined, + }, + ); + await awaitVerifiedAcpxProviderOwnership(guardian); + const providerPid = Number.parseInt(await waitForFile(pidFile), 10); + const guardianExit = once(guardian, "exit"); + process.kill(providerPid, "SIGSTOP"); + process.kill(guardian.pid!, "SIGSTOP"); + try { + expect(guardian.kill("SIGKILL")).toBe(true); + // Retry synchronously while the resumed guardian has not yet processed + // owner-pipe EOF. Each retry wakes the exact guardian; the guardian + // remains alive to reap the whole provider group itself. + expect(guardian.kill("SIGKILL")).toBe(true); + await guardianExit; + await waitUntil(() => !processAlive(providerPid)); + } finally { + if (processAlive(providerPid)) { + // The stopped provider still pins this exact PID. Resume it only for + // failure cleanup so guardian-pipe EOF can make it self-reap. + process.kill(providerPid, "SIGCONT"); + } + if (guardian.exitCode === null && guardian.signalCode === null) { + process.kill(guardian.pid!, "SIGCONT"); + guardian.kill("SIGKILL"); + await guardianExit.catch(() => undefined); + } + await Promise.all(fences.map(closeServer)); + } + }, + ); }); async function expectOutput( @@ -1161,6 +1583,129 @@ async function expectFailure( expect(stderr).toContain(expected); } +async function persistentInstallationFixture() { + const fixture = await installationFixture(); + const command = [ + "#!/usr/bin/env node", + 'const fs = require("node:fs");', + "fs.writeFileSync(process.env.PAPERCLIP_PROVIDER_PID_FILE, String(process.pid));", + "setInterval(() => undefined, 1_000);", + ].join("\n"); + await writeFile(fixture.commandPath, command); + return { + ...fixture, + command, + profile: { + ...fixture.profile, + commandDigest: `sha256:${createHash("sha256").update(command).digest("hex")}`, + }, + }; +} + +async function childMessage( + child: ChildProcess, + type: string, +): Promise> { + return await new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`Timed out waiting for child message ${type}`)), + 5_000, + ); + const onExit = (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timer); + reject(new Error(`Child exited before ${type}: ${code ?? signal}`)); + }; + child.once("exit", onExit); + child.on("message", (message) => { + if ( + typeof message !== "object" || + message === null || + (message as { type?: unknown }).type !== type + ) + return; + clearTimeout(timer); + child.off("exit", onExit); + resolve(message as Record); + }); + }); +} + +async function waitForFile(path: string): Promise { + let value = ""; + await waitUntilAsync(async () => { + try { + value = await readFile(path, "utf8"); + return value.length > 0; + } catch { + return false; + } + }); + return value; +} + +function processAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +async function processStopped(pid: number): Promise { + try { + const status = await readFile(`/proc/${pid}/status`, "utf8"); + return /^State:\s+T/m.test(status); + } catch { + return false; + } +} + +async function listenOnLoopback(port = 0): Promise { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen( + { host: "127.0.0.1", port, exclusive: true, reusePort: false }, + resolve, + ); + }); + return server; +} + +async function closeServer(server: Server): Promise { + if (!server.listening) return; + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); +} + +async function canBindLoopbackPort(port: number): Promise { + try { + const server = await listenOnLoopback(port); + await closeServer(server); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "EADDRINUSE") return false; + throw error; + } +} + +async function waitUntil(predicate: () => boolean): Promise { + await waitUntilAsync(async () => predicate()); +} + +async function waitUntilAsync( + predicate: () => Promise, +): Promise { + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if (await predicate()) return; + await new Promise((resolve) => setTimeout(resolve, 20)); + } + throw new Error("Timed out waiting for subprocess state"); +} + async function installationFixture() { const root = await mkdtemp(join(tmpdir(), "paperclip-acpx-installation-")); temporaryDirectories.push(root); @@ -1206,6 +1751,7 @@ async function installationFixture() { runtimeDirectory, serverPackageJsonPath, runtimePackageJsonPath, + paths, resolve(packageName: string): string { const resolved = paths.get(packageName); if (!resolved) throw new Error(`unexpected package ${packageName}`); diff --git a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts index e49706e776..e628af617d 100644 --- a/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts +++ b/packages/paperclip-runner/src/drivers/acpx/installation-integrity.ts @@ -22,7 +22,7 @@ import { relative, resolve, } from "node:path"; -import type { Writable } from "node:stream"; +import type { Readable, Writable } from "node:stream"; import type { QualifiedAcpxProfile } from "./qualified-profiles.js"; @@ -32,6 +32,169 @@ const COMMAND_SOURCE_FD = 3; const COMMAND_DIRECTORY_FD = 4; const DEPENDENCY_ANCESTOR_FD_START = 5; const MAX_DEPENDENCY_ANCESTORS = 64; +const PROVIDER_WATCHDOG_HANDSHAKE_TIMEOUT_MS = 2_000; +const PROVIDER_GUARDIAN_HANDSHAKE_TIMEOUT_MS = 5_000; + +const PROVIDER_LIFETIME_WATCHDOG_SOURCE = ` +const fs = require("node:fs"); +let reaped = false; +const reap = () => { + if (reaped) return; + reaped = true; + try { + // Resolve the watchdog's current group at signal-delivery time. The live + // watchdog itself pins that identity until this atomic reap. + process.kill(0, "SIGKILL"); + } catch { + try { + process.kill(process.pid, "SIGKILL"); + } catch { + process.exit(1); + } + } +}; +const owner = fs.createReadStream("", { fd: 3, autoClose: false }); +owner.once("end", reap); +owner.once("error", reap); +owner.resume(); +try { + fs.writeSync(4, "armed\\n"); +} catch { + reap(); +} +`; + +export const PROVIDER_LIFETIME_GUARDIAN_SOURCE = ` +const fs = require("node:fs"); +const { spawn } = require("node:child_process"); +const WATCHDOG_SOURCE = ${JSON.stringify(PROVIDER_LIFETIME_WATCHDOG_SOURCE)}; +const dependencyAncestorCount = Number.parseInt(process.argv[4], 10); +if (!Number.isSafeInteger(dependencyAncestorCount) || dependencyAncestorCount < 0 || dependencyAncestorCount > ${MAX_DEPENDENCY_ANCESTORS}) throw new Error("ACPX provider dependency ancestry is invalid"); +const OWNER_FD = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount; +const OWNERSHIP_FD = OWNER_FD + 1; +const PROVIDER_EXIT_FD = OWNERSHIP_FD + 1; +const CREDENTIAL_FENCE_FD_START = PROVIDER_EXIT_FD + 1; +const dependencyAncestorFds = Array.from({ length: dependencyAncestorCount }, (_, index) => ${DEPENDENCY_ANCESTOR_FD_START} + index); +const PROVIDER_GUARDIAN_FD = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount; +let provider; +let watchdog; +let reaped = false; +let shutdownStarted = false; +const reap = () => { + if (reaped) return; + reaped = true; + // This sentinel is the provider group's leader. It remains alive until this + // one atomic signal, pinning the numeric group identity against PID reuse. + process.kill(-process.pid, "SIGKILL"); +}; +const owner = fs.createReadStream("", { fd: OWNER_FD, autoClose: false }); +owner.once("end", reap); +owner.once("error", reap); +owner.resume(); +// Fail before provider code exists unless both inherited quorum fences are live. +fs.fstatSync(CREDENTIAL_FENCE_FD_START); +fs.fstatSync(CREDENTIAL_FENCE_FD_START + 1); +const shutdown = () => { + if (shutdownStarted || reaped) return; + shutdownStarted = true; + try { + provider?.kill("SIGTERM"); + } catch { + reap(); + return; + } + setTimeout(reap, 1_000); +}; +process.on("SIGTERM", shutdown); +process.on("SIGINT", shutdown); +process.on("SIGHUP", shutdown); +const startProvider = () => { + if (provider || reaped || shutdownStarted) return; + try { + provider = spawn( + process.execPath, + ["--eval", process.argv[1], ...process.argv.slice(2)], + { + cwd: process.cwd(), + detached: false, + env: process.env, + shell: false, + // The provider observes this guardian-owned pipe directly. Kernel EOF + // therefore revokes it even when SIGKILL/OOM prevents our JS reap path. + // It also inherits both quorum fences until that self-reap completes. + stdio: [0, 1, 2, ${COMMAND_SOURCE_FD}, ${COMMAND_DIRECTORY_FD}, ...dependencyAncestorFds, "pipe", PROVIDER_EXIT_FD, CREDENTIAL_FENCE_FD_START, CREDENTIAL_FENCE_FD_START + 1], + windowsHide: true, + }, + ); + provider.once("error", reap); + provider.once("exit", reap); + provider.once("spawn", () => { + try { + if (reaped || shutdownStarted) { + reap(); + return; + } + // The provider now owns the only child-side copy. Parent-side EOF is an + // independent kernel observation of provider exit even if this guardian + // is killed before it can reap the group. + fs.closeSync(PROVIDER_EXIT_FD); + fs.writeSync(OWNERSHIP_FD, "owned\\n"); + } catch { + reap(); + } + }); + } catch { + reap(); + } +}; +try { + // A credential-free peer in this same process group reaps the group through + // its live identity if this guardian is killed before it can run its reap. + // Its private owner pipe reaches kernel EOF on guardian death even while the + // provider is stopped and unable to process its own guardian-loss callback. + watchdog = spawn(process.execPath, ["--eval", WATCHDOG_SOURCE], { + cwd: process.cwd(), + detached: false, + env: {}, + shell: false, + stdio: ["ignore", "ignore", "ignore", "pipe", "pipe"], + windowsHide: true, + }); + const watchdogOwnerPipe = watchdog.stdio[3]; + const watchdogReady = watchdog.stdio[4]; + if (watchdogOwnerPipe == null) throw new Error("ACPX provider lifetime watchdog omitted its owner pipe"); + if (watchdogReady == null) throw new Error("ACPX provider lifetime watchdog omitted its readiness pipe"); + watchdogOwnerPipe.once("error", reap); + watchdog.once("error", reap); + watchdog.once("exit", reap); + let watchdogOutput = ""; + let watchdogArmed = false; + const watchdogReadyTimeout = setTimeout(reap, ${PROVIDER_WATCHDOG_HANDSHAKE_TIMEOUT_MS}); + watchdogReadyTimeout.unref(); + const rejectUnarmedWatchdog = () => { + if (!watchdogArmed) reap(); + }; + watchdogReady.once("error", rejectUnarmedWatchdog); + watchdogReady.once("close", rejectUnarmedWatchdog); + watchdogReady.on("data", (chunk) => { + watchdogOutput += chunk.toString(); + if (watchdogOutput.length > 64) { + reap(); + return; + } + if (!watchdogOutput.includes("armed\\n")) return; + watchdogArmed = true; + clearTimeout(watchdogReadyTimeout); + watchdogReady.removeAllListeners("data"); + startProvider(); + }); +} catch { + reap(); +} +`; + +const providerGuardianOwnership = new WeakMap>(); +const providerExitProof = new WeakMap>(); export type AcpxPackageJsonResolver = (packageName: string) => string; @@ -51,10 +214,69 @@ export interface VerifiedAcpxCommandLease { spawn( args?: readonly string[], options?: SpawnOptionsWithoutStdio, + lifetime?: VerifiedAcpxProviderLifetime, ): ChildProcess; close(): Promise; } +export interface VerifiedAcpxProviderLifetime { + /** Two listening sockets that fence the canonical Codex credential home. */ + credentialFenceFds: readonly [number, number]; + /** Validate the guardian before provider admission can succeed. */ + activateCredentialFenceOwner(pid: number): Promise; +} + +/** Fail closed where verified provider-group ownership cannot be guaranteed. */ +export function assertVerifiedAcpxProviderPlatform( + platform: NodeJS.Platform, +): void { + if (platform === "win32") { + throw new Error( + "The production ACPX runtime is unavailable on Windows because verified provider launch requires atomic no-follow file opening", + ); + } +} + +/** Reap only the group the live provider belongs to at signal-delivery time. */ +export function reapCurrentProviderProcessGroup( + kill: (pid: number, signal: NodeJS.Signals) => unknown, + currentPid: number, + exit: (code: number) => unknown, +): void { + try { + // POSIX pid zero names the caller's current process group. Unlike a saved + // guardian PGID, the kernel resolves this ownership at the instant of the + // signal, so a dead guardian's recycled identifier can never be targeted. + kill(0, "SIGKILL"); + } catch { + try { + // The caller's own live PID cannot be recycled out from under it. This + // fallback still revokes the provider if whole-group signaling fails. + kill(currentPid, "SIGKILL"); + } catch { + exit(1); + } + } +} + +/** Wait until the verified wrapper has armed owner-death and credential fencing. */ +export async function awaitVerifiedAcpxProviderOwnership( + child: ChildProcess, +): Promise { + await (providerGuardianOwnership.get(child) ?? Promise.resolve()); +} + +/** Wait for kernel EOF on the descriptor held only by the provider process. */ +export async function awaitVerifiedAcpxProviderExit( + child: ChildProcess, +): Promise { + const exitProof = providerExitProof.get(child); + if (!exitProof) { + throw new Error("ACPX provider exit proof is unavailable"); + } + await exitProof; +} + interface VerifiedAcpxCommandIdentity { device: string; inode: string; @@ -77,6 +299,8 @@ type AcpxCommandFormat = "commonjs" | "module"; const COMMONJS_SNAPSHOT_BOOTSTRAP = snapshotBootstrap("commonjs"); const MODULE_SNAPSHOT_BOOTSTRAP = snapshotBootstrap("module"); +const GUARDED_COMMONJS_SNAPSHOT_BOOTSTRAP = snapshotBootstrap("commonjs", true); +const GUARDED_MODULE_SNAPSHOT_BOOTSTRAP = snapshotBootstrap("module", true); /** Resolve and verify every installed artifact bound by a qualified profile. */ export async function verifyQualifiedAcpxInstallation( @@ -564,43 +788,124 @@ function commandLease( spawn( args: readonly string[] = [], options: SpawnOptionsWithoutStdio = {}, + lifetime?: VerifiedAcpxProviderLifetime, ): ChildProcess { if (consumed) throw new Error("Verified ACPX command lease is closed"); consumed = true; let child: ChildProcess; try { + const guarded = lifetime !== undefined; + if (guarded) assertVerifiedAcpxProviderPlatform(process.platform); + const providerBootstrap = guarded + ? format === "module" + ? GUARDED_MODULE_SNAPSHOT_BOOTSTRAP + : GUARDED_COMMONJS_SNAPSHOT_BOOTSTRAP + : format === "module" + ? MODULE_SNAPSHOT_BOOTSTRAP + : COMMONJS_SNAPSHOT_BOOTSTRAP; + const providerOwnershipFd = + DEPENDENCY_ANCESTOR_FD_START + dependencyAncestors.length + 1; + const providerExitFd = providerOwnershipFd + 1; + if ( + guarded && + (!Array.isArray(lifetime.credentialFenceFds) || + lifetime.credentialFenceFds.length !== 2 || + lifetime.credentialFenceFds.some( + (fd) => !Number.isSafeInteger(fd) || fd < 0, + ) || + lifetime.credentialFenceFds[0] === lifetime.credentialFenceFds[1] || + typeof lifetime.activateCredentialFenceOwner !== "function") + ) { + throw new Error("ACPX provider credential fence is invalid"); + } child = spawnChildProcess( process.execPath, - [ - // Keep resolved module URLs on the retained descriptor paths so - // the hook can distinguish them from ordinary host ancestry. - "--preserve-symlinks", - "--eval", - format === "module" - ? MODULE_SNAPSHOT_BOOTSTRAP - : COMMONJS_SNAPSHOT_BOOTSTRAP, - commandDirectoryPath, - commandName, - String(dependencyAncestors.length), - String(serverDependencyAncestorCount), - serverPackageFormat, - JSON.stringify(dependencyAncestorFormats), - ...args, - ], + guarded + ? [ + // Keep resolved module URLs on the retained descriptor paths + // so the hook can distinguish them from host ancestry. + "--preserve-symlinks", + "--eval", + PROVIDER_LIFETIME_GUARDIAN_SOURCE, + providerBootstrap, + commandDirectoryPath, + commandName, + String(dependencyAncestors.length), + String(serverDependencyAncestorCount), + serverPackageFormat, + JSON.stringify(dependencyAncestorFormats), + ...args, + ] + : [ + "--preserve-symlinks", + "--eval", + providerBootstrap, + commandDirectoryPath, + commandName, + String(dependencyAncestors.length), + String(serverDependencyAncestorCount), + serverPackageFormat, + JSON.stringify(dependencyAncestorFormats), + ...args, + ], { ...options, + // In production this process is a persistent sentinel and group + // leader. It arms owner-death before spawning provider code, keeps + // both credential quorum listeners inherited, and pins the PGID + // until its single whole-group reap. + detached: process.platform !== "win32", env: sanitizedNodeEnvironment(options.env), shell: false, - stdio: [ - "pipe", - "pipe", - "pipe", - "pipe", - commandDirectory.fd, - ...dependencyAncestors.map((handle) => handle.fd), - ], + stdio: guarded + ? [ + "pipe", + "pipe", + "pipe", + "pipe", + commandDirectory.fd, + ...dependencyAncestors.map((handle) => handle.fd), + "pipe", + "pipe", + "pipe", + ...lifetime.credentialFenceFds, + ] + : [ + "pipe", + "pipe", + "pipe", + "pipe", + commandDirectory.fd, + ...dependencyAncestors.map((handle) => handle.fd), + ], }, ); + if (guarded) { + const guardianOwnerPipe = child.stdio[ + providerOwnershipFd - 1 + ] as Writable | null; + if (guardianOwnerPipe === null) { + throw new Error( + "ACPX provider lifetime guardian omitted its owner pipe", + ); + } + protectProviderGroupKill(child, guardianOwnerPipe); + const exitProof = providerExitHandshake(child, providerExitFd); + void exitProof.catch(() => undefined); + providerExitProof.set(child, exitProof); + const guardianPid = child.pid!; + const ownership = Promise.all([ + providerOwnershipHandshake(child, providerOwnershipFd), + Promise.resolve().then(() => + lifetime.activateCredentialFenceOwner(guardianPid), + ), + ]).then(() => undefined); + // Session construction can reject before the adapter reaches its + // explicit ownership await. Observe that early rejection now while + // preserving it for the admission boundary. + void ownership.catch(() => undefined); + providerGuardianOwnership.set(child, ownership); + } } catch (error) { verifiedBytes.fill(0); releaseDirectoriesBestEffort(); @@ -624,6 +929,133 @@ function commandLease( }; } +function protectProviderGroupKill( + child: ChildProcess, + guardianOwnerPipe: Writable, +): void { + const signalGuardian = child.kill.bind(child); + let groupReaped = false; + let revocationStarted = false; + child.once("exit", () => { + groupReaped = true; + }); + child.kill = (signal?: NodeJS.Signals | number): boolean => { + if (signal !== "SIGKILL" && signal !== 9) { + return signalGuardian(signal); + } + if (groupReaped) return false; + if (!revocationStarted) { + revocationStarted = true; + // Revocation closes the retained parent-to-guardian owner pipe. Resume + // the exact direct child as well: SIGCONT is harmless for a running + // guardian and lets a stopped guardian observe EOF and reap its own + // still-pinned group. Do not mark the group reaped until exit is seen. + guardianOwnerPipe.destroy(); + } + try { + // Every retry wakes the exact live guardian so it can observe the owner + // pipe EOF and reap the whole group itself. Never SIGKILL the guardian: + // a stopped real provider could otherwise survive after the adapter + // forgets the only process that still pins its group identity. + signalGuardian("SIGCONT"); + } catch { + // The pipe close remains the primary revocation operation. Retained + // cleanup keeps waiting for observed guardian exit and may retry wakeup. + } + return true; + }; +} + +function providerOwnershipHandshake( + child: ChildProcess, + ownershipFd: number, +): Promise { + const output = (child.stdio as Array)[ + ownershipFd + ] as Readable | null | undefined; + if (output == null) { + return Promise.reject( + new Error("ACPX provider lifetime guardian omitted its ownership pipe"), + ); + } + return new Promise((resolve, reject) => { + let settled = false; + let buffered = ""; + const finish = (error?: Error): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + child.off("error", onError); + child.off("close", onClose); + output.off("data", onData); + if (error) reject(error); + else resolve(); + }; + const onError = (): void => + finish(new Error("ACPX provider lifetime guardian failed to start")); + const onClose = (): void => + finish( + new Error( + "ACPX provider lifetime guardian exited before ownership transfer", + ), + ); + const onData = (chunk: Buffer | string): void => { + buffered += chunk.toString(); + if (buffered.includes("owned\n")) finish(); + }; + const timer = setTimeout( + () => + finish( + new Error("ACPX provider lifetime guardian ownership timed out"), + ), + PROVIDER_GUARDIAN_HANDSHAKE_TIMEOUT_MS, + ); + timer.unref(); + child.once("error", onError); + child.once("close", onClose); + output.on("data", onData); + }); +} + +function providerExitHandshake( + child: ChildProcess, + providerExitFd: number, +): Promise { + const output = (child.stdio as Array)[ + providerExitFd + ] as Readable | null | undefined; + if (output == null) { + return Promise.reject( + new Error("ACPX provider lifetime proof pipe was not created"), + ); + } + return new Promise((resolve, reject) => { + let settled = false; + const finish = (error?: Error): void => { + if (settled) return; + settled = true; + output.off("end", onEnd); + output.off("close", onClose); + output.off("error", onError); + if (error) reject(error); + else resolve(); + }; + const onEnd = (): void => finish(); + const onClose = (): void => + finish( + output.readableEnded + ? undefined + : new Error("ACPX provider lifetime proof pipe closed before EOF"), + ); + const onError = (): void => + finish(new Error("ACPX provider lifetime proof pipe failed")); + output.once("end", onEnd); + output.once("close", onClose); + output.once("error", onError); + output.resume(); + }); +} + export function sanitizedNodeEnvironment( environment: NodeJS.ProcessEnv | undefined, ): NodeJS.ProcessEnv { @@ -650,7 +1082,7 @@ export function sanitizedNodeEnvironment( return sanitized; } -function snapshotBootstrap(format: AcpxCommandFormat): string { +function snapshotBootstrap(format: AcpxCommandFormat, guarded = false): string { return [ 'const fs = require("node:fs");', 'const { isBuiltin, registerHooks } = require("node:module");', @@ -666,6 +1098,24 @@ function snapshotBootstrap(format: AcpxCommandFormat): string { `if (!Number.isSafeInteger(dependencyAncestorCount) || dependencyAncestorCount < 0 || dependencyAncestorCount > ${MAX_DEPENDENCY_ANCESTORS}) throw new Error("ACPX provider dependency ancestry is invalid");`, 'if (!Number.isSafeInteger(serverDependencyAncestorCount) || serverDependencyAncestorCount < 0 || serverDependencyAncestorCount > dependencyAncestorCount) throw new Error("ACPX provider package ancestry is invalid");', 'if ((serverPackageFormat !== "module" && serverPackageFormat !== "commonjs") || !Array.isArray(dependencyAncestorFormats) || dependencyAncestorFormats.length !== dependencyAncestorCount || dependencyAncestorFormats.some((value) => value !== "module" && value !== "commonjs")) throw new Error("ACPX provider package formats are invalid");', + ...(guarded + ? [ + `const guardianFd = ${DEPENDENCY_ANCESTOR_FD_START} + dependencyAncestorCount;`, + 'const guardian = fs.createReadStream("", { fd: guardianFd, autoClose: false });', + `const reapCurrentProviderProcessGroup = ${reapCurrentProviderProcessGroup.toString()};`, + "const killProviderProcess = process.kill.bind(process);", + "const providerProcessId = process.pid;", + "const exitProviderProcess = process.exit.bind(process);", + "let guardianLost = false;", + "const reapOnGuardianLoss = () => { if (guardianLost) return; guardianLost = true; reapCurrentProviderProcessGroup(killProviderProcess, providerProcessId, exitProviderProcess); };", + 'guardian.once("end", reapOnGuardianLoss);', + 'guardian.once("error", reapOnGuardianLoss);', + "guardian.resume();", + "fs.fstatSync(guardianFd + 1);", + "fs.fstatSync(guardianFd + 2);", + "fs.fstatSync(guardianFd + 3);", + ] + : []), "const commandPath = resolve(commandDirectory, commandName);", `const guardSnapshotModuleLookup = ${guardSnapshotModuleLookup.toString()};`, `const directory = process.platform === "linux" ? "/proc/self/fd/${COMMAND_DIRECTORY_FD}" : commandDirectory;`, diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index 17baf4c64d..9653979079 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -413,7 +413,6 @@ describe("ACPX runtime host", () => { .mockRejectedValueOnce(new Error("first admission cleanup failed")) .mockImplementationOnce(() => retryClose), }); - await expect( AcpxRuntimeHost.open( { @@ -464,6 +463,79 @@ describe("ACPX runtime host", () => { await contender.close(); }); + it("bounds post-handshake model verification and cleans the runtime", async () => { + const fixture = await hostFixture(); + const runtime = runtimePort({ + getStatus: () => new Promise(() => undefined), + }); + const dependencies = fixture.dependencies({ + openRuntime: async () => runtime, + }); + dependencies.admissionVerificationTimeoutMs = 1; + + await expect( + AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "approve-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }, + dependencies, + ), + ).rejects.toThrow("admission verification exceeded its deadline"); + expect(runtime.close).toHaveBeenCalledOnce(); + expect(fixture.commandClose).toHaveBeenCalledOnce(); + }); + + it("bounds post-handshake cleanup while retaining its exact owner", async () => { + const fixture = await hostFixture(); + let finishRuntimeClose!: () => void; + const runtimeClose = new Promise((resolve) => { + finishRuntimeClose = resolve; + }); + const runtime = runtimePort({ + getStatus: () => new Promise(() => undefined), + onClose: () => runtimeClose, + }); + const dependencies = fixture.dependencies({ + openRuntime: async () => runtime, + }); + let retainedAdmissionCleanup: Promise | null = null; + dependencies.retainAdmissionCleanup = (cleanup) => { + retainedAdmissionCleanup = cleanup; + }; + dependencies.admissionVerificationTimeoutMs = 1; + dependencies.admissionCleanupTimeoutMs = 1; + + await expect( + AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "approve-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }, + dependencies, + ), + ).rejects.toThrow("initialization and cleanup failed"); + expect(runtime.close).toHaveBeenCalledOnce(); + expect(fixture.commandClose).toHaveBeenCalledOnce(); + expect(retainedAdmissionCleanup).not.toBeNull(); + let cleanupSettled = false; + void retainedAdmissionCleanup!.finally(() => { + cleanupSettled = true; + }); + await new Promise((resolve) => setTimeout(resolve, 1)); + expect(cleanupSettled).toBe(false); + + finishRuntimeClose(); + await retainedAdmissionCleanup; + expect(cleanupSettled).toBe(true); + }); + it("retains credential ownership when runtime shutdown fails until retry succeeds", async () => { const fixture = await hostFixture(); let failClose = true; @@ -871,6 +943,8 @@ describe("ACPX runtime host", () => { const credentialAdmission = deferred<{ path: string; mode: "inline_json"; + lifetimeFenceFds: readonly [number, number]; + activateLifetimeOwner(pid: number): Promise; close(): Promise; }>(); const cleanupFailure = new Error("transient credential cleanup failure"); @@ -908,6 +982,8 @@ describe("ACPX runtime host", () => { credentialAdmission.resolve({ path: lateCredentialPath, mode: "inline_json", + lifetimeFenceFds: [42, 43], + activateLifetimeOwner: async () => undefined, close: lateCredentialClose, }); @@ -1043,6 +1119,8 @@ describe("ACPX runtime host", () => { stageCredential: async () => ({ path: join(fixture.root, "auth.json"), mode: "inline_json", + lifetimeFenceFds: [42, 43], + activateLifetimeOwner: async () => undefined, close: credentialClose, }), }, diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts index 070e861627..bdc061469c 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts @@ -43,6 +43,15 @@ import { import type { AcpxExpectedSessionIdentity } from "./sidecar-protocol.js"; export const ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS = 2_000; +const RUNTIME_ADMISSION_VERIFICATION_TIMEOUT_MS = 8_000; +const activeRuntimeHostCleanupOwners = new Set>(); + +class AcpxRuntimeAdmissionTimeoutError extends Error { + constructor() { + super("ACPX runtime admission verification exceeded its deadline"); + this.name = "AcpxRuntimeAdmissionTimeoutError"; + } +} const ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS = 8; const ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS = 10; @@ -88,6 +97,10 @@ export interface AcpxRuntimePortOpenOptions { permissionMode: NativeAcpxPermissionMode; permissionPolicy: ReturnType; launchEnvironment: Readonly; + /** Kernel credential-home quorum inherited by the provider sentinel. */ + credentialFenceFds: readonly [number, number] | null; + /** Validate the guardian while the credential-home quorum is held. */ + activateCredentialFenceOwner: ((pid: number) => Promise) | null; systemInstructions: string; /** Revalidate a pinned recovery workspace at the provider spawn boundary. */ assertWorkspaceHeld?: () => void; @@ -101,12 +114,6 @@ export interface AcpxRuntimePortOpenOptions { retainFailedAdmissionCleanup(cleanup: Promise): void; } -export interface AcpxRetainedCleanupFailure { - resource: "credential" | "command" | "runtime" | "tool_bridge"; - attempt: number; - error: unknown; -} - export interface AcpxMcpServerBinding { name: string; url: string; @@ -116,6 +123,12 @@ export interface AcpxMcpServerBinding { export type AcpxSemanticToolSession = Omit; +export interface AcpxRetainedCleanupFailure { + resource: "credential" | "command" | "runtime" | "tool_bridge"; + attempt: number; + error: unknown; +} + export interface AcpxRuntimeHostDependencies { verifyInstallation?: ( profile: QualifiedAcpxProfile, @@ -123,6 +136,16 @@ export interface AcpxRuntimeHostDependencies { /** Internal test seam for aborting credential acquisition. */ stageCredential?: typeof stageManagedCodexCredential; openRuntime(options: AcpxRuntimePortOpenOptions): Promise; + /** Internal test seam for the post-handshake admission deadline. */ + admissionVerificationTimeoutMs?: number; + /** Internal test seam for failed-admission cleanup. */ + admissionCleanupTimeoutMs?: number; + /** + * Transfers failed-admission cleanup ownership to the embedding lifecycle. + * The callback receives the exact aggregate cleanup attempt before the + * bounded admission wait can return. + */ + retainAdmissionCleanup?: (cleanup: Promise) => void; /** * Required observability channel for resources acquired after admission was * aborted. Implementations must not throw from this callback. @@ -148,7 +171,6 @@ export interface OpenAcpxRuntimeHostOptions { semanticTools?: AcpxSemanticToolSession; } -const activeRuntimeHostCleanupOwners = new Set>(); const RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS = 10; const RETAINED_CLEANUP_RETRY_MAX_DELAY_MS = 1_000; @@ -254,6 +276,12 @@ export class AcpxRuntimeHost { let toolBridge: RunnerToolBridge | null = null; let runtime: AcpxRuntimePort | null = null; let pendingRuntimeOwnsCredential = false; + const admissionVerificationTimeoutMs = + dependencies.admissionVerificationTimeoutMs ?? + RUNTIME_ADMISSION_VERIFICATION_TIMEOUT_MS; + const admissionCleanupTimeoutMs = + dependencies.admissionCleanupTimeoutMs ?? + RUNTIME_ADMISSION_VERIFICATION_TIMEOUT_MS; let failedAdmissionCleanupTransferred = false; let resolveFailedAdmissionCleanupTransfer!: () => void; const failedAdmissionCleanupTransfer = new Promise((resolve) => { @@ -340,6 +368,11 @@ export class AcpxRuntimeHost { binding.permissionMode, ), launchEnvironment: sandbox.launchEnvironment, + credentialFenceFds: credential?.lifetimeFenceFds ?? null, + activateCredentialFenceOwner: + typeof credential?.activateLifetimeOwner === "function" + ? credential.activateLifetimeOwner.bind(credential) + : null, systemInstructions: boundedInstructions(options.systemInstructions), ...(options.assertWorkspaceHeld === undefined ? {} @@ -379,12 +412,14 @@ export class AcpxRuntimeHost { }); }, }); - await runAbortableAdmissionStage(options.signal, () => - requireVerifiedAcpxModel(runtime!, profile), - ); const runtimeIdentity = await runAbortableAdmissionStage( options.signal, - () => runtime!.identity(), + () => + boundedRuntimeAdmissionVerification( + runtime!, + profile, + admissionVerificationTimeoutMs, + ), ); const observedIdentity: AcpxExpectedSessionIdentity = { kind: "acpx", @@ -411,14 +446,16 @@ export class AcpxRuntimeHost { toolBridge, }); } catch (error) { - const cleanupError = await cleanupRuntimeResources( + const cleanup = cleanupRuntimeResources( runtime, toolBridge, pendingRuntimeOwnsCredential ? null : credential, command, "ACPX runtime initialization failed", ); - if (cleanupError) { + retainRuntimeHostCleanup(cleanup); + void cleanup.then((cleanupError) => { + if (!cleanupError) return; retainFailedAcpxAdmissionCleanup({ runtime, toolBridge, @@ -426,8 +463,30 @@ export class AcpxRuntimeHost { command, reason: "ACPX runtime initialization failed", }); + }); + dependencies.retainAdmissionCleanup?.( + cleanup.then((cleanupError) => { + if (cleanupError) throw cleanupError; + }), + ); + const cleanupOutcome = await awaitRuntimeHostCleanupWithin( + cleanup, + admissionCleanupTimeoutMs, + ); + if (cleanupOutcome === "deferred") { throw new AggregateError( - [error, ...cleanupError.errors], + [ + error, + new Error( + "ACPX runtime initialization cleanup exceeded its shutdown timeout", + ), + ], + "ACPX runtime initialization and cleanup failed", + ); + } + if (cleanupOutcome) { + throw new AggregateError( + [error, ...cleanupOutcome.errors], "ACPX runtime initialization and cleanup failed", ); } @@ -468,9 +527,7 @@ export class AcpxRuntimeHost { text, requestId, ...(input.signal ? { signal: input.signal } : {}), - ...(input.onElicitation - ? { onElicitation: input.onElicitation } - : {}), + ...(input.onElicitation ? { onElicitation: input.onElicitation } : {}), }); this.#activeTurn = turn; void turn.result @@ -531,11 +588,15 @@ export class AcpxRuntimeHost { this.#command, reason, ); + if (cleanupError) errors.push(...cleanupError.errors); if (!cleanupError) { + // Runtime, credential, and command ownership has been relinquished even + // when the provider never acknowledged turn cancellation. Preserve that + // cancellation error for this caller, but make later close calls + // idempotently observe the successfully closed host. if (this.#activeTurn === activeTurn) this.#activeTurn = null; this.#closed = true; } - if (cleanupError) errors.push(...cleanupError.errors); if (errors.length > 0) { throw new AggregateError(errors, "ACPX runtime cleanup failed"); } @@ -557,8 +618,8 @@ async function acquireAbortableAdmissionResource(input: { acquire: () => Promise; resource: AcpxRetainedCleanupFailure["resource"]; releaseLate: (resource: T) => Promise; - reportFailure: (failure: AcpxRetainedCleanupFailure) => void; onAbortedPending?: (pending: Promise) => void; + reportFailure: (failure: AcpxRetainedCleanupFailure) => void; }): Promise { if (input.signal === undefined) return await input.acquire(); input.signal.throwIfAborted(); @@ -611,53 +672,6 @@ function raceAdmissionWithAbort( }); } -async function releaseRetainedAdmissionResource(input: { - resource: T; - resourceKind: AcpxRetainedCleanupFailure["resource"]; - release: (resource: T) => Promise; - reportFailure: (failure: AcpxRetainedCleanupFailure) => void; -}): Promise { - let attempt = 0; - let retryDelayMs = RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS; - for (;;) { - attempt += 1; - try { - await input.release(input.resource); - return; - } catch (error) { - try { - input.reportFailure({ - resource: input.resourceKind, - attempt, - error, - }); - } catch { - // The required reporter is observational. A broken reporter must not - // relinquish ownership of the resource that still needs cleanup. - } - await waitForRetainedCleanupRetry(retryDelayMs); - retryDelayMs = Math.min( - retryDelayMs * 2, - RETAINED_CLEANUP_RETRY_MAX_DELAY_MS, - ); - } - } -} - -async function waitForRetainedCleanupRetry(delayMs: number): Promise { - await new Promise((resolve) => { - const timer = setTimeout(resolve, delayMs); - timer.unref?.(); - }); -} - -function retainRuntimeHostCleanup(cleanup: Promise): void { - activeRuntimeHostCleanupOwners.add(cleanup); - void cleanup - .finally(() => activeRuntimeHostCleanupOwners.delete(cleanup)) - .catch(() => undefined); -} - function retainAbortedRuntimeAdmissionCleanup(input: { pendingRuntime: Promise; credential: ManagedCodexCredentialLease | null; @@ -775,6 +789,101 @@ async function waitForAdmissionCleanupRetry(delayMs: number): Promise { }); } +async function releaseRetainedAdmissionResource(input: { + resource: T; + resourceKind: AcpxRetainedCleanupFailure["resource"]; + release: (resource: T) => Promise; + reportFailure: (failure: AcpxRetainedCleanupFailure) => void; +}): Promise { + let attempt = 0; + let retryDelayMs = RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS; + for (;;) { + attempt += 1; + try { + await input.release(input.resource); + return; + } catch (error) { + try { + input.reportFailure({ + resource: input.resourceKind, + attempt, + error, + }); + } catch { + // The required reporter is observational. A broken reporter must not + // relinquish ownership of the resource that still needs cleanup. + } + await waitForRetainedCleanupRetry(retryDelayMs); + retryDelayMs = Math.min( + retryDelayMs * 2, + RETAINED_CLEANUP_RETRY_MAX_DELAY_MS, + ); + } + } +} + +async function waitForRetainedCleanupRetry(delayMs: number): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, delayMs); + timer.unref?.(); + }); +} + +function retainRuntimeHostCleanup(cleanup: Promise): void { + activeRuntimeHostCleanupOwners.add(cleanup); + void cleanup.then( + () => activeRuntimeHostCleanupOwners.delete(cleanup), + () => activeRuntimeHostCleanupOwners.delete(cleanup), + ); +} + +async function awaitRuntimeHostCleanupWithin( + cleanup: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + cleanup, + new Promise<"deferred">((resolve) => { + timer = setTimeout( + () => resolve("deferred"), + Math.max(1, Math.floor(timeoutMs)), + ); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +async function boundedRuntimeAdmissionVerification( + runtime: AcpxRuntimePort, + profile: QualifiedAcpxProfile, + timeoutMs: number, +): Promise { + const verification = Promise.resolve().then(async () => { + await requireVerifiedAcpxModel(runtime, profile); + return await runtime.identity(); + }); + let timer: ReturnType | undefined; + try { + return await Promise.race([ + verification, + new Promise((_resolve, reject) => { + timer = setTimeout( + () => reject(new AcpxRuntimeAdmissionTimeoutError()), + timeoutMs, + ); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + async function boundedCancellation( cancellation: Promise, ): Promise { @@ -794,6 +903,7 @@ async function boundedCancellation( }), ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS, ); + timer.unref(); }), ]); if (timer) clearTimeout(timer); diff --git a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs index 5f079eb8f7..114185371f 100644 --- a/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs +++ b/packages/paperclip-runner/test/acpx-codex-package-contract.test.mjs @@ -37,6 +37,12 @@ test("the runner pins only the Codex ACPX production dependencies", () => { ); }); +test("the package exposes only the reviewed Codex ACPX sidecar binary", () => { + assert.deepEqual(runnerPackage.bin, { + "paperclip-runner-acpx-sidecar": "./dist/cli/acpx-runtime-sidecar.js", + }); +}); + test("old and new pnpm configuration both apply the exact runtime patches", () => { assert.equal( rootPackage.pnpm.patchedDependencies["acpx@0.13.1"],