diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts new file mode 100644 index 0000000000..1806909fcf --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts @@ -0,0 +1,1483 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { AcpRuntimeEvent } from "acpx/runtime"; + +import { + PRP_BLOCK_TOOL_NAME, + PRP_COMPLETION_TOOL_NAME, +} from "../../contracts/completion-result.js"; +import type { PrpEvent } from "../../protocol/replay-contract.js"; +import { validatePrpEvent } from "../../protocol/replay-contract.js"; +import { + CodexAcpxDriver, + type CodexAcpxDriverDependencies, + type CodexAcpxDriverOptions, +} from "./codex-acpx-driver.js"; +import type { + AcpxRuntimeTurn, + OpenAcpxRuntimeHostOptions, +} from "./runtime-host.js"; + +describe("Codex ACPX harness driver", () => { + it("rejects a pre-aborted open before starting host admission", async () => { + const fixture = driverFixture(); + const controller = new AbortController(); + const cancellation = new Error("session open cancelled"); + controller.abort(cancellation); + + await expect( + fixture.driver.openSession({ + runId: "run-pre-abort", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + signal: controller.signal, + }), + ).rejects.toBe(cancellation); + + expect(fixture.openHost).not.toHaveBeenCalled(); + }); + + it("forwards the exact open signal to host admission", async () => { + const fixture = driverFixture(); + const controller = new AbortController(); + const session = await fixture.driver.openSession({ + runId: "run-open-signal", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + signal: controller.signal, + }); + + expect(fixture.hostOptions?.signal).toBe(controller.signal); + await session.close({ reason: "signal forwarding verified" }); + }); + + it("closes and quarantines a host that resolves after open is aborted", async () => { + const hostAdmission = deferred>(); + const fixture = driverFixture( + {}, + { + openHost: () => hostAdmission.promise, + closeSettlementTimeoutMs: 5, + }, + ); + fixture.host.close.mockRejectedValueOnce( + new Error("late host close failed once"), + ); + const controller = new AbortController(); + const cancellation = new Error("host admission cancelled"); + const opening = fixture.driver.openSession({ + runId: "run-late-host", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + signal: controller.signal, + }); + await vi.waitFor(() => expect(fixture.openHost).toHaveBeenCalledOnce()); + + controller.abort(cancellation); + await expect(opening).rejects.toBe(cancellation); + hostAdmission.resolve(fixture.host); + + await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2)); + expect(fixture.host.close).toHaveBeenNthCalledWith(1, { + reason: "Codex ACPX host resolved after admission was aborted", + }); + expect(fixture.host.close).toHaveBeenNthCalledWith(2, { + reason: expect.stringContaining("quarantined cleanup recovery"), + }); + }); + + it("quarantines failed cleanup after session construction fails", async () => { + const quarantineRecovery = deferred(); + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const identity = fixture.host.identity(); + fixture.host.identity = vi + .fn() + .mockReturnValueOnce({ + ...identity, + normalizedSessionId: "different-session", + }) + .mockReturnValue(identity); + fixture.host.close + .mockRejectedValueOnce(new Error("initial cleanup failed")) + .mockImplementationOnce(() => quarantineRecovery.promise) + .mockResolvedValue(undefined); + + await expect( + fixture.driver.openSession({ + runId: "run-construction-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }), + ).rejects.toThrow("Codex ACPX host returned a different session identity"); + await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2)); + + const nextAdmission = fixture.driver.openSession({ + runId: "run-after-construction-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + await Promise.resolve(); + expect(fixture.openHost).toHaveBeenCalledOnce(); + + quarantineRecovery.resolve(); + const session = await nextAdmission; + expect(fixture.openHost).toHaveBeenCalledTimes(2); + expect(fixture.host.close).toHaveBeenNthCalledWith(2, { + reason: expect.stringContaining("quarantined cleanup recovery"), + }); + await session.close({ reason: "construction cleanup recovered" }); + }); + + it("advertises only the implemented Codex production surface", async () => { + const fixture = driverFixture(); + const descriptor = await fixture.driver.descriptor(); + + expect(descriptor).toMatchObject({ + kind: "acpx_runtime", + displayName: "Codex via ACPX", + capabilities: { + resume: false, + interruption: true, + dynamicTools: true, + runtimeRequestResolution: false, + }, + runtimeContextCapabilities: { + instructions: "native", + skills: "unsupported", + mcp: "native", + }, + }); + await expect( + fixture.driver.validateConfig({ + agent: "claude", + model: "claude-sonnet-5", + permissionMode: "approve-reads", + }), + ).resolves.toMatchObject({ + ok: false, + issues: [{ code: "unsupported_agent" }], + }); + }); + + it("maps one turn, dispatches tools, and commits one semantic result", async () => { + const dynamicToolHandler = vi.fn(async () => ({ title: "Document" })); + const fixture = driverFixture({ dynamicToolHandler }); + const session = await fixture.driver.openSession({ + runId: "run-1", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const terminalEvents = collectUntil(session.events(), "turn.completed"); + + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + const bridgeHandler = fixture.hostOptions!.semanticTools!.handler; + await expect( + bridgeHandler({ + tool: "documents.read", + callId: "tool-1", + arguments: { id: "doc-1" }, + signal: new AbortController().signal, + }), + ).resolves.toEqual({ title: "Document" }); + expect(dynamicToolHandler).toHaveBeenCalledWith({ + tool: "documents.read", + callId: "tool-1", + providerSessionId: "agent-1", + turnId, + arguments: { id: "doc-1" }, + signal: expect.any(AbortSignal), + }); + + await expect( + bridgeHandler({ + tool: PRP_COMPLETION_TOOL_NAME, + callId: "finish-1", + arguments: completedResult(), + signal: new AbortController().signal, + }), + ).resolves.toEqual({ accepted: true }); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + + const events = await terminalEvents; + expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true); + expect(events.map((event) => event.eventType)).toEqual( + expect.arrayContaining([ + "turn.submitted", + "turn.accepted", + "turn.started", + "item.delta", + "tool.execution.started", + "run.result.proposed", + "item.completed", + "turn.completed", + ]), + ); + expect( + events.findIndex((event) => event.eventType === "run.result.proposed"), + ).toBeLessThan( + events.findIndex((event) => event.eventType === "turn.completed"), + ); + await expect(session.snapshot()).resolves.toMatchObject({ + driverKind: "acpx_runtime", + activeTurnId: null, + providerIdentity: { + kind: "acpx", + agentSessionId: "agent-1", + }, + semanticResult: { + callId: "finish-1", + turnId, + result: { reportedWorkDisposition: "done" }, + }, + }); + await session.close({ reason: "complete" }); + await session.close({ reason: "idempotent close" }); + expect(fixture.host.close).toHaveBeenCalledOnce(); + }); + + it("rejects terminal disposition drift and bounds interruption", async () => { + const fixture = driverFixture(); + const session = await fixture.driver.openSession({ + runId: "run-2", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + const bridgeHandler = fixture.hostOptions!.semanticTools!.handler; + + await expect( + bridgeHandler({ + tool: PRP_BLOCK_TOOL_NAME, + callId: "block-1", + arguments: completedResult(), + signal: new AbortController().signal, + }), + ).rejects.toThrow("does not match"); + await session.interrupt({ turnId, reason: "user cancelled" }); + expect(fixture.host.interruptActiveTurn).toHaveBeenCalledWith( + "user cancelled", + ); + await expect(session.interrupt({ turnId: "stale-turn" })).rejects.toThrow( + "is not the active turn", + ); + await session.close({ reason: "cancelled" }); + }); + + it("emits an interrupted terminal before closing an active stream", async () => { + const fixture = driverFixture(); + const session = await fixture.driver.openSession({ + runId: "run-close", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const terminalEvents = collectUntil(session.events(), "turn.interrupted"); + await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + + await Promise.all([ + session.close({ reason: "operator shutdown" }), + session.close({ reason: "duplicate shutdown" }), + ]); + + await expect(terminalEvents).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ eventType: "turn.interrupted" }), + ]), + ); + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })], + }); + expect(fixture.host.close).toHaveBeenCalledOnce(); + }); + + it("classifies a pump rejection during close as interrupted", async () => { + const runtimeEventFailure = deferred(); + const fixture = driverFixture( + {}, + { runtimeEventFailure: runtimeEventFailure.promise }, + ); + const session = await fixture.driver.openSession({ + runId: "run-close-pump-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const terminalEvents = collectUntil(session.events(), "turn.interrupted"); + await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + + const closing = session.close({ reason: "operator shutdown" }); + runtimeEventFailure.reject( + new Error("provider stream closed during shutdown"), + ); + await expect(closing).resolves.toBeUndefined(); + + const emitted = await terminalEvents; + expect( + emitted.filter((event) => event.eventType === "turn.interrupted"), + ).toHaveLength(1); + expect(emitted.some((event) => event.eventType === "turn.failed")).toBe( + false, + ); + }); + + it("reports a host close timeout while retaining the exact cleanup", async () => { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const hostClose = deferred(); + fixture.host.close.mockImplementation(() => hostClose.promise); + const session = await fixture.driver.openSession({ + runId: "run-close-timeout", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const terminalEvents = collectUntil(session.events(), "turn.interrupted"); + await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + + await expect( + session.close({ reason: "runtime close stalled" }), + ).rejects.toThrow("host cleanup exceeded its shutdown timeout"); + expect(fixture.host.close).toHaveBeenCalledOnce(); + await expect(terminalEvents).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ eventType: "turn.interrupted" }), + ]), + ); + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + terminalTurns: [expect.objectContaining({ turnId: expect.any(String) })], + }); + + fixture.finishTurn({ status: "cancelled", stopReason: "session_closed" }); + hostClose.resolve(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(fixture.host.close).toHaveBeenCalledOnce(); + }); + + it("reconciles the retained close when it settles after the wait bound", async () => { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const retainedClose = deferred(); + fixture.host.close.mockImplementation(() => retainedClose.promise); + const session = await fixture.driver.openSession({ + runId: "run-close-recovery", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + await expect( + session.close({ reason: "runtime close stalled" }), + ).rejects.toThrow("host cleanup exceeded its shutdown timeout"); + retainedClose.resolve(); + await expect( + session.close({ reason: "observe retained completion" }), + ).resolves.toBeUndefined(); + expect(fixture.host.close).toHaveBeenCalledOnce(); + }); + + it("does not spin while the exact host cleanup remains pending", async () => { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const stalledClose = deferred(); + fixture.host.close.mockImplementation(() => stalledClose.promise); + const session = await fixture.driver.openSession({ + runId: "run-close-stalled-recovery", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + await expect( + session.close({ reason: "runtime close never settles" }), + ).rejects.toThrow("host cleanup exceeded its shutdown timeout"); + expect(fixture.host.close).toHaveBeenCalledOnce(); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(fixture.host.close).toHaveBeenCalledOnce(); + }); + + it("blocks admission while the first retained host close is still pending", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const stalledClose = deferred(); + fixture.host.close.mockImplementation(() => stalledClose.promise); + const session = await fixture.driver.openSession({ + runId: "run-close-pending-admission", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "runtime close remains pending" }), + ).rejects.toThrow("host cleanup exceeded its shutdown timeout"); + await vi.advanceTimersByTimeAsync(1); + await closing; + const admission = fixture.driver.openSession({ + runId: "run-before-pending-close-settles", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const blocked = expect(admission).rejects.toThrow( + "exceeded the admission grace", + ); + // Drive the implementation's complete admission grace instead of + // duplicating its derived shutdown bound in this test. Later transport + // layers may add another bounded cleanup phase without weakening the + // invariant exercised here: no replacement host opens while the exact + // retained close is still pending. + await vi.runAllTimersAsync(); + await blocked; + expect(fixture.openHost).toHaveBeenCalledOnce(); + stalledClose.resolve(); + await vi.runAllTimersAsync(); + } finally { + vi.useRealTimers(); + } + }); + + it("retains autonomous host cleanup recovery through repeated failure", async () => { + const fixture = driverFixture( + {}, + { + closeSettlementTimeoutMs: 1, + }, + ); + fixture.host.close + .mockRejectedValueOnce(new Error("transient cleanup failure")) + .mockRejectedValueOnce(new Error("second cleanup failure")) + .mockResolvedValueOnce(undefined); + const session = await fixture.driver.openSession({ + runId: "run-close-permanent-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const diagnosticEvents = collectUntil( + session.events(), + "harness.diagnostic", + ); + + await expect( + session.close({ reason: "runtime close initially failed" }), + ).rejects.toThrow("transient cleanup failure"); + await expect(diagnosticEvents).resolves.toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: "harness.diagnostic", + payload: expect.objectContaining({ + code: "acpx_host_cleanup_deferred", + }), + }), + ]), + ); + await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(3)); + expect(fixture.host.close).toHaveBeenLastCalledWith({ + reason: "runtime close initially failed (automatic cleanup recovery 2)", + }); + await expect( + session.close({ reason: "observe recovered cleanup" }), + ).resolves.toBeUndefined(); + expect(fixture.host.close).toHaveBeenCalledTimes(3); + }); + + it("keeps quarantined cleanup scheduled and lets admission await its exact owner", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockRejectedValue( + new Error("persistent cleanup failure"), + ); + const session = await fixture.driver.openSession({ + runId: "run-close-retry-bound", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "runtime close persistently failed" }), + ).rejects.toThrow("persistent cleanup failure"); + await vi.advanceTimersByTimeAsync(1); + await closing; + await vi.advanceTimersByTimeAsync(10); + expect(fixture.host.close).toHaveBeenCalledTimes(7); + await vi.advanceTimersByTimeAsync(59_000); + expect(fixture.host.close).toHaveBeenCalledTimes(7); + + fixture.host.close.mockImplementation(({ reason }) => + reason.includes("scheduled quarantined cleanup recovery") + ? // Exercise the complete production host bound: two seconds for + // active-turn cancellation plus six seconds for protocol/TERM/KILL. + new Promise((resolve) => setTimeout(resolve, 8_500)) + : Promise.resolve(), + ); + await vi.advanceTimersToNextTimerAsync(); + let admissionSettled = false; + const admission = fixture.driver.openSession({ + runId: "run-after-recovery", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + void admission.then( + () => { + admissionSettled = true; + }, + () => { + admissionSettled = true; + }, + ); + expect(fixture.host.close).toHaveBeenCalledTimes(8); + expect(fixture.host.close).toHaveBeenLastCalledWith({ + reason: + "runtime close persistently failed (scheduled quarantined cleanup recovery)", + }); + await vi.advanceTimersByTimeAsync(8_499); + expect(admissionSettled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + await expect(admission).resolves.toBeDefined(); + expect(fixture.host.close).toHaveBeenCalledTimes(8); + } finally { + vi.useRealTimers(); + } + }); + + it("runs a full admission batch after an inherited scheduled attempt fails", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockRejectedValue( + new Error("persistent cleanup failure"), + ); + const session = await fixture.driver.openSession({ + runId: "run-scheduled-owner-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "exhaust cleanup before scheduled recovery" }), + ).rejects.toThrow("persistent cleanup failure"); + await vi.advanceTimersByTimeAsync(1); + await closing; + await vi.advanceTimersByTimeAsync(10); + + let scheduledAttempt = 0; + let admissionAttempt = 0; + fixture.host.close.mockImplementation(({ reason }) => { + if (reason.includes("scheduled quarantined cleanup recovery")) { + scheduledAttempt += 1; + return new Promise((_resolve, reject) => { + setTimeout( + () => reject(new Error("scheduled cleanup failed")), + 8_000, + ); + }); + } + if (reason.includes("quarantined cleanup admission recovery")) { + admissionAttempt += 1; + const attempt = admissionAttempt; + return new Promise((resolve, reject) => { + setTimeout(() => { + if (attempt < 3) reject(new Error("admission cleanup failed")); + else resolve(); + }, 8_000); + }); + } + return Promise.reject(new Error("unexpected cleanup reason")); + }); + + await vi.advanceTimersToNextTimerAsync(); + const admission = fixture.driver.openSession({ + runId: "run-after-scheduled-owner-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + let admissionSettled = false; + void admission + .finally(() => { + admissionSettled = true; + }) + .catch(() => undefined); + // The inherited attempt plus the replacement three-attempt batch takes + // just over 32 seconds with this fixture. The production grace must cover + // that complete bounded owner chain rather than expiring at 27 seconds. + await vi.advanceTimersByTimeAsync(32_001); + expect(admissionSettled).toBe(false); + await vi.advanceTimersByTimeAsync(1); + const admitted = await admission; + + expect(scheduledAttempt).toBe(1); + expect(admissionAttempt).toBe(3); + expect(fixture.openHost).toHaveBeenCalledTimes(2); + fixture.host.close.mockResolvedValue(undefined); + await admitted.close({ + reason: "complete after scheduled owner recovery", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("lets admission observe a complete bounded multi-attempt recovery", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + let quarantineAttempt = 0; + fixture.host.close.mockImplementation(({ reason }) => { + if (!reason.includes("quarantined cleanup recovery")) { + return Promise.reject(new Error("persistent cleanup failure")); + } + quarantineAttempt += 1; + const attempt = quarantineAttempt; + return new Promise((resolve, reject) => { + setTimeout(() => { + if (attempt < 3) reject(new Error("transient quarantine failure")); + else resolve(); + }, 8_000); + }); + }); + const session = await fixture.driver.openSession({ + runId: "run-multi-attempt-recovery", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "runtime close persistently failed" }), + ).rejects.toThrow("persistent cleanup failure"); + await vi.advanceTimersByTimeAsync(1); + await closing; + await vi.advanceTimersByTimeAsync(4); + expect(fixture.host.close).toHaveBeenCalledTimes(5); + + let admissionSettled = false; + const admission = fixture.driver.openSession({ + runId: "run-after-multi-attempt-recovery", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + void admission.then( + () => { + admissionSettled = true; + }, + () => { + admissionSettled = true; + }, + ); + await vi.advanceTimersByTimeAsync(23_000); + expect(admissionSettled).toBe(false); + expect(fixture.host.close).toHaveBeenCalledTimes(7); + await vi.advanceTimersByTimeAsync(2_000); + await expect(admission).resolves.toBeDefined(); + expect(quarantineAttempt).toBe(3); + } finally { + vi.useRealTimers(); + } + }); + + it("awaits quarantine recovery installed by an exhausted retained owner", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockImplementation(({ reason }) => { + if (reason.includes("quarantined cleanup recovery")) { + return new Promise((resolve) => setTimeout(resolve, 2)); + } + return Promise.reject(new Error("bounded host recovery failed")); + }); + const session = await fixture.driver.openSession({ + runId: "run-replacement-cleanup-owner", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const closing = expect( + session.close({ reason: "host cleanup must be replaced" }), + ).rejects.toThrow("bounded host recovery failed"); + await vi.advanceTimersByTimeAsync(1); + await closing; + + const admission = fixture.driver.openSession({ + runId: "run-after-replacement-cleanup", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + await vi.advanceTimersByTimeAsync(10); + const admitted = await admission; + expect(fixture.openHost).toHaveBeenCalledTimes(2); + fixture.host.close.mockResolvedValue(undefined); + await admitted.close({ reason: "complete after replacement cleanup" }); + } finally { + vi.useRealTimers(); + } + }); + + it("gives replacement quarantine recovery a fresh admission grace", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockImplementation( + ({ reason }) => + new Promise((resolve, reject) => { + setTimeout(() => { + if (reason.includes("quarantined cleanup recovery")) resolve(); + else reject(new Error("bounded autonomous cleanup failed")); + }, 8_000); + }), + ); + const session = await fixture.driver.openSession({ + runId: "run-owner-phase-grace", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "exhaust cleanup near admission deadline" }), + ).rejects.toThrow("host cleanup exceeded its shutdown timeout"); + await vi.advanceTimersByTimeAsync(1); + await closing; + + let admissionSettled = false; + const admission = fixture.driver.openSession({ + runId: "run-after-owner-phase-grace", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + void admission + .finally(() => { + admissionSettled = true; + }) + .catch(() => undefined); + + // The inherited autonomous owner consumes about 32 seconds before it + // installs a distinct bounded quarantine recovery. That replacement + // owner must not inherit the nearly exhausted admission deadline. + await vi.advanceTimersByTimeAsync(35_001); + expect(admissionSettled).toBe(false); + expect(fixture.openHost).toHaveBeenCalledOnce(); + + await vi.advanceTimersByTimeAsync(5_002); + const admitted = await admission; + expect(fixture.openHost).toHaveBeenCalledTimes(2); + fixture.host.close.mockResolvedValue(undefined); + await admitted.close({ reason: "complete after replacement owner" }); + } finally { + vi.useRealTimers(); + } + }); + + it("reschedules host cleanup after an admission batch is exhausted", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockRejectedValue( + new Error("persistent cleanup failure"), + ); + const session = await fixture.driver.openSession({ + runId: "run-transient-admission-cleanup", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "exhaust cleanup before admission" }), + ).rejects.toThrow("persistent cleanup failure"); + await vi.advanceTimersByTimeAsync(1); + await closing; + await vi.advanceTimersByTimeAsync(10); + + let admissionAttempt = 0; + let scheduledAttempt = 0; + fixture.host.close.mockImplementation(({ reason }) => { + if (reason.includes("quarantined cleanup admission recovery")) { + admissionAttempt += 1; + return Promise.reject(new Error("admission cleanup failed")); + } + if (reason.includes("scheduled quarantined cleanup recovery")) { + scheduledAttempt += 1; + return Promise.resolve(); + } + return Promise.reject(new Error("unexpected cleanup reason")); + }); + + const admission = expect( + fixture.driver.openSession({ + runId: "run-after-transient-admission-cleanup", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }), + ).rejects.toThrow("quarantined host cleanup remains incomplete"); + await vi.advanceTimersByTimeAsync(3); + await admission; + + expect(admissionAttempt).toBe(3); + expect(scheduledAttempt).toBe(0); + await vi.advanceTimersByTimeAsync(60_000); + expect(scheduledAttempt).toBe(1); + + const admitted = await fixture.driver.openSession({ + runId: "run-after-autonomous-cleanup", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + expect(fixture.openHost).toHaveBeenCalledTimes(2); + fixture.host.close.mockResolvedValue(undefined); + await admitted.close({ reason: "complete after admission retry" }); + } finally { + vi.useRealTimers(); + } + }); + + it("restores another host's retry timer when admission recovery times out", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + const hostA = { + ...fixture.host, + close: vi.fn(async () => { + throw new Error("host A cleanup failed"); + }), + }; + const hostB = { + ...fixture.host, + close: vi.fn(async () => { + throw new Error("host B cleanup failed"); + }), + }; + fixture.openHost + .mockReset() + .mockResolvedValueOnce(hostB) + .mockResolvedValueOnce(hostA) + .mockResolvedValue(fixture.host); + + const sessionB = await fixture.driver.openSession({ + runId: "run-quarantined-host-b", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const sessionA = await fixture.driver.openSession({ + runId: "run-quarantined-host-a", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closingB = expect( + sessionB.close({ reason: "quarantine host B" }), + ).rejects.toThrow("host B cleanup failed"); + await vi.advanceTimersByTimeAsync(1); + await closingB; + await vi.advanceTimersByTimeAsync(10); + expect(hostB.close).toHaveBeenCalledTimes(7); + + // Offset the autonomous retry timers so host B can own an inherited + // scheduled attempt while host A still has a dormant retry timer. + await vi.advanceTimersByTimeAsync(100); + const closingA = expect( + sessionA.close({ reason: "quarantine host A" }), + ).rejects.toThrow("host A cleanup failed"); + await vi.advanceTimersByTimeAsync(1); + await closingA; + await vi.advanceTimersByTimeAsync(10); + expect(hostA.close).toHaveBeenCalledTimes(7); + + let hostAAdmissionAttempts = 0; + let hostAScheduledAttempts = 0; + hostA.close.mockImplementation(({ reason }) => { + if (reason.includes("quarantined cleanup admission recovery")) { + hostAAdmissionAttempts += 1; + return Promise.reject(new Error("host A admission cleanup failed")); + } + if (reason.includes("scheduled quarantined cleanup recovery")) { + hostAScheduledAttempts += 1; + return Promise.resolve(); + } + return Promise.reject(new Error("unexpected host A cleanup reason")); + }); + let hostBAdmissionAttempts = 0; + hostB.close.mockImplementation(({ reason }) => { + if (reason.includes("scheduled quarantined cleanup recovery")) { + return new Promise((_resolve, reject) => { + setTimeout( + () => reject(new Error("host B scheduled cleanup failed")), + 1, + ); + }); + } + if (reason.includes("quarantined cleanup admission recovery")) { + hostBAdmissionAttempts += 1; + return new Promise(() => undefined); + } + return Promise.reject(new Error("unexpected host B cleanup reason")); + }); + + await vi.advanceTimersToNextTimerAsync(); + expect(hostB.close).toHaveBeenLastCalledWith({ + reason: expect.stringContaining( + "scheduled quarantined cleanup recovery", + ), + }); + expect(hostAScheduledAttempts).toBe(0); + + const admission = expect( + fixture.driver.openSession({ + runId: "run-after-multi-host-quarantine", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }), + ).rejects.toThrow("exceeded the admission grace"); + await vi.advanceTimersByTimeAsync(40_000); + await admission; + + expect(hostAAdmissionAttempts).toBe(3); + expect(hostBAdmissionAttempts).toBe(1); + expect(hostAScheduledAttempts).toBe(0); + + await vi.advanceTimersByTimeAsync(60_000); + expect(hostAScheduledAttempts).toBe(1); + } finally { + vi.useRealTimers(); + } + }); + + it("fails admission within a finite grace when quarantined cleanup never settles", async () => { + vi.useFakeTimers(); + try { + const fixture = driverFixture({}, { closeSettlementTimeoutMs: 1 }); + fixture.host.close.mockRejectedValue( + new Error("persistent cleanup failure"), + ); + const session = await fixture.driver.openSession({ + runId: "run-close-never-settles", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + + const closing = expect( + session.close({ reason: "runtime close persistently failed" }), + ).rejects.toThrow("persistent cleanup failure"); + await vi.advanceTimersByTimeAsync(1); + await closing; + await vi.advanceTimersByTimeAsync(10); + expect(fixture.host.close).toHaveBeenCalledTimes(7); + + fixture.host.close.mockImplementation(() => new Promise(() => {})); + let admissionSettled = false; + const admission = fixture.driver.openSession({ + runId: "run-after-stalled-quarantine", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + void admission + .finally(() => { + admissionSettled = true; + }) + .catch(() => undefined); + await vi.advanceTimersByTimeAsync(34_999); + expect(admissionSettled).toBe(false); + await vi.advanceTimersByTimeAsync(2); + await expect(admission).rejects.toThrow("exceeded the admission grace"); + expect(fixture.host.close).toHaveBeenCalledTimes(8); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds lagging streams without introducing source sequence gaps", async () => { + const fixture = driverFixture( + {}, + { + runtimeEvents: Array.from({ length: 1_100 }, (_, index) => ({ + type: "text_delta" as const, + text: `chunk-${index}`, + stream: "output" as const, + })), + }, + ); + const session = await fixture.driver.openSession({ + runId: "run-bounds", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + await session.startTurn({ + message: { role: "user", text: "Produce many events." }, + }); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + + await vi.waitFor(async () => { + const snapshot = await session.snapshot(); + expect(snapshot.terminalTurns).toHaveLength(1); + }); + await vi.waitFor(async () => { + const snapshot = await session.snapshot(); + expect(snapshot.terminalTurns).toHaveLength(1); + const transcript = await session.transcript!(); + expect(transcript.eventCount).toBeGreaterThan(1_024); + expect(transcript.complete).toBe(false); + expect(transcript.events.length).toBeLessThanOrEqual(1_024); + expect(transcript.omissionReason).toBe("retention_limit"); + }); + await expect( + session.startTurn({ + message: { + role: "user", + text: "Do not overtake the lagging consumer.", + }, + }), + ).rejects.toThrow("event consumer must drain"); + + await session.close({ reason: "bounds verified" }); + const iterator = session.events()[Symbol.asyncIterator](); + const retained: PrpEvent[] = []; + for (;;) { + const next = await iterator.next(); + if (next.done) break; + retained.push(next.value); + } + expect(retained.length).toBeLessThanOrEqual(512); + expect(retained).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: "harness.diagnostic", + payload: expect.objectContaining({ + code: "event_stream_retention_limit", + }), + }), + ]), + ); + expect( + retained.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + expect(retained.map((event) => event.sourceSeq)).toEqual( + Array.from({ length: retained.length }, (_, index) => index + 1), + ); + }); + + it("retains a committed semantic proposal under terminal queue pressure", async () => { + const fixture = driverFixture({}, { maxBufferedEvents: 6 }); + const session = await fixture.driver.openSession({ + runId: "run-semantic-bounds", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Complete the task." }, + }); + await expect( + fixture.hostOptions!.semanticTools!.handler({ + tool: PRP_COMPLETION_TOOL_NAME, + callId: "finish-bounded", + arguments: completedResult(), + signal: new AbortController().signal, + }), + ).resolves.toEqual({ accepted: true }); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + + await vi.waitFor(async () => { + await expect(session.snapshot()).resolves.toMatchObject({ + terminalTurns: [expect.objectContaining({ turnId })], + }); + }); + await session.close({ reason: "bounded result verified" }); + const retained: PrpEvent[] = []; + for await (const event of session.events()) retained.push(event); + + const proposalIndex = retained.findIndex( + (event) => event.eventType === "run.result.proposed", + ); + const terminalIndex = retained.findIndex( + (event) => event.eventType === "turn.completed", + ); + expect(proposalIndex).toBeGreaterThanOrEqual(0); + expect(terminalIndex).toBeGreaterThan(proposalIndex); + expect(retained).toHaveLength(6); + }); + + it("keeps a full-queue terminal pending until it can be streamed", async () => { + const dynamicToolHandler = vi.fn(async () => ({ title: "late result" })); + const fixture = driverFixture( + { dynamicToolHandler }, + { + closeSettlementTimeoutMs: 1, + maxBufferedEvents: 6, + terminalEventReserve: 0, + runtimeEvents: Array.from({ length: 8 }, (_, index) => ({ + type: "text_delta" as const, + text: `pressure-${index}`, + stream: "output" as const, + })), + }, + ); + const session = await fixture.driver.openSession({ + runId: "run-terminal-capacity-failure", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Fill the complete event queue." }, + }); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + + await vi.waitFor(async () => { + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: turnId, + terminalTurns: [], + }); + const transcript = await session.transcript!(); + expect(transcript.complete).toBe(false); + expect(transcript.eventCount).toBe(14); + }); + + const bridgeHandler = fixture.hostOptions!.semanticTools!.handler; + await expect( + bridgeHandler({ + tool: PRP_COMPLETION_TOOL_NAME, + callId: "late-completion", + arguments: completedResult(), + signal: new AbortController().signal, + }), + ).rejects.toThrow("awaiting canonical terminal-event retention"); + await expect( + bridgeHandler({ + tool: "documents.read", + callId: "late-dynamic-tool", + arguments: { id: "doc-after-terminal" }, + signal: new AbortController().signal, + }), + ).rejects.toThrow("awaiting canonical terminal-event retention"); + expect(dynamicToolHandler).not.toHaveBeenCalled(); + await expect( + session.interrupt({ turnId, reason: "too late" }), + ).rejects.toThrow("awaiting canonical terminal-event retention"); + expect(fixture.host.interruptActiveTurn).not.toHaveBeenCalled(); + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: turnId, + semanticResult: null, + terminalTurns: [], + }); + fixture.host.close + .mockRejectedValueOnce(new Error("initial host cleanup failed")) + .mockResolvedValueOnce(undefined); + await expect( + session.close({ reason: "queue is still full" }), + ).rejects.toThrow("before its turn.completed event is retained"); + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: turnId, + semanticResult: null, + terminalTurns: [], + }); + await vi.waitFor(() => expect(fixture.host.close).toHaveBeenCalledTimes(2)); + expect(fixture.host.close).toHaveBeenLastCalledWith({ + reason: "queue is still full (automatic cleanup recovery 1)", + }); + + const iterator = session.events()[Symbol.asyncIterator](); + const pressureEvents: PrpEvent[] = []; + for (let index = 0; index < 6; index += 1) { + const next = await iterator.next(); + expect(next.done).toBe(false); + if (!next.done) pressureEvents.push(next.value); + } + expect(pressureEvents).toHaveLength(6); + expect(isCanonicalTurnTerminal(pressureEvents, turnId)).toHaveLength(0); + + await session.close({ reason: "capacity became available" }); + const settlementEvents: PrpEvent[] = []; + for (;;) { + const next = await iterator.next(); + if (next.done) break; + settlementEvents.push(next.value); + } + expect(isCanonicalTurnTerminal(settlementEvents, turnId)).toEqual([ + expect.objectContaining({ + eventType: "turn.completed", + turnId, + }), + ]); + await expect(session.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + terminalTurns: [expect.objectContaining({ turnId })], + }); + }); + + it("rejects turn admission when only terminal reserve capacity remains", async () => { + const fixture = driverFixture({}, { maxBufferedEvents: 6 }); + const session = await fixture.driver.openSession({ + runId: "run-admission-bounds", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + await session.startTurn({ + message: { role: "user", text: "Fill the regular event capacity." }, + }); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + await vi.waitFor(async () => { + await expect(session.snapshot()).resolves.toMatchObject({ + terminalTurns: [ + expect.objectContaining({ turnId: expect.any(String) }), + ], + }); + }); + const iterator = session.events()[Symbol.asyncIterator](); + await iterator.next(); + await iterator.next(); + + await expect( + session.startTurn({ + message: { role: "user", text: "Must wait for the remaining events." }, + }), + ).rejects.toThrow("event consumer must drain"); + expect(fixture.host.startTurn).toHaveBeenCalledOnce(); + await session.close({ reason: "admission bound verified" }); + }); + + it("redacts a complete Authorization credential from events and transcripts", async () => { + const fixture = driverFixture(); + const session = await fixture.driver.openSession({ + runId: "run-redaction", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + fixture.host.startTurn.mockImplementationOnce(() => { + throw new Error( + "Authorization: Bearer first-secret second-secret, code=denied", + ); + }); + const terminalEvents = collectUntil(session.events(), "turn.failed"); + + await expect( + session.startTurn({ + message: { role: "user", text: "Do not expose provider credentials." }, + }), + ).rejects.toThrow("Authorization"); + + const serializedEvents = JSON.stringify(await terminalEvents); + const serializedTranscript = JSON.stringify(await session.transcript!()); + for (const serialized of [serializedEvents, serializedTranscript]) { + expect(serialized).toContain("[REDACTED]"); + expect(serialized).not.toContain("Bearer"); + expect(serialized).not.toContain("first-secret"); + expect(serialized).not.toContain("second-secret"); + } + await session.close({ reason: "redaction verified" }); + }); + + it("preserves every terminal when the bounded consumer keeps draining", async () => { + const fixture = driverFixture({}, { maxBufferedEvents: 6 }); + const session = await fixture.driver.openSession({ + runId: "run-critical-bounds", + normalizedSessionId: "session-1", + workingDirectory: "/workspace", + }); + const terminalTurnIds: string[] = []; + for (let index = 0; index < 6; index += 1) { + const terminalEvents = collectUntil(session.events(), "turn.completed"); + const { turnId } = await session.startTurn({ + message: { role: "user", text: `Complete turn ${index}.` }, + }); + terminalTurnIds.push(turnId); + fixture.finishTurn({ status: "completed", stopReason: "end_turn" }); + const emitted = await terminalEvents; + expect(emitted.at(-1)).toMatchObject({ + eventType: "turn.completed", + turnId, + }); + } + await expect( + session.startTurn({ + message: { role: "user", text: "Exceed the bounded turn limit." }, + }), + ).rejects.toThrow("bounded session turn limit"); + + await session.close({ reason: "critical bounds verified" }); + await expect(session.snapshot()).resolves.toMatchObject({ + terminalTurns: terminalTurnIds.map((turnId) => + expect.objectContaining({ turnId }), + ), + }); + }); +}); + +function driverFixture( + overrides: Partial = {}, + fixtureOptions: { + runtimeEvents?: readonly AcpRuntimeEvent[]; + runtimeEventFailure?: Promise; + closeSettlementTimeoutMs?: number; + maxBufferedEvents?: number; + terminalEventReserve?: number; + openHost?: NonNullable; + } = {}, +): { + driver: CodexAcpxDriver; + host: ReturnType; + openHost: ReturnType; + hostOptions: OpenAcpxRuntimeHostOptions | null; + finishTurn(result: Awaited): void; +} { + const result = deferred>(); + const turn: AcpxRuntimeTurn = { + requestId: "provider-turn-1", + promptStarted: Promise.resolve(), + events: { + async *[Symbol.asyncIterator]() { + yield* fixtureOptions.runtimeEvents ?? [ + { + type: "text_delta" as const, + text: "Task complete.", + stream: "output" as const, + }, + { + type: "tool_call" as const, + toolCallId: "provider-tool-1", + title: "Read", + kind: "read" as const, + status: "pending", + tag: "tool_call", + text: "Reading", + }, + ]; + if (fixtureOptions.runtimeEventFailure) { + await fixtureOptions.runtimeEventFailure; + } + }, + }, + result: result.promise, + cancel: vi.fn(async () => undefined), + closeStream: vi.fn(async () => undefined), + }; + const host = fakeHost(turn, () => + result.resolve({ status: "cancelled", stopReason: "session_closed" }), + ); + let hostOptions: OpenAcpxRuntimeHostOptions | null = null; + const openHost = vi.fn( + fixtureOptions.openHost ?? + (async (options: OpenAcpxRuntimeHostOptions) => { + hostOptions = options; + return host; + }), + ); + const dependencies: CodexAcpxDriverDependencies = { + openHost, + closeSettlementTimeoutMs: fixtureOptions.closeSettlementTimeoutMs, + maxBufferedEvents: fixtureOptions.maxBufferedEvents, + terminalEventReserve: fixtureOptions.terminalEventReserve, + }; + const driver = new CodexAcpxDriver( + { + runtimeDirectory: "/runtime", + model: "gpt-5.6-sol", + permissionMode: "approve-reads", + dynamicTools: [ + { + name: "documents.read", + inputSchema: { type: "object" }, + }, + ], + now: () => new Date("2026-08-27T12:00:00.000Z"), + ...overrides, + }, + dependencies, + ); + return { + driver, + host, + openHost, + get hostOptions() { + return hostOptions; + }, + finishTurn: result.resolve, + }; +} + +function fakeHost(turn: AcpxRuntimeTurn, onClose: () => void) { + return { + identity: () => ({ + schema: "paperclip.runner.acpx-identity.v1" as const, + normalizedSessionId: "session-1", + acpxRecordId: "record-1", + backendSessionId: "backend-1", + agentSessionId: "agent-1", + profileDigest: `sha256:${"a".repeat(64)}`, + workspaceDigest: `sha256:${"b".repeat(64)}`, + requestedModel: "gpt-5.6-sol", + effectiveModel: "gpt-5.6-sol", + permissionMode: "approve-reads" as const, + }), + binding: () => ({ + normalizedSessionId: "session-1", + workspacePath: "/workspace", + workspaceDigest: `sha256:${"b".repeat(64)}`, + runtimeRoot: "/runtime/acpx/session-1", + profileDigest: `sha256:${"a".repeat(64)}`, + requestedModel: "gpt-5.6-sol", + effectiveModel: "gpt-5.6-sol", + permissionMode: "approve-reads" as const, + profileSessionKey: "paperclip-session", + }), + status: vi.fn(async () => ({ + agentSessionId: "agent-1", + models: { + currentModelId: "gpt-5.6-sol", + availableModelIds: ["gpt-5.6-sol"], + }, + })), + startTurn: vi.fn(() => turn), + interruptActiveTurn: vi.fn(async () => undefined), + close: vi.fn(async () => { + onClose(); + }), + }; +} + +async function collectUntil( + events: AsyncIterable, + terminalType: PrpEvent["eventType"], +): Promise { + const collected: PrpEvent[] = []; + for await (const event of events) { + collected.push(event); + if (event.eventType === terminalType) return collected; + } + throw new Error(`Event stream closed before ${terminalType}`); +} + +function completedResult() { + return { + schema: "paperclip.run_result.v1" as const, + reportedWorkDisposition: "done" as const, + summary: "The task is complete.", + completionClaim: { + contractRevision: "codex-acpx-test-v1", + objectiveSatisfied: true, + criteria: [], + remainingWork: [], + }, + evidence: [], + verification: [ + { commandOrCheck: "Codex ACPX driver test", status: "passed" as const }, + ], + attentionRequests: [], + artifacts: [], + }; +} + +function isCanonicalTurnTerminal(events: PrpEvent[], turnId: string) { + return events.filter( + (event) => + event.turnId === turnId && + (event.eventType === "turn.completed" || + event.eventType === "turn.failed" || + event.eventType === "turn.interrupted"), + ); +} + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((settle, fail) => { + resolve = settle; + reject = fail; + }); + return { promise, reject, resolve }; +} diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts new file mode 100644 index 0000000000..f28f669a94 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.ts @@ -0,0 +1,1403 @@ +import { createHash, randomBytes } from "node:crypto"; + +import type { AcpRuntimeEvent } from "acpx/runtime"; + +import { + PRP_BLOCK_TOOL_NAME, + PRP_COMPLETION_TOOL_NAME, +} from "../../contracts/completion-result.js"; +import { + HarnessCapabilityUnavailableError, + HarnessStaleTurnError, + type HarnessDriver, + type HarnessDriverConfigValidation, + type HarnessDriverDescriptor, + type HarnessSession, + type HarnessTranscriptSnapshot, + type OpenHarnessSessionInput, + type PersistedHarnessSession, +} from "../../contracts/harness-driver.js"; +import type { NativeAcpxPermissionMode } from "../../contracts/native-execution.js"; +import type { NativeUserMessage } from "../../contracts/types.js"; +import type { + PrpEvent, + PrpStructuredRunResult, +} from "../../protocol/replay-contract.js"; +import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; +import { + canonicalProviderEventsFromAcpxRuntimeEvent, + createAcpxToolEventNormalizer, +} from "../../provider-events.js"; +import { + canonicalRunnerToolName, + type RunnerToolCall, +} from "../runner-tool-bridge.js"; +import { + DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS, + openCodexAcpxRuntime, +} from "./codex-runtime-adapter.js"; +import { + acpxDriverDescriptor, + validateAcpxDriverConfig, +} from "./driver-profile.js"; +import { + ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS, + AcpxRuntimeHost, + type AcpxRuntimeTurn, + type OpenAcpxRuntimeHostOptions, +} from "./runtime-host.js"; + +const MAX_BUFFERED_EVENTS = 512; +const TERMINAL_EVENT_RESERVE = 3; +const TURN_START_EVENT_COUNT = 3; +const MAX_TRANSCRIPT_EVENTS = 1_024; +const MAX_TRANSCRIPT_BYTES = 8 * 1024 * 1024; +const CLOSE_TURN_SETTLEMENT_TIMEOUT_MS = 2_000; +const MAX_AUTONOMOUS_HOST_CLOSE_RETRIES = 3; +const MAX_QUARANTINED_HOST_CLOSE_RETRIES = 3; +const QUARANTINED_HOST_CLOSE_RETRY_MS = 60_000; +const MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS = 1_000; +// Host shutdown first bounds active-turn cancellation and then performs the +// adapter's bounded protocol/TERM/KILL cleanup. Bound each ownership phase that +// admission observes: an inherited owner may install a distinct replacement +// recovery, which receives its own finite grace rather than inheriting an +// almost-expired deadline. Cover every attempt and bounded inter-attempt delay, +// plus one second of scheduling margin. +const QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS = + ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS + + DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS; +const MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS = 1; +const QUARANTINED_HOST_ADMISSION_GRACE_MS = + (MAX_INHERITED_QUARANTINED_HOST_CLOSE_ATTEMPTS + + MAX_QUARANTINED_HOST_CLOSE_RETRIES) * + QUARANTINED_HOST_CLOSE_ATTEMPT_BOUND_MS + + (MAX_QUARANTINED_HOST_CLOSE_RETRIES - 1) * + MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS + + 1_000; + +export interface CodexAcpxDynamicToolCall { + tool: string; + callId: string; + providerSessionId: string; + turnId: string; + arguments: unknown; + signal: AbortSignal; +} + +export interface CodexAcpxDriverOptions { + runtimeDirectory: string; + model: string; + permissionMode?: NativeAcpxPermissionMode; + systemInstructions?: string; + environment?: NodeJS.ProcessEnv; + managedCodexCredentialSourcePath?: string; + dynamicTools?: readonly Readonly>[]; + dynamicToolHandler?: (call: CodexAcpxDynamicToolCall) => Promise; + now?: () => Date; +} + +interface CodexAcpxHost { + identity(): ReturnType; + binding(): ReturnType; + status(): ReturnType; + startTurn( + input: Parameters[0], + ): AcpxRuntimeTurn; + interruptActiveTurn(reason: string): Promise; + close(input: { reason: string }): Promise; +} + +interface QuarantinedHostCleanup { + host: CodexAcpxHost; + reason: string; + attempt: Promise | null; + recovery: Promise | null; + recoveryMaxAttempts: number | null; + timer: ReturnType | null; +} + +export interface CodexAcpxDriverDependencies { + openHost?: (options: OpenAcpxRuntimeHostOptions) => Promise; + /** Internal test seam; production uses the fixed close-settlement bound. */ + closeSettlementTimeoutMs?: number; + /** Internal test seam; production uses the fixed event-retention bound. */ + maxBufferedEvents?: number; + /** Internal test seam; production reserves fixed terminal-event capacity. */ + terminalEventReserve?: number; +} + +/** Codex-only HarnessDriver backed by the admitted ACPX runtime host. */ +export class CodexAcpxDriver implements HarnessDriver { + readonly #options: CodexAcpxDriverOptions; + readonly #openHost: NonNullable; + readonly #closeSettlementTimeoutMs: number; + readonly #maxBufferedEvents: number; + readonly #terminalEventReserve: number; + readonly #cleanupOwners = new Set>(); + readonly #quarantinedHostCleanups = new Set(); + + constructor( + options: CodexAcpxDriverOptions, + dependencies: CodexAcpxDriverDependencies = {}, + ) { + this.#options = { + ...options, + ...(options.environment + ? { environment: { ...options.environment } } + : {}), + ...(options.dynamicTools + ? { dynamicTools: structuredClone(options.dynamicTools) } + : {}), + }; + this.#openHost = + dependencies.openHost ?? + ((hostOptions) => + AcpxRuntimeHost.open(hostOptions, { + openRuntime: openCodexAcpxRuntime, + reportRetainedCleanupFailure: reportRetainedAcpxCleanupFailure, + } as Parameters[1] & { + reportRetainedCleanupFailure: typeof reportRetainedAcpxCleanupFailure; + })); + this.#closeSettlementTimeoutMs = + dependencies.closeSettlementTimeoutMs ?? CLOSE_TURN_SETTLEMENT_TIMEOUT_MS; + this.#terminalEventReserve = Math.max( + 0, + Math.floor(dependencies.terminalEventReserve ?? TERMINAL_EVENT_RESERVE), + ); + this.#maxBufferedEvents = Math.max( + this.#terminalEventReserve + TURN_START_EVENT_COUNT, + Math.floor(dependencies.maxBufferedEvents ?? MAX_BUFFERED_EVENTS), + ); + } + + async descriptor(): Promise { + const descriptor = acpxDriverDescriptor("codex"); + return { + ...descriptor, + displayName: "Codex via ACPX", + runtimeContextCapabilities: { + instructions: "native", + skills: "unsupported", + mcp: "native", + }, + capabilities: { + ...descriptor.capabilities, + resume: false, + runtimeRequestResolution: false, + runtimeRequestHandoff: false, + unsupported: [ + "resume", + "steering", + "runtimeRequestResolution", + "runtimeRequestHandoff", + "goals", + "threadLineage", + ], + }, + }; + } + + async validateConfig(value: unknown): Promise { + const validation = validateAcpxDriverConfig(value); + if (!validation.ok || validation.config.agent === "codex") + return validation; + return { + ok: false, + config: null, + issues: [ + { + path: "agent", + code: "unsupported_agent", + message: "The production ACPX driver currently supports Codex only.", + }, + ], + }; + } + + async openSession(input: OpenHarnessSessionInput): Promise { + await runAbortableDriverAdmission(input.signal, () => + this.#retryQuarantinedHostCleanups(), + ); + let session: CodexAcpxSession | null = null; + const host = await this.#openHostForAdmission( + { + runtimeDirectory: this.#options.runtimeDirectory, + normalizedSessionId: input.normalizedSessionId, + workingDirectory: input.workingDirectory, + agent: "codex", + model: this.#options.model, + permissionMode: this.#options.permissionMode ?? "approve-reads", + systemInstructions: this.#options.systemInstructions, + environment: this.#options.environment, + managedCodexCredentialSourcePath: + this.#options.managedCodexCredentialSourcePath, + ...(input.signal === undefined ? {} : { signal: input.signal }), + semanticTools: { + tools: this.#options.dynamicTools ?? [], + handler: (call) => { + if (!session) { + throw new Error("Codex ACPX session is not ready for tool calls"); + } + return session.dispatchTool(call); + }, + }, + }, + input.signal, + ); + try { + input.signal?.throwIfAborted(); + session = new CodexAcpxSession({ + host, + input, + dynamicToolHandler: this.#options.dynamicToolHandler, + now: this.#options.now ?? (() => new Date()), + closeSettlementTimeoutMs: this.#closeSettlementTimeoutMs, + maxBufferedEvents: this.#maxBufferedEvents, + terminalEventReserve: this.#terminalEventReserve, + retainCleanup: (cleanup) => this.#retainCleanup(cleanup), + quarantineCleanup: (hostToRetain, reason) => + this.#quarantineHostCleanup(hostToRetain, reason), + }); + return session; + } catch (error) { + const reason = "Codex ACPX session initialization failed"; + const cleanup = Promise.resolve().then(() => host.close({ reason })); + this.#retainCleanup(cleanup); + void cleanup.catch(() => this.#quarantineHostCleanup(host, reason)); + await settleWithin(cleanup, this.#closeSettlementTimeoutMs).catch( + () => undefined, + ); + throw error; + } + } + + async #openHostForAdmission( + options: OpenAcpxRuntimeHostOptions, + signal: AbortSignal | undefined, + ): Promise { + if (signal === undefined) return await this.#openHost(options); + signal.throwIfAborted(); + const opening = Promise.resolve().then(() => this.#openHost(options)); + try { + return await raceDriverAdmissionWithAbort(opening, signal); + } catch (error) { + if (signal.aborted) { + const lateHostCleanup = opening.then( + (host) => this.#closeLateAdmissionHost(host), + () => undefined, + ); + this.#retainCleanup(lateHostCleanup); + } + throw error; + } + } + + async #closeLateAdmissionHost(host: CodexAcpxHost): Promise { + const reason = "Codex ACPX host resolved after admission was aborted"; + const cleanup = Promise.resolve().then(() => host.close({ reason })); + this.#retainCleanup(cleanup); + try { + await cleanup; + } catch { + this.#quarantineHostCleanup(host, reason); + } + } + + #retainCleanup(cleanup: Promise): void { + this.#cleanupOwners.add(cleanup); + void cleanup + .finally(() => this.#cleanupOwners.delete(cleanup)) + .catch(() => undefined); + } + + #quarantineHostCleanup(host: CodexAcpxHost, reason: string): void { + if ( + [...this.#quarantinedHostCleanups].some((entry) => entry.host === host) + ) { + return; + } + const cleanup: QuarantinedHostCleanup = { + host, + reason, + attempt: null, + recovery: null, + recoveryMaxAttempts: null, + timer: null, + }; + this.#quarantinedHostCleanups.add(cleanup); + this.#startQuarantinedHostCleanupRecovery( + cleanup, + MAX_QUARANTINED_HOST_CLOSE_RETRIES, + "quarantined cleanup recovery", + ); + } + + #startQuarantinedHostCleanupRecovery( + cleanup: QuarantinedHostCleanup, + maxAttempts: number, + reason: string, + ): Promise { + if (cleanup.recovery) return cleanup.recovery; + const recovery = (async () => { + for ( + let attemptCount = 0; + attemptCount < maxAttempts && + this.#quarantinedHostCleanups.has(cleanup); + attemptCount += 1 + ) { + // Start the first retry immediately so admission's finite grace applies + // to provider cleanup rather than to this rate-limit delay. Only later + // attempts wait, preserving sequential bounded retry behavior. + if (attemptCount > 0) { + await waitForCleanupRetry( + Math.max( + 1, + Math.min( + MAX_QUARANTINED_HOST_ATTEMPT_RETRY_DELAY_MS, + this.#closeSettlementTimeoutMs, + ), + ), + ); + } + const attempt = Promise.resolve().then(() => + cleanup.host.close({ + reason: `${cleanup.reason} (${reason})`, + }), + ); + cleanup.attempt = attempt; + try { + await attempt; + this.#quarantinedHostCleanups.delete(cleanup); + } catch { + // Retain the quarantine after this finite, sequential retry batch. + } finally { + if (cleanup.attempt === attempt) cleanup.attempt = null; + } + } + })(); + cleanup.recovery = recovery; + cleanup.recoveryMaxAttempts = maxAttempts; + this.#retainCleanup(recovery); + void recovery + .finally(() => { + if (cleanup.recovery === recovery) { + cleanup.recovery = null; + cleanup.recoveryMaxAttempts = null; + } + this.#scheduleQuarantinedHostCleanup(cleanup); + }) + .catch(() => undefined); + return recovery; + } + + #scheduleQuarantinedHostCleanup(cleanup: QuarantinedHostCleanup): void { + if ( + !this.#quarantinedHostCleanups.has(cleanup) || + cleanup.recovery || + cleanup.attempt || + cleanup.timer + ) { + return; + } + // A quarantined host remains actively owned, but recovery is rate-limited: + // each entry holds at most one unref'd timer and one sequential close. + // Admission may pull that timer forward; it never creates a parallel try. + cleanup.timer = setTimeout(() => { + cleanup.timer = null; + if (!this.#quarantinedHostCleanups.has(cleanup)) return; + this.#startQuarantinedHostCleanupRecovery( + cleanup, + 1, + "scheduled quarantined cleanup recovery", + ); + }, QUARANTINED_HOST_CLOSE_RETRY_MS); + cleanup.timer.unref?.(); + } + + async #retryQuarantinedHostCleanups(): Promise { + const observedOwners = new Set>(); + const acceleratedCleanups = new Set(); + try { + while (true) { + // A close that is still pending has not entered quarantine yet, but it + // owns the same provider resources. Observe every retained owner and any + // replacement quarantine recovery it installs before admitting a host. + const cleanupOwners = new Set>(this.#cleanupOwners); + for (const cleanup of this.#quarantinedHostCleanups) { + if (cleanup.timer) { + clearTimeout(cleanup.timer); + cleanup.timer = null; + } + if (cleanup.recovery) { + // A scheduled autonomous owner has only one attempt. Admission + // observes it first, then starts one complete bounded batch if that + // attempt fails. A pre-existing full batch already consumed that + // allowance and must never be duplicated. + if ( + (cleanup.recoveryMaxAttempts ?? 0) >= + MAX_QUARANTINED_HOST_CLOSE_RETRIES + ) { + acceleratedCleanups.add(cleanup); + } + cleanupOwners.add(cleanup.recovery); + } else if (!acceleratedCleanups.has(cleanup)) { + acceleratedCleanups.add(cleanup); + cleanupOwners.add( + this.#startQuarantinedHostCleanupRecovery( + cleanup, + MAX_QUARANTINED_HOST_CLOSE_RETRIES, + "quarantined cleanup admission recovery", + ), + ); + } + } + const replacementOwners = [...cleanupOwners].filter( + (owner) => !observedOwners.has(owner), + ); + if (replacementOwners.length === 0) break; + replacementOwners.forEach((owner) => observedOwners.add(owner)); + if ( + !(await settlesWithin( + Promise.all( + replacementOwners.map((owner) => owner.catch(() => undefined)), + ), + QUARANTINED_HOST_ADMISSION_GRACE_MS, + )) + ) { + throw new Error( + "Codex ACPX cannot open a new session because quarantined host cleanup exceeded the admission grace", + ); + } + } + } finally { + // Admission accelerates autonomous timers, but it must never strand a + // different quarantined host if observing any owner exhausts the grace. + for (const cleanup of this.#quarantinedHostCleanups) { + this.#scheduleQuarantinedHostCleanup(cleanup); + } + } + if ( + this.#cleanupOwners.size > 0 || + this.#quarantinedHostCleanups.size > 0 + ) { + throw new Error( + "Codex ACPX cannot open a new session while quarantined host cleanup remains incomplete", + ); + } + } +} + +class CodexAcpxSession implements HarnessSession { + readonly #host: CodexAcpxHost; + readonly #input: OpenHarnessSessionInput; + readonly #dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"]; + readonly #now: () => Date; + readonly #closeSettlementTimeoutMs: number; + readonly #maxBufferedEvents: number; + readonly #terminalEventReserve: number; + readonly #events: AsyncQueue; + readonly #retainCleanup: (cleanup: Promise) => void; + readonly #quarantineCleanup: (host: CodexAcpxHost, reason: string) => void; + readonly #transcript: Array<{ event: PrpEvent; bytes: number }> = []; + readonly #terminalTurns = new Map(); + readonly #sourceInstanceId: string; + #sourceSequence = 0; + #activeTurnId: string | null = null; + #semanticResult: PrpStructuredRunResult | null = null; + #semanticFingerprint: string | null = null; + #semanticCallId: string | null = null; + #semanticTurnId: string | null = null; + #usage: Record | null = null; + #assistantText = ""; + #closed = false; + #closingStarted = false; + #eventStreamClosed = false; + #activePump: Promise | null = null; + #closePromise: Promise | null = null; + #hostClosePromise: Promise | null = null; + #hostCloseRecoveryPromise: Promise | null = null; + #hostClosed = false; + #transcriptBytes = 0; + #transcriptEventCount = 0; + #transcriptOmitted = false; + #eventStreamOmitted = false; + #pendingTerminal: { + turnId: string; + fingerprint: string; + eventType: "turn.completed" | "turn.failed" | "turn.interrupted"; + payload: Record; + } | null = null; + + constructor(input: { + host: CodexAcpxHost; + input: OpenHarnessSessionInput; + dynamicToolHandler?: CodexAcpxDriverOptions["dynamicToolHandler"]; + now: () => Date; + closeSettlementTimeoutMs: number; + maxBufferedEvents: number; + terminalEventReserve: number; + retainCleanup: (cleanup: Promise) => void; + quarantineCleanup: (host: CodexAcpxHost, reason: string) => void; + }) { + const identity = input.host.identity(); + if (identity.normalizedSessionId !== input.input.normalizedSessionId) { + throw new Error("Codex ACPX host returned a different session identity"); + } + this.#host = input.host; + this.#input = structuredClone(input.input); + this.#dynamicToolHandler = input.dynamicToolHandler; + this.#now = input.now; + this.#closeSettlementTimeoutMs = input.closeSettlementTimeoutMs; + this.#maxBufferedEvents = input.maxBufferedEvents; + this.#terminalEventReserve = input.terminalEventReserve; + this.#retainCleanup = input.retainCleanup; + this.#quarantineCleanup = input.quarantineCleanup; + this.#events = new AsyncQueue(input.maxBufferedEvents); + this.#sourceInstanceId = stableId( + "paperclip-acpx", + input.input.normalizedSessionId, + ); + } + + ids() { + const identity = this.#host.identity(); + return { + driverSessionId: identity.acpxRecordId, + providerSessionId: identity.agentSessionId, + displayId: identity.agentSessionId, + }; + } + + events(): AsyncIterable { + return this.#events; + } + + async startTurn(input: { + message: NativeUserMessage; + }): Promise<{ turnId: string }> { + this.#assertOpen(); + if (this.#activeTurnId) { + throw new Error("Codex ACPX session already has an active turn"); + } + if ( + !this.#events.hasCapacity( + this.#terminalEventReserve + TURN_START_EVENT_COUNT, + ) + ) { + throw new HarnessCapabilityUnavailableError( + "turn.start", + "the event consumer must drain the previous turn before another turn can start", + ); + } + if (this.#terminalTurns.size >= this.#maxBufferedEvents) { + throw new HarnessCapabilityUnavailableError( + "turn.start", + "the bounded session turn limit was reached; open a new session", + ); + } + const turnId = `turn-${randomBytes(12).toString("hex")}`; + this.#activeTurnId = turnId; + this.#assistantText = ""; + this.#emit("turn.submitted", { text: input.message.text }, { turnId }); + this.#emit("turn.accepted", { turnId }, { turnId }); + this.#emit("turn.started", { status: "inProgress" }, { turnId }); + let turn: AcpxRuntimeTurn; + try { + turn = this.#host.startTurn({ + text: input.message.text, + requestId: `${safeId(this.#input.runId, "run")}:${turnId}`, + }); + } catch (error) { + this.#publishTerminal( + turnId, + canonicalJson({ status: "failed" }), + "turn.failed", + { status: "failed", error: { message: safeMessage(error) } }, + ); + throw error; + } + const pump = this.#pumpTurn(turnId, turn); + this.#activePump = pump; + void pump + .finally(() => { + if (this.#activePump === pump) this.#activePump = null; + }) + .catch(() => undefined); + return { turnId }; + } + + async interrupt(input: { turnId?: string; reason?: string }): Promise { + this.#assertOpen(); + if (input.turnId && input.turnId !== this.#activeTurnId) { + throw new HarnessStaleTurnError(input.turnId); + } + if (this.#pendingTerminal) { + throw new HarnessCapabilityUnavailableError( + "interruption", + "the active turn is awaiting canonical terminal-event retention", + ); + } + if (!this.#activeTurnId) { + throw new HarnessCapabilityUnavailableError( + "interruption", + "there is no active Codex ACPX turn", + ); + } + await this.#host.interruptActiveTurn(input.reason ?? "interrupted"); + } + + async dispatchTool(call: RunnerToolCall): Promise { + this.#assertOpen(); + if (this.#pendingTerminal) { + throw new HarnessCapabilityUnavailableError( + "tool.dispatch", + "the active turn is awaiting canonical terminal-event retention", + ); + } + const turnId = this.#activeTurnId; + if (!turnId) { + throw new Error("Codex ACPX tool call is not bound to an active turn"); + } + const tool = canonicalRunnerToolName(call.tool); + if (tool === PRP_COMPLETION_TOOL_NAME || tool === PRP_BLOCK_TOOL_NAME) { + const validation = validatePrpStructuredRunResult(call.arguments); + if (!validation.ok) throw new Error("Invalid semantic run result"); + const blocked = validation.result.reportedWorkDisposition === "blocked"; + if ( + (tool === PRP_BLOCK_TOOL_NAME && !blocked) || + (tool === PRP_COMPLETION_TOOL_NAME && blocked) + ) { + throw new Error( + "Semantic result disposition does not match the terminal tool", + ); + } + const fingerprint = canonicalJson(validation.result); + if ( + this.#semanticFingerprint !== null && + this.#semanticFingerprint !== fingerprint + ) { + throw new Error("A different semantic result is already committed"); + } + if (this.#semanticFingerprint === null) { + if ( + !this.#emit("run.result.proposed", validation.result, { + turnId, + itemId: call.callId, + }) + ) { + throw new HarnessCapabilityUnavailableError( + "run.result.proposed", + "the event consumer must drain provider events before a semantic result can be accepted", + ); + } + this.#semanticResult = structuredClone(validation.result); + this.#semanticFingerprint = fingerprint; + this.#semanticCallId = call.callId; + this.#semanticTurnId = turnId; + } + return { accepted: true }; + } + if (!this.#dynamicToolHandler) { + throw new Error(`Unsupported Paperclip operation ${tool}`); + } + return await this.#dynamicToolHandler({ + tool, + callId: call.callId, + providerSessionId: this.#host.identity().agentSessionId, + turnId, + arguments: structuredClone(call.arguments), + signal: call.signal, + }); + } + + async read(): Promise> { + return { + identity: this.#host.identity(), + binding: this.#host.binding(), + status: await this.#host.status(), + }; + } + + async reconcile(): Promise> { + const identity = this.#host.identity(); + const status = await this.#host.status(); + if ( + status.agentSessionId && + status.agentSessionId !== identity.agentSessionId + ) { + throw new Error("ACPX reconciliation changed the provider session"); + } + return { identity, status }; + } + + async usage(): Promise | null> { + return this.#usage === null ? null : structuredClone(this.#usage); + } + + async transcript(): Promise { + return { + schema: "paperclip-runner/harness-transcript/v1", + complete: !this.#transcriptOmitted, + eventCount: this.#transcriptEventCount, + events: structuredClone(this.#transcript.map(({ event }) => event)), + omissionReason: this.#transcriptOmitted ? "retention_limit" : null, + }; + } + + async snapshot(): Promise { + const identity = this.#host.identity(); + return { + driverKind: "acpx_runtime", + driverSessionId: identity.acpxRecordId, + providerSessionId: identity.agentSessionId, + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + activeTurnId: this.#activeTurnId, + lastSourceSequence: this.#sourceSequence, + providerIdentity: { + kind: "acpx", + normalizedSessionId: identity.normalizedSessionId, + acpxRecordId: identity.acpxRecordId, + backendSessionId: identity.backendSessionId, + agentSessionId: identity.agentSessionId, + profileDigest: identity.profileDigest, + workspaceDigest: identity.workspaceDigest, + requestedModel: identity.requestedModel, + effectiveModel: identity.effectiveModel, + permissionMode: identity.permissionMode, + }, + semanticResult: + this.#semanticResult && + this.#semanticFingerprint && + this.#semanticTurnId + ? { + result: structuredClone(this.#semanticResult), + fingerprint: this.#semanticFingerprint, + callId: this.#semanticCallId, + turnId: this.#semanticTurnId, + } + : null, + terminalTurns: [...this.#terminalTurns].map( + ([terminalTurnId, fingerprint]) => ({ + turnId: terminalTurnId, + fingerprint, + }), + ), + }; + } + + async close(input: { reason: string }): Promise { + if (this.#closePromise) return await this.#closePromise; + if (this.#closed) return; + this.#closingStarted = true; + const closePromise = this.#finishClose(input.reason); + this.#closePromise = closePromise; + try { + await closePromise; + this.#closed = true; + } catch (error) { + this.#scheduleHostCloseRecovery(input.reason); + throw error; + } finally { + if (this.#closePromise === closePromise) this.#closePromise = null; + } + } + + async #finishClose(reason: string): Promise { + const closingTurnId = this.#activeTurnId; + const pump = this.#activePump; + const hostClose = + this.#hostClosePromise ?? this.#startHostClose({ reason }); + let hostCloseError: unknown = null; + try { + await settleWithin(hostClose, this.#closeSettlementTimeoutMs); + } catch (error) { + hostCloseError = error; + } + if (pump) { + await settleWithin( + pump.catch(() => undefined), + this.#closeSettlementTimeoutMs, + ).catch(() => undefined); + } + if (closingTurnId && !this.#terminalTurns.has(closingTurnId)) { + const pendingTerminal = this.#pendingTerminal; + if (pendingTerminal?.turnId === closingTurnId) { + this.#publishTerminal( + pendingTerminal.turnId, + pendingTerminal.fingerprint, + pendingTerminal.eventType, + pendingTerminal.payload, + ); + } else { + this.#publishTerminal( + closingTurnId, + canonicalJson({ status: "interrupted" }), + "turn.interrupted", + { status: "interrupted", stopReason: "session_closed" }, + ); + } + } + if (hostCloseError) { + this.#emit( + "harness.diagnostic", + { + code: "acpx_host_cleanup_deferred", + message: + "ACPX host cleanup exceeded its caller wait bound; the driver retains the exact cleanup until it settles.", + }, + closingTurnId ? { turnId: closingTurnId } : {}, + 0, + ); + } + this.#eventStreamClosed = true; + this.#events.close(); + if (hostCloseError) { + throw hostCloseError; + } + } + + #startHostClose(input: { reason: string }): Promise { + const closePromise = this.#host.close(input).then(() => { + this.#hostClosed = true; + }); + this.#hostClosePromise = closePromise; + this.#retainCleanup(closePromise); + return closePromise; + } + + #scheduleHostCloseRecovery(reason: string): void { + if (this.#hostClosed || this.#hostCloseRecoveryPromise) return; + const failedOrPendingClose = this.#hostClosePromise; + if (!failedOrPendingClose) return; + // Never overlap the exact cleanup that exceeded the caller's wait bound. + // Once an attempt rejects, keep one delay-bounded recovery owner alive for + // a finite retry budget. A permanently pending attempt remains the sole + // owner while the runtime adapter independently terminates its children; + // repeated terminal failures settle instead of creating an immortal loop. + // Host cleanup does not logically close the harness session: #finishClose + // still owns terminal publication and event-stream closure after recovery. + const recovery = (async () => { + let attempt = failedOrPendingClose; + let retryCount = 0; + while (!this.#hostClosed) { + try { + await attempt; + return; + } catch (error) { + if (retryCount >= MAX_AUTONOMOUS_HOST_CLOSE_RETRIES) { + this.#quarantineCleanup(this.#host, reason); + throw error; + } + retryCount += 1; + } + await waitForCleanupRetry( + Math.max(1, Math.min(1_000, this.#closeSettlementTimeoutMs)), + ); + if (this.#hostClosed) { + return; + } + if (this.#hostClosePromise === attempt) { + this.#hostClosePromise = null; + } + attempt = this.#startHostClose({ + reason: `${reason} (automatic cleanup recovery ${retryCount})`, + }); + } + })(); + this.#hostCloseRecoveryPromise = recovery; + this.#retainCleanup(recovery); + void recovery + .finally(() => { + if (this.#hostCloseRecoveryPromise === recovery) { + this.#hostCloseRecoveryPromise = null; + } + }) + .catch(() => undefined); + } + + async #pumpTurn(turnId: string, turn: AcpxRuntimeTurn): Promise { + try { + let index = 0; + const normalizeToolEvent = + createAcpxToolEventNormalizer(); + for await (const event of turn.events) { + this.#mapRuntimeEvent(normalizeToolEvent(event), turnId, ++index); + } + const result = await turn.result; + if (this.#terminalTurns.has(turnId)) return; + if (result.status === "completed") { + const finalText = this.#assistantText.trim(); + if (finalText) { + this.#emit( + "item.completed", + { kind: "agentMessage", channel: "final", text: finalText }, + { turnId, itemId: `${turnId}:final-answer` }, + ); + } + this.#publishTerminal( + turnId, + canonicalJson({ + status: "completed", + semanticResult: this.#semanticFingerprint, + }), + "turn.completed", + { status: "completed", stopReason: result.stopReason ?? null }, + ); + } else if (result.status === "cancelled") { + this.#publishTerminal( + turnId, + canonicalJson({ status: "interrupted" }), + "turn.interrupted", + { + status: "interrupted", + stopReason: result.stopReason ?? "cancelled", + }, + ); + } else { + this.#emit( + "provider.notice.recorded", + { + schema: "paperclip.provider.notice.v1", + noticeId: `${turnId}:failure`, + severity: "error", + category: "acpx_turn_failed", + scope: "turn", + recoverable: result.error.retryable ?? false, + userActionable: true, + summary: safeMessage(result.error.message), + }, + { turnId, itemId: `${turnId}:failure` }, + ); + this.#publishTerminal( + turnId, + canonicalJson({ status: "failed" }), + "turn.failed", + { + status: "failed", + error: { + code: result.error.code ?? null, + message: safeMessage(result.error.message), + }, + }, + ); + } + } catch (error) { + if (this.#terminalTurns.has(turnId)) return; + if (error instanceof TerminalEventCapacityError) throw error; + if (this.#closed || this.#closingStarted) { + this.#publishTerminal( + turnId, + canonicalJson({ status: "interrupted" }), + "turn.interrupted", + { status: "interrupted", stopReason: "session_closed" }, + ); + } else { + this.#publishTerminal( + turnId, + canonicalJson({ status: "failed" }), + "turn.failed", + { status: "failed", error: { message: safeMessage(error) } }, + ); + } + } + } + + #publishTerminal( + turnId: string, + fingerprint: string, + eventType: "turn.completed" | "turn.failed" | "turn.interrupted", + payload: Record, + ): void { + if (this.#terminalTurns.has(turnId)) return; + const pendingTerminal = this.#pendingTerminal; + if ( + pendingTerminal !== null && + (pendingTerminal.turnId !== turnId || + pendingTerminal.fingerprint !== fingerprint || + pendingTerminal.eventType !== eventType || + canonicalJson(pendingTerminal.payload) !== canonicalJson(payload)) + ) { + throw new Error( + `Codex ACPX terminal disposition changed while ${pendingTerminal.turnId} awaited stream capacity`, + ); + } + const terminal = pendingTerminal ?? { + turnId, + fingerprint, + eventType, + payload: structuredClone(payload), + }; + this.#pendingTerminal = terminal; + if (!this.#emit(terminal.eventType, terminal.payload, { turnId })) { + // The persisted terminal index is the driver's settlement authority. + // Never advance it unless the matching canonical event is already in + // the bounded stream. A caller may drain and retry close, but it cannot + // observe a settled turn whose terminal fact was omitted. + throw new TerminalEventCapacityError(turnId, eventType); + } + this.#terminalTurns.set(turnId, fingerprint); + this.#pendingTerminal = null; + if (this.#activeTurnId === turnId) this.#activeTurnId = null; + } + + #mapRuntimeEvent( + event: AcpRuntimeEvent, + turnId: string, + index: number, + ): void { + const fallbackItemId = `${turnId}:acp:${index}`; + if (event.type === "text_delta") { + const output = boundedText(event.text, 64 * 1024); + if (event.stream !== "thought" && event.tag !== "agent_thought_chunk") { + this.#assistantText = boundedText( + `${this.#assistantText}${output}`, + 256 * 1024, + ); + } + this.#emit( + "item.delta", + { + kind: + event.stream === "thought" || event.tag === "agent_thought_chunk" + ? "thinking" + : "agent_message", + text: output, + }, + { turnId, itemId: fallbackItemId }, + ); + } + if (event.type === "status" && event.tag === "usage_update") { + this.#usage = boundedRecord({ + cumulative: event.breakdown, + cost: event.cost, + }); + } + for (const canonical of canonicalProviderEventsFromAcpxRuntimeEvent( + event, + fallbackItemId, + turnId, + )) { + this.#emit(canonical.eventType, canonical.payload, { + turnId, + itemId: canonical.itemId, + }); + } + } + + #emit( + eventType: PrpEvent["eventType"], + payload: Record, + refs: { turnId?: string; itemId?: string } = {}, + reservedAfter = isTerminalEvent(eventType) + ? 0 + : eventType === "run.result.proposed" + ? Math.max(0, this.#terminalEventReserve - 1) + : this.#terminalEventReserve, + ): boolean { + if (this.#eventStreamClosed) return false; + if (this.#eventStreamOmitted && isTerminalEvent(eventType)) { + this.#eventStreamOmitted = false; + const recordedOmission = this.#emit( + "harness.diagnostic", + { + code: "event_stream_retention_limit", + message: + "Earlier provider events were omitted because the consumer exceeded the bounded event buffer.", + }, + refs, + 1, + ); + if (!recordedOmission) this.#eventStreamOmitted = true; + } + const sourceSeq = this.#sourceSequence + 1; + const event: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: `${this.#sourceInstanceId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: this.#sourceInstanceId, + sourceKind: "runner", + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + ...(refs.turnId ? { turnId: refs.turnId } : {}), + ...(refs.itemId ? { itemId: refs.itemId } : {}), + eventType, + schemaVersion: 1, + priority: eventType === "run.result.proposed" ? 0 : 1, + emittedAt: this.#now().toISOString(), + payload: structuredClone(payload), + }; + if (!this.#events.push(event, reservedAfter)) { + this.#eventStreamOmitted = true; + this.#transcriptEventCount += 1; + this.#transcriptOmitted = true; + return false; + } + this.#sourceSequence = sourceSeq; + this.#retainTranscriptEvent(event); + return true; + } + + #retainTranscriptEvent(event: PrpEvent): void { + this.#transcriptEventCount += 1; + const retained = structuredClone(event); + let bytes: number; + try { + bytes = Buffer.byteLength(JSON.stringify(retained)); + } catch { + this.#transcriptOmitted = true; + return; + } + if (bytes > MAX_TRANSCRIPT_BYTES) { + this.#transcriptOmitted = true; + return; + } + this.#transcript.push({ event: retained, bytes }); + this.#transcriptBytes += bytes; + while ( + this.#transcript.length > MAX_TRANSCRIPT_EVENTS || + this.#transcriptBytes > MAX_TRANSCRIPT_BYTES + ) { + const omitted = this.#transcript.shift(); + if (omitted) this.#transcriptBytes -= omitted.bytes; + this.#transcriptOmitted = true; + } + } + + #assertOpen(): void { + if (this.#closed || this.#closingStarted) { + throw new Error("Codex ACPX session is closing or closed"); + } + } +} + +function waitForCleanupRetry(delayMs: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, delayMs); + timer.unref(); + }); +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +function boundedRecord(value: unknown): Record { + const serialized = JSON.stringify(value); + if (!serialized || Buffer.byteLength(serialized) > 64 * 1024) { + return { omitted: true, reason: "payload_limit" }; + } + const parsed: unknown = JSON.parse(serialized); + return typeof parsed === "object" && parsed !== null && !Array.isArray(parsed) + ? (parsed as Record) + : {}; +} + +function boundedText(value: string, maxBytes: number): string { + const bytes = Buffer.from(value); + return bytes.length <= maxBytes + ? value + : bytes.subarray(bytes.length - maxBytes).toString("utf8"); +} + +function safeId(value: string, fallback: string): string { + const candidate = value.replace(/[^A-Za-z0-9._:-]/g, "-").slice(0, 160); + return /^[A-Za-z0-9]/.test(candidate) ? candidate : fallback; +} + +function stableId(prefix: string, value: string): string { + const readable = safeId(value, "session").slice(0, 80); + const suffix = createHash("sha256").update(value).digest("hex").slice(0, 16); + return `${prefix}-${readable}-${suffix}`; +} + +function safeMessage(error: unknown): string { + const message = error instanceof Error ? error.message : String(error); + return message + .replace( + /(\bauthorization\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\r\n,}\]]*)/gi, + "$1[REDACTED]", + ) + .replace( + /(\b(?:key|token|secret|password)\s*[:=]\s*)(?:"(?:\\.|[^"\r\n])*"|'(?:\\.|[^'\r\n])*'|[^\s,}\]]+)/gi, + "$1[REDACTED]", + ) + .slice(0, 4_000); +} + +function reportRetainedAcpxCleanupFailure(input: { + resource: "credential" | "command" | "runtime"; + attempt: number; + error: unknown; +}): void { + const errorName = input.error instanceof Error ? input.error.name : "Error"; + process.emitWarning( + JSON.stringify({ + schema: "paperclip.runner.retained_cleanup_failure.v1", + resource: input.resource, + attempt: input.attempt, + errorName, + }), + { + code: "PAPERCLIP_ACPX_RETAINED_CLEANUP_FAILURE", + type: "PaperclipRunnerCleanupWarning", + }, + ); +} + +function isTerminalEvent(eventType: PrpEvent["eventType"]): boolean { + return ( + eventType === "turn.completed" || + eventType === "turn.failed" || + eventType === "turn.interrupted" + ); +} + +class TerminalEventCapacityError extends Error { + constructor(turnId: string, eventType: PrpEvent["eventType"]) { + super( + `Codex ACPX cannot settle ${turnId} before its ${eventType} event is retained`, + ); + this.name = "TerminalEventCapacityError"; + } +} + +class AsyncQueue implements AsyncIterable { + readonly #items: T[] = []; + readonly #waiters: Array<(result: IteratorResult) => void> = []; + readonly #maxItems: number; + #closed = false; + + constructor(maxItems: number) { + this.#maxItems = maxItems; + } + + hasCapacity(requiredItems: number): boolean { + return ( + requiredItems <= + this.#waiters.length + this.#maxItems - this.#items.length + ); + } + + push(item: T, reservedAfter = 0): boolean { + if (this.#closed) return false; + // Admission and insertion are one synchronous operation so every regular + // event leaves the terminal lane intact. In particular, a lagging + // consumer cannot occupy terminal capacity between a check and a push. + if (!this.hasCapacity(1 + reservedAfter)) return false; + const waiter = this.#waiters.shift(); + if (waiter) { + waiter({ done: false, value: item }); + return true; + } + if (this.#items.length >= this.#maxItems) return false; + this.#items.push(item); + return true; + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + for (const waiter of this.#waiters.splice(0)) { + waiter({ done: true, value: undefined }); + } + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: () => { + const item = this.#items.shift(); + if (item !== undefined) { + return Promise.resolve({ done: false, value: item }); + } + if (this.#closed) { + return Promise.resolve({ done: true, value: undefined }); + } + return new Promise((resolve) => this.#waiters.push(resolve)); + }, + }; + } +} + +async function settleWithin( + promise: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + reject(new Error("ACPX host cleanup exceeded its shutdown timeout")); + }, timeoutMs); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} + +async function runAbortableDriverAdmission( + signal: AbortSignal | undefined, + operation: () => Promise, +): Promise { + if (signal === undefined) return await operation(); + signal.throwIfAborted(); + return await raceDriverAdmissionWithAbort( + Promise.resolve().then(operation), + signal, + ); +} + +function raceDriverAdmissionWithAbort( + pending: Promise, + signal: AbortSignal, +): Promise { + return new Promise((resolve, reject) => { + let settled = false; + const settle = (operation: () => void): void => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + operation(); + }; + const onAbort = (): void => settle(() => reject(signal.reason)); + signal.addEventListener("abort", onAbort, { once: true }); + if (signal.aborted) { + onAbort(); + return; + } + void pending.then( + (value) => settle(() => resolve(value)), + (error: unknown) => settle(() => reject(error)), + ); + }); +} + +async function settlesWithin( + promise: Promise, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } +} diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts index 4bea8aad78..0e89953bfe 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts @@ -32,14 +32,23 @@ describe("Codex ACPX runtime adapter", () => { controller.abort(cancellation); const command = fakeCommand(); const createRuntime = vi.fn(); + const retainFailedAdmissionCleanup = vi.fn(); await expect( openCodexAcpxRuntime( - { ...openOptions(command), signal: controller.signal }, + { + ...openOptions(command), + signal: controller.signal, + retainFailedAdmissionCleanup, + }, { createRuntime }, ), ).rejects.toBe(cancellation); + expect(retainFailedAdmissionCleanup).toHaveBeenCalledOnce(); + await expect( + retainFailedAdmissionCleanup.mock.calls[0]?.[0], + ).resolves.toBeUndefined(); expect(createRuntime).not.toHaveBeenCalled(); expect(command.spawn).not.toHaveBeenCalled(); }); @@ -248,6 +257,807 @@ describe("Codex ACPX runtime adapter", () => { }); }); + it("never overlaps a retained protocol close that has not settled", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + const runtimeClose = new Promise(() => {}); + vi.mocked(runtime.close) + .mockReturnValueOnce(runtimeClose) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + + const firstClose = expect( + port.close({ reason: "runtime close stalled" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + expect(runtime.close).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(2_000); + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + await firstClose; + + // Repeated callers inherit the same bounded observation. A permanently + // pending exact close remains the sole protocol attempt for this handle. + expect(runtime.close).toHaveBeenCalledOnce(); + const secondClose = expect( + port.close({ reason: "idempotent terminal close" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await vi.advanceTimersByTimeAsync(2_000); + await secondClose; + expect(runtime.close).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it("allows a fresh close after a retained attempt rejects late", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectRuntimeClose!: (error: unknown) => void; + const runtimeClose = new Promise((_resolve, reject) => { + rejectRuntimeClose = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(runtimeClose) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + + const firstClose = expect( + port.close({ reason: "runtime close stalled" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(2_000); + await firstClose; + expect(child.kill).toHaveBeenCalledWith("SIGTERM"); + + const protocolFailure = new Error("late protocol close failure"); + rejectRuntimeClose(protocolFailure); + // The late-settlement observer schedules reconciliation asynchronously. + // Wait for that fresh attempt instead of racing another caller against + // the already-settled retained failure. + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + await expect( + port.close({ reason: "retry after retained failure" }), + ).resolves.toBeUndefined(); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("reconciles a retained close only after its late failure settles", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectFirstClose!: (error: unknown) => void; + const firstRuntimeClose = new Promise((_resolve, reject) => { + rejectFirstClose = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(firstRuntimeClose) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + + const firstClose = expect( + port.close({ reason: "first protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(2_000); + await firstClose; + + const inheritedClose = expect( + port.close({ reason: "observe pending protocol close" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await vi.advanceTimersByTimeAsync(2_000); + await inheritedClose; + expect(runtime.close).toHaveBeenCalledOnce(); + + rejectFirstClose(new Error("older protocol close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + await expect( + port.close({ reason: "observe reconciled cleanup" }), + ).resolves.toBeUndefined(); + expect(runtime.close).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("shares a pending close across concurrent callers before reconciliation", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectFirstClose!: (error: unknown) => void; + const firstRuntimeClose = new Promise((_resolve, reject) => { + rejectFirstClose = reject; + }); + let resolveFreshClose!: () => void; + const freshRuntimeClose = new Promise((resolve) => { + resolveFreshClose = resolve; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(firstRuntimeClose) + .mockReturnValueOnce(freshRuntimeClose) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + }); + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + + const firstClose = expect( + port.close({ reason: "first protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(2_000); + await firstClose; + + const inheritedClose = expect( + port.close({ reason: "observe pending protocol close" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + expect(runtime.close).toHaveBeenCalledOnce(); + rejectFirstClose(new Error("retained protocol close failed")); + await inheritedClose; + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + + resolveFreshClose(); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(2); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + await expect( + port.close({ reason: "observe reconciled cleanup" }), + ).resolves.toBeUndefined(); + expect(runtime.close).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let late settlements bypass the reconciliation retry bound", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockRejectedValueOnce(new Error("reconciliation 3 failed")); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + }); + + const firstClose = expect( + port.close({ reason: "first protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(2_000); + await firstClose; + rejectInitialClose(new Error("initial protocol close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + } + }); + + it("keeps timed-out reconciliation failures in their originating budget", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + const reconciliationRejectors: Array<(error: unknown) => void> = []; + const reconciliationAttempts = Array.from( + { length: 3 }, + () => + new Promise((_resolve, reject) => { + reconciliationRejectors.push(reject); + }), + ); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockReturnValueOnce(reconciliationAttempts[0]!) + .mockReturnValueOnce(reconciliationAttempts[1]!) + .mockReturnValueOnce(reconciliationAttempts[2]!) + .mockResolvedValueOnce(undefined); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const firstClose = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await firstClose; + rejectInitialClose(new Error("external close failed late")); + + for (let index = 0; index < reconciliationRejectors.length; index += 1) { + await vi.waitFor(() => + expect(runtime.close).toHaveBeenCalledTimes(index + 2), + ); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: `ACPX late protocol cleanup reconciliation ${index + 1}`, + discardPersistentState: false, + }); + const overlappingExternalClose = + index === 0 + ? expect( + port.close({ + reason: "external observer of reconciliation", + }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed") + : null; + // The external observer coalesces onto reconciliation attempt one. It + // must not relabel that immutable attempt as an external generation. + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + if (overlappingExternalClose) await overlappingExternalClose; + reconciliationRejectors[index]!( + new Error(`reconciliation ${index + 1} failed late`), + ); + } + + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + expect(runtime.close).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + } + }); + + it("renews one budget when an external close joins the final reconciliation", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let rejectFinalReconciliation!: (error: unknown) => void; + const finalReconciliation = new Promise((_resolve, reject) => { + rejectFinalReconciliation = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockReturnValueOnce(finalReconciliation) + .mockResolvedValueOnce(undefined); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const initialObserver = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await initialObserver; + rejectInitialClose(new Error("external close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + + const finalObserver = expect( + port.close({ reason: "external observer of final reconciliation" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await finalObserver; + expect(runtime.close).toHaveBeenCalledTimes(4); + + rejectFinalReconciliation( + new Error("final reconciliation failed late"), + ); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + + it("batches coalesced external closes when the final reconciliation fails", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let rejectFinalReconciliation!: (error: unknown) => void; + const finalReconciliation = new Promise((_resolve, reject) => { + rejectFinalReconciliation = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockReturnValueOnce(finalReconciliation) + .mockRejectedValueOnce(new Error("renewed reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("renewed reconciliation 2 failed")) + .mockRejectedValueOnce(new Error("renewed reconciliation 3 failed")); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const initialObserver = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await initialObserver; + rejectInitialClose(new Error("external close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + + const coalescedObservers = ["first", "second", "third"].map((label) => + expect( + port.close({ reason: `${label} external observer` }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"), + ); + rejectFinalReconciliation( + new Error("final reconciliation failed with joined observers"), + ); + expect(runtime.close).toHaveBeenCalledTimes(4); + await Promise.all(coalescedObservers); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(7)); + expect( + vi + .mocked(runtime.close) + .mock.calls.slice(4) + .map(([input]) => input.reason), + ).toEqual([ + "ACPX late protocol cleanup reconciliation 1", + "ACPX late protocol cleanup reconciliation 2", + "ACPX late protocol cleanup reconciliation 3", + ]); + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + expect(runtime.close).toHaveBeenCalledTimes(7); + } finally { + vi.useRealTimers(); + } + }); + + it("consumes an external intent when a timed-out final reconciliation succeeds late", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let resolveFinalReconciliation!: () => void; + const finalReconciliation = new Promise((resolve) => { + resolveFinalReconciliation = resolve; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockReturnValueOnce(finalReconciliation); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const initialObserver = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await initialObserver; + rejectInitialClose(new Error("external close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + + const finalObserver = expect( + port.close({ + reason: "external observer of successful reconciliation", + }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await finalObserver; + expect(runtime.close).toHaveBeenCalledTimes(4); + + resolveFinalReconciliation(); + await vi.advanceTimersByTimeAsync(0); + await Promise.resolve(); + expect(runtime.close).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + } + }); + + it("renews an external intent when bounded protocol success cannot terminate the provider", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let resolveFinalReconciliation!: () => void; + const finalReconciliation = new Promise((resolve) => { + resolveFinalReconciliation = resolve; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockReturnValueOnce(finalReconciliation) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + child.kill = vi.fn(() => true); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + runtimeCloseTimeoutMs: 1, + }); + + const initialObserver = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await initialObserver; + rejectInitialClose(new Error("external close failed late")); + for ( + let turn = 0; + turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4; + turn += 1 + ) { + await Promise.resolve(); + } + expect(runtime.close).toHaveBeenCalledTimes(4); + + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + const finalObserver = expect( + port.close({ reason: "external observer of failed process cleanup" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + resolveFinalReconciliation(); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(4_000); + await finalObserver; + + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + child.signalCode = "SIGKILL"; + child.emit("exit", null, "SIGKILL"); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + + it("renews an external intent when late protocol success follows process cleanup failure", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let resolveFinalReconciliation!: () => void; + const finalReconciliation = new Promise((resolve) => { + resolveFinalReconciliation = resolve; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockReturnValueOnce(finalReconciliation) + .mockResolvedValueOnce(undefined); + const child = fakeChild(); + child.kill = vi.fn(() => true); + const command = fakeCommand(); + vi.mocked(command.spawn).mockReturnValue(child); + let runtimeOptions: AcpRuntimeOptions | undefined; + const port = await openCodexAcpxRuntime(openOptions(command), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: (options) => { + runtimeOptions = options; + return runtime; + }, + runtimeCloseTimeoutMs: 1, + }); + + const initialObserver = expect( + port.close({ reason: "external protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await initialObserver; + rejectInitialClose(new Error("external close failed late")); + for ( + let turn = 0; + turn < 50 && vi.mocked(runtime.close).mock.calls.length < 4; + turn += 1 + ) { + await Promise.resolve(); + } + expect(runtime.close).toHaveBeenCalledTimes(4); + + runtimeOptions?.spawnAgent?.({ + command: "ignored", + args: ["--stdio"], + options: {}, + }); + const finalObserver = expect( + port.close({ reason: "external observer of failed process cleanup" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(4_000); + await finalObserver; + expect(runtime.close).toHaveBeenCalledTimes(4); + + child.signalCode = "SIGKILL"; + child.emit("exit", null, "SIGKILL"); + resolveFinalReconciliation(); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(5); + } finally { + vi.useRealTimers(); + } + }); + + it("shares one reconciliation budget across repeated bounded observers", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectRetainedClose!: (error: unknown) => void; + const retainedClose = new Promise((_resolve, reject) => { + rejectRetainedClose = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(retainedClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockRejectedValueOnce(new Error("reconciliation 3 failed")); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + for (const reason of [ + "first observer", + "second observer", + "third observer", + ]) { + const close = expect(port.close({ reason })).rejects.toThrow( + "ACPX runtime and provider cleanup failed", + ); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await close; + } + expect(runtime.close).toHaveBeenCalledOnce(); + rejectRetainedClose(new Error("retained close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + await vi.advanceTimersByTimeAsync(0); + expect(runtime.close).toHaveBeenCalledTimes(4); + } finally { + vi.useRealTimers(); + } + }); + + it("gives each late failure generation a bounded reconciliation budget", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let rejectInitialClose!: (error: unknown) => void; + const initialClose = new Promise((_resolve, reject) => { + rejectInitialClose = reject; + }); + let rejectNewerClose!: (error: unknown) => void; + const newerClose = new Promise((_resolve, reject) => { + rejectNewerClose = reject; + }); + vi.mocked(runtime.close) + .mockReturnValueOnce(initialClose) + .mockRejectedValueOnce(new Error("reconciliation 1 failed")) + .mockRejectedValueOnce(new Error("reconciliation 2 failed")) + .mockRejectedValueOnce(new Error("reconciliation 3 failed")) + .mockReturnValueOnce(newerClose) + .mockResolvedValueOnce(undefined); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const firstClose = expect( + port.close({ reason: "first protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await firstClose; + rejectInitialClose(new Error("initial close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(4)); + await vi.advanceTimersByTimeAsync(0); + + const secondClose = expect( + port.close({ reason: "newer protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await secondClose; + expect(runtime.close).toHaveBeenCalledTimes(5); + + rejectNewerClose(new Error("newer close failed late")); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(6)); + expect(runtime.close).toHaveBeenLastCalledWith({ + handle: HANDLE, + reason: "ACPX late protocol cleanup reconciliation 1", + discardPersistentState: false, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("marks a retained protocol close terminal when it succeeds late", async () => { + vi.useFakeTimers(); + try { + const runtime = fakeRuntime(); + let resolveRetainedClose!: () => void; + const retainedClose = new Promise((resolve) => { + resolveRetainedClose = resolve; + }); + vi.mocked(runtime.close).mockReturnValueOnce(retainedClose); + const port = await openCodexAcpxRuntime(openOptions(fakeCommand()), { + createRegistry: () => registry(), + createStore: () => store(), + createRuntime: () => runtime, + runtimeCloseTimeoutMs: 1, + }); + + const close = expect( + port.close({ reason: "protocol close stalls" }), + ).rejects.toThrow("ACPX runtime and provider cleanup failed"); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(1); + await close; + expect(runtime.close).toHaveBeenCalledOnce(); + + resolveRetainedClose(); + await vi.advanceTimersByTimeAsync(0); + await expect( + port.close({ reason: "observe late success" }), + ).resolves.toBeUndefined(); + expect(runtime.close).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + it("maps prompt turns to the admitted ACPX handle", async () => { const runtime = fakeRuntime(); const turn = { @@ -566,11 +1376,14 @@ describe("Codex ACPX runtime adapter", () => { .mockResolvedValueOnce(undefined); const controller = new AbortController(); const cancellation = new Error("runtime admission cancelled"); + const retainedAdmissionCleanups: Promise[] = []; const opening = openCodexAcpxRuntime( { ...openOptions(fakeCommand()), signal: controller.signal, + retainFailedAdmissionCleanup: (cleanup) => + retainedAdmissionCleanups.push(cleanup), }, { createRegistry: () => registry(), @@ -584,9 +1397,11 @@ describe("Codex ACPX runtime adapter", () => { controller.abort(cancellation); await expect(opening).rejects.toBe(cancellation); + expect(retainedAdmissionCleanups).toHaveLength(1); resolveHandshake?.(HANDLE); await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + await expect(retainedAdmissionCleanups[0]).resolves.toBeUndefined(); expect(vi.mocked(runtime.close).mock.calls[0]?.[0]).toEqual({ handle: HANDLE, reason: "ACPX runtime admission aborted", @@ -631,7 +1446,9 @@ describe("Codex ACPX runtime adapter", () => { controller.abort(cancellation); await expect(opening).rejects.toBe(cancellation); - expect(retainedCleanups).toHaveLength(1); + // The exact late-handshake owner and the host-facing aggregate proof are + // distinct retained promises over the same cleanup obligation. + expect(retainedCleanups).toHaveLength(2); resolveHandshake?.(HANDLE); await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce()); @@ -649,7 +1466,7 @@ describe("Codex ACPX runtime adapter", () => { expect(runtime.close).toHaveBeenCalledOnce(); resolveClose?.(); - await retainedCleanups[0]; + await expect(Promise.all(retainedCleanups)).resolves.toBeDefined(); expect(cleanupSettled).toBe(true); expect(runtime.close).toHaveBeenCalledOnce(); }); @@ -892,13 +1709,22 @@ describe("Codex ACPX runtime adapter", () => { } as never); await expect(opening).rejects.toBeInstanceOf(Error); - expect(retainedCleanups).toHaveLength(2); + // Retain the pending handshake, the discovered runtime handle cleanup, + // and their host-facing aggregate proof until the same obligation settles. + expect(retainedCleanups).toHaveLength(3); + const settledCleanups = new Set>(); + for (const cleanup of retainedCleanups) { + void cleanup + .finally(() => settledCleanups.add(cleanup)) + .catch(() => undefined); + } await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(5)); - await expect(retainedCleanups[0]).resolves.toBeUndefined(); + await vi.waitFor(() => expect(settledCleanups.size).toBe(1)); expect(runtime.close).toHaveBeenCalledTimes(5); rejectHandshake?.(new Error("test handshake stopped")); - await expect(retainedCleanups[1]).rejects.toThrow("test handshake stopped"); + await vi.waitFor(() => expect(settledCleanups.size).toBe(3)); + await expect(Promise.all(retainedCleanups)).resolves.toBeDefined(); }); it("aggregates asynchronous provider signal errors after a failed handshake", async () => { @@ -1267,6 +2093,7 @@ function openOptions( }, systemInstructions: "Use Paperclip tools.", mcpServers: [], + retainFailedAdmissionCleanup: vi.fn(), }; } diff --git a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts index 01a15540a8..3c4508be88 100644 --- a/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts +++ b/packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.ts @@ -24,6 +24,22 @@ const VERIFIED_COMMAND_SENTINEL = "paperclip-verified-acpx-command"; const DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS = 2_000; const RETAINED_ADMISSION_CLEANUP_RETRY_MIN_MS = 10; const RETAINED_ADMISSION_CLEANUP_RETRY_MAX_MS = 30_000; +const PROVIDER_TERM_EXIT_TIMEOUT_MS = 2_000; +const PROVIDER_KILL_EXIT_TIMEOUT_MS = 2_000; +const MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS = 3; +// Production shutdown waits for the protocol close bound before beginning the +// sequential TERM/KILL verification windows. Keep this exported package-local +// bound aligned with the implementation so admission can include the complete +// provider cleanup path instead of accounting for only part of it. +export const DEFAULT_CODEX_ACPX_RUNTIME_SHUTDOWN_BOUND_MS = + DEFAULT_RUNTIME_CLOSE_TIMEOUT_MS + + PROVIDER_TERM_EXIT_TIMEOUT_MS + + PROVIDER_KILL_EXIT_TIMEOUT_MS; +// A close may outlive its caller-facing wait bound. Keep every exact attempt +// owned until it settles. A handle never starts a second protocol close while +// the first remains unresolved; late failure can start bounded reconciliation +// only after the exact attempt reaches a terminal outcome. +const activeRuntimeCleanupOwners = new Set>(); const activeCodexRuntimeCleanupOwners = new Set>(); class AcpxRuntimeCloseTimeoutError extends Error { @@ -53,12 +69,19 @@ export async function openCodexAcpxRuntime( options: AcpxRuntimePortOpenOptions, dependencies: CodexAcpxRuntimeDependencies = {}, ): Promise { + if (options.signal?.aborted) { + // The host may have already transferred its staged credential to this + // pending admission before this microtask begins. No adapter resources + // exist yet, so publish an already-complete cleanup proof before preserving + // the caller's exact abort reason. + options.retainFailedAdmissionCleanup(Promise.resolve()); + throw options.signal.reason; + } if (options.profile.agent !== "codex") { throw new Error( "The production ACPX runtime currently supports Codex only", ); } - options.signal?.throwIfAborted(); // The verified-command boundary already refuses to mint a Windows command // lease, because Node cannot pin its executable there. Repeat the platform // gate at this lower boundary so alternate host wiring cannot launch a @@ -81,6 +104,7 @@ export async function openCodexAcpxRuntime( const baseStore = createStore({ stateDir: options.stateDirectory }); let failedHandshakeHandle: AcpRuntimeHandle | null = null; let admissionCleanup: RuntimeAdmissionCleanup | null = null; + let abortedHandshakeCleanup: Promise | null = null; const retainedCleanupOwners = new WeakSet>(); const retainCleanup = (cleanup: Promise): void => { if (retainedCleanupOwners.has(cleanup)) { @@ -236,14 +260,19 @@ export async function openCodexAcpxRuntime( handle = await raceRuntimeHandshakeWithAbort(handshake, options.signal); } catch (error) { if (options.signal.aborted) { - retainCleanup( - handshake.then((lateHandle) => + abortedHandshakeCleanup = handshake.then( + (lateHandle) => admissionCleanup!.runRetained( lateHandle, "ACPX runtime admission aborted", ), - ), + () => + admissionCleanup!.runRetained( + failedHandshakeHandle, + "ACPX runtime admission aborted", + ), ); + retainCleanup(abortedHandshakeCleanup); } throw error; } @@ -252,12 +281,30 @@ export async function openCodexAcpxRuntime( options.signal.throwIfAborted(); } } catch (error) { + const cleanupHandle = handle ?? failedHandshakeHandle; const cleanupErrors = await admissionCleanup.run( - handle ?? failedHandshakeHandle, + cleanupHandle, options.signal?.aborted ? "ACPX runtime admission aborted" : "ACPX session handshake failed", ); + const retainedCleanup = + cleanupErrors.length === 0 + ? Promise.resolve() + : admissionCleanup.runRetained( + cleanupHandle, + options.signal?.aborted + ? "ACPX runtime admission aborted" + : "ACPX session handshake failed", + ); + const cleanupProof = + abortedHandshakeCleanup === null + ? retainedCleanup + : Promise.all([retainedCleanup, abortedHandshakeCleanup]).then( + () => undefined, + ); + options.retainFailedAdmissionCleanup(cleanupProof); + retainCleanup(cleanupProof); if (cleanupErrors.length > 0) { throw new AggregateError( [error, ...cleanupErrors], @@ -277,13 +324,23 @@ export async function openCodexAcpxRuntime( runtime, handle, requireIdentity(handle), - admissionCleanup, + children, + runtimeCloseTimeoutMs, ); } catch (error) { const cleanupErrors = await admissionCleanup.run( handle, "ACPX runtime identity validation failed", ); + const cleanupProof = + cleanupErrors.length === 0 + ? Promise.resolve() + : admissionCleanup.runRetained( + handle, + "ACPX runtime identity validation failed", + ); + options.retainFailedAdmissionCleanup(cleanupProof); + retainCleanup(cleanupProof); if (cleanupErrors.length > 0) { throw new AggregateError( [error, ...cleanupErrors], @@ -371,12 +428,12 @@ class RuntimeAdmissionCleanup { const targetKey = this.#resolveTargetKey(rawTargetKey, handle); const existing = this.#registeredTargets.get(targetKey); if (existing !== undefined) { - if (handle !== null) { - existing.handle = - existing.handle === null - ? handle + existing.handle = + existing.handle === null + ? handle + : handle === null + ? existing.handle : preferRuntimeAdmissionCleanupHandle(existing.handle, handle); - } this.#targetAliases.set(rawTargetKey, targetKey); return existing.cleanup!; } @@ -407,8 +464,7 @@ class RuntimeAdmissionCleanup { const fallbackHandle = this.#registeredTargets.get(fallbackTargetKey)?.handle; if ( - fallbackHandle !== undefined && - fallbackHandle !== null && + fallbackHandle != null && nonEmptyRuntimeIdentity(fallbackHandle.acpxRecordId) === undefined && sameRuntimeAdmissionCleanupOwner(fallbackHandle, handle) ) { @@ -590,9 +646,232 @@ function runtimePort( runtime: AcpRuntime, handle: AcpRuntimeHandle, identity: AcpxRuntimePortIdentity, - admissionCleanup: RuntimeAdmissionCleanup, + children: SpawnedChildSet, + runtimeCloseTimeoutMs: number, ): AcpxRuntimePort { - return { + type RuntimeCloseAttempt = { + readonly outcome: Promise; + readonly reconciliationGeneration: number; + readonly origin: + | { readonly kind: "external" } + | { + readonly kind: "reconciliation"; + readonly generation: number; + readonly attemptNumber: number; + }; + pendingExternalIntent: boolean; + }; + let runtimeClosed = false; + let runtimeCloseAttempt: RuntimeCloseAttempt | undefined; + let lateReconciliationOwner: Promise | undefined; + // Each independently observed late failure receives a bounded reconciliation + // budget. Exhausting retries for an older generation must not prevent a newer + // late failure from acquiring its own recovery owner. + let lateReconciliationAttemptGeneration = 0; + let lateReconciliationAttempts = 0; + let lateFailureGeneration = 0; + let reconciledLateFailureGeneration = 0; + const watchedReleasedAttempts = new Set(); + + const hasUnreconciledLateFailure = (): boolean => + reconciledLateFailureGeneration < lateFailureGeneration; + + const consumePendingExternalIntent = ( + attempt: RuntimeCloseAttempt, + failed: boolean, + ): boolean => { + const pending = attempt.pendingExternalIntent; + attempt.pendingExternalIntent = false; + return ( + pending && + failed && + attempt.origin.kind === "reconciliation" && + attempt.origin.attemptNumber >= + MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS + ); + }; + + const scheduleLateFailureReconciliation = (): void => { + if ( + runtimeCloseAttempt || + lateReconciliationOwner || + !hasUnreconciledLateFailure() + ) { + return; + } + if (lateReconciliationAttemptGeneration !== lateFailureGeneration) { + lateReconciliationAttemptGeneration = lateFailureGeneration; + lateReconciliationAttempts = 0; + } + if ( + lateReconciliationAttempts >= + MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS + ) { + return; + } + const attemptGeneration = lateFailureGeneration; + const attemptNumber = lateReconciliationAttempts + 1; + lateReconciliationAttempts = attemptNumber; + let retry = false; + const reconciliation = closeRuntime({ + reason: `ACPX late protocol cleanup reconciliation ${attemptNumber}`, + reconciliation: { + generation: attemptGeneration, + attemptNumber, + }, + }).then( + () => { + if (hasUnreconciledLateFailure()) { + retry = + lateFailureGeneration === attemptGeneration && + attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS; + } + }, + () => { + retry = + lateFailureGeneration === attemptGeneration && + attemptNumber < MAX_LATE_RUNTIME_CLEANUP_RECONCILIATION_ATTEMPTS; + }, + ); + const owner = reconciliation.finally(() => { + if (lateReconciliationOwner === owner) + lateReconciliationOwner = undefined; + if (retry || hasUnreconciledLateFailure()) { + queueMicrotask(scheduleLateFailureReconciliation); + } + }); + lateReconciliationOwner = owner; + retainRuntimeCleanupOwner(owner); + }; + + const watchPendingAttempt = ( + attempt: RuntimeCloseAttempt, + processCleanupSucceeded: boolean, + ): void => { + if (watchedReleasedAttempts.has(attempt)) return; + watchedReleasedAttempts.add(attempt); + void attempt.outcome.then((error) => { + watchedReleasedAttempts.delete(attempt); + if (runtimeCloseAttempt === attempt) runtimeCloseAttempt = undefined; + const renewForExternalIntent = consumePendingExternalIntent( + attempt, + error !== null || !processCleanupSucceeded, + ); + if (renewForExternalIntent) lateFailureGeneration += 1; + if (error === null) { + if (processCleanupSucceeded) { + reconciledLateFailureGeneration = Math.max( + reconciledLateFailureGeneration, + attempt.reconciliationGeneration, + ); + runtimeClosed = !hasUnreconciledLateFailure(); + } + scheduleLateFailureReconciliation(); + return; + } + // A newer successful close cannot erase an older outcome that had not + // settled yet. Re-open cleanup state and autonomously create a bounded + // reconciliation generation so the late failure is not suppression-only. + // An autonomous failure remains charged to the budget of the generation + // that created it. If external callers coalesced onto the exhausted final + // attempt, their single batched intent creates exactly one new generation; + // joins on earlier attempts are satisfied by the remaining same-generation + // retries. + if (attempt.origin.kind === "external") { + lateFailureGeneration += 1; + } + runtimeClosed = false; + scheduleLateFailureReconciliation(); + }); + }; + + async function closeRuntime(input: { + reason: string; + reconciliation?: { + generation: number; + attemptNumber: number; + }; + }): Promise { + if (runtimeClosed) return; + if ( + runtimeCloseAttempt?.origin.kind === "reconciliation" && + input.reconciliation === undefined + ) { + // Preserve the attempt's autonomous origin while remembering that one or + // more external callers requested a fresh cleanup observation. The exact + // outcome consumes this bit, so coalesced callers cannot mint generations + // independently. + runtimeCloseAttempt.pendingExternalIntent = true; + } + if (!runtimeCloseAttempt) { + // A close can reconcile only failures already known when its protocol + // attempt begins. A released older attempt may reject while this one is + // in flight; that later generation must trigger a subsequent close. + runtimeCloseAttempt = { + outcome: ownedRuntimeCloseOutcome(runtime, handle, input.reason), + reconciliationGeneration: + input.reconciliation?.generation ?? lateFailureGeneration, + origin: + input.reconciliation === undefined + ? { kind: "external" } + : { + kind: "reconciliation", + generation: input.reconciliation.generation, + attemptNumber: input.reconciliation.attemptNumber, + }, + pendingExternalIntent: false, + }; + } + const observedAttempt = runtimeCloseAttempt; + const processCleanup = terminateChildrenAfterCloseBound( + observedAttempt.outcome, + children, + runtimeCloseTimeoutMs, + ); + // The caller may stop waiting, but the exact ACPX protocol cleanup stays + // owned and remains this handle's sole close attempt until it settles. + // Provider termination still proceeds at the deadline. + const [closeError, processErrors] = await Promise.all([ + boundedCloseOutcome(observedAttempt.outcome, runtimeCloseTimeoutMs), + processCleanup, + ]); + if (closeError instanceof AcpxRuntimeCloseTimeoutError) { + watchPendingAttempt(observedAttempt, processErrors.length === 0); + } else { + if ( + consumePendingExternalIntent( + observedAttempt, + closeError !== null || processErrors.length > 0, + ) + ) { + lateFailureGeneration += 1; + } + if (runtimeCloseAttempt === observedAttempt) { + runtimeCloseAttempt = undefined; + } + } + if (processErrors.length === 0 && closeError === null) { + reconciledLateFailureGeneration = Math.max( + reconciledLateFailureGeneration, + observedAttempt.reconciliationGeneration, + ); + runtimeClosed = !hasUnreconciledLateFailure(); + } else { + runtimeClosed = false; + } + scheduleLateFailureReconciliation(); + if (closeError !== null || processErrors.length > 0) { + const errors = [closeError, ...processErrors].filter( + (error): error is unknown => error !== null, + ); + throw new AggregateError( + errors, + "ACPX runtime and provider cleanup failed", + ); + } + } + + const port: AcpxRuntimePort = { async identity() { return structuredClone(identity); }, @@ -622,16 +901,9 @@ function runtimePort( ...(input.signal ? { signal: input.signal } : {}), }); }, - async close(input) { - const errors = await admissionCleanup.run(handle, input.reason); - if (errors.length > 0) { - throw new AggregateError( - errors, - "ACPX runtime and provider cleanup failed", - ); - } - }, + close: closeRuntime, }; + return port; } function runtimeCloseOutcome( @@ -663,6 +935,70 @@ async function closeOutcomeWithin( return outcome; } +function ownedRuntimeCloseOutcome( + runtime: AcpRuntime, + handle: AcpRuntimeHandle, + reason: string, +): Promise { + const cleanup = Promise.resolve() + .then(() => + runtime.close({ handle, reason, discardPersistentState: false }), + ) + .then( + () => null, + (error: unknown) => error, + ); + return retainRuntimeCleanupOwner(cleanup); +} + +function retainRuntimeCleanupOwner(cleanup: Promise): Promise { + activeRuntimeCleanupOwners.add(cleanup); + void cleanup + .finally(() => activeRuntimeCleanupOwners.delete(cleanup)) + .catch(() => undefined); + return cleanup; +} + +async function terminateChildrenAfterCloseBound( + closeOutcome: Promise, + children: SpawnedChildSet, + timeoutMs: number, +): Promise { + let timer: ReturnType | undefined; + try { + await Promise.race([ + closeOutcome.then(() => undefined), + new Promise((resolve) => { + timer = setTimeout(resolve, Math.max(1, Math.floor(timeoutMs))); + timer.unref(); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + return await children.terminate(); +} + +async function boundedCloseOutcome( + closeOutcome: Promise, + timeoutMs: number, +): Promise { + const boundedTimeoutMs = Math.max(1, timeoutMs); + let timer: ReturnType | undefined; + const outcome = await Promise.race([ + closeOutcome.then((error) => ({ error })), + new Promise<{ error: unknown }>((resolve) => { + timer = setTimeout( + () => resolve({ error: new AcpxRuntimeCloseTimeoutError() }), + boundedTimeoutMs, + ); + timer.unref(); + }), + ]); + if (timer) clearTimeout(timer); + return outcome.error; +} + function delay(timeoutMs: number): Promise { return new Promise((resolve) => { const timer = setTimeout(resolve, timeoutMs); @@ -703,7 +1039,7 @@ class SpawnedChildSet { const terminateOutcome = await signalAndWaitForExit( tracked, "SIGTERM", - 2_000, + PROVIDER_TERM_EXIT_TIMEOUT_MS, ); if (terminateOutcome.error !== undefined) { pushUnique(errors, terminateOutcome.error); @@ -712,7 +1048,7 @@ class SpawnedChildSet { const killOutcome = await signalAndWaitForExit( tracked, "SIGKILL", - 2_000, + PROVIDER_KILL_EXIT_TIMEOUT_MS, ); if (killOutcome.error !== undefined) { pushUnique(errors, killOutcome.error); diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts index 5a35c44fa5..22cd50e916 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts @@ -4,6 +4,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { openCodexAcpxRuntime } from "./codex-runtime-adapter.js"; import { stageManagedCodexCredential } from "./codex-credentials.js"; import type { VerifiedAcpxCommandLease, @@ -14,6 +15,7 @@ import { AcpxRuntimeHost, type AcpxRuntimeHostDependencies, type AcpxRuntimePort, + type AcpxRuntimePortOpenOptions, type AcpxRuntimeTurn, } from "./runtime-host.js"; @@ -59,6 +61,65 @@ describe("ACPX runtime host", () => { expect(fixture.commandClose).not.toHaveBeenCalled(); }); + it("scrubs credentials when abort wins before the adapter body starts", async () => { + const fixture = await hostFixture(); + const controller = new AbortController(); + const cancellation = new Error("runtime admission cancelled before entry"); + const createRuntime = vi.fn(); + let credentialHome = ""; + const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => { + credentialHome = options.launchEnvironment.CODEX_HOME!; + // The host has scheduled its openRuntime callback and transferred the + // staged credential to that pending admission. Abort before entering the + // adapter so its pre-entry path must publish a completed cleanup proof. + controller.abort(cancellation); + return openCodexAcpxRuntime(options, { createRuntime }); + }); + + await expect( + AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "deny-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + signal: controller.signal, + }, + fixture.dependencies({ openRuntime }), + ), + ).rejects.toBe(cancellation); + + expect(openRuntime).toHaveBeenCalledOnce(); + expect(createRuntime).not.toHaveBeenCalled(); + expect(fixture.commandClose).toHaveBeenCalledOnce(); + const authPath = join(credentialHome, "auth.json"); + const contender = await vi.waitFor(() => + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ); + await contender.close(); + await expect(readFile(authPath)).rejects.toMatchObject({ code: "ENOENT" }); + + const retryRuntime = runtimePort(); + const retryHost = await AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "deny-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }, + fixture.dependencies({ openRuntime: async () => retryRuntime }), + ); + await retryHost.close({ reason: "retry admission complete" }); + expect(retryRuntime.close).toHaveBeenCalledOnce(); + }); + it("composes admission, isolation, model verification, and cleanup", async () => { const fixture = await hostFixture(); let capturedEnvironment: Readonly = {}; @@ -301,6 +362,77 @@ describe("ACPX runtime host", () => { expect(fixture.commandClose).toHaveBeenCalledOnce(); }); + it("retains failed admission cleanup until provider shutdown permits credential scrub", async () => { + const fixture = await hostFixture(); + let authPath = ""; + let credentialHome = ""; + let resolveRetryClose!: () => void; + const retryClose = new Promise((resolve) => { + resolveRetryClose = resolve; + }); + const runtime = runtimePort({ + getStatus: async () => ({ + models: { + currentModelId: "wrong-model", + availableModelIds: ["wrong-model"], + }, + }), + onClose: vi + .fn() + .mockRejectedValueOnce(new Error("first admission cleanup failed")) + .mockImplementationOnce(() => retryClose), + }); + + await expect( + AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "approve-all", + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: + '{"owner":"failed-admission"}', + }, + }, + fixture.dependencies({ + openRuntime: async (options) => { + credentialHome = options.launchEnvironment.CODEX_HOME!; + authPath = join(credentialHome, "auth.json"); + return runtime; + }, + }), + ), + ).rejects.toThrow(/initialization and cleanup failed/); + + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledTimes(2)); + await expect(readFile(authPath, "utf8")).resolves.toContain( + "failed-admission", + ); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + + resolveRetryClose(); + await vi.waitFor(async () => { + await expect(readFile(authPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + const contender = await stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }); + await contender.close(); + }); + it("retains credential ownership when runtime shutdown fails until retry succeeds", async () => { const fixture = await hostFixture(); let failClose = true; @@ -402,6 +534,70 @@ describe("ACPX runtime host", () => { await contender.close(); }); + it("retains the exact pending cleanup while independent resources close", async () => { + const fixture = await hostFixture(); + const firstClose = new Promise(() => undefined); + const runtime = runtimePort({ + onClose: vi + .fn() + .mockImplementationOnce(() => firstClose) + .mockResolvedValueOnce(undefined), + }); + const host = await AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "approve-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }, + fixture.dependencies({ openRuntime: async () => runtime }), + ); + + const first = host.close({ reason: "first close stalls" }); + await vi.waitFor(() => expect(runtime.close).toHaveBeenCalledOnce()); + const second = host.close({ reason: "same pending owner" }); + let settled = false; + void Promise.all([first, second]).finally(() => { + settled = true; + }); + await Promise.resolve(); + expect(settled).toBe(false); + expect(runtime.close).toHaveBeenCalledOnce(); + expect(fixture.commandClose).toHaveBeenCalledOnce(); + }); + + it("retries only after the exact close outcome settles with failure", async () => { + const fixture = await hostFixture(); + const runtime = runtimePort({ + onClose: vi + .fn() + .mockRejectedValueOnce(new Error("runtime close failed")) + .mockResolvedValueOnce(undefined), + }); + const host = await AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "approve-all", + environment: { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}" }, + }, + fixture.dependencies({ openRuntime: async () => runtime }), + ); + + await expect(host.close({ reason: "first close" })).rejects.toThrow( + /cleanup failed/, + ); + await expect( + host.close({ reason: "fresh attempt" }), + ).resolves.toBeUndefined(); + await expect( + host.close({ reason: "ownership released" }), + ).resolves.toBeUndefined(); + expect(runtime.close).toHaveBeenCalledTimes(2); + }); + it("admits one bounded turn and cancels it before shutdown", async () => { const fixture = await hostFixture(); const turn = runtimeTurn(); @@ -429,6 +625,9 @@ describe("ACPX runtime host", () => { host.startTurn({ text: "Concurrent", requestId: "turn-2" }), ).toThrow("already has an active turn"); + await host.interruptActiveTurn("user interrupt"); + expect(turn.cancel).toHaveBeenCalledWith({ reason: "user interrupt" }); + await host.close({ reason: "shutdown" }); expect(turn.cancel).toHaveBeenCalledWith({ reason: "shutdown" }); expect(runtime.close).toHaveBeenCalledOnce(); @@ -693,13 +892,23 @@ describe("ACPX runtime host", () => { expect(fixture.commandClose).not.toHaveBeenCalled(); }); - it("forwards cancellation and closes a runtime that resolves after abort", async () => { + it("retains managed credentials until an aborted late runtime is closed", async () => { const fixture = await hostFixture(); const runtimeAdmission = deferred(); - const lateRuntime = runtimePort(); + const retryClose = deferred(); + const lateRuntime = runtimePort({ + onClose: vi + .fn() + .mockRejectedValueOnce(new Error("late runtime close failed")) + .mockImplementationOnce(() => retryClose.promise), + }); let receivedSignal: AbortSignal | undefined; + let credentialHome = ""; + let bridgeUrl = ""; const openRuntime = vi.fn((options) => { receivedSignal = options.signal; + credentialHome = options.launchEnvironment.CODEX_HOME!; + bridgeUrl = options.mcpServers[0]!.url; return runtimeAdmission.promise; }); const controller = new AbortController(); @@ -714,6 +923,10 @@ describe("ACPX runtime host", () => { PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: "{}", }, signal: controller.signal, + semanticTools: { + tools: [], + handler: async () => ({ ok: true }), + }, }, fixture.dependencies({ openRuntime }), ); @@ -723,13 +936,93 @@ describe("ACPX runtime host", () => { controller.abort(cancellation); await expect(opening).rejects.toBe(cancellation); expect(fixture.commandClose).toHaveBeenCalledOnce(); - runtimeAdmission.resolve(lateRuntime); + await expect(fetch(bridgeUrl)).rejects.toThrow(); + const authPath = join(credentialHome, "auth.json"); + await expect(readFile(authPath, "utf8")).resolves.toBe("{}"); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); - await vi.waitFor(() => - expect(lateRuntime.close).toHaveBeenCalledWith({ - reason: "ACPX runtime admission aborted", + runtimeAdmission.resolve(lateRuntime); + await vi.waitFor(() => expect(lateRuntime.close).toHaveBeenCalledTimes(2)); + expect(lateRuntime.close).toHaveBeenNthCalledWith(1, { + reason: "ACPX runtime admission aborted", + }); + await expect(readFile(authPath, "utf8")).resolves.toBe("{}"); + await expect( + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, + }), + ).rejects.toThrow("already has an active lease"); + + retryClose.resolve(undefined); + await vi.waitFor(async () => { + await expect(readFile(authPath)).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + // File removal precedes kernel lease release. Wait for the lease itself so + // this assertion cannot race between those two ordered cleanup steps. + const contender = await vi.waitFor(() => + stageManagedCodexCredential({ + agentHomeDirectory: credentialHome, + environment: { + PAPERCLIP_ACPX_CODEX_AUTH_JSON_SECRET: '{"owner":"contender"}', + }, }), ); + await contender.close(); + }); + + it("scrubs credentials after rejected runtime cleanup is proven", async () => { + const fixture = await hostFixture(); + const runtimeAdmission = deferred(); + const providerCleanup = deferred(); + const credentialClose = vi.fn(async () => undefined); + const controller = new AbortController(); + const cancellation = new Error("runtime admission cancelled"); + const openRuntime = vi.fn((options: AcpxRuntimePortOpenOptions) => { + options.retainFailedAdmissionCleanup(providerCleanup.promise); + return runtimeAdmission.promise; + }); + const opening = AcpxRuntimeHost.open( + { + ...fixture.options, + agent: "codex", + model: "gpt-5.6-sol", + permissionMode: "deny-all", + signal: controller.signal, + }, + { + ...fixture.dependencies({ openRuntime }), + stageCredential: async () => ({ + path: join(fixture.root, "auth.json"), + mode: "inline_json", + close: credentialClose, + }), + }, + ); + await vi.waitFor(() => expect(openRuntime).toHaveBeenCalledOnce()); + + controller.abort(cancellation); + await expect(opening).rejects.toBe(cancellation); + const cleanupFailure = new Error("provider survived forced cleanup"); + runtimeAdmission.reject(new AggregateError([cancellation, cleanupFailure])); + await new Promise((resolve) => setImmediate(resolve)); + + expect(credentialClose).not.toHaveBeenCalled(); + expect(fixture.commandClose).toHaveBeenCalledOnce(); + + providerCleanup.resolve(undefined); + await vi.waitFor(() => expect(credentialClose).toHaveBeenCalledOnce()); }); }); @@ -824,10 +1117,13 @@ async function hostFixture() { function deferred(): { promise: Promise; resolve(value: T): void; + reject(reason: unknown): void; } { let resolve!: (value: T) => void; - const promise = new Promise((settle) => { + let reject!: (reason: unknown) => void; + const promise = new Promise((settle, fail) => { resolve = settle; + reject = fail; }); - return { promise, resolve }; + return { promise, resolve, reject }; } diff --git a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts index 86c200744c..81e02b42c4 100644 --- a/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts +++ b/packages/paperclip-runner/src/drivers/acpx/runtime-host.ts @@ -38,7 +38,11 @@ import { } from "./runtime-sandbox.js"; import type { AcpxExpectedSessionIdentity } from "./sidecar-protocol.js"; -const TURN_CANCELLATION_TIMEOUT_MS = 2_000; +export const ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS = 2_000; + +const ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS = 8; +const ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS = 10; +const ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS = 1_000; export interface AcpxRuntimePortIdentity { acpxRecordId: string; @@ -83,6 +87,11 @@ export interface AcpxRuntimePortOpenOptions { /** Abort provider admission and clean any runtime that resolves too late. */ signal?: AbortSignal; mcpServers: readonly AcpxMcpServerBinding[]; + /** + * Transfer the provider cleanup proof before a failed open settles. The host + * keeps credentials fenced until this exact cleanup succeeds. + */ + retainFailedAdmissionCleanup(cleanup: Promise): void; } export interface AcpxRetainedCleanupFailure { @@ -134,6 +143,35 @@ const activeRuntimeHostCleanupOwners = new Set>(); const RETAINED_CLEANUP_RETRY_INITIAL_DELAY_MS = 10; const RETAINED_CLEANUP_RETRY_MAX_DELAY_MS = 1_000; +interface RetainedRejectedRuntimeAdmission { + readonly credential: ManagedCodexCredentialLease; + cleanup: Promise; +} + +// A rejected provider open is not itself proof that provider processes are +// gone. Retain the credential with the adapter's exact cleanup proof and scrub +// it only after that proof succeeds. Terminal cleanup failure stays inert and +// fenced instead of exposing the credential to a replacement admission. +const retainedRejectedRuntimeAdmissions = + new Set(); + +interface RetainedAcpxAdmissionCleanup { + readonly runtime: AcpxRuntimePort | null; + readonly toolBridge: RunnerToolBridge | null; + readonly credential: ManagedCodexCredentialLease | null; + readonly command: VerifiedAcpxCommandLease | null; + readonly reason: string; + recovery: Promise | null; + timer: ReturnType | null; +} + +// An admission failure has no AcpxRuntimeHost instance for a caller to close. +// Retain those resources here until provider shutdown succeeds and the staged +// credential can consequently be scrubbed. Each recovery batch is finite and +// sequential; an unref'd timer rate-limits later autonomous attempts without +// allowing the failed admission's ownership to become unreachable. +const retainedAcpxAdmissionCleanups = new Set(); + export class AcpxRuntimeHost { readonly #runtime: AcpxRuntimePort; readonly #binding: AcpxRecoveryBinding; @@ -205,6 +243,37 @@ export class AcpxRuntimeHost { let credential: ManagedCodexCredentialLease | null = null; let toolBridge: RunnerToolBridge | null = null; let runtime: AcpxRuntimePort | null = null; + let pendingRuntimeOwnsCredential = false; + let failedAdmissionCleanupTransferred = false; + let resolveFailedAdmissionCleanupTransfer!: () => void; + const failedAdmissionCleanupTransfer = new Promise((resolve) => { + resolveFailedAdmissionCleanupTransfer = resolve; + }); + const retainFailedAdmissionCleanup = (cleanup: Promise): void => { + if (failedAdmissionCleanupTransferred) return; + failedAdmissionCleanupTransferred = true; + pendingRuntimeOwnsCredential = credential !== null; + resolveFailedAdmissionCleanupTransfer(); + if (credential === null) { + retainRuntimeHostCleanup(cleanup); + return; + } + const retained: RetainedRejectedRuntimeAdmission = { + credential, + cleanup: Promise.resolve(), + }; + const ownedCleanup = cleanup.then(async () => { + await cleanupAbortedRuntimeAdmission( + null, + retained.credential, + "ACPX rejected runtime admission cleanup confirmed", + ); + retainedRejectedRuntimeAdmissions.delete(retained); + }); + retained.cleanup = ownedCleanup; + retainedRejectedRuntimeAdmissions.add(retained); + retainRuntimeHostCleanup(ownedCleanup); + }; try { const sandbox = await runAbortableAdmissionStage(options.signal, () => prepareAcpxRuntimeSandbox({ @@ -272,6 +341,7 @@ export class AcpxRuntimeHost { }, ] : [], + retainFailedAdmissionCleanup, }), resource: "runtime", releaseLate: (lateRuntime) => @@ -280,6 +350,19 @@ export class AcpxRuntimeHost { }), reportFailure: (failure) => dependencies.reportRetainedCleanupFailure(failure), + onAbortedPending: (pendingRuntime) => { + // A provider may already be running even though openRuntime has not + // returned its port. Keep its credential lease with that exact + // admission while the ordinary catch path revokes tools and releases + // the consumed command snapshot without delaying cancellation. + pendingRuntimeOwnsCredential = credential !== null; + retainAbortedRuntimeAdmissionCleanup({ + pendingRuntime, + credential, + reason: "ACPX runtime admission aborted", + failedAdmissionCleanupTransfer, + }); + }, }); await runAbortableAdmissionStage(options.signal, () => requireVerifiedAcpxModel(runtime!, profile), @@ -316,11 +399,18 @@ export class AcpxRuntimeHost { const cleanupError = await cleanupRuntimeResources( runtime, toolBridge, - credential, + pendingRuntimeOwnsCredential ? null : credential, command, "ACPX runtime initialization failed", ); if (cleanupError) { + retainFailedAcpxAdmissionCleanup({ + runtime, + toolBridge, + credential: pendingRuntimeOwnsCredential ? null : credential, + command, + reason: "ACPX runtime initialization failed", + }); throw new AggregateError( [error, ...cleanupError.errors], "ACPX runtime initialization and cleanup failed", @@ -346,6 +436,10 @@ export class AcpxRuntimeHost { return Object.freeze({ ...this.#sandbox.persistedEnvironment }); } + async status(): Promise { + return structuredClone(await this.#runtime.getStatus()); + } + startTurn(input: AcpxRuntimeTurnInput): AcpxRuntimeTurn { if (this.#closed || this.#closingStarted) { throw new Error("ACPX runtime host is closing"); @@ -374,9 +468,20 @@ export class AcpxRuntimeHost { return turn; } + async interruptActiveTurn(reason: string): Promise { + const turn = this.#activeTurn; + if (!turn) throw new Error("ACPX runtime host has no active turn"); + const cancellationError = await boundedCancellation( + turn.cancel({ reason: boundedReason(reason) }), + ); + if (cancellationError) throw cancellationError; + } + async close(input: { reason: string }): Promise { if (this.#closed) return; - if (this.#closePromise) return await this.#closePromise; + if (this.#closePromise) { + return await this.#closePromise; + } this.#closingStarted = true; const closePromise = this.#close(boundedReason(input.reason)); this.#closePromise = closePromise; @@ -435,6 +540,7 @@ async function acquireAbortableAdmissionResource(input: { resource: AcpxRetainedCleanupFailure["resource"]; releaseLate: (resource: T) => Promise; reportFailure: (failure: AcpxRetainedCleanupFailure) => void; + onAbortedPending?: (pending: Promise) => void; }): Promise { if (input.signal === undefined) return await input.acquire(); input.signal.throwIfAborted(); @@ -443,16 +549,20 @@ async function acquireAbortableAdmissionResource(input: { return await raceAdmissionWithAbort(pending, input.signal); } catch (error) { if (input.signal.aborted) { - retainRuntimeHostCleanup( - pending.then((resource) => - releaseRetainedAdmissionResource({ - resource, - resourceKind: input.resource, - release: input.releaseLate, - reportFailure: input.reportFailure, - }), - ), - ); + if (input.onAbortedPending) { + input.onAbortedPending(pending); + } else { + retainRuntimeHostCleanup( + pending.then((resource) => + releaseRetainedAdmissionResource({ + resource, + resourceKind: input.resource, + release: input.releaseLate, + reportFailure: input.reportFailure, + }), + ), + ); + } } throw error; } @@ -530,6 +640,123 @@ function retainRuntimeHostCleanup(cleanup: Promise): void { .catch(() => undefined); } +function retainAbortedRuntimeAdmissionCleanup(input: { + pendingRuntime: Promise; + credential: ManagedCodexCredentialLease | null; + reason: string; + failedAdmissionCleanupTransfer: Promise; +}): void { + const cleanup = input.pendingRuntime.then( + (runtime) => + cleanupAbortedRuntimeAdmission(runtime, input.credential, input.reason), + () => input.failedAdmissionCleanupTransfer, + ); + retainRuntimeHostCleanup(cleanup); +} + +async function cleanupAbortedRuntimeAdmission( + runtime: AcpxRuntimePort | null, + credential: ManagedCodexCredentialLease | null, + reason: string, +): Promise { + const cleanupError = await cleanupRuntimeResources( + runtime, + null, + credential, + null, + reason, + ); + if (!cleanupError) return; + retainFailedAcpxAdmissionCleanup({ + runtime, + toolBridge: null, + credential, + command: null, + reason, + }); +} + +function retainFailedAcpxAdmissionCleanup(input: { + runtime: AcpxRuntimePort | null; + toolBridge: RunnerToolBridge | null; + credential: ManagedCodexCredentialLease | null; + command: VerifiedAcpxCommandLease | null; + reason: string; +}): void { + const cleanup: RetainedAcpxAdmissionCleanup = { + ...input, + recovery: null, + timer: null, + }; + retainedAcpxAdmissionCleanups.add(cleanup); + startRetainedAcpxAdmissionCleanup(cleanup); +} + +function startRetainedAcpxAdmissionCleanup( + cleanup: RetainedAcpxAdmissionCleanup, +): Promise { + if (cleanup.recovery) return cleanup.recovery; + const recovery = (async () => { + let retryDelayMs = ACPX_ADMISSION_CLEANUP_RETRY_DELAY_MS; + for ( + let attempt = 1; + attempt <= ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS; + attempt += 1 + ) { + const cleanupError = await cleanupRuntimeResources( + cleanup.runtime, + cleanup.toolBridge, + cleanup.credential, + cleanup.command, + `${cleanup.reason} (automatic cleanup recovery ${attempt})`, + ); + if (!cleanupError) { + retainedAcpxAdmissionCleanups.delete(cleanup); + if (cleanup.timer) clearTimeout(cleanup.timer); + cleanup.timer = null; + return; + } + if (attempt < ACPX_ADMISSION_CLEANUP_BATCH_ATTEMPTS) { + await waitForAdmissionCleanupRetry(retryDelayMs); + retryDelayMs = Math.min(retryDelayMs * 2, 1_000); + } + } + })(); + cleanup.recovery = recovery; + void recovery + .finally(() => { + if (cleanup.recovery === recovery) cleanup.recovery = null; + scheduleRetainedAcpxAdmissionCleanup(cleanup); + }) + .catch(() => undefined); + return recovery; +} + +function scheduleRetainedAcpxAdmissionCleanup( + cleanup: RetainedAcpxAdmissionCleanup, +): void { + if ( + !retainedAcpxAdmissionCleanups.has(cleanup) || + cleanup.recovery || + cleanup.timer + ) { + return; + } + cleanup.timer = setTimeout(() => { + cleanup.timer = null; + if (!retainedAcpxAdmissionCleanups.has(cleanup)) return; + startRetainedAcpxAdmissionCleanup(cleanup); + }, ACPX_ADMISSION_CLEANUP_RESCHEDULE_MS); + cleanup.timer.unref?.(); +} + +async function waitForAdmissionCleanupRetry(delayMs: number): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, delayMs); + timer.unref?.(); + }); +} + async function boundedCancellation( cancellation: Promise, ): Promise { @@ -547,7 +774,7 @@ async function boundedCancellation( "ACPX turn cancellation exceeded its shutdown timeout", ), }), - TURN_CANCELLATION_TIMEOUT_MS, + ACPX_TURN_CANCELLATION_SHUTDOWN_BOUND_MS, ); }), ]); diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 60c04b33ab..3cb1e0521e 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -40,7 +40,9 @@ const result: PrpStructuredRunResult = { completionClaim: { contractRevision: "1", objectiveSatisfied: true, - criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + criteria: [ + { criterionId: "objective", status: "satisfied", evidenceRefs: [] }, + ], remainingWork: [], }, evidence: [], @@ -63,8 +65,16 @@ const yieldedResult: PrpStructuredRunResult = { completionClaim: { contractRevision: "1", objectiveSatisfied: false, - criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], - remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + criteria: [ + { + criterionId: "objective", + status: "unknown", + evidenceRefs: ["interaction:pending"], + }, + ], + remainingWork: [ + { description: "Resume after the response.", blocksCompletion: true }, + ], }, evidence: [{ ref: "interaction:pending" }], verification: [], @@ -93,8 +103,17 @@ const input: NativeExecutionInputV1 = { prompt: "# PAP-RECOVERY: Recover native work", workMode: "standard", }, - workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, - session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 }, + workspace: { + cwd: "/workspace", + repoUrl: null, + repoRef: null, + branchName: null, + }, + session: { + normalizedSessionId: identity.sessionId, + driverKind: "codex_app_server", + protocolVersion: 1, + }, provider: { kind: "codex", model: null }, completionContract: { id: "contract-recovery", @@ -163,7 +182,9 @@ function highestContiguous(events: PrpEvent[]): number { describe("executeNativeSession recovery", () => { it("keeps governed-wait discovery synchronous", () => { - type GovernedWaitResolver = NonNullable; + type GovernedWaitResolver = NonNullable< + ExecuteNativeSessionOptions["resolveGovernedWait"] + >; const resolver: GovernedWaitResolver = () => null; // An async resolver could retain control-plane mutation authority after // execution settles, so the public boundary rejects it at compile time. @@ -174,6 +195,938 @@ describe("executeNativeSession recovery", () => { void asynchronousResolver; }); + it("preserves durable success while quarantined cleanup stays bounded", async () => { + vi.useFakeTimers(); + try { + let executionCloseCount = 0; + let quarantineAttempt = 0; + const close = vi.fn(({ reason }: { reason: string }) => { + if (reason === "native session quarantined cleanup recovery") { + quarantineAttempt += 1; + const attempt = quarantineAttempt; + return new Promise((resolve, reject) => { + setTimeout(() => { + if (attempt < 3) + reject(new Error("transient quarantine failure")); + else resolve(); + }, 6_500); + }); + } + if (reason === "native session execution complete") { + executionCloseCount += 1; + if (executionCloseCount > 1) return Promise.resolve(); + } + return Promise.reject(new Error("persistent close failure")); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + await expect(execute()).resolves.toMatchObject({ result }); + expect(close).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(3_000); + expect(close).toHaveBeenCalledTimes(5); + + // Admission inherits the already-running three-attempt recovery and + // waits through its bounded attempts instead of timing out after one. + const recoveredExecution = execute(); + let admissionSettled = false; + void recoveredExecution.then( + () => { + admissionSettled = true; + }, + () => { + admissionSettled = true; + }, + ); + await vi.advanceTimersByTimeAsync(20_000); + expect(admissionSettled).toBe(false); + expect(openSession).toHaveBeenCalledTimes(1); + expect(close).toHaveBeenCalledTimes(7); + await vi.advanceTimersByTimeAsync(2_000); + await expect(recoveredExecution).resolves.toMatchObject({ result }); + expect(quarantineAttempt).toBe(3); + expect(close).toHaveBeenCalledTimes(8); + expect(openSession).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("awaits the quarantine owner that replaces an exhausted close recovery", async () => { + vi.useFakeTimers(); + try { + let executionCloseCount = 0; + const close = vi.fn(({ reason }: { reason: string }) => { + if (reason === "native session execution complete") { + executionCloseCount += 1; + return executionCloseCount === 1 + ? Promise.reject(new Error("initial close failed")) + : Promise.resolve(); + } + if ( + reason.startsWith( + "native session cleanup recovery after close failure", + ) + ) { + return Promise.reject(new Error("bounded close recovery failed")); + } + if (reason === "native session quarantined cleanup recovery") { + return new Promise((resolve) => setTimeout(resolve, 50)); + } + return Promise.resolve(); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + await expect(execute()).resolves.toMatchObject({ result }); + const admitted = execute(); + await vi.advanceTimersByTimeAsync(3_100); + await expect(admitted).resolves.toMatchObject({ result }); + expect(openSession).toHaveBeenCalledTimes(2); + expect(close).toHaveBeenCalledWith({ + reason: "native session quarantined cleanup recovery", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("covers maximum-duration retained and replacement recovery phases", async () => { + vi.useFakeTimers(); + try { + let executionCloseCount = 0; + let retainedRecoveryAttempt = 0; + let quarantineRecoveryAttempt = 0; + const close = vi.fn(({ reason }: { reason: string }) => { + if (reason === "native session execution complete") { + executionCloseCount += 1; + if (executionCloseCount > 1) return Promise.resolve(); + return new Promise((_resolve, reject) => { + setTimeout(() => reject(new Error("initial close failed")), 6_900); + }); + } + if ( + reason.startsWith( + "native session cleanup recovery after close failure", + ) + ) { + retainedRecoveryAttempt += 1; + return new Promise((_resolve, reject) => { + setTimeout( + () => reject(new Error("bounded retained recovery failed")), + 6_900, + ); + }); + } + if (reason === "native session quarantined cleanup recovery") { + quarantineRecoveryAttempt += 1; + const attempt = quarantineRecoveryAttempt; + return new Promise((resolve, reject) => { + setTimeout(() => { + if (attempt < 3) + reject(new Error("transient quarantine recovery failure")); + else resolve(); + }, 6_500); + }); + } + return Promise.resolve(); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + const firstExecution = execute(); + await vi.advanceTimersByTimeAsync(100); + await expect(firstExecution).resolves.toMatchObject({ result }); + const admitted = execute(); + let admissionSettled = false; + void admitted.then( + () => { + admissionSettled = true; + }, + () => { + admissionSettled = true; + }, + ); + + // The initial close plus three near-bound retries exceed the old 23s + // admission grace but remain within the configured 31s owner phase. + await vi.advanceTimersByTimeAsync(23_100); + expect(retainedRecoveryAttempt).toBe(2); + expect(quarantineRecoveryAttempt).toBe(0); + expect(admissionSettled).toBe(false); + expect(openSession).toHaveBeenCalledOnce(); + + await vi.advanceTimersByTimeAsync(7_600); + expect(retainedRecoveryAttempt).toBe(3); + expect(quarantineRecoveryAttempt).toBe(1); + expect(admissionSettled).toBe(false); + expect(openSession).toHaveBeenCalledOnce(); + + // The replacement then receives its complete three-attempt bound rather + // than inheriting only the remainder of the retained owner's deadline. + await vi.advanceTimersByTimeAsync(21_600); + await expect(admitted).resolves.toMatchObject({ result }); + expect(quarantineRecoveryAttempt).toBe(3); + expect(openSession).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("runs a full admission batch after an inherited scheduled attempt fails", async () => { + vi.useFakeTimers(); + try { + let executionCloseCount = 0; + let scheduledRecoveryAttempt = 0; + let admissionRecoveryAttempt = 0; + const close = vi.fn(({ reason }: { reason: string }) => { + if (reason === "native session execution complete") { + executionCloseCount += 1; + return executionCloseCount === 1 + ? Promise.reject(new Error("initial close failed")) + : Promise.resolve(); + } + if ( + reason.startsWith( + "native session cleanup recovery after close failure", + ) + ) { + return Promise.reject(new Error("bounded retained recovery failed")); + } + if (reason === "native session quarantined cleanup recovery") { + return Promise.reject( + new Error("bounded quarantine recovery failed"), + ); + } + if ( + reason === "native session scheduled quarantined cleanup recovery" + ) { + scheduledRecoveryAttempt += 1; + return new Promise((_resolve, reject) => { + setTimeout( + () => reject(new Error("scheduled cleanup failed")), + 6_500, + ); + }); + } + if (reason === "native session quarantined admission recovery") { + admissionRecoveryAttempt += 1; + const attempt = admissionRecoveryAttempt; + return new Promise((resolve, reject) => { + setTimeout(() => { + if (attempt === 1) + reject(new Error("transient admission cleanup failure")); + else resolve(); + }, 6_500); + }); + } + return Promise.resolve(); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + await expect(execute()).resolves.toMatchObject({ result }); + // Exhaust retained and initial quarantine batches, then enter the slow + // autonomous one-attempt recovery scheduled sixty seconds later. + await vi.advanceTimersByTimeAsync(65_001); + expect(scheduledRecoveryAttempt).toBe(1); + + const admitted = execute(); + let admissionSettled = false; + void admitted.then( + () => { + admissionSettled = true; + }, + () => { + admissionSettled = true; + }, + ); + await vi.advanceTimersByTimeAsync(6_600); + expect(admissionRecoveryAttempt).toBe(1); + expect(admissionSettled).toBe(false); + expect(openSession).toHaveBeenCalledOnce(); + + await vi.advanceTimersByTimeAsync(14_100); + await expect(admitted).resolves.toMatchObject({ result }); + expect(admissionRecoveryAttempt).toBe(2); + expect(openSession).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it("stops scheduled cleanup after the quarantine lifetime budget", async () => { + vi.useFakeTimers(); + try { + let executionCloseCount = 0; + let admissionAttempt = 0; + let scheduledAttempt = 0; + const lifetimeIdentity = { + ...identity, + companyId: "company-cleanup-lifetime", + }; + const close = vi.fn(({ reason }: { reason: string }) => { + if (reason === "native session execution complete") { + executionCloseCount += 1; + return executionCloseCount === 1 + ? Promise.reject(new Error("initial close failed")) + : Promise.resolve(); + } + if ( + reason.startsWith( + "native session cleanup recovery after close failure", + ) + ) { + return Promise.reject(new Error("bounded close recovery failed")); + } + if (reason === "native session quarantined cleanup recovery") { + return Promise.reject(new Error("quarantined close recovery failed")); + } + if (reason === "native session quarantined admission recovery") { + admissionAttempt += 1; + return Promise.reject(new Error("admission cleanup failed")); + } + if ( + reason === "native session scheduled quarantined cleanup recovery" + ) { + scheduledAttempt += 1; + return Promise.reject(new Error("scheduled cleanup failed")); + } + return Promise.resolve(); + }); + const session: NativeSession = { + identity: () => lifetimeIdentity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity: lifetimeIdentity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input: { + ...input, + binding: { + ...input.binding, + companyId: lifetimeIdentity.companyId, + }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + await expect(execute()).resolves.toMatchObject({ result }); + await vi.advanceTimersByTimeAsync(6_000); + + const recoveredExecution = expect(execute()).rejects.toThrow( + "prior session cleanup remains incomplete", + ); + await vi.advanceTimersByTimeAsync(2_100); + await recoveredExecution; + + expect(admissionAttempt).toBe(3); + expect(scheduledAttempt).toBe(0); + await vi.advanceTimersByTimeAsync(180_000); + expect(scheduledAttempt).toBe(3); + await vi.advanceTimersByTimeAsync(300_000); + expect(scheduledAttempt).toBe(3); + expect(openSession).toHaveBeenCalledOnce(); + } finally { + vi.useRealTimers(); + } + }); + + it("fails admission within a bound when quarantined close never settles", async () => { + vi.useFakeTimers(); + try { + let releaseBlockedClose = () => {}; + let blockClose = false; + const close = vi.fn(() => { + if (blockClose) { + return new Promise((resolve) => { + releaseBlockedClose = resolve; + }); + } + return Promise.reject(new Error("persistent close failure")); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + await expect(execute()).resolves.toMatchObject({ result }); + await vi.advanceTimersByTimeAsync(6_000); + blockClose = true; + const blockedAdmission = execute(); + const blockedResult = expect(blockedAdmission).rejects.toThrow( + "prior session cleanup exceeded the admission grace", + ); + await vi.advanceTimersByTimeAsync(31_100); + await blockedResult; + expect(openSession).toHaveBeenCalledOnce(); + + const isolatedSession: NativeSession = { + ...session, + identity: () => ({ ...identity, companyId: "company-isolated" }), + close: vi.fn(async () => undefined), + }; + const isolatedOpenSession = vi.fn(async () => isolatedSession); + await expect( + executeNativeSession({ + input: { + ...input, + binding: { + ...input.binding, + companyId: "company-isolated", + }, + }, + backend: { ...backend, openSession: isolatedOpenSession }, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ result }); + expect(isolatedOpenSession).toHaveBeenCalledOnce(); + + blockClose = false; + releaseBlockedClose(); + close.mockResolvedValue(undefined); + await vi.runAllTimersAsync(); + await expect(execute()).resolves.toMatchObject({ result }); + } finally { + vi.useRealTimers(); + } + }); + + it("quarantines a pending first close before another provider session can open", async () => { + vi.useFakeTimers(); + try { + let releaseClose = () => {}; + const pendingClose = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(() => pendingClose); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const openSession = vi.fn(async () => session); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + const firstExecution = execute(); + await vi.advanceTimersByTimeAsync(100); + await expect(firstExecution).resolves.toMatchObject({ result }); + expect(close).toHaveBeenCalledOnce(); + + const blockedAdmission = execute(); + const blockedResult = expect(blockedAdmission).rejects.toThrow( + "prior session cleanup exceeded the admission grace", + ); + await vi.advanceTimersByTimeAsync(31_100); + await blockedResult; + expect(openSession).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + + releaseClose(); + await vi.runAllTimersAsync(); + } finally { + vi.useRealTimers(); + } + }); + it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => { const digest = "0".repeat(64); const context = { @@ -217,27 +1170,33 @@ describe("executeNativeSession recovery", () => { const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input: { - ...input, - schema: "paperclip.native-execution-input.v3", - executionMode: "default", - planningContext: null, - runtimeContext: { - ...context, - aggregateDigest: canonicalNativeRuntimeContextDigest(context), + await expect( + executeNativeSession({ + input: { + ...input, + schema: "paperclip.native-execution-input.v3", + executionMode: "default", + planningContext: null, + runtimeContext: { + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }, }, - }, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).rejects.toThrow("does not natively realize instructions, skills, mcp"); + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).rejects.toThrow("does not natively realize instructions, skills, mcp"); expect(openSession).not.toHaveBeenCalled(); }); @@ -266,31 +1225,38 @@ describe("executeNativeSession recovery", () => { }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, + async appendEvent() { + throw new Error("unexpected event"); + }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).rejects.toBe(providerFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).rejects.toBe(providerFailure); - expect(openSession).toHaveBeenCalledWith(expect.objectContaining({ - identity, - workingDirectory: input.workspace.cwd, - signal: expect.any(AbortSignal), - })); + expect(openSession).toHaveBeenCalledWith( + expect.objectContaining({ + identity, + workingDirectory: input.workspace.cwd, + signal: expect.any(AbortSignal), + }), + ); expect(openRun).not.toHaveBeenCalled(); }); - it("bounds fresh session bootstrap and closes a session returned after timeout", async () => { + it("retains late bootstrap cleanup through the next admission", async () => { vi.useFakeTimers(); + let releaseClose = () => {}; try { let resolveBootstrap = (_value: NativeSession) => {}; const stalledBootstrap = new Promise((resolve) => { @@ -304,33 +1270,57 @@ describe("executeNativeSession recovery", () => { const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); - const close = vi.fn(async () => { markCloseStarted(); }); + const closeReleased = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(async () => { + markCloseStarted(); + await closeReleased; + }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { - return { resume: false, typedEvents: true, steering: false, interruption: true }; + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + }; }, async *events() {}, - async startTurn() { throw new Error("late fresh session must not start"); }, - async result() { return null; }, - async snapshot() { throw new Error("late fresh session must not snapshot"); }, + async startTurn() { + throw new Error("late fresh session must not start"); + }, + async result() { + return null; + }, + async snapshot() { + throw new Error("late fresh session must not snapshot"); + }, close, }; let bootstrapSignal: AbortSignal | undefined; - const openSession = vi.fn((bootstrapInput: { - identity: NativeRunIdentity; - workingDirectory?: string; - signal?: AbortSignal; - }) => { - bootstrapSignal = bootstrapInput.signal; - bootstrapInput.signal?.addEventListener( - "abort", - () => resolveBootstrap(lateSession), - { once: true }, - ); - markBootstrapStarted(); - return stalledBootstrap; - }); + let bootstrapCount = 0; + const openSession = vi.fn( + (bootstrapInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + bootstrapCount += 1; + if (bootstrapCount > 1) { + throw new Error("replacement bootstrap launched"); + } + bootstrapSignal = bootstrapInput.signal; + bootstrapInput.signal?.addEventListener( + "abort", + () => resolveBootstrap(lateSession), + { once: true }, + ); + markBootstrapStarted(); + return stalledBootstrap; + }, + ); const openRun = vi.fn(async () => undefined); const backend: NativeSessionBackend = { async descriptor() { @@ -345,19 +1335,24 @@ describe("executeNativeSession recovery", () => { }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - - const execution = executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - timeoutMs: 5, - }); + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + }); + const execution = execute(); const rejection = expect(execution).rejects.toThrow( "native session bootstrap timed out after 5ms", ); @@ -373,7 +1368,25 @@ describe("executeNativeSession recovery", () => { expect(close).toHaveBeenCalledWith({ reason: "native session bootstrap timed out", }); + + // Even after the detached disposer exhausts its short settlement grace, + // the exact close remains admission-visible until it releases provider + // resources. A replacement bootstrap cannot start concurrently. + await vi.advanceTimersByTimeAsync(101); + const blockedAdmission = execute(); + const blockedAdmissionRejection = expect(blockedAdmission).rejects.toThrow( + "replacement bootstrap launched", + ); + await vi.advanceTimersByTimeAsync(1_000); + expect(openSession).toHaveBeenCalledOnce(); + + releaseClose(); + await vi.advanceTimersByTimeAsync(0); + await blockedAdmissionRejection; + expect(openSession).toHaveBeenCalledTimes(2); + expect(openRun).not.toHaveBeenCalled(); } finally { + releaseClose(); vi.useRealTimers(); } }); @@ -384,12 +1397,23 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: false, typedEvents: true, steering: false, interruption: true }; + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + }; }, async *events() {}, - async startTurn() { throw new Error("unexpected turn"); }, - async result() { return null; }, - async snapshot() { throw snapshotFailure; }, + async startTurn() { + throw new Error("unexpected turn"); + }, + async result() { + return null; + }, + async snapshot() { + throw snapshotFailure; + }, close, }; const backend: NativeSessionBackend = { @@ -401,29 +1425,37 @@ describe("executeNativeSession recovery", () => { capabilities: await session.capabilities(), }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; const retainedSessions: Array = []; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - keepSessionOpen: true, - onSession(current) { - retainedSessions.push(current); - if (current === null) throw new Error("owner notification failed"); - }, - })).rejects.toBe(snapshotFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + keepSessionOpen: true, + onSession(current) { + retainedSessions.push(current); + if (current === null) throw new Error("owner notification failed"); + }, + }), + ).rejects.toBe(snapshotFailure); expect(retainedSessions).toEqual([session, null]); expect(close).toHaveBeenCalledOnce(); @@ -445,14 +1477,13 @@ describe("executeNativeSession recovery", () => { }); let checkpointSignal: AbortSignal | undefined; let controlPlaneCheckpointCount = 0; - const checkpointSession: NonNullable = async ( - _snapshot, - checkpointOptions, - ) => { + const checkpointSession: NonNullable< + ControlPlanePort["checkpointSession"] + > = async (_snapshot, checkpointOptions) => { controlPlaneCheckpointCount += 1; if ( - stalledBoundary === "control-plane checkpoint" - && controlPlaneCheckpointCount === 2 + stalledBoundary === "control-plane checkpoint" && + controlPlaneCheckpointCount === 2 ) { checkpointSignal = checkpointOptions?.signal; markCheckpointStalled(); @@ -460,21 +1491,22 @@ describe("executeNativeSession recovery", () => { } }; let ownerCheckpointCount = 0; - const onCheckpoint: NonNullable = async ( - _snapshot, - checkpointOptions, - ) => { + const onCheckpoint: NonNullable< + ExecuteNativeSessionOptions["onCheckpoint"] + > = async (_snapshot, checkpointOptions) => { ownerCheckpointCount += 1; if ( - stalledBoundary === "owner checkpoint" - && ownerCheckpointCount === 2 + stalledBoundary === "owner checkpoint" && + ownerCheckpointCount === 2 ) { checkpointSignal = checkpointOptions?.signal; markCheckpointStalled(); await never; } }; - const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled" })); + const startTurn = vi.fn(async () => ({ + turnId: "turn-checkpoint-stalled", + })); const close = vi.fn(async () => { releaseStream(); }); @@ -493,7 +1525,9 @@ describe("executeNativeSession recovery", () => { await streamReleased; }, startTurn, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -517,13 +1551,17 @@ describe("executeNativeSession recovery", () => { capabilities: await session.capabilities(), }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const openRun = vi.fn(async () => undefined); const port: ControlPlanePort = { openRun, checkpointSession, - async appendEvent() { throw new Error("unexpected event"); }, + async appendEvent() { + throw new Error("unexpected event"); + }, async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, @@ -596,8 +1634,12 @@ describe("executeNativeSession recovery", () => { structuredResult: true, }; }, - async *events() { yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, async result() { resultCalls += 1; if (stalledBoundary === "provider result") { @@ -630,7 +1672,9 @@ describe("executeNativeSession recovery", () => { capabilities: await session.capabilities(), }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -644,8 +1688,8 @@ describe("executeNativeSession recovery", () => { }, async appendEvent(event, operationOptions) { if ( - stalledBoundary === "final event append" - && (event as PrpEvent).sourceKind === "control_plane" + stalledBoundary === "final event append" && + (event as PrpEvent).sourceKind === "control_plane" ) { stalledSignal = operationOptions?.signal; markFinalizationStalled(); @@ -711,10 +1755,7 @@ describe("executeNativeSession recovery", () => { }, ); - it.each([ - "final event append", - "run completion", - ] as const)( + it.each(["final event append", "run completion"] as const)( "confirms durable completion when %s commits before its acknowledgement stalls", async (stalledBoundary) => { vi.useFakeTimers(); @@ -739,9 +1780,15 @@ describe("executeNativeSession recovery", () => { structuredResult: true, }; }, - async *events() { yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, async snapshot() { return { backendKind: "mock", @@ -765,22 +1812,25 @@ describe("executeNativeSession recovery", () => { capabilities: await session.capabilities(), }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent(event, operationOptions) { const appended = structuredClone(event as PrpEvent); - const existing = events.find((candidate) => - candidate.sourceInstanceId === appended.sourceInstanceId - && candidate.sourceSeq === appended.sourceSeq + const existing = events.find( + (candidate) => + candidate.sourceInstanceId === appended.sourceInstanceId && + candidate.sourceSeq === appended.sourceSeq, ); if (existing === undefined) events.push(appended); if ( - stalledBoundary === "final event append" - && appended.sourceKind === "control_plane" - && !stalledOnce + stalledBoundary === "final event append" && + appended.sourceKind === "control_plane" && + !stalledOnce ) { stalledOnce = true; stalledSignal = operationOptions?.signal; @@ -788,7 +1838,8 @@ describe("executeNativeSession recovery", () => { return await never; } const sourceEvents = events.filter( - (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId, + (candidate) => + candidate.sourceInstanceId === appended.sourceInstanceId, ); return { cursor: events.length, @@ -801,9 +1852,11 @@ describe("executeNativeSession recovery", () => { (event) => event.sourceInstanceId === replay.sourceInstanceId, ); return { - events: structuredClone(sourceEvents.filter( - (event) => event.sourceSeq > replay.afterSourceSeq, - )), + events: structuredClone( + sourceEvents.filter( + (event) => event.sourceSeq > replay.afterSourceSeq, + ), + ), highestContiguousSourceSeq: highestContiguous(sourceEvents), }; }, @@ -842,8 +1895,9 @@ describe("executeNativeSession recovery", () => { }); expect(stalledSignal?.aborted).toBe(true); expect(durableCompletion).toMatchObject({ result, terminal }); - expect(events.filter((event) => event.sourceKind === "control_plane")) - .toHaveLength(2); + expect( + events.filter((event) => event.sourceKind === "control_plane"), + ).toHaveLength(2); } finally { vi.useRealTimers(); } @@ -878,9 +1932,15 @@ describe("executeNativeSession recovery", () => { structuredResult: true, }; }, - async *events() { yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, async snapshot(snapshotOptions) { if (stalledBoundary === "provider snapshot" && runCompleted) { stalledSignal = snapshotOptions?.signal; @@ -916,7 +1976,9 @@ describe("executeNativeSession recovery", () => { capabilities: await session.capabilities(), }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const completeRun = vi.fn(async () => { runCompleted = true; @@ -924,7 +1986,10 @@ describe("executeNativeSession recovery", () => { const port: ControlPlanePort = { async openRun() {}, async checkpointSession(_snapshot, operationOptions) { - if (stalledBoundary === "post-completion checkpoint" && runCompleted) { + if ( + stalledBoundary === "post-completion checkpoint" && + runCompleted + ) { stalledSignal = operationOptions?.signal; markEnrichmentStalled(); await never; @@ -955,7 +2020,8 @@ describe("executeNativeSession recovery", () => { onSession: (current) => retainedSessions.push(current), }); await enrichmentStalled; - if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false); + if (stalledSignal !== undefined) + expect(stalledSignal.aborted).toBe(false); await vi.advanceTimersByTimeAsync(10); @@ -966,7 +2032,8 @@ describe("executeNativeSession recovery", () => { driverVersion: "1", usage: null, }); - if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true); + if (stalledSignal !== undefined) + expect(stalledSignal.aborted).toBe(true); expect(completeRun).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); expect(retainedSessions).toEqual([session, null]); @@ -978,22 +2045,36 @@ describe("executeNativeSession recovery", () => { it("contains a consumer rejection when starting the turn fails first", async () => { let markAppendStarted = () => {}; - const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); let releaseAppend = () => {}; - const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + const appendReleased = new Promise((resolve) => { + releaseAppend = resolve; + }); let appendCommitted = false; const close = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.started"); }, - async *events() { yield runnerEvent(1, "turn.started"); }, async startTurn() { await appendStarted; throw new Error("start turn failed"); }, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1014,10 +2095,18 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -1027,9 +2116,13 @@ describe("executeNativeSession recovery", () => { await Promise.race([ appendReleased, new Promise((_resolve, reject) => { - const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); + const rejectAbort = () => + reject(options?.signal.reason ?? new Error("append aborted")); if (options?.signal.aborted) rejectAbort(); - else options?.signal.addEventListener("abort", rejectAbort, { once: true }); + else + options?.signal.addEventListener("abort", rejectAbort, { + once: true, + }); }), ]); appendCommitted = true; @@ -1039,7 +2132,9 @@ describe("executeNativeSession recovery", () => { disposition: "committed" as const, }; }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -1061,42 +2156,59 @@ describe("executeNativeSession recovery", () => { it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => { let markAppendStarted = () => {}; - const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + const appendStarted = new Promise((resolve) => { + markAppendStarted = resolve; + }); let releaseAppend = () => {}; - const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + const appendReleased = new Promise((resolve) => { + releaseAppend = resolve; + }); let releaseTeardown = () => {}; - const teardownReleased = new Promise((resolve) => { releaseTeardown = resolve; }); + const teardownReleased = new Promise((resolve) => { + releaseTeardown = resolve; + }); const iteratorTeardown = vi.fn(); let appendCommitted = false; - const appendEvent = vi.fn(async ( - _event: PrpEvent, - options?: { signal: AbortSignal }, - ) => { - markAppendStarted(); - await Promise.race([ - appendReleased, - new Promise((_resolve, reject) => { - const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); - if (options?.signal.aborted) rejectAbort(); - else options?.signal.addEventListener("abort", rejectAbort, { once: true }); - }), - ]); - appendCommitted = true; - return { - cursor: 1, - highestContiguousSourceSeq: 1, - disposition: "committed" as const, - }; - }); + const appendEvent = vi.fn( + async (_event: PrpEvent, options?: { signal: AbortSignal }) => { + markAppendStarted(); + await Promise.race([ + appendReleased, + new Promise((_resolve, reject) => { + const rejectAbort = () => + reject(options?.signal.reason ?? new Error("append aborted")); + if (options?.signal.aborted) rejectAbort(); + else + options?.signal.addEventListener("abort", rejectAbort, { + once: true, + }); + }), + ]); + appendCommitted = true; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed" as const, + }; + }, + ); const cancel = vi.fn(() => { releaseTeardown(); return { cleanup: Promise.resolve() }; }); - const close = vi.fn(async () => { releaseTeardown(); }); + const close = vi.fn(async () => { + releaseTeardown(); + }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() { try { @@ -1106,9 +2218,13 @@ describe("executeNativeSession recovery", () => { await teardownReleased; } }, - async startTurn() { return { turnId: "turn-recovery" }; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, cancel, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1129,16 +2245,26 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, appendEvent, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -1151,7 +2277,9 @@ describe("executeNativeSession recovery", () => { timeoutMs: 1, keepSessionOpen: true, }); - const rejection = expect(execution).rejects.toThrow("native session timed out"); + const rejection = expect(execution).rejects.toThrow( + "native session timed out", + ); await appendStarted; await vi.waitFor(() => expect(iteratorTeardown).toHaveBeenCalledOnce()); await rejection; @@ -1166,19 +2294,33 @@ describe("executeNativeSession recovery", () => { it("closes a failed session while retaining an uncancellable event read", async () => { let releaseStream = () => {}; - const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); - const close = vi.fn(async () => { releaseStream(); }); + const streamReleased = new Promise((resolve) => { + releaseStream = resolve; + }); + const close = vi.fn(async () => { + releaseStream(); + }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; }, async *events() { await streamReleased; yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return null; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1199,56 +2341,86 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - timeoutMs: 1, - keepSessionOpen: true, - })).rejects.toThrow("native session timed out"); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }), + ).rejects.toThrow("native session timed out"); expect(close).toHaveBeenCalledOnce(); }); it("commits cancellation before bounding failed provider cleanup", async () => { let releaseStream = () => {}; - const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); + const streamReleased = new Promise((resolve) => { + releaseStream = resolve; + }); let releaseCancellation = () => {}; - const cancellationReleased = new Promise((resolve) => { releaseCancellation = resolve; }); + const cancellationReleased = new Promise((resolve) => { + releaseCancellation = resolve; + }); const interrupt = vi.fn(() => cancellationReleased); let cancellationCommitted = false; const cancel = vi.fn(() => { cancellationCommitted = true; return { cleanup: cancellationReleased }; }); - const close = vi.fn(async () => { releaseStream(); }); + const close = vi.fn(async () => { + releaseStream(); + }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() { await streamReleased; yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, interrupt, cancel, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1269,16 +2441,28 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -1301,18 +2485,32 @@ describe("executeNativeSession recovery", () => { it("bounds failure when iterator teardown and provider close never settle", async () => { const never = new Promise(() => undefined); - const close = vi.fn(() => never); + let releaseClose = () => {}; + const pendingClose = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(() => pendingClose); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; }, async *events() { await never; yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return null; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1333,43 +2531,74 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - timeoutMs: 1, - keepSessionOpen: true, - })).rejects.toThrow("native session timed out"); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }), + ).rejects.toThrow("native session timed out"); expect(close).toHaveBeenCalledOnce(); + releaseClose(); + await pendingClose; }); it("preserves durable success when provider close never settles", async () => { - const never = new Promise(() => undefined); - const close = vi.fn(() => never); + let releaseClose = () => {}; + const pendingClose = new Promise((resolve) => { + releaseClose = resolve; + }); + const close = vi.fn(() => pendingClose); const completeRun = vi.fn(async () => undefined); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; }, - async *events() { yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { return { backendKind: "mock", @@ -1390,10 +2619,18 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { @@ -1407,35 +2644,53 @@ describe("executeNativeSession recovery", () => { disposition: "committed", }; }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, completeRun, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ result, terminal }); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ result, terminal }); expect(completeRun).toHaveBeenCalledOnce(); expect(close).toHaveBeenCalledOnce(); + releaseClose(); + await pendingClose; }); it("closes after a synchronous governed-wait probe returns no result", async () => { const resolveGovernedWait = vi.fn(() => null); const lifecycle: string[] = []; - const close = vi.fn(async () => { lifecycle.push("closed"); }); + const close = vi.fn(async () => { + lifecycle.push("closed"); + }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() { yield runnerEvent(1, "item.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return null; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1456,18 +2711,32 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, async checkpointSession() {}, async appendEvent() { - return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; @@ -1494,20 +2763,32 @@ describe("executeNativeSession recovery", () => { lifecycle.push("cancelled"); return { cleanup: new Promise(() => undefined) }; }); - const close = vi.fn(async () => { lifecycle.push("closed"); }); + const close = vi.fn(async () => { + lifecycle.push("closed"); + }); const events: PrpEvent[] = []; const retainedSessions: Array = []; const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() { yield runnerEvent(1, "item.completed"); }, - async startTurn() { return { turnId: "turn-recovery" }; }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, cancel, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -1528,10 +2809,18 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -1544,7 +2833,9 @@ describe("executeNativeSession recovery", () => { disposition: "committed", }; }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -1582,7 +2873,13 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, openSession, @@ -1597,18 +2894,24 @@ describe("executeNativeSession recovery", () => { }; }, checkpointSession, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).rejects.toThrow("native_session_checkpoint_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(checkpointSession).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); @@ -1620,13 +2923,25 @@ describe("executeNativeSession recovery", () => { const existingSession: NativeSession = { identity: () => ({ ...identity, companyId: "other-company" }), async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, attachRun, async *events() {}, - async startTurn() { return { turnId: "unexpected" }; }, - async result() { return null; }, - async snapshot() { throw new Error("unexpected snapshot"); }, + async startTurn() { + return { turnId: "unexpected" }; + }, + async result() { + return null; + }, + async snapshot() { + throw new Error("unexpected snapshot"); + }, async close() {}, }; const backend: NativeSessionBackend = { @@ -1635,26 +2950,40 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "existing-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected open"); }, + async openSession() { + throw new Error("unexpected open"); + }, }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - existingSession, - })).rejects.toThrow("native_session_attach_binding_mismatch"); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + }), + ).rejects.toThrow("native_session_attach_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(attachRun).not.toHaveBeenCalled(); }); @@ -1673,42 +3002,66 @@ describe("executeNativeSession recovery", () => { const existingSession: NativeSession = { identity: () => structuredClone(retainedIdentity), async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, attachRun, async *events() {}, startTurn, - async result() { return null; }, - async snapshot() { throw new Error("unexpected snapshot"); }, + async result() { + return null; + }, + async snapshot() { + throw new Error("unexpected snapshot"); + }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", - name: "existing-backend", + name: "attachment-failure-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected open"); }, + async openSession() { + throw new Error("unexpected open"); + }, }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - existingSession, - onSession, - })).rejects.toBe(attachmentFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + }), + ).rejects.toBe(attachmentFailure); expect(attachRun).toHaveBeenCalledWith({ identity }); expect(retainedIdentity).toEqual(identity); expect(onSession).toHaveBeenCalledOnce(); @@ -1732,42 +3085,66 @@ describe("executeNativeSession recovery", () => { const existingSession: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, attachRun, async *events() {}, startTurn, - async result() { return null; }, - async snapshot() { throw new Error("unexpected snapshot"); }, + async result() { + return null; + }, + async snapshot() { + throw new Error("unexpected snapshot"); + }, close, }; const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", - name: "existing-backend", + name: "control-plane-admission-failure-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected open"); }, + async openSession() { + throw new Error("unexpected open"); + }, }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - existingSession, - onSession, - })).rejects.toBe(admissionFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + }), + ).rejects.toBe(admissionFailure); expect(attachRun).toHaveBeenCalledWith({ identity }); expect(openRun).toHaveBeenCalledOnce(); expect(onSession).toHaveBeenCalledOnce(); @@ -1787,7 +3164,13 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, openSession, @@ -1801,21 +3184,27 @@ describe("executeNativeSession recovery", () => { identity: { ...identity, sessionId: "other-session" }, }; }, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; - await expect(executeNativeSession({ - input: { - ...input, - session: { ...input.session, normalizedSessionId: null }, - }, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + await expect( + executeNativeSession({ + input: { + ...input, + session: { ...input.session, normalizedSessionId: null }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).rejects.toThrow("native_session_checkpoint_binding_mismatch"); expect(openRun).not.toHaveBeenCalled(); expect(openSession).not.toHaveBeenCalled(); }); @@ -1838,14 +3227,22 @@ describe("executeNativeSession recovery", () => { recovered: false as const, reason: "provider session no longer exists", })); - const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); }); + const openSession = vi.fn(async () => { + throw new Error("replacement is forbidden"); + }); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, openSession, @@ -1853,20 +3250,28 @@ describe("executeNativeSession recovery", () => { }; const port: ControlPlanePort = { openRun, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, completeRun, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).rejects.toThrow( + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).rejects.toThrow( "native_session_recovery_failed: provider session no longer exists", ); @@ -1921,15 +3326,25 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected replacement"); }, + async openSession() { + throw new Error("unexpected replacement"); + }, recoverSession, }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, + async appendEvent() { + throw new Error("unexpected event"); + }, replayEvents, async completeRun() {}, }; @@ -1983,10 +3398,16 @@ describe("executeNativeSession recovery", () => { pendingRuntimeRequests: [], lineage: [], }; - let resolveRecovery = (_value: { recovered: true; session: NativeSession }) => {}; - const stalledRecovery = new Promise<{ recovered: true; session: NativeSession }>( - (resolve) => { resolveRecovery = resolve; }, - ); + let resolveRecovery = (_value: { + recovered: true; + session: NativeSession; + }) => {}; + const stalledRecovery = new Promise<{ + recovered: true; + session: NativeSession; + }>((resolve) => { + resolveRecovery = resolve; + }); let markRecoveryStarted = () => {}; const recoveryStarted = new Promise((resolve) => { markRecoveryStarted = resolve; @@ -1995,21 +3416,37 @@ describe("executeNativeSession recovery", () => { const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); - const close = vi.fn(async () => { markCloseStarted(); }); + const close = vi.fn(async () => { + markCloseStarted(); + }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + }; }, async *events() {}, - async startTurn() { throw new Error("late recovery session must not start"); }, - async result() { return null; }, - async snapshot() { return structuredClone(checkpoint); }, + async startTurn() { + throw new Error("late recovery session must not start"); + }, + async result() { + return null; + }, + async snapshot() { + return structuredClone(checkpoint); + }, close, }; let recoverySignal: AbortSignal | undefined; const recoverSession = vi.fn( - (_checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }) => { + ( + _checkpoint: PersistedNativeSession, + recoveryOptions: { signal: AbortSignal }, + ) => { recoverySignal = recoveryOptions.signal; recoveryOptions.signal.addEventListener( "abort", @@ -2028,16 +3465,27 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + }, }; }, - async openSession() { throw new Error("unexpected replacement"); }, + async openSession() { + throw new Error("unexpected replacement"); + }, recoverSession, }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -2099,33 +3547,48 @@ describe("executeNativeSession recovery", () => { const closeStarted = new Promise((resolve) => { markCloseStarted = resolve; }); - const close = vi.fn(async () => { markCloseStarted(); }); + const close = vi.fn(async () => { + markCloseStarted(); + }); const lateSession: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + }; }, async *events() {}, - async startTurn() { throw new Error("late replacement session must not start"); }, - async result() { return null; }, - async snapshot() { return structuredClone(checkpoint); }, + async startTurn() { + throw new Error("late replacement session must not start"); + }, + async result() { + return null; + }, + async snapshot() { + return structuredClone(checkpoint); + }, close, }; let replacementSignal: AbortSignal | undefined; - const openReplacementSession = vi.fn((replacementInput: { - identity: NativeRunIdentity; - workingDirectory?: string; - signal?: AbortSignal; - }) => { - replacementSignal = replacementInput.signal; - replacementInput.signal?.addEventListener( - "abort", - () => resolveReplacement(lateSession), - { once: true }, - ); - markReplacementStarted(); - return stalledReplacement; - }); + const openReplacementSession = vi.fn( + (replacementInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + replacementSignal = replacementInput.signal; + replacementInput.signal?.addEventListener( + "abort", + () => resolveReplacement(lateSession), + { once: true }, + ); + markReplacementStarted(); + return stalledReplacement; + }, + ); const openRun = vi.fn(async () => undefined); const onSession = vi.fn(); const backend: NativeSessionBackend = { @@ -2134,10 +3597,17 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "replacement-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + }, }; }, - async openSession() { throw new Error("replacement seam must be used"); }, + async openSession() { + throw new Error("replacement seam must be used"); + }, async recoverSession() { return { recovered: false, reason: "provider session is missing" }; }, @@ -2145,8 +3615,12 @@ describe("executeNativeSession recovery", () => { }; const port: ControlPlanePort = { openRun, - async appendEvent() { throw new Error("unexpected event"); }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async appendEvent() { + throw new Error("unexpected event"); + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, async completeRun() {}, }; @@ -2198,27 +3672,41 @@ describe("executeNativeSession recovery", () => { const openRun = vi.fn(async () => undefined); const checkpointSession = vi.fn(async () => undefined); const onCheckpoint = vi.fn(async () => undefined); - const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { - expect(recoveryCheckpoint.cursor).toBe("1"); - throw recoveryFailure; - }); + const recoverSession = vi.fn( + async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + throw recoveryFailure; + }, + ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected replacement"); }, + async openSession() { + throw new Error("unexpected replacement"); + }, recoverSession, }; const port: ControlPlanePort = { openRun, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, checkpointSession, - async appendEvent() { throw new Error("unexpected event"); }, + async appendEvent() { + throw new Error("unexpected event"); + }, async replayEvents(replay) { return { events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], @@ -2228,14 +3716,16 @@ describe("executeNativeSession recovery", () => { async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - onCheckpoint, - })).rejects.toBe(recoveryFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + }), + ).rejects.toBe(recoveryFailure); expect(recoverSession).toHaveBeenCalledOnce(); expect(openRun).not.toHaveBeenCalled(); @@ -2256,7 +3746,9 @@ describe("executeNativeSession recovery", () => { lineage: [], }; const admissionFailure = new Error("control-plane admission rejected"); - const openRun = vi.fn(async () => { throw admissionFailure; }); + const openRun = vi.fn(async () => { + throw admissionFailure; + }); const checkpointSession = vi.fn(async () => undefined); const onCheckpoint = vi.fn(async () => undefined); const onSession = vi.fn(); @@ -2264,35 +3756,61 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() {}, - async startTurn() { throw new Error("unexpected turn"); }, - async result() { return null; }, - async snapshot() { throw new Error("unexpected snapshot"); }, + async startTurn() { + throw new Error("unexpected turn"); + }, + async result() { + return null; + }, + async snapshot() { + throw new Error("unexpected snapshot"); + }, close, }; - const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { - expect(recoveryCheckpoint.cursor).toBe("1"); - return { recovered: true as const, session }; - }); + const recoverSession = vi.fn( + async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + return { recovered: true as const, session }; + }, + ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("unexpected replacement"); }, + async openSession() { + throw new Error("unexpected replacement"); + }, recoverSession, }; const port: ControlPlanePort = { openRun, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, checkpointSession, - async appendEvent() { throw new Error("unexpected event"); }, + async appendEvent() { + throw new Error("unexpected event"); + }, async replayEvents(replay) { return { events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], @@ -2302,15 +3820,17 @@ describe("executeNativeSession recovery", () => { async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - onCheckpoint, - onSession, - })).rejects.toBe(admissionFailure); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + onSession, + }), + ).rejects.toBe(admissionFailure); expect(recoverSession).toHaveBeenCalledOnce(); expect(openRun).toHaveBeenCalledOnce(); @@ -2356,16 +3876,30 @@ describe("executeNativeSession recovery", () => { }; const bySource = new Map(); const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" })); - const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); }); + const openSession = vi.fn(async () => { + throw new Error("must recover the provider session"); + }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield terminalEvent; }, - async *events() { yield terminalEvent; }, startTurn, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, - async snapshot() { return structuredClone(providerSnapshot); }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return structuredClone(providerSnapshot); + }, async close() {}, }; const backend: NativeSessionBackend = { @@ -2374,15 +3908,25 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, openSession, - async recoverSession() { return { recovered: true, session }; }, + async recoverSession() { + return { recovered: true, session }; + }, }; const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, async appendEvent(event) { const list = bySource.get(event.sourceInstanceId) ?? []; @@ -2397,20 +3941,27 @@ describe("executeNativeSession recovery", () => { async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { - events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + events: structuredClone( + list.filter((event) => event.sourceSeq > replay.afterSourceSeq), + ), highestContiguousSourceSeq: highestContiguous(list), }; }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery" }); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ + turnId: "turn-recovery", + providerSessionId: "provider-recovery", + }); expect(openSession).not.toHaveBeenCalled(); expect(startTurn).not.toHaveBeenCalled(); }); @@ -2441,39 +3992,74 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield terminalEvent; + }, + async startTurn() { + return { turnId: "turn-recovery" }; + }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; }, - async *events() { yield terminalEvent; }, - async startTurn() { return { turnId: "turn-recovery" }; }, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, async snapshot() { - return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null }; + return { + ...checkpoint, + cursor: String(terminalSequence), + activeTurnId: null, + }; }, async close() {}, }; - const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { - expect(recoveryCheckpoint.cursor).toBe(expectedCursor); - return { recovered: true, session }; - }); + const recoverSession = vi.fn( + async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe(expectedCursor); + return { recovered: true, session }; + }, + ); const backend: NativeSessionBackend = { async descriptor() { return { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("must recover the provider session"); }, + async openSession() { + throw new Error("must recover the provider session"); + }, recoverSession, }; const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, - async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, + async checkpointSession(snapshot) { + checkpoints.push(structuredClone(snapshot)); + }, async appendEvent(event) { - const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; - if (target.some((existing) => existing.sourceSeq === event.sourceSeq)) { + const target = + event.sourceInstanceId === "runner-recovery" + ? runnerEvents + : controlEvents; + if ( + target.some((existing) => existing.sourceSeq === event.sourceSeq) + ) { throw new Error(`native_event_replay_conflict:${event.sourceSeq}`); } target.push(structuredClone(event)); @@ -2484,26 +4070,36 @@ describe("executeNativeSession recovery", () => { }; }, async replayEvents(replay) { - const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; + const source = + replay.sourceInstanceId === "runner-recovery" + ? runnerEvents + : controlEvents; const events = source .filter((event) => event.sourceSeq > replay.afterSourceSeq) .sort((left, right) => left.sourceSeq - right.sourceSeq) .slice(0, replay.limit); - return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source) }; + return { + events: structuredClone(events), + highestContiguousSourceSeq: highestContiguous(source), + }; }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ turnId: "turn-recovery" }); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ turnId: "turn-recovery" }); expect(recoverSession).toHaveBeenCalledOnce(); - expect(runnerEvents.some((event) => event.sourceSeq === terminalSequence)).toBe(true); + expect( + runnerEvents.some((event) => event.sourceSeq === terminalSequence), + ).toBe(true); if (checkpointCursor === "12") { expect(checkpoints[0]).toMatchObject({ cursor: "41" }); } @@ -2531,12 +4127,26 @@ describe("executeNativeSession recovery", () => { const replacementSession: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { + return { turnId: "turn-replacement" }; + }, + async result() { + return { result, terminal, turnId: "turn-replacement" }; + }, + async snapshot() { + return structuredClone(replacementSnapshot); }, - async *events() { yield runnerEvent(1, "turn.completed"); }, - async startTurn() { return { turnId: "turn-replacement" }; }, - async result() { return { result, terminal, turnId: "turn-replacement" }; }, - async snapshot() { return structuredClone(replacementSnapshot); }, async close() {}, }; const recoverSession = vi.fn(async () => ({ @@ -2551,34 +4161,52 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "replacement-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("replacement seam must be used"); }, + async openSession() { + throw new Error("replacement seam must be used"); + }, recoverSession, openReplacementSession, }; const events: PrpEvent[] = []; const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, async appendEvent(event) { events.push(structuredClone(event)); - return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" }; + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; }, - async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-replacement", - controlPlaneInstanceId: "control-replacement", - onContinuityBreak, - })).resolves.toMatchObject({ providerSessionId: "provider-new" }); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-replacement", + controlPlaneInstanceId: "control-replacement", + onContinuityBreak, + }), + ).resolves.toMatchObject({ providerSessionId: "provider-new" }); expect(recoverSession).toHaveBeenCalledOnce(); expect(openReplacementSession).toHaveBeenCalledOnce(); @@ -2599,7 +4227,9 @@ describe("executeNativeSession recovery", () => { providerSessionId: "provider-recovery", cursor: "1", activeTurnId: null, - terminalTurns: [{ turnId: "turn-work", fingerprint: "terminal-fingerprint" }], + terminalTurns: [ + { turnId: "turn-work", fingerprint: "terminal-fingerprint" }, + ], dispositionOnlyRecoveryConsumed: true, dispositionOnlyRecoveryTurnId: "turn-missing-disposition", pendingRuntimeRequests: [], @@ -2629,12 +4259,24 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield terminalEvent; }, - async *events() { yield terminalEvent; }, startTurn, - async result() { return { result, terminal, turnId: "turn-continuation" }; }, - async snapshot() { return structuredClone(recoveredSnapshot); }, + async result() { + return { result, terminal, turnId: "turn-continuation" }; + }, + async snapshot() { + return structuredClone(recoveredSnapshot); + }, async close() {}, }; const backend: NativeSessionBackend = { @@ -2643,26 +4285,42 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("must recover the provider session"); }, - async recoverSession() { return { recovered: true, session }; }, + async openSession() { + throw new Error("must recover the provider session"); + }, + async recoverSession() { + return { recovered: true, session }; + }, }; const bySource = new Map(); const replayedPages: PrpEvent[][] = []; - const replayEvents = vi.fn(async (replay: Parameters[0]) => { - const list = bySource.get(replay.sourceInstanceId) ?? []; - const events = structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)); - replayedPages.push(events); - return { - events, - highestContiguousSourceSeq: highestContiguous(list), - }; - }); + const replayEvents = vi.fn( + async (replay: Parameters[0]) => { + const list = bySource.get(replay.sourceInstanceId) ?? []; + const events = structuredClone( + list.filter((event) => event.sourceSeq > replay.afterSourceSeq), + ); + replayedPages.push(events); + return { + events, + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + ); const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, async appendEvent(event) { const list = bySource.get(event.sourceInstanceId) ?? []; @@ -2678,13 +4336,18 @@ describe("executeNativeSession recovery", () => { async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" }); + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ + turnId: "turn-continuation", + providerSessionId: "provider-recovery", + }); expect(startTurn).toHaveBeenCalledOnce(); expect(replayEvents).toHaveBeenCalledWith({ runId: identity.runId, @@ -2708,19 +4371,23 @@ describe("executeNativeSession recovery", () => { recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined; startTurn.mockClear(); bySource.clear(); - bySource.set("runner-recovery", [{ - ...terminalEvent, - sourceEventId: "runner-recovery:stale-terminal", - turnId: "turn-stale-unbound", - }]); + bySource.set("runner-recovery", [ + { + ...terminalEvent, + sourceEventId: "runner-recovery:stale-terminal", + turnId: "turn-stale-unbound", + }, + ]); - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }), + ).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery", }); @@ -2770,12 +4437,24 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield terminalEvent; }, - async *events() { yield terminalEvent; }, startTurn, - async result() { return null; }, - async snapshot() { return structuredClone(recoveredSnapshot); }, + async result() { + return null; + }, + async snapshot() { + return structuredClone(recoveredSnapshot); + }, async close() {}, }; const events: PrpEvent[] = []; @@ -2785,24 +4464,42 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("must recover the provider session"); }, - async recoverSession() { return { recovered: true, session }; }, + async openSession() { + throw new Error("must recover the provider session"); + }, + async recoverSession() { + return { recovered: true, session }; + }, }; const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession(snapshot) { if ( - snapshot.terminalTurns?.some((turn) => turn.turnId === "turn-disposition") - && !dispositionTerminalCommitted - ) prematureDispositionCheckpoint = true; + snapshot.terminalTurns?.some( + (turn) => turn.turnId === "turn-disposition", + ) && + !dispositionTerminalCommitted + ) + prematureDispositionCheckpoint = true; }, async appendEvent(event) { events.push(structuredClone(event)); - if (event.eventType === "turn.completed" && event.turnId === "turn-disposition") { + if ( + event.eventType === "turn.completed" && + event.turnId === "turn-disposition" + ) { dispositionTerminalCommitted = true; } return { @@ -2813,24 +4510,29 @@ describe("executeNativeSession recovery", () => { }, async replayEvents(replay) { return { - events: structuredClone(events.filter((event) => - event.sourceInstanceId === replay.sourceInstanceId - && event.sourceSeq > replay.afterSourceSeq - )), + events: structuredClone( + events.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, + ), + ), highestContiguousSourceSeq: highestContiguous(events), }; }, async completeRun() {}, }; - await expect(executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - resolveMissingResult: async () => result, - })).resolves.toMatchObject({ + await expect( + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult: async () => result, + }), + ).resolves.toMatchObject({ result, turnId: "turn-disposition", }); @@ -2896,11 +4598,21 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield structuredClone(terminalEvent); }, - async *events() { yield structuredClone(terminalEvent); }, startTurn, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { ...structuredClone(checkpoint), @@ -2924,21 +4636,36 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("must recover the provider session"); }, - async recoverSession() { return { recovered: true, session }; }, + async openSession() { + throw new Error("must recover the provider session"); + }, + async recoverSession() { + return { recovered: true, session }; + }, }; const bySource = new Map([ - ["runner-recovery", [ - structuredClone(originalTaskProposal), - structuredClone(originalTaskTerminal), - ]], + [ + "runner-recovery", + [ + structuredClone(originalTaskProposal), + structuredClone(originalTaskTerminal), + ], + ], ]); const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, async appendEvent(event) { const list = bySource.get(event.sourceInstanceId) ?? []; @@ -2953,24 +4680,27 @@ describe("executeNativeSession recovery", () => { async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { - events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + events: structuredClone( + list.filter((event) => event.sourceSeq > replay.afterSourceSeq), + ), highestContiguousSourceSeq: highestContiguous(list), }; }, async completeRun() {}, }; - const execute = () => executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - resolveMissingResult: async ({ terminalEvent: replayed }) => { - expect(replayed).toEqual(terminalEvent); - return result; - }, - }); + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult: async ({ terminalEvent: replayed }) => { + expect(replayed).toEqual(terminalEvent); + return result; + }, + }); await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", @@ -2997,11 +4727,9 @@ describe("executeNativeSession recovery", () => { resultProposalEvent, terminalEvent, ]); - expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([ - "run.result.accepted", - "run.terminal", - ]); - + expect( + bySource.get("control-recovery")?.map((event) => event.eventType), + ).toEqual(["run.result.accepted", "run.terminal"]); }); it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => { @@ -3029,20 +4757,33 @@ describe("executeNativeSession recovery", () => { pendingRuntimeRequests: [], lineage: [], }; - const recoveredCheckpoint: PersistedNativeSession = structuredClone(checkpoint); + const recoveredCheckpoint: PersistedNativeSession = + structuredClone(checkpoint); const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); - const events = vi.fn(() => (async function* () { - throw new Error("checkpoint fallback must not consume provider events"); - })()); + const events = vi.fn(() => + (async function* () { + throw new Error("checkpoint fallback must not consume provider events"); + })(), + ); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, events, startTurn, - async result() { return null; }, - async snapshot() { return structuredClone(recoveredCheckpoint); }, + async result() { + return null; + }, + async snapshot() { + return structuredClone(recoveredCheckpoint); + }, async close() {}, }; const backend: NativeSessionBackend = { @@ -3051,11 +4792,21 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { throw new Error("must recover the provider session"); }, - async recoverSession() { return { recovered: true, session }; }, + async openSession() { + throw new Error("must recover the provider session"); + }, + async recoverSession() { + return { recovered: true, session }; + }, }; const bySource = new Map([ ["runner-recovery", [workProposal, workTerminal]], @@ -3063,7 +4814,9 @@ describe("executeNativeSession recovery", () => { const completeRun = vi.fn(async () => undefined); const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, async checkpointSession() {}, async appendEvent(event) { const list = bySource.get(event.sourceInstanceId) ?? []; @@ -3078,36 +4831,39 @@ describe("executeNativeSession recovery", () => { async replayEvents(replay) { const list = bySource.get(replay.sourceInstanceId) ?? []; return { - events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + events: structuredClone( + list.filter((event) => event.sourceSeq > replay.afterSourceSeq), + ), highestContiguousSourceSeq: highestContiguous(list), }; }, completeRun, }; - const execute = () => executeNativeSession({ - input, - backend, - controlPlane: port, - runnerInstanceId: "runner-recovery", - controlPlaneInstanceId: "control-recovery", - resolveMissingResult: async ({ turnId, terminalEvent }) => { - expect(turnId).toBe("turn-disposition"); - expect(terminalEvent).toMatchObject({ - sourceInstanceId: "control-recovery", - sourceKind: "control_plane", - runId: identity.runId, - normalizedSessionId: identity.sessionId, - turnId: "turn-disposition", - eventType: "turn.completed", - payload: { - recovery: "checkpointed_resultless_disposition", - terminalFingerprint: "disposition-terminal", - }, - }); - return result; - }, - }); + const execute = () => + executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult: async ({ turnId, terminalEvent }) => { + expect(turnId).toBe("turn-disposition"); + expect(terminalEvent).toMatchObject({ + sourceInstanceId: "control-recovery", + sourceKind: "control_plane", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-disposition", + eventType: "turn.completed", + payload: { + recovery: "checkpointed_resultless_disposition", + terminalFingerprint: "disposition-terminal", + }, + }); + return result; + }, + }); await expect(execute()).resolves.toMatchObject({ result, turnId: "turn-disposition", @@ -3119,10 +4875,9 @@ describe("executeNativeSession recovery", () => { workProposal, workTerminal, ]); - expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([ - "run.result.accepted", - "run.terminal", - ]); + expect( + bySource.get("control-recovery")?.map((event) => event.eventType), + ).toEqual(["run.result.accepted", "run.terminal"]); recoveredCheckpoint.terminalTurns![1]!.fingerprint = "conflicting-terminal"; await expect(execute()).rejects.toThrow( @@ -3143,7 +4898,9 @@ describe("executeNativeSession recovery", () => { semanticResult: result, terminal, activeTurnId: "turn-recovery", - terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }], + terminalTurns: [ + { turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }, + ], pendingRuntimeRequests: [], lineage: [], }; @@ -3152,17 +4909,29 @@ describe("executeNativeSession recovery", () => { const completeRun = vi.fn(async () => undefined); const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); const openSession = vi.fn(async () => { - throw new Error("a recovered run must not open a second provider session"); + throw new Error( + "a recovered run must not open a second provider session", + ); }); const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; }, async *events() {}, startTurn, - async result() { return { result, terminal, turnId: "turn-recovery" }; }, - async snapshot() { return structuredClone(checkpoint); }, + async result() { + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return structuredClone(checkpoint); + }, async close() {}, }; const recoverSession = vi.fn(async () => ({ recovered: true, session })); @@ -3172,7 +4941,13 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "recovery-backend", version: "1", - capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, openSession, @@ -3180,8 +4955,12 @@ describe("executeNativeSession recovery", () => { }; const port: ControlPlanePort = { async openRun() {}, - async loadSessionCheckpoint() { return structuredClone(checkpoint); }, - async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async loadSessionCheckpoint() { + return structuredClone(checkpoint); + }, + async checkpointSession(snapshot) { + checkpoints.push(structuredClone(snapshot)); + }, async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); return { @@ -3191,8 +4970,13 @@ describe("executeNativeSession recovery", () => { }; }, async replayEvents(replay) { - const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq); - return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) }; + const replayed = events.filter( + (event) => event.sourceSeq > replay.afterSourceSeq, + ); + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(events), + }; }, completeRun, }; @@ -3208,11 +4992,20 @@ describe("executeNativeSession recovery", () => { expect(openSession).not.toHaveBeenCalled(); expect(recoverSession).toHaveBeenCalledOnce(); expect(startTurn).not.toHaveBeenCalled(); - expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]); + expect(events.map((event) => event.eventType)).toEqual([ + "run.result.accepted", + "run.terminal", + ]); expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]); expect(completeRun).toHaveBeenCalledOnce(); - expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 }); - expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal }); + expect(completed).toMatchObject({ + nativeEventCount: 1, + highestContiguousSourceSeq: 2, + }); + expect(checkpoints.at(-1)).toMatchObject({ + semanticResult: result, + terminal, + }); }); it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => { @@ -3238,8 +5031,16 @@ describe("executeNativeSession recovery", () => { completionClaim: { contractRevision: "1", objectiveSatisfied: false, - criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], - remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + criteria: [ + { + criterionId: "objective", + status: "unknown", + evidenceRefs: ["interaction:pending"], + }, + ], + remainingWork: [ + { description: "Resume after the response.", blocksCompletion: true }, + ], }, evidence: [{ ref: "interaction:pending" }], verification: [], @@ -3257,11 +5058,23 @@ describe("executeNativeSession recovery", () => { const session: NativeSession = { identity: () => identity, async capabilities() { - return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield terminalEvent; + }, + async startTurn() { + return { turnId: "turn-waiting" }; + }, + async result() { + return null; }, - async *events() { yield terminalEvent; }, - async startTurn() { return { turnId: "turn-waiting" }; }, - async result() { return null; }, async snapshot() { return { backendKind: "mock", @@ -3282,10 +5095,18 @@ describe("executeNativeSession recovery", () => { kind: "mock", name: "governed-wait-backend", version: "1", - capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -3299,10 +5120,15 @@ describe("executeNativeSession recovery", () => { }; }, async replayEvents(replay) { - const replayed = events.filter((event) => - event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + const replayed = events.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, ); - return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(replayed), + }; }, completeRun, }; @@ -3316,10 +5142,16 @@ describe("executeNativeSession recovery", () => { resolveMissingResult, }); - expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent }); + expect(resolveMissingResult).toHaveBeenCalledWith({ + turnId: "turn-waiting", + terminalEvent, + }); expect(completed).toMatchObject({ result: yielded, - terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" }, + terminal: { + runTerminalState: "succeeded", + reportedWorkDisposition: "yielded", + }, turnId: "turn-waiting", }); expect(completeRun).toHaveBeenCalledWith( @@ -3501,13 +5333,18 @@ describe("executeNativeSession recovery", () => { try { const questionSet = { schema: "paperclip.question_set.v1" as const, - questions: [{ - id: "region", - prompt: "Which region?", - required: true, - answerMode: "single_select" as const, - options: [{ id: "us", label: "US" }, { id: "eu", label: "Europe" }], - }], + questions: [ + { + id: "region", + prompt: "Which region?", + required: true, + answerMode: "single_select" as const, + options: [ + { id: "us", label: "US" }, + { id: "eu", label: "Europe" }, + ], + }, + ], }; const request = { schema: "paperclip.runtime_request.v2", @@ -3521,24 +5358,39 @@ describe("executeNativeSession recovery", () => { turnId: "turn-waiting", itemId: "input-1", }; - const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting" }; - const expired = { ...runnerEvent(2, "runtime_request.expired", { - requestId: "input-1", - requestKind: "runtime", + const created = { + ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting", - itemId: "input-1", - reason: "durable_handoff", - replayAllowed: false, - requestType: "input", - request, - }), turnId: "turn-waiting" }; - const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting" }; + }; + const expired = { + ...runnerEvent(2, "runtime_request.expired", { + requestId: "input-1", + requestKind: "runtime", + turnId: "turn-waiting", + itemId: "input-1", + reason: "durable_handoff", + replayAllowed: false, + requestType: "input", + request, + }), + turnId: "turn-waiting", + }; + const interrupted = { + ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), + turnId: "turn-waiting", + }; let releaseHandoff!: () => void; - const handedOff = new Promise((resolve) => { releaseHandoff = resolve; }); + const handedOff = new Promise((resolve) => { + releaseHandoff = resolve; + }); let releaseCancelled!: () => void; - const cancelled = new Promise((resolve) => { releaseCancelled = resolve; }); + const cancelled = new Promise((resolve) => { + releaseCancelled = resolve; + }); let releaseCreated!: () => void; - const createdCommitted = new Promise((resolve) => { releaseCreated = resolve; }); + const createdCommitted = new Promise((resolve) => { + releaseCreated = resolve; + }); const handoffRuntimeRequest = vi.fn(() => { releaseHandoff(); return { result: "handed_off" as const, cleanup: Promise.resolve() }; @@ -3567,10 +5419,14 @@ describe("executeNativeSession recovery", () => { await cancelled; yield interrupted; }, - async startTurn() { return { turnId: "turn-waiting" }; }, + async startTurn() { + return { turnId: "turn-waiting" }; + }, handoffRuntimeRequest, cancel, - async result() { return null; }, + async result() { + return null; + }, async snapshot() { return { backendKind: "mock", @@ -3601,7 +5457,9 @@ describe("executeNativeSession recovery", () => { }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -3609,7 +5467,10 @@ describe("executeNativeSession recovery", () => { async appendEvent(event) { events.push(structuredClone(event as PrpEvent)); if (event.eventType === "runtime_request.created") releaseCreated(); - const sourceEvents = events.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + const sourceEvents = events.filter( + (candidate) => + candidate.sourceInstanceId === event.sourceInstanceId, + ); return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), @@ -3617,10 +5478,15 @@ describe("executeNativeSession recovery", () => { }; }, async replayEvents(replay) { - const replayed = events.filter((event) => - event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + const replayed = events.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, ); - return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(replayed), + }; }, async completeRun() {}, }; @@ -3649,7 +5515,9 @@ describe("executeNativeSession recovery", () => { signal: expect.any(AbortSignal), }); expect(cancel).toHaveBeenCalledOnce(); - expect(events.map((event) => event.eventType)).toContain("runtime_request.expired"); + expect(events.map((event) => event.eventType)).toContain( + "runtime_request.expired", + ); } finally { vi.useRealTimers(); } @@ -3665,12 +5533,14 @@ describe("executeNativeSession recovery", () => { prompt: "Which region?", input: { schema: "paperclip.question_set.v1", - questions: [{ - id: "region", - prompt: "Which region?", - required: true, - answerMode: "text", - }], + questions: [ + { + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }, + ], }, origin: { adapter: "mock" }, turnId: "turn-stalled", @@ -3715,7 +5585,9 @@ describe("executeNativeSession recovery", () => { markHandoffStarted(); return { result: "handed_off", - cleanup: new Promise((resolve) => { releaseHandoff = resolve; }), + cleanup: new Promise((resolve) => { + releaseHandoff = resolve; + }), }; }, cancel() { @@ -3807,7 +5679,10 @@ describe("executeNativeSession recovery", () => { }, async *events() { try { - yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal" }; + yield { + ...runnerEvent(1, "turn.completed"), + turnId: "turn-terminal", + }; } finally { teardownStarted(); await new Promise((resolve) => { @@ -3891,12 +5766,14 @@ describe("executeNativeSession recovery", () => { prompt: "Which region?", input: { schema: "paperclip.question_set.v1", - questions: [{ - id: "region", - prompt: "Which region?", - required: true, - answerMode: "text", - }], + questions: [ + { + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }, + ], }, origin: { adapter: "mock" }, turnId: "turn-terminal", @@ -4030,19 +5907,24 @@ describe("executeNativeSession recovery", () => { prompt: "Which region?", input: { schema: "paperclip.question_set.v1", - questions: [{ - id: "region", - prompt: "Which region?", - required: true, - answerMode: "text", - }], + questions: [ + { + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }, + ], }, origin: { adapter: "mock" }, turnId: "turn-live", itemId: "input-live", }; const providerEvents = [ - { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live" }, + { + ...runnerEvent(1, "runtime_request.created", { request }), + turnId: "turn-live", + }, { ...runnerEvent(2, "runtime_request.resolved", { requestId: "input-live", @@ -4081,10 +5963,16 @@ describe("executeNativeSession recovery", () => { runtimeRequestHandoff: true, }; }, - async *events() { yield* providerEvents; }, - async startTurn() { return { turnId: "turn-live" }; }, + async *events() { + yield* providerEvents; + }, + async startTurn() { + return { turnId: "turn-live" }; + }, handoffRuntimeRequest, - async result() { return { result, terminal, turnId: "turn-live" }; }, + async result() { + return { result, terminal, turnId: "turn-live" }; + }, async snapshot() { return { backendKind: "mock", @@ -4115,7 +6003,9 @@ describe("executeNativeSession recovery", () => { }, }; }, - async openSession() { return session; }, + async openSession() { + return session; + }, }; const port: ControlPlanePort = { async openRun() {}, @@ -4126,7 +6016,9 @@ describe("executeNativeSession recovery", () => { await settlementAppendReleased; } appended.push(structuredClone(event as PrpEvent)); - const sourceEvents = appended.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + const sourceEvents = appended.filter( + (candidate) => candidate.sourceInstanceId === event.sourceInstanceId, + ); return { cursor: appended.length, highestContiguousSourceSeq: highestContiguous(sourceEvents), @@ -4134,17 +6026,26 @@ describe("executeNativeSession recovery", () => { }; }, async replayEvents(replay) { - const replayed = appended.filter((event) => - event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + const replayed = appended.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, ); - return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(replayed), + }; }, async completeRun() {}, }; const originalSetTimeout = globalThis.setTimeout; let queuedHandoffCallback: (() => void) | null = null; - const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback, delay, ...args) => { + const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation((( + callback, + delay, + ...args + ) => { if (delay === 123_456) { queuedHandoffCallback = () => callback(...args); const handle = originalSetTimeout(() => undefined, 60_000); @@ -4172,7 +6073,9 @@ describe("executeNativeSession recovery", () => { queuedHandoffCallback?.(); await Promise.resolve(); expect(handoffRuntimeRequest).not.toHaveBeenCalled(); - expect(appended.some((event) => event.eventType === "runtime_request.expired")).toBe(false); + expect( + appended.some((event) => event.eventType === "runtime_request.expired"), + ).toBe(false); } finally { releaseSettlementAppend(); timeoutSpy.mockRestore(); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 31d89f3ddf..cf608b941d 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -4,17 +4,66 @@ import type { CheckpointControlPlaneSessionOptions, ControlPlanePort, } from "./contracts/control-plane-port.js"; -import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contracts/native-execution.js"; -import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js"; -import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js"; +import type { + NativeExecutionInput, + NativeSessionExecutionResult, +} from "./contracts/native-execution.js"; +import { + buildNativeModelEnvelope, + parseNativeExecutionInput, +} from "./contracts/native-execution.js"; +import type { + NativeSession, + NativeSessionBackend, +} from "./contracts/native-session-backend.js"; import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; -import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "./protocol/replay-contract.js"; import { parsePaperclipQuestionSet } from "./contracts/question-set.js"; export const DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS = 120_000; const OPTIONAL_SESSION_CANCELLATION_GRACE_MS = 100; const FAILED_OPERATION_SETTLEMENT_GRACE_MS = 100; const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000; +type NativeSessionCleanupDomain = string; + +const failedSessionCleanupOwners = new Map< + Promise, + NativeSessionCleanupDomain +>(); +const FAILED_SESSION_CLOSE_RETRY_MS = 1_000; +const MAX_FAILED_SESSION_CLOSE_RETRIES = 3; +const MAX_QUARANTINED_SESSION_CLOSE_RETRIES = 3; +const MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS = 9; +const QUARANTINED_SESSION_CLOSE_RETRY_MS = 60_000; +// Admission can inherit the initial close plus all three retained retries. +// Keep the wait finite while covering every bounded close attempt and all +// three production retry delays instead of timing out midway through recovery. +const QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS = 7_000; +const QUARANTINED_SESSION_ADMISSION_GRACE_MS = + (MAX_FAILED_SESSION_CLOSE_RETRIES + 1) * + QUARANTINED_SESSION_CLOSE_ATTEMPT_BOUND_MS + + MAX_FAILED_SESSION_CLOSE_RETRIES * FAILED_SESSION_CLOSE_RETRY_MS; +// Admission can observe a retained close recovery and then the one quarantine +// recovery that retained owner installs when it exhausts its retries. Give +// each owner generation a complete finite grace, while rejecting any +// unexpected third generation instead of permitting an unbounded owner chain. +const MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES = 2; + +interface QuarantinedSessionCleanup { + session: NativeSession; + domain: NativeSessionCleanupDomain; + automaticAttempts: number; + attempt: Promise | null; + recovery: Promise | null; + recoveryMaxAttempts: number | null; + timer: ReturnType | null; +} + +const quarantinedSessionCleanups = new Set(); export interface ExecuteNativeSessionOptions { input: NativeExecutionInput; @@ -23,6 +72,8 @@ export interface ExecuteNativeSessionOptions { runnerInstanceId: string; controlPlaneInstanceId: string; timeoutMs?: number; + /** Abort admission while waiting for prior cleanup in the same domain. */ + signal?: AbortSignal; /** Internal test seam; production bounds checkpoint persistence to 30 seconds. */ checkpointTimeoutMs?: number; /** Internal test seam; production uses the fixed 120-second platform policy. */ @@ -68,18 +119,43 @@ export interface ExecuteNativeSessionOptions { } function isTurnTerminal(event: PrpEvent): boolean { - return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType); + return [ + "turn.completed", + "turn.failed", + "turn.interrupted", + "turn.cancelled", + ].includes(event.eventType); } -function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["reportedWorkDisposition"]): PrpTerminalState { - const states = event.eventType === "turn.completed" - ? { turnTerminalState: "completed" as const, runTerminalState: "succeeded" as const } - : event.eventType === "turn.failed" - ? { turnTerminalState: "failed" as const, runTerminalState: "failed" as const } - : event.eventType === "turn.interrupted" - ? { turnTerminalState: "interrupted" as const, runTerminalState: "cancelled" as const } - : { turnTerminalState: "cancelled" as const, runTerminalState: "cancelled" as const }; - return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition }; +function terminalFromEvent( + event: PrpEvent, + disposition: PrpTerminalState["reportedWorkDisposition"], +): PrpTerminalState { + const states = + event.eventType === "turn.completed" + ? { + turnTerminalState: "completed" as const, + runTerminalState: "succeeded" as const, + } + : event.eventType === "turn.failed" + ? { + turnTerminalState: "failed" as const, + runTerminalState: "failed" as const, + } + : event.eventType === "turn.interrupted" + ? { + turnTerminalState: "interrupted" as const, + runTerminalState: "cancelled" as const, + } + : { + turnTerminalState: "cancelled" as const, + runTerminalState: "cancelled" as const, + }; + return { + schema: "paperclip.prp.terminal.v1", + ...states, + reportedWorkDisposition: disposition, + }; } async function attemptOptionalSessionCancellation( @@ -104,21 +180,48 @@ async function attemptOptionalSessionCancellation( if (await settlesWithin(attempts, OPTIONAL_SESSION_CANCELLATION_GRACE_MS)) { return null; } - cancellationAbort.abort(new Error("native session cancellation grace expired")); + cancellationAbort.abort( + new Error("native session cancellation grace expired"), + ); return { settlement: attempts }; } -async function settlesWithin(operation: Promise, timeoutMs: number): Promise { +async function settlesWithin( + operation: Promise, + timeoutMs: number, + signal?: AbortSignal, +): Promise { + signal?.throwIfAborted(); let graceTimer: ReturnType | undefined; - const settled = await Promise.race([ - operation.then(() => true), - new Promise((resolve) => { - graceTimer = setTimeout(() => resolve(false), timeoutMs); - graceTimer.unref?.(); - }), - ]); - if (graceTimer !== undefined) clearTimeout(graceTimer); - return settled; + let removeAbort = () => {}; + try { + const aborted = signal + ? new Promise((_resolve, reject) => { + const onAbort = () => + reject( + signal.reason ?? + new Error("native session cleanup admission aborted"), + ); + if (signal.aborted) { + onAbort(); + } else { + signal.addEventListener("abort", onAbort, { once: true }); + removeAbort = () => signal.removeEventListener("abort", onAbort); + } + }) + : new Promise(() => undefined); + return await Promise.race([ + operation.then(() => true), + new Promise((resolve) => { + graceTimer = setTimeout(() => resolve(false), timeoutMs); + graceTimer.unref?.(); + }), + aborted, + ]); + } finally { + if (graceTimer !== undefined) clearTimeout(graceTimer); + removeAbort(); + } } async function runAbortableOperationWithin(input: { @@ -152,7 +255,8 @@ async function runAbortableOperationWithin(input: { operation, new Promise((_resolve, reject) => { timer = setTimeout(() => { - const error = input.timeoutError?.() ?? new Error(input.timeoutMessage); + const error = + input.timeoutError?.() ?? new Error(input.timeoutMessage); timedOut = true; timeoutError = error; reject(error); @@ -174,14 +278,54 @@ async function runAbortableOperationWithin(input: { async function disposeUnadmittedSession( session: NativeSession, reason: string, + cleanupDomain: NativeSessionCleanupDomain, ): Promise { // A provider may ignore abort and return a session after its caller has // already timed out. That session was never published through onSession, so // close it without clearing ownership that a later execution may establish. - const closeSettlement = Promise.allSettled([ - Promise.resolve().then(() => session.close({ reason })), - ]); - await settlesWithin(closeSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); + // Retain slow or failed cleanup in the same admission gate as an admitted + // session: the absence of an onSession publication does not mean provider + // resources have already been released. + const cleanup = retainUnadmittedSessionCleanup( + session, + reason, + cleanupDomain, + ); + await settlesWithin( + Promise.allSettled([cleanup]), + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); +} + +function retainUnadmittedSessionCleanup( + session: NativeSession, + reason: string, + cleanupDomain: NativeSessionCleanupDomain, +): Promise { + const cleanup = (async () => { + let attempt = Promise.resolve().then(() => session.close({ reason })); + let retryCount = 0; + while (true) { + try { + await attempt; + return; + } catch (error) { + if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) { + quarantineSessionCleanup(session, cleanupDomain); + throw error; + } + retryCount += 1; + await waitForSessionCloseRetry(); + attempt = Promise.resolve().then(() => + session.close({ + reason: `${reason} cleanup recovery (${retryCount})`, + }), + ); + } + } + })(); + retainFailedSessionCleanupOwner(cleanup, cleanupDomain); + return cleanup; } class NativeSessionFinalizationTimeoutError extends Error { @@ -198,7 +342,8 @@ async function finalizeWithin(input: { return runAbortableOperationWithin({ ...input, timeoutMessage: `native session finalization timed out after ${input.timeoutMs}ms`, - timeoutError: () => new NativeSessionFinalizationTimeoutError(input.timeoutMs), + timeoutError: () => + new NativeSessionFinalizationTimeoutError(input.timeoutMs), }); } @@ -223,13 +368,14 @@ async function quarantineRetainedSession( session: NativeSession, onSession: ExecuteNativeSessionOptions["onSession"], reason: string, + cleanupDomain: NativeSessionCleanupDomain, ): Promise { // Eviction and provider cleanup are independent obligations. Keep both // observed so a throwing owner callback cannot prevent close from starting, // and a broken provider cannot keep the failed execution pending forever. const quarantineSettlement = Promise.allSettled([ Promise.resolve().then(() => onSession?.(null)), - Promise.resolve().then(() => session.close({ reason })), + retainUnadmittedSessionCleanup(session, reason, cleanupDomain), ]); await settlesWithin( quarantineSettlement, @@ -251,8 +397,9 @@ async function persistCheckpointWithin(input: { const externalAbortFailure = externalSignal ? new Promise((_resolve, reject) => { const abort = () => { - const reason = externalSignal.reason - ?? new Error("native session checkpoint aborted"); + const reason = + externalSignal.reason ?? + new Error("native session checkpoint aborted"); checkpointAbort.abort(reason); reject(reason); }; @@ -260,7 +407,8 @@ async function persistCheckpointWithin(input: { abort(); } else { externalSignal.addEventListener("abort", abort, { once: true }); - removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + removeExternalAbort = () => + externalSignal.removeEventListener("abort", abort); } }) : new Promise(() => undefined); @@ -271,8 +419,10 @@ async function persistCheckpointWithin(input: { checkpointOptions, ); if (checkpointAbort.signal.aborted) { - throw checkpointAbort.signal.reason - ?? new Error("native session checkpoint aborted"); + throw ( + checkpointAbort.signal.reason ?? + new Error("native session checkpoint aborted") + ); } await input.onCheckpoint?.(input.snapshot, checkpointOptions); })(); @@ -300,6 +450,226 @@ async function persistCheckpointWithin(input: { } } +function waitForSessionCloseRetry(): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, FAILED_SESSION_CLOSE_RETRY_MS); + timer.unref?.(); + }); +} +function retainFailedSessionCleanupOwner( + cleanup: Promise, + cleanupDomain: NativeSessionCleanupDomain, +): void { + failedSessionCleanupOwners.set(cleanup, cleanupDomain); + void cleanup + .finally(() => failedSessionCleanupOwners.delete(cleanup)) + .catch(() => undefined); +} + +function quarantineSessionCleanup( + session: NativeSession, + cleanupDomain: NativeSessionCleanupDomain, +): void { + if ( + [...quarantinedSessionCleanups].some((entry) => entry.session === session) + ) { + return; + } + const cleanup: QuarantinedSessionCleanup = { + session, + domain: cleanupDomain, + automaticAttempts: 0, + attempt: null, + recovery: null, + recoveryMaxAttempts: null, + timer: null, + }; + quarantinedSessionCleanups.add(cleanup); + startQuarantinedSessionCleanupRecovery( + cleanup, + MAX_QUARANTINED_SESSION_CLOSE_RETRIES, + "native session quarantined cleanup recovery", + ); +} + +function startQuarantinedSessionCleanupRecovery( + cleanup: QuarantinedSessionCleanup, + maxAttempts: number, + reason: string, +): Promise { + if (cleanup.recovery) return cleanup.recovery; + const remainingAttempts = + MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS - + cleanup.automaticAttempts; + const boundedMaxAttempts = Math.min(maxAttempts, remainingAttempts); + if (boundedMaxAttempts <= 0) return Promise.resolve(); + const recovery = (async () => { + for ( + let attemptCount = 0; + attemptCount < boundedMaxAttempts && + quarantinedSessionCleanups.has(cleanup); + attemptCount += 1 + ) { + // The first retry starts immediately so an admission-triggered recovery + // receives the complete close grace. Later attempts retain the bounded + // delay that prevents a hot retry loop. + if (attemptCount > 0) await waitForSessionCloseRetry(); + cleanup.automaticAttempts += 1; + const attempt = Promise.resolve().then(() => + cleanup.session.close({ + reason, + }), + ); + cleanup.attempt = attempt; + try { + await attempt; + quarantinedSessionCleanups.delete(cleanup); + } catch { + // Retain the quarantine after this finite, sequential retry batch. + } finally { + if (cleanup.attempt === attempt) cleanup.attempt = null; + } + } + })(); + cleanup.recovery = recovery; + cleanup.recoveryMaxAttempts = boundedMaxAttempts; + failedSessionCleanupOwners.set(recovery, cleanup.domain); + void recovery + .finally(() => { + failedSessionCleanupOwners.delete(recovery); + if (cleanup.recovery === recovery) { + cleanup.recovery = null; + cleanup.recoveryMaxAttempts = null; + } + scheduleQuarantinedSessionCleanup(cleanup); + }) + .catch(() => undefined); + return recovery; +} + +function scheduleQuarantinedSessionCleanup( + cleanup: QuarantinedSessionCleanup, +): void { + if ( + !quarantinedSessionCleanups.has(cleanup) || + cleanup.recovery || + cleanup.attempt || + cleanup.timer || + cleanup.automaticAttempts >= + MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS + ) { + return; + } + // Keep cleanup live without a hot retry loop: one unref'd timer and one + // sequential close are the maximum background work owned by each entry. + // A later admission may accelerate, but never overlap, the scheduled try. + cleanup.timer = setTimeout(() => { + cleanup.timer = null; + if (!quarantinedSessionCleanups.has(cleanup)) return; + startQuarantinedSessionCleanupRecovery( + cleanup, + 1, + "native session scheduled quarantined cleanup recovery", + ); + }, QUARANTINED_SESSION_CLOSE_RETRY_MS); + cleanup.timer.unref?.(); +} + +async function retryQuarantinedSessionCleanups( + cleanupDomain: NativeSessionCleanupDomain, + signal?: AbortSignal, +): Promise { + // A close attempt becomes admission-visible as soon as the runtime retains + // its exact cleanup owner. It may not have rejected yet, so it may not have + // entered the retry quarantine below. Observe both states through the same + // finite gate to prevent a later execution from opening a second provider + // session while the first close still owns provider resources. + const observedOwners = new Set>(); + const acceleratedCleanups = new Set(); + let observedOwnerPhases = 0; + while (true) { + signal?.throwIfAborted(); + const cleanupOwners = new Set>( + [...failedSessionCleanupOwners] + .filter(([, domain]) => domain === cleanupDomain) + .map(([owner]) => owner), + ); + for (const cleanup of quarantinedSessionCleanups) { + if (cleanup.domain !== cleanupDomain) continue; + if (cleanup.timer) { + clearTimeout(cleanup.timer); + cleanup.timer = null; + } + if (cleanup.recovery) { + // An autonomous scheduled owner receives one attempt. Admission must + // observe it without mistaking it for the complete three-attempt + // admission batch; if that attempt fails, the next bounded owner + // phase accelerates one full batch. Existing full recoveries already + // consumed that allowance and are never duplicated. + if ( + (cleanup.recoveryMaxAttempts ?? 0) >= + MAX_QUARANTINED_SESSION_CLOSE_RETRIES + ) { + acceleratedCleanups.add(cleanup); + } + cleanupOwners.add(cleanup.recovery); + } else if (!acceleratedCleanups.has(cleanup)) { + acceleratedCleanups.add(cleanup); + if ( + cleanup.automaticAttempts < + MAX_QUARANTINED_SESSION_AUTOMATIC_CLOSE_ATTEMPTS + ) { + cleanupOwners.add( + startQuarantinedSessionCleanupRecovery( + cleanup, + MAX_QUARANTINED_SESSION_CLOSE_RETRIES, + "native session quarantined admission recovery", + ), + ); + } + } + } + const replacementOwners = [...cleanupOwners].filter( + (owner) => !observedOwners.has(owner), + ); + if (replacementOwners.length === 0) break; + replacementOwners.forEach((owner) => observedOwners.add(owner)); + observedOwnerPhases += 1; + if ( + observedOwnerPhases > MAX_QUARANTINED_SESSION_ADMISSION_OWNER_PHASES || + !(await settlesWithin( + Promise.all( + replacementOwners.map((owner) => owner.catch(() => undefined)), + ), + QUARANTINED_SESSION_ADMISSION_GRACE_MS, + signal, + )) + ) { + throw new Error( + "native_session_cleanup_quarantined: prior session cleanup exceeded the admission grace", + ); + } + } + if ( + [...failedSessionCleanupOwners.values()].some( + (domain) => domain === cleanupDomain, + ) || + [...quarantinedSessionCleanups].some( + (cleanup) => cleanup.domain === cleanupDomain, + ) + ) { + // Admission may have pulled a scheduled retry forward and observed a + // complete recovery batch that still failed. Restore autonomous ownership + // before rejecting so cleanup cannot remain dormant until another run. + for (const cleanup of quarantinedSessionCleanups) { + if (cleanup.domain !== cleanupDomain) continue; + scheduleQuarantinedSessionCleanup(cleanup); + } + throw new Error( + "native_session_cleanup_quarantined: prior session cleanup remains incomplete", + ); + } +} async function consumeTurn( session: NativeSession, controlPlane: ControlPlanePort, @@ -328,7 +698,9 @@ async function consumeTurn( const externalAbortFailure = externalSignal ? new Promise((_resolve, reject) => { const abort = () => { - const reason = externalSignal.reason ?? new Error("native event consumption aborted"); + const reason = + externalSignal.reason ?? + new Error("native event consumption aborted"); stopConsumer = true; appendAbort.abort(reason); reject(reason); @@ -337,7 +709,8 @@ async function consumeTurn( abort(); } else { externalSignal.addEventListener("abort", abort, { once: true }); - removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + removeExternalAbort = () => + externalSignal.removeEventListener("abort", abort); } }) : new Promise(() => undefined); @@ -347,128 +720,155 @@ async function consumeTurn( inputTimers.delete(requestId); }; const consumer = (async () => { - let eventCount = 0; - let highestContiguousSourceSeq = 0; - let governedResult: PrpStructuredRunResult | null = null; - while (true) { - const next = await eventIterator.next(); - if (stopConsumer) throw new Error("native event consumer stopped"); - if (next.done) throw new Error("native event stream closed before a turn terminal fact"); - const event = next.value; - const payload = event.payload as Record; - const settlingRequestId = - ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"] - .includes(event.eventType) - && typeof payload.requestId === "string" - ? payload.requestId - : null; - // Beginning settlement revokes the expiry timer's handoff authority. - // appendEvent may remain pending across the live-window deadline; if - // the timer stayed live until the receipt returned, both settlement - // and a durable handoff could commit for the same request. - if (settlingRequestId !== null) clearInputTimer(settlingRequestId); - const receipt = await controlPlane.appendEvent(event, { - signal: appendAbort.signal, - }); - if (stopConsumer) throw new Error("native event consumer stopped"); - eventCount += receipt.disposition === "committed" ? 1 : 0; - highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); - const request = payload.request && typeof payload.request === "object" && !Array.isArray(payload.request) - ? payload.request as Record - : null; - if ( - receipt.disposition === "committed" - && event.eventType === "runtime_request.created" - && request?.schema === "paperclip.runtime_request.v2" - && request.type === "input" - && typeof request.requestId === "string" - && typeof request.turnId === "string" - ) { + let eventCount = 0; + let highestContiguousSourceSeq = 0; + let governedResult: PrpStructuredRunResult | null = null; + while (true) { + const next = await eventIterator.next(); + if (stopConsumer) throw new Error("native event consumer stopped"); + if (next.done) + throw new Error( + "native event stream closed before a turn terminal fact", + ); + const event = next.value; + const payload = event.payload as Record; + const settlingRequestId = + [ + "runtime_request.resolved", + "runtime_request.cancelled", + "runtime_request.expired", + ].includes(event.eventType) && typeof payload.requestId === "string" + ? payload.requestId + : null; + // Beginning settlement revokes the expiry timer's handoff authority. + // appendEvent may remain pending across the live-window deadline; if + // the timer stayed live until the receipt returned, both settlement + // and a durable handoff could commit for the same request. + if (settlingRequestId !== null) clearInputTimer(settlingRequestId); + const receipt = await controlPlane.appendEvent(event, { + signal: appendAbort.signal, + }); + if (stopConsumer) throw new Error("native event consumer stopped"); + eventCount += receipt.disposition === "committed" ? 1 : 0; + highestContiguousSourceSeq = Math.max( + highestContiguousSourceSeq, + receipt.highestContiguousSourceSeq, + ); + const request = + payload.request && + typeof payload.request === "object" && + !Array.isArray(payload.request) + ? (payload.request as Record) + : null; + if ( + receipt.disposition === "committed" && + event.eventType === "runtime_request.created" && + request?.schema === "paperclip.runtime_request.v2" && + request.type === "input" && + typeof request.requestId === "string" && + typeof request.turnId === "string" + ) { + try { + parsePaperclipQuestionSet(request.input); + const requestId = request.requestId; + const turnId = request.turnId; + clearInputTimer(requestId); + const inputTimer = setTimeout(() => { + // Clearing a timeout does not revoke a callback that is already + // queued. The map entry is the per-request authority token. + if (inputTimers.get(requestId) !== inputTimer) return; + inputTimers.delete(requestId); + // A timer callback can already be queued when teardown clears + // its handle. Re-check the live-turn authority inside the + // callback before starting or registering durable work. + if (stopConsumer || appendAbort.signal.aborted) return; + if (session.handoffRuntimeRequest === undefined) { + rejectHandoff?.( + new Error("native_runtime_request_handoff_unavailable"), + ); + return; + } + let handoffCleanup: Promise; try { - parsePaperclipQuestionSet(request.input); - const requestId = request.requestId; - const turnId = request.turnId; - clearInputTimer(requestId); - const inputTimer = setTimeout(() => { - // Clearing a timeout does not revoke a callback that is already - // queued. The map entry is the per-request authority token. - if (inputTimers.get(requestId) !== inputTimer) return; - inputTimers.delete(requestId); - // A timer callback can already be queued when teardown clears - // its handle. Re-check the live-turn authority inside the - // callback before starting or registering durable work. - if (stopConsumer || appendAbort.signal.aborted) return; - if (session.handoffRuntimeRequest === undefined) { - rejectHandoff?.(new Error("native_runtime_request_handoff_unavailable")); - return; - } - let handoffCleanup: Promise; - try { - const handoff = session.handoffRuntimeRequest({ - requestId, - turnId, - reason: "durable_handoff", - signal: appendAbort.signal, - }); - // Durable handoff mutation is synchronous. The returned - // promise owns provider interruption only, so it can remain - // observed without acquiring authority to delay or reverse - // a provider terminal fact. - handoffCleanup = handoff.cleanup; - } catch (error) { - rejectHandoff?.(error); - return; - } - handoffCleanupOperations.add(handoffCleanup); - void handoffCleanup - .catch((error) => { - if (!stopConsumer && !appendAbort.signal.aborted) { - rejectHandoff?.(error); - } - }) - .finally(() => handoffCleanupOperations.delete(handoffCleanup)); - }, runtimeInputLiveWindowMs); - inputTimer.unref?.(); - inputTimers.set(requestId, inputTimer); - } catch { - // Invalid structured inputs remain rejected by the driver and never become durable questions. - } - } - if (governedResult === null && resolveGovernedWait) { - if (appendAbort.signal.aborted) { - throw appendAbort.signal.reason ?? new Error("native event consumption aborted"); - } - governedResult = resolveGovernedWait({ - turnId: event.turnId ?? null, - event, - }); - if (governedResult !== null && !isTurnTerminal(event)) { - if (session.cancel === undefined) { - throw new Error("native_governed_wait_cancellation_unavailable"); - } - // A governed result is already durable. Commit cancellation - // synchronously, then stop consuming provider output now rather - // than waiting for an abort-insensitive cleanup or terminal event. - // The returned promise owns cleanup only and remains observed in - // finally, where its wait is bounded and the session quarantined. - const cancellation = session.cancel({ - reason: "Paperclip parked this turn on a durable governed interaction.", + const handoff = session.handoffRuntimeRequest({ + requestId, + turnId, + reason: "durable_handoff", signal: appendAbort.signal, }); - governedCancellationCommitted = true; - const cleanup = cancellation.cleanup; - governedCleanupOperations.add(cleanup); - void cleanup - .catch(() => quarantineSession()) - .finally(() => governedCleanupOperations.delete(cleanup)); - return { event, eventCount, highestContiguousSourceSeq, governedResult }; + // Durable handoff mutation is synchronous. The returned + // promise owns provider interruption only, so it can remain + // observed without acquiring authority to delay or reverse + // a provider terminal fact. + handoffCleanup = handoff.cleanup; + } catch (error) { + rejectHandoff?.(error); + return; } - } - if (isTurnTerminal(event)) { - return { event, eventCount, highestContiguousSourceSeq, governedResult }; - } + handoffCleanupOperations.add(handoffCleanup); + void handoffCleanup + .catch((error) => { + if (!stopConsumer && !appendAbort.signal.aborted) { + rejectHandoff?.(error); + } + }) + .finally(() => handoffCleanupOperations.delete(handoffCleanup)); + }, runtimeInputLiveWindowMs); + inputTimer.unref?.(); + inputTimers.set(requestId, inputTimer); + } catch { + // Invalid structured inputs remain rejected by the driver and never become durable questions. } - })(); + } + if (governedResult === null && resolveGovernedWait) { + if (appendAbort.signal.aborted) { + throw ( + appendAbort.signal.reason ?? + new Error("native event consumption aborted") + ); + } + governedResult = resolveGovernedWait({ + turnId: event.turnId ?? null, + event, + }); + if (governedResult !== null && !isTurnTerminal(event)) { + if (session.cancel === undefined) { + throw new Error("native_governed_wait_cancellation_unavailable"); + } + // A governed result is already durable. Commit cancellation + // synchronously, then stop consuming provider output now rather + // than waiting for an abort-insensitive cleanup or terminal event. + // The returned promise owns cleanup only and remains observed in + // finally, where its wait is bounded and the session quarantined. + const cancellation = session.cancel({ + reason: + "Paperclip parked this turn on a durable governed interaction.", + signal: appendAbort.signal, + }); + governedCancellationCommitted = true; + const cleanup = cancellation.cleanup; + governedCleanupOperations.add(cleanup); + void cleanup + .catch(() => quarantineSession()) + .finally(() => governedCleanupOperations.delete(cleanup)); + return { + event, + eventCount, + highestContiguousSourceSeq, + governedResult, + }; + } + } + if (isTurnTerminal(event)) { + return { + event, + eventCount, + highestContiguousSourceSeq, + governedResult, + }; + } + } + })(); // A timeout can win the race while an iterator is still waiting for data. // Observe any later consumer rejection so it cannot become process-fatal. void consumer.catch(() => undefined); @@ -494,8 +894,15 @@ async function consumeTurn( // cancellation has already committed synchronously. Bound only the // authority-free provider cleanup while keeping its outcome observed. if (governedCleanupOperations.size > 0) { - const governedCleanupSettlement = Promise.allSettled([...governedCleanupOperations]); - if (!(await settlesWithin(governedCleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS))) { + const governedCleanupSettlement = Promise.allSettled([ + ...governedCleanupOperations, + ]); + if ( + !(await settlesWithin( + governedCleanupSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + )) + ) { deferredGovernedCleanupSettlement = governedCleanupSettlement; } } @@ -504,20 +911,23 @@ async function consumeTurn( session, "Native session event consumption failed.", ); - deferredSessionCancellationSettlement = deferredCancellation?.settlement ?? null; + deferredSessionCancellationSettlement = + deferredCancellation?.settlement ?? null; } throw error; } finally { stopConsumer = true; for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer); inputTimers.clear(); - const activeHandoffCleanupSettlement = handoffCleanupOperations.size > 0 - ? Promise.allSettled([...handoffCleanupOperations]) - : null; - const activeGovernedCleanupSettlement = deferredGovernedCleanupSettlement === null - && governedCleanupOperations.size > 0 - ? Promise.allSettled([...governedCleanupOperations]) - : null; + const activeHandoffCleanupSettlement = + handoffCleanupOperations.size > 0 + ? Promise.allSettled([...handoffCleanupOperations]) + : null; + const activeGovernedCleanupSettlement = + deferredGovernedCleanupSettlement === null && + governedCleanupOperations.size > 0 + ? Promise.allSettled([...governedCleanupOperations]) + : null; if (!consumptionFailed && !appendAbort.signal.aborted) { // A provider terminal or synchronous governed cancellation revokes // live-turn authority. Handoff state was already committed; abort only @@ -537,9 +947,15 @@ async function consumeTurn( const passiveTeardownSettlement = Promise.allSettled([ iteratorTeardown, consumer, - ...(activeHandoffCleanupSettlement ? [activeHandoffCleanupSettlement] : []), - ...(activeGovernedCleanupSettlement ? [activeGovernedCleanupSettlement] : []), - ...(deferredGovernedCleanupSettlement ? [deferredGovernedCleanupSettlement] : []), + ...(activeHandoffCleanupSettlement + ? [activeHandoffCleanupSettlement] + : []), + ...(activeGovernedCleanupSettlement + ? [activeGovernedCleanupSettlement] + : []), + ...(deferredGovernedCleanupSettlement + ? [deferredGovernedCleanupSettlement] + : []), ...(deferredSessionCancellationSettlement ? [deferredSessionCancellationSettlement] : []), @@ -556,7 +972,10 @@ async function consumeTurn( passiveTeardownSettlement, Promise.resolve().then(closeFailedSession), ]); - await settlesWithin(cleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); + await settlesWithin( + cleanupSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); } else { // Iterator and provider cleanup own no control-plane mutation authority. // A slow subscription or cleanup remains observed and is released by the @@ -611,7 +1030,10 @@ async function reconcileRecoveryCursor(input: { persistedHighWater = Math.max(persistedHighWater, pageHighWater); afterSourceSeq = pageHighWater; } - if (persistedHighWater === checkpointHighWater && input.checkpoint.cursor === String(checkpointHighWater)) { + if ( + persistedHighWater === checkpointHighWater && + input.checkpoint.cursor === String(checkpointHighWater) + ) { return input.checkpoint; } return { ...input.checkpoint, cursor: String(persistedHighWater) }; @@ -639,9 +1061,12 @@ async function replayCheckpointedTurnTerminal(input: { // Disposition recovery owns the newest durable provider terminal. An // older task turn may also have a valid proposal, but selecting it would // finalize stale work and strand the actual recovery terminal. - const terminal = [...terminals] - .sort((left, right) => right.sourceSeq - left.sourceSeq)[0] ?? null; - const proposalSequence = latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0; + const terminal = + [...terminals].sort( + (left, right) => right.sourceSeq - left.sourceSeq, + )[0] ?? null; + const proposalSequence = + latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0; return terminal === null ? null : { @@ -663,11 +1088,9 @@ async function replayCheckpointedTurnTerminal(input: { if ( event.turnId && isTurnTerminal(event) && - ( - input.expectedTurnId !== undefined && input.expectedTurnId !== null - ? event.turnId === input.expectedTurnId - : !priorTerminalTurnIds.has(event.turnId) - ) + (input.expectedTurnId !== undefined && input.expectedTurnId !== null + ? event.turnId === input.expectedTurnId + : !priorTerminalTurnIds.has(event.turnId)) ) { terminals.push(structuredClone(event)); } @@ -690,30 +1113,37 @@ function checkpointedResultlessDispositionFallback(input: { }): PrpEvent | null { const turnId = input.persisted.dispositionOnlyRecoveryTurnId; if ( - !input.persisted.dispositionOnlyRecoveryConsumed - || input.persisted.semanticResult - || input.persisted.activeTurnId - || typeof turnId !== "string" - || turnId.length === 0 - ) return null; - const persistedTerminal = input.persisted.terminalTurns?.filter( - (terminal) => terminal.turnId === turnId, - ) ?? []; - if (persistedTerminal.length !== 1 || persistedTerminal[0]!.fingerprint.length === 0) { + !input.persisted.dispositionOnlyRecoveryConsumed || + input.persisted.semanticResult || + input.persisted.activeTurnId || + typeof turnId !== "string" || + turnId.length === 0 + ) + return null; + const persistedTerminal = + input.persisted.terminalTurns?.filter( + (terminal) => terminal.turnId === turnId, + ) ?? []; + if ( + persistedTerminal.length !== 1 || + persistedTerminal[0]!.fingerprint.length === 0 + ) { return null; } const recoveredTurnId = input.recovered.dispositionOnlyRecoveryTurnId; - const recoveredTerminal = input.recovered.terminalTurns?.filter( - (terminal) => terminal.turnId === recoveredTurnId, - ) ?? []; + const recoveredTerminal = + input.recovered.terminalTurns?.filter( + (terminal) => terminal.turnId === recoveredTurnId, + ) ?? []; if ( - !input.recovered.dispositionOnlyRecoveryConsumed - || input.recovered.semanticResult - || input.recovered.activeTurnId - || recoveredTurnId !== turnId - || recoveredTerminal.length !== 1 - || recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint - || canonicalJson(input.recovered.identity) !== canonicalJson(input.persisted.identity) + !input.recovered.dispositionOnlyRecoveryConsumed || + input.recovered.semanticResult || + input.recovered.activeTurnId || + recoveredTurnId !== turnId || + recoveredTerminal.length !== 1 || + recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint || + canonicalJson(input.recovered.identity) !== + canonicalJson(input.persisted.identity) ) { throw new Error("native_disposition_recovery_checkpoint_conflict"); } @@ -741,43 +1171,57 @@ function checkpointedResultlessDispositionFallback(input: { * Package-owned normalized session loop. Paperclip supplies persistence and * authority through ControlPlanePort; provider/session behavior stays here. */ -export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise { +export async function executeNativeSession( + options: ExecuteNativeSessionOptions, +): Promise { const input = parseNativeExecutionInput(options.input); const descriptor = await options.backend.descriptor(); + const cleanupDomain = JSON.stringify([ + input.binding.companyId, + descriptor.kind, + descriptor.name, + ]); + await retryQuarantinedSessionCleanups(cleanupDomain, options.signal); if ("runtimeContext" in input) { const capabilities = descriptor.runtimeContextCapabilities; - const unsupported = (["instructions", "skills", "mcp"] as const).filter((key) => capabilities?.[key] !== "native"); - if (unsupported.length) throw new Error(`native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`); + const unsupported = (["instructions", "skills", "mcp"] as const).filter( + (key) => capabilities?.[key] !== "native", + ); + if (unsupported.length) + throw new Error( + `native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`, + ); } let persistedSession = options.existingSession ? null - : options.persistedSession ?? await options.controlPlane.loadSessionCheckpoint?.() ?? null; + : (options.persistedSession ?? + (await options.controlPlane.loadSessionCheckpoint?.()) ?? + null); if ( - persistedSession - && ( - persistedSession.identity.runId !== input.binding.runId - || persistedSession.identity.companyId !== input.binding.companyId - || persistedSession.identity.issueId !== input.binding.issueId - || persistedSession.identity.agentId !== input.binding.agentId - || input.session.normalizedSessionId === null - || persistedSession.identity.sessionId !== input.session.normalizedSessionId - ) - ) throw new Error("native_session_checkpoint_binding_mismatch"); + persistedSession && + (persistedSession.identity.runId !== input.binding.runId || + persistedSession.identity.companyId !== input.binding.companyId || + persistedSession.identity.issueId !== input.binding.issueId || + persistedSession.identity.agentId !== input.binding.agentId || + input.session.normalizedSessionId === null || + persistedSession.identity.sessionId !== input.session.normalizedSessionId) + ) + throw new Error("native_session_checkpoint_binding_mismatch"); const existingIdentity = options.existingSession?.identity() ?? null; if ( - existingIdentity - && ( - existingIdentity.companyId !== input.binding.companyId - || existingIdentity.issueId !== input.binding.issueId - || existingIdentity.agentId !== input.binding.agentId - || input.session.normalizedSessionId === null - || existingIdentity.sessionId !== input.session.normalizedSessionId - ) - ) throw new Error("native_session_attach_binding_mismatch"); - const normalizedSessionId = persistedSession?.identity.sessionId - ?? existingIdentity?.sessionId - ?? input.session.normalizedSessionId - ?? randomUUID(); + existingIdentity && + (existingIdentity.companyId !== input.binding.companyId || + existingIdentity.issueId !== input.binding.issueId || + existingIdentity.agentId !== input.binding.agentId || + input.session.normalizedSessionId === null || + existingIdentity.sessionId !== input.session.normalizedSessionId) + ) + throw new Error("native_session_attach_binding_mismatch"); + const normalizedSessionId = + persistedSession?.identity.sessionId ?? + existingIdentity?.sessionId ?? + input.session.normalizedSessionId ?? + randomUUID(); const identity = { runId: input.binding.runId, sessionId: normalizedSessionId, @@ -811,6 +1255,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) options.existingSession, options.onSession, "native session attachment failed", + cleanupDomain, ); throw error; } @@ -826,13 +1271,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) persistedSession = await runAbortableOperationWithin({ timeoutMs: recoveryTimeoutMs, timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`, - operation: (signal) => reconcileRecoveryCursor({ - controlPlane: options.controlPlane, - checkpoint: recoveryCheckpoint, - runId: input.binding.runId, - sourceInstanceId: options.runnerInstanceId, - signal, - }), + operation: (signal) => + reconcileRecoveryCursor({ + controlPlane: options.controlPlane, + checkpoint: recoveryCheckpoint, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + signal, + }), }); reconciledRecoveryCheckpoint = persistedSession; const providerRecoveryCheckpoint = persistedSession; @@ -844,28 +1290,35 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) ? await runAbortableOperationWithin({ timeoutMs: recoveryTimeoutMs, timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`, - operation: (signal) => options.backend.recoverSession!( - providerRecoveryCheckpoint, - { signal }, - ), + operation: (signal) => + options.backend.recoverSession!(providerRecoveryCheckpoint, { + signal, + }), onLateResolution: async (lateRecovery) => { if (lateRecovery.session) { await disposeUnadmittedSession( lateRecovery.session, "native session provider recovery timed out", + cleanupDomain, ); } }, }) - : { recovered: false as const, reason: "driver does not support recovery" }; + : { + recovered: false as const, + reason: "driver does not support recovery", + }; if (!recovery.recovered || !recovery.session) { if (!replacementAllowed) { - throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`); + throw new Error( + `native_session_recovery_failed: ${recovery.reason ?? "unknown"}`, + ); } continuityBreak = { reason: recovery.reason ?? "provider session is no longer recoverable", previousDriverSessionId: providerRecoveryCheckpoint.sessionId, - previousProviderSessionId: providerRecoveryCheckpoint.providerSessionId ?? null, + previousProviderSessionId: + providerRecoveryCheckpoint.providerSessionId ?? null, }; const replacementInput = { identity, @@ -883,10 +1336,12 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) ) : options.backend.openSession(abortableReplacementInput); }, - onLateResolution: (lateSession) => disposeUnadmittedSession( - lateSession, - "native session replacement bootstrap timed out", - ), + onLateResolution: (lateSession) => + disposeUnadmittedSession( + lateSession, + "native session replacement bootstrap timed out", + cleanupDomain, + ), }); } else { session = recovery.session; @@ -905,14 +1360,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) session = await runAbortableOperationWithin({ timeoutMs: bootstrapTimeoutMs, timeoutMessage: `native session bootstrap timed out after ${bootstrapTimeoutMs}ms`, - operation: (signal) => options.backend.openSession({ - ...bootstrapInput, - signal, - }), - onLateResolution: (lateSession) => disposeUnadmittedSession( - lateSession, - "native session bootstrap timed out", - ), + operation: (signal) => + options.backend.openSession({ + ...bootstrapInput, + signal, + }), + onLateResolution: (lateSession) => + disposeUnadmittedSession( + lateSession, + "native session bootstrap timed out", + cleanupDomain, + ), }); } try { @@ -930,6 +1388,7 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) session, options.onSession, "native control-plane run admission failed", + cleanupDomain, ); } throw error; @@ -945,14 +1404,66 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) // Owner notification cannot prevent provider cleanup. } }; + let failedCleanupDeferred = false; + let sessionCloseRecoveryPromise: Promise | null = null; + const retainFailedCleanup = (cleanup: Promise) => { + failedCleanupDeferred = true; + quarantineSession(); + retainFailedSessionCleanupOwner(cleanup, cleanupDomain); + }; + const startSessionClose = (reason: string) => { + const attempt = session.close({ reason }); + sessionClosePromise = attempt; + return attempt; + }; const closeSession = () => { if (sessionClosePromise === null) { quarantineSession(); - sessionClosePromise = session.close({ - reason: "native session execution complete", - }); + const firstAttempt = startSessionClose( + "native session execution complete", + ); + // Preserve the first close outcome for its caller. If an exact attempt + // fails, retain one ordered, delay-bounded production recovery within a + // finite retry budget. A still-pending attempt is never overlapped or + // replaced, and repeated terminal failure cannot create an immortal loop. + const recovery = (async () => { + let attempt = firstAttempt; + let retryCount = 0; + while (true) { + try { + await attempt; + return; + } catch (error) { + if (retryCount >= MAX_FAILED_SESSION_CLOSE_RETRIES) { + quarantineSessionCleanup(session, cleanupDomain); + throw error; + } + retryCount += 1; + await waitForSessionCloseRetry(); + if (sessionClosePromise === attempt) { + sessionClosePromise = null; + } + attempt = startSessionClose( + `native session cleanup recovery after close failure (${retryCount})`, + ); + } + } + })(); + sessionCloseRecoveryPromise = recovery; + retainFailedCleanup(recovery); + void recovery + .finally(() => { + if (sessionCloseRecoveryPromise === recovery) { + sessionCloseRecoveryPromise = null; + } + }) + .catch(() => undefined); } - return sessionClosePromise; + const activeClose = sessionClosePromise; + if (activeClose === null) { + throw new Error("native session cleanup lost its active close attempt"); + } + return activeClose; }; let executionSucceeded = false; try { @@ -960,8 +1471,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) // callback fails, the finally block below still quarantines and closes the // provider session. options.onSession?.(session); - const checkpointTimeoutMs = options.checkpointTimeoutMs - ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS; + const checkpointTimeoutMs = + options.checkpointTimeoutMs ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS; const persistCheckpoint = ( snapshot: PersistedNativeSession, externalSignal?: AbortSignal, @@ -987,14 +1498,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) }; const recoveredSnapshot = await session.snapshot(); const recoveredActiveTurnId = recovered - ? recoveredSnapshot.activeTurnId ?? null - : persistedSession?.activeTurnId ?? null; + ? (recoveredSnapshot.activeTurnId ?? null) + : (persistedSession?.activeTurnId ?? null); const adoptedDispositionTerminal = Boolean( - recovered - && recoveredSnapshot.dispositionOnlyRecoveryConsumed - && !recoveredActiveTurnId - && (recoveredSnapshot.terminalTurns?.length ?? 0) - > (persistedSession?.terminalTurns?.length ?? 0) + recovered && + recoveredSnapshot.dispositionOnlyRecoveryConsumed && + !recoveredActiveTurnId && + (recoveredSnapshot.terminalTurns?.length ?? 0) > + (persistedSession?.terminalTurns?.length ?? 0), ); if (continuityBreak) { await options.onContinuityBreak?.({ @@ -1020,16 +1531,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) highestContiguousSourceSeq: 0, governedResult: null as PrpStructuredRunResult | null, }; - const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal - ? recoveredSnapshot - : persistedSession; - let completed = completionSnapshot?.semanticResult && completionSnapshot.terminal - ? { - result: completionSnapshot.semanticResult, - terminal: completionSnapshot.terminal, - turnId: completionSnapshot.activeTurnId ?? null, - } - : null; + const completionSnapshot = + recoveredSnapshot.semanticResult && recoveredSnapshot.terminal + ? recoveredSnapshot + : persistedSession; + let completed = + completionSnapshot?.semanticResult && completionSnapshot.terminal + ? { + result: completionSnapshot.semanticResult, + terminal: completionSnapshot.terminal, + turnId: completionSnapshot.activeTurnId ?? null, + } + : null; if (!completed) { // A recovered driver is authoritative about whether a provider turn is // still active. In particular, drivers normalize the checkpoint race @@ -1038,45 +1551,48 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) // resurrects that terminal turn and waits forever for an event that was // already consumed. const dispositionRecoveryWasSubmitted = Boolean( - recovered - && persistedSession?.dispositionOnlyRecoveryConsumed - && !recoveredSnapshot.semanticResult - && !recoveredActiveTurnId + recovered && + persistedSession?.dispositionOnlyRecoveryConsumed && + !recoveredSnapshot.semanticResult && + !recoveredActiveTurnId, ); const dispositionRecoveryTurnId = - persistedSession?.dispositionOnlyRecoveryTurnId - ?? recoveredSnapshot.dispositionOnlyRecoveryTurnId - ?? null; + persistedSession?.dispositionOnlyRecoveryTurnId ?? + recoveredSnapshot.dispositionOnlyRecoveryTurnId ?? + null; const recoveredDispositionTurnObserved = Boolean( - dispositionRecoveryTurnId - && recoveredSnapshot.terminalTurns?.some( + dispositionRecoveryTurnId && + recoveredSnapshot.terminalTurns?.some( (terminal) => terminal.turnId === dispositionRecoveryTurnId, - ) + ), ); const dispositionRecoveryStillOwned = Boolean( - dispositionRecoveryWasSubmitted - && recoveredSnapshot.dispositionOnlyRecoveryConsumed - && dispositionRecoveryTurnId !== null - && recoveredDispositionTurnObserved + dispositionRecoveryWasSubmitted && + recoveredSnapshot.dispositionOnlyRecoveryConsumed && + dispositionRecoveryTurnId !== null && + recoveredDispositionTurnObserved, ); const replayedDisposition = dispositionRecoveryWasSubmitted && dispositionRecoveryTurnId !== null - ? await replayCheckpointedTurnTerminal({ - controlPlane: options.controlPlane, - runId: input.binding.runId, - sourceInstanceId: options.runnerInstanceId, - priorTerminalTurnIds: (persistedSession?.terminalTurns ?? []) - .map((terminal) => terminal.turnId), - expectedTurnId: dispositionRecoveryTurnId, - }) - : null; + ? await replayCheckpointedTurnTerminal({ + controlPlane: options.controlPlane, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + priorTerminalTurnIds: (persistedSession?.terminalTurns ?? []).map( + (terminal) => terminal.turnId, + ), + expectedTurnId: dispositionRecoveryTurnId, + }) + : null; // Durable replay remains authoritative. If it has no terminal, an exact // consumed marker bound to the same terminal fingerprint in both // checkpoints proves provider completion without reconstructing provider // output. Give only that non-provider fact to control-plane policy; a // mismatch fails closed and a null policy result fails finalization. const dispositionFallback = - dispositionRecoveryWasSubmitted && replayedDisposition === null && persistedSession + dispositionRecoveryWasSubmitted && + replayedDisposition === null && + persistedSession ? checkpointedResultlessDispositionFallback({ persisted: persistedSession, recovered: recoveredSnapshot, @@ -1085,26 +1601,29 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) : null; const checkpointedDispositionTerminal = replayedDisposition !== null || dispositionFallback !== null; - const recoveryTerminal = replayedDisposition?.terminal ?? dispositionFallback; + const recoveryTerminal = + replayedDisposition?.terminal ?? dispositionFallback; const consumptionAbort = new AbortController(); - const consuming = recoveryTerminal === null - ? consumeTurn( - session, - options.controlPlane, - options.timeoutMs ?? 900_000, - options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, - closeSession, - quarantineSession, - options.resolveGovernedWait, - consumptionAbort.signal, - ) - : Promise.resolve({ - event: recoveryTerminal, - eventCount: 0, - highestContiguousSourceSeq: - replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq, - governedResult: null, - }); + const consuming = + recoveryTerminal === null + ? consumeTurn( + session, + options.controlPlane, + options.timeoutMs ?? 900_000, + options.runtimeInputLiveWindowMs ?? + DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, + closeSession, + quarantineSession, + options.resolveGovernedWait, + consumptionAbort.signal, + ) + : Promise.resolve({ + event: recoveryTerminal, + eventCount: 0, + highestContiguousSourceSeq: + replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq, + governedResult: null, + }); // Event consumption must begin before startTurn so an eager provider cannot // outrun us. Observe its rejection immediately, though: if startTurn or // checkpointing fails first, the outer finally closes the session and the @@ -1113,20 +1632,18 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) void consuming.catch(() => undefined); try { if ( - !recovered - || ( - !recoveredActiveTurnId - && !adoptedDispositionTerminal - && !checkpointedDispositionTerminal - && !dispositionRecoveryStillOwned - ) + !recovered || + (!recoveredActiveTurnId && + !adoptedDispositionTerminal && + !checkpointedDispositionTerminal && + !dispositionRecoveryStillOwned) ) { const modelEnvelope = buildNativeModelEnvelope(input); const dispositionOnlyRecovery = Boolean( recovered && !recoveredSnapshot.semanticResult && (persistedSession?.terminalTurns?.length ?? 0) > 0 && - !recoveredActiveTurnId + !recoveredActiveTurnId, ); if (dispositionOnlyRecovery) { modelEnvelope.task.prompt = [ @@ -1138,7 +1655,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) } await session.startTurn({ message: { role: "user", text: JSON.stringify(modelEnvelope) }, - requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default", + requestedCollaborationMode: + "executionMode" in input ? input.executionMode : "default", }); await checkpoint(); } @@ -1162,21 +1680,24 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) if (settledCompletion === null) { const terminalEvent = consumed.event; if (terminalEvent === null) { - throw new Error("native_finalization_missing: session returned no terminal event"); + throw new Error( + "native_finalization_missing: session returned no terminal event", + ); } - settledCompletion = consumed.governedResult === null - ? await session.result() - : { - result: consumed.governedResult, - terminal: { - schema: "paperclip.prp.terminal.v1", - turnTerminalState: "completed", - runTerminalState: "succeeded", - reportedWorkDisposition: - consumed.governedResult.reportedWorkDisposition, - }, - turnId: terminalEvent.turnId ?? null, - }; + settledCompletion = + consumed.governedResult === null + ? await session.result() + : { + result: consumed.governedResult, + terminal: { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: + consumed.governedResult.reportedWorkDisposition, + }, + turnId: terminalEvent.turnId ?? null, + }; signal.throwIfAborted(); if (settledCompletion === null && options.resolveMissingResult) { const recoveredResult = await options.resolveMissingResult({ @@ -1203,12 +1724,17 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) completed = settledCompletion; } if (settledCompletion === null) { - throw new Error("native_finalization_missing: session returned no semantic result"); + throw new Error( + "native_finalization_missing: session returned no semantic result", + ); } let terminal: PrpTerminalState; if (consumed.governedResult !== null) { terminal = settledCompletion.terminal; - } else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) { + } else if ( + completionSnapshot?.semanticResult && + completionSnapshot.terminal + ) { terminal = completionSnapshot.terminal; } else { terminal = terminalFromEvent( @@ -1216,10 +1742,11 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) settledCompletion.result.reportedWorkDisposition, ); } - const eventTurnId = settledCompletion.turnId - ?? persistedSession?.activeTurnId - ?? persistedSession?.terminalTurns?.at(-1)?.turnId - ?? consumed.event?.turnId; + const eventTurnId = + settledCompletion.turnId ?? + persistedSession?.activeTurnId ?? + persistedSession?.terminalTurns?.at(-1)?.turnId ?? + consumed.event?.turnId; const controlEvent = ( sourceSeq: number, eventType: PrpEvent["eventType"], @@ -1261,22 +1788,30 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({ timeoutMs: finalizationTimeoutMs, operation: async (signal) => { - const controlReplay = await options.controlPlane.replayEvents({ - runId: input.binding.runId, - sourceInstanceId: options.controlPlaneInstanceId, - afterSourceSeq: 0, - limit: 10, - }, { signal }); + const controlReplay = await options.controlPlane.replayEvents( + { + runId: input.binding.runId, + sourceInstanceId: options.controlPlaneInstanceId, + afterSourceSeq: 0, + limit: 10, + }, + { signal }, + ); signal.throwIfAborted(); - const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event])); + const replayBySequence = new Map( + controlReplay.events.map((event) => [event.sourceSeq, event]), + ); for (const existing of controlReplay.events) { - const expected = preparedFinalization.expectedControlEvents[existing.sourceSeq - 1]; + const expected = + preparedFinalization.expectedControlEvents[existing.sourceSeq - 1]; if ( - expected === undefined - || existing.eventType !== expected.eventType - || canonicalJson(existing.payload) !== canonicalJson(expected.payload) + expected === undefined || + existing.eventType !== expected.eventType || + canonicalJson(existing.payload) !== canonicalJson(expected.payload) ) { - throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`); + throw new Error( + `native_control_event_replay_conflict:${existing.sourceSeq}`, + ); } } if (!baselineControlReplayCaptured) { @@ -1287,8 +1822,8 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) } else { for (const existing of controlReplay.events) { if ( - !baselineControlEventSequences.has(existing.sourceSeq) - && !accountedControlEventSequences.has(existing.sourceSeq) + !baselineControlEventSequences.has(existing.sourceSeq) && + !accountedControlEventSequences.has(existing.sourceSeq) ) { accountedControlEventSequences.add(existing.sourceSeq); consumed.eventCount += 1; @@ -1301,11 +1836,13 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) ); for (const event of preparedFinalization.expectedControlEvents) { if (replayBySequence.has(event.sourceSeq)) continue; - const receipt = await options.controlPlane.appendEvent(event, { signal }); + const receipt = await options.controlPlane.appendEvent(event, { + signal, + }); signal.throwIfAborted(); if ( - receipt.disposition === "committed" - && !accountedControlEventSequences.has(event.sourceSeq) + receipt.disposition === "committed" && + !accountedControlEventSequences.has(event.sourceSeq) ) { accountedControlEventSequences.add(event.sourceSeq); consumed.eventCount += 1; @@ -1315,13 +1852,16 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) receipt.highestContiguousSourceSeq, ); } - await options.controlPlane.completeRun({ - result: preparedFinalization.completed.result, - terminal: preparedFinalization.terminal, - turnId: preparedFinalization.completed.turnId, - callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`, - callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`, - }, { signal }); + await options.controlPlane.completeRun( + { + result: preparedFinalization.completed.result, + terminal: preparedFinalization.terminal, + turnId: preparedFinalization.completed.turnId, + callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`, + callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`, + }, + { signal }, + ); return { result: preparedFinalization.completed.result, terminal: preparedFinalization.terminal, @@ -1351,13 +1891,14 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) }; await persistCheckpoint(completedSnapshot, signal); signal.throwIfAborted(); - const usage = await session.usage?.() ?? null; + const usage = (await session.usage?.()) ?? null; signal.throwIfAborted(); return { providerSessionId: snapshot.providerSessionId ?? null, - driverVersion: typeof usage?.driverVersion === "string" - ? usage.driverVersion - : descriptor.version, + driverVersion: + typeof usage?.driverVersion === "string" + ? usage.driverVersion + : descriptor.version, usage, }; }, @@ -1376,7 +1917,10 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) executionSucceeded = true; return { ...durableExecutionResult, ...enrichment }; } finally { - if (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) { + if ( + (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) && + !failedCleanupDeferred + ) { // A provider that ignores close must not keep execution pending forever. // closeSession removes it from the caller before invoking the backend; // retain observation of the promise, but bound the final join. Provider