diff --git a/packages/paperclip-runner/src/contracts/durable-recovery.ts b/packages/paperclip-runner/src/contracts/durable-recovery.ts index 5cd271b687..b04d41c1e4 100644 --- a/packages/paperclip-runner/src/contracts/durable-recovery.ts +++ b/packages/paperclip-runner/src/contracts/durable-recovery.ts @@ -37,7 +37,23 @@ export interface DurableRecoveryProcessedCommand { commandId: string; controllerSeq: number; commandDigest: string; - status: "completed" | "failed" | "rejected"; + /** + * The runner's command journal is persisted before a command's effect and + * re-persisted after recovery, so a recovered trace carries the whole + * lifecycle, not just the settled end of it: + * + * - `pending` — journaled, effect not yet confirmed. Written by + * `DurableState::begin_command` and durable from that moment. + * - `indeterminate` — the crash-recovery verdict. + * `DurableState::reconcile_pending_commands` promotes every `pending` + * entry on load and saves the state back, so the command is never + * executed twice. Terminal. + * + * Both are values a consumer can read off `processedCommands`; a union that + * omits them tells the compiler a state the runner routinely writes is + * impossible. + */ + status: "pending" | "completed" | "failed" | "rejected" | "indeterminate"; logicalEffectCount: number; result: Record; } @@ -79,7 +95,13 @@ export interface DurableRecoveryCoreCommand { type: string; issuedAt: string; payload: Record; - status: "pending" | "completed" | "failed" | "rejected"; + /** + * `indeterminate` is the runner's crash-recovery verdict: the command was + * journaled but its effect was never confirmed, so the runner will not + * execute it a second time. It is terminal, like the other non-pending + * statuses. + */ + status: "pending" | "completed" | "failed" | "rejected" | "indeterminate"; result: Record | null; } diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index af7aae9b87..90995500e1 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -672,4 +672,98 @@ describe.sequential("DurablePrpControlPlane", () => { rmSync(root, { recursive: true, force: true }); } }); + + it("keeps a recovered runner attached when it reports an indeterminate command", async () => { + const root = mkdtempSync(resolve(tmpdir(), "paperclip-prp-indeterminate-")); + const controlPlane = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + }); + try { + await controlPlane.start(); + const journaled = controlPlane.queueCommand( + "semantic_tool.result", + { callId: "call-1" }, + "command-tool-1", + ); + controlPlane.queueCommand( + "turn.interrupt", + { turnId: "turn-1" }, + "command-interrupt-1", + ); + const client = await authenticate( + controlPlane, + controlPlane.issueBootstrapTicket(), + ); + expect(client?.welcome.payload).toMatchObject({ + pendingCommands: [ + expect.objectContaining({ commandId: "command-tool-1" }), + ], + }); + + // Exactly what runnerd replays after it is killed between journaling a + // command and confirming its effect. Its durable contract promotes such a + // command to `indeterminate` so that it is never executed twice. + const indeterminateResult = { + protocol: "paperclip.runner", + version: 1, + kind: "command_result", + payload: { + commandId: journaled.commandId, + commandType: journaled.type, + controllerSeq: journaled.controllerSeq, + status: "indeterminate", + result: { + code: "execution_indeterminate", + message: + "runner recovered after journaling this command; it will not execute twice", + }, + }, + }; + sendSecure(client!, indeterminateResult); + + // The authority has to accept that terminal status and hand out the next + // command. Closing the connection instead strands the runner in a silent + // reconnect loop that never re-reports its provider identity. + await expect(receiveSecure(client!)).resolves.toMatchObject({ + kind: "command", + payload: { commandId: "command-interrupt-1" }, + }); + expect(controlPlane.store.state.commands).toMatchObject([ + { commandId: "command-tool-1", status: "indeterminate" }, + { commandId: "command-interrupt-1", status: "pending" }, + ]); + + // The runner replays its journal on every reconnect, so the same + // indeterminate result arrives again. It has to be absorbed as a + // duplicate rather than treated as a conflicting result. + sendSecure(client!, indeterminateResult); + await expect(receiveSecure(client!)).resolves.toMatchObject({ + kind: "command", + payload: { commandId: "command-interrupt-1" }, + }); + expect(controlPlane.store.state.duplicateCommandResults).toBe(1); + + client?.socket.destroy(); + await controlPlane.stop(); + + // That result is now persisted, so a control plane restarted over the + // same directory has to be able to read its own state back. + const restarted = new DurablePrpControlPlane({ + stateDirectory: root, + identity, + expectedRunnerVersion, + expectedRunnerDigest, + }); + expect(restarted.store.state.commands).toMatchObject([ + { commandId: "command-tool-1", status: "indeterminate" }, + { commandId: "command-interrupt-1", status: "pending" }, + ]); + } finally { + await controlPlane.stop(); + rmSync(root, { recursive: true, force: true }); + } + }); }); diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index e7cb078e5d..408ba27940 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -380,9 +380,13 @@ function isStoredCoreState( commandTypes.has(command.type) && typeof command.issuedAt === "string" && isRecord(command.payload) && - ["pending", "completed", "failed", "rejected"].includes( - String(command.status), - ) && + [ + "pending", + "completed", + "failed", + "rejected", + "indeterminate", + ].includes(String(command.status)) && (command.result === null || isRecord(command.result)), ) ) { @@ -1641,10 +1645,16 @@ export class DurablePrpControlPlane { return; } const status = result.status; + // `indeterminate` is terminal too: a runner that crashed between journaling + // a command and confirming its effect reports it on recovery and will not + // execute it again. Rejecting it closes the connection, and since the + // runner replays the same result on every reconnect, the session never + // recovers. if ( status !== "completed" && status !== "failed" && - status !== "rejected" + status !== "rejected" && + status !== "indeterminate" ) { connection.close(); return; diff --git a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts index 6a9920573a..dda8167292 100644 --- a/packages/paperclip-runner/src/control-plane/prp-transport-types.ts +++ b/packages/paperclip-runner/src/control-plane/prp-transport-types.ts @@ -14,7 +14,13 @@ export interface DurableRecoveryCoreCommand { type: string; issuedAt: string; payload: Record; - status: "pending" | "completed" | "failed" | "rejected"; + /** + * `indeterminate` is the runner's crash-recovery verdict: the command was + * journaled but its effect was never confirmed, so the runner will not + * execute it a second time. It is terminal, like the other non-pending + * statuses. + */ + status: "pending" | "completed" | "failed" | "rejected" | "indeterminate"; result: Record | null; }