diff --git a/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json new file mode 100644 index 0000000000..670fc5b980 --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json @@ -0,0 +1,29 @@ +{ + "schema": "paperclip-runner/harness-driver-conformance-fixture/v1", + "validConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "semantic-tool-success" + }, + "invalidConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "unknown" + }, + "completionMessage": "Run deterministic semantic-tool conformance.", + "interruptMessage": "[conformance:interrupt] Keep the turn active until interrupted.", + "requiredCapabilities": [ + "resume", + "typedEvents", + "interruption", + "structuredResult", + "read", + "reconciliation", + "usage", + "dynamicTools" + ], + "unsupportedFeatures": [ + "steering", + "runtimeRequestResolution", + "goals", + "threadLineage" + ] +} diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index 8575322f47..9400ab615a 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -121,6 +121,12 @@ "expectation": "accept", "compatibilityCase": "cross-language-input" }, + { + "path": "fixtures/evals/harness-driver-conformance.json", + "sha256": "ce84eb64faa6af78e17a156cde352a951db7c108c8819a20109b732b23a5bec5", + "expectation": "accept", + "compatibilityCase": "canonical" + }, { "path": "fixtures/local-runner/scripts/duplicate-terminal.json", "sha256": "56928a21318ea4b390c65fb34f0d3114760619d79301f4d535391a8566b693ab", diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts new file mode 100644 index 0000000000..689800e54e --- /dev/null +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -0,0 +1,451 @@ +import { describe, expect, it } from "vitest"; + +import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import type { PrpEvent, PrpStructuredRunResult } from "../protocol/replay-contract.js"; +import { HarnessDriverBackend } from "./harness-driver-backend.js"; + +const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Backend adapter completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "fake", status: "passed" }], + attentionRequests: [], + artifacts: [], +}; + +const providerIdentity = { + kind: "acpx" as const, + normalizedSessionId: "session-1", + acpxRecordId: "driver-1", + backendSessionId: "backend-1", + agentSessionId: "provider-1", + profileDigest: "sha256:profile", + workspaceDigest: "sha256:workspace", + requestedModel: "claude-sonnet-4-20250514", + effectiveModel: "claude-sonnet-4-20250514", +}; + +function prpEvent(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `fake:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "fake", + sourceKind: "runner", + runId: "run-1", + normalizedSessionId: "session-1", + turnId: "turn-1", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: `2026-08-09T00:00:0${sourceSeq}.000Z`, + payload, + }; +} + +const runtimeResolutions: unknown[] = []; + +class FakeHarnessSession implements HarnessSession { + ids() { return { driverSessionId: "driver-1", providerSessionId: "provider-1" }; } + async *events() { + yield prpEvent(1, "run.result.proposed", result); + yield prpEvent(2, "turn.completed", { status: "completed" }); + } + async startTurn() { return { turnId: "turn-1" }; } + async resolveRuntimeRequest(input: unknown) { + runtimeResolutions.push(structuredClone(input)); + } + async snapshot(): Promise { + return { + driverKind: "fake", + driverSessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + semanticResult: { result, fingerprint: "fingerprint", turnId: "turn-1" }, + lastSourceSequence: 2, + }; + } + async close() {} +} + +const driver: HarnessDriver = { + async descriptor() { + return { + kind: "fake", + displayName: "Fake harness", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + async openSession() { return new FakeHarnessSession(); }, +}; + +describe("HarnessDriverBackend", () => { + it("rejects and closes a provider session without a durable provider identity", async () => { + let closed = false; + class MissingProviderIdentitySession extends FakeHarnessSession { + override ids() { + return { + driverSessionId: "driver-missing-provider", + providerSessionId: null, + }; + } + + override async close() { + closed = true; + } + } + const incompleteDriver: HarnessDriver = { + ...driver, + async openSession() { + return new MissingProviderIdentitySession(); + }, + }; + const backend = new HarnessDriverBackend(incompleteDriver); + + await expect(backend.openSession({ + identity: { + runId: "run-incomplete", + sessionId: "session-incomplete", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + })).rejects.toThrow( + "provider_initialize_protocol_error: provider=fake stage=session.open missing durable provider session identity", + ); + expect(closed).toBe(true); + }); + + it("rejects and closes a recovered session without a durable provider identity", async () => { + let closed = false; + class MissingRecoveredIdentitySession extends FakeHarnessSession { + override ids() { + return { + driverSessionId: "driver-missing-recovered-provider", + providerSessionId: null, + }; + } + + override async close() { + closed = true; + } + } + const incompleteDriver: HarnessDriver = { + ...driver, + async recoverSession() { + return { + recovered: true, + session: new MissingRecoveredIdentitySession(), + }; + }, + }; + const backend = new HarnessDriverBackend(incompleteDriver); + + await expect(backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + identity: { + runId: "run-recover-incomplete", + sessionId: "session-recover-incomplete", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + cursor: "0", + }, { + signal: new AbortController().signal, + })).rejects.toThrow( + "provider_initialize_protocol_error: provider=fake stage=session.recover missing durable provider session identity", + ); + expect(closed).toBe(true); + }); + + it("normalizes harness events, result, terminal, and snapshot", async () => { + const backend = new HarnessDriverBackend(driver); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + expect(events).toHaveLength(2); + await expect(session.result()).resolves.toMatchObject({ result, turnId: "turn-1", terminal: { runTerminalState: "succeeded" } }); + await expect(session.snapshot()).resolves.toMatchObject({ + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + }); + }); + + it("passes the persisted harness driver kind through recovery", async () => { + let recoveredDriverKind: string | null = null; + let recoveredProviderIdentity: PersistedHarnessSession["providerIdentity"]; + const recoveryDriver: HarnessDriver = { + ...driver, + async recoverSession(snapshot) { + recoveredDriverKind = snapshot.driverKind; + recoveredProviderIdentity = snapshot.providerIdentity; + return { recovered: true, session: new FakeHarnessSession() }; + }, + }; + const backend = new HarnessDriverBackend(recoveryDriver); + const recovery = await backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { + signal: new AbortController().signal, + }); + expect(recovery.recovered).toBe(true); + expect(recoveredDriverKind).toBe("fake"); + expect(recoveredProviderIdentity).toEqual(providerIdentity); + }); + + it("forwards the native bootstrap signal to the harness driver", async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const backend = new HarnessDriverBackend({ + ...driver, + async openSession(input) { + receivedSignal = input.signal; + return new FakeHarnessSession(); + }, + }); + + await backend.openSession({ + identity: { runId: "run-signal", sessionId: "session-signal", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + signal: controller.signal, + }); + + expect(receivedSignal).toBe(controller.signal); + }); + + it("forwards the native recovery signal to the harness driver", async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const backend = new HarnessDriverBackend({ + ...driver, + async recoverSession(_snapshot, options) { + receivedSignal = options.signal; + return { recovered: true, session: new FakeHarnessSession() }; + }, + }); + + const recovery = await backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + identity: { runId: "run-recover-signal", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { + signal: controller.signal, + }); + + expect(recovery.recovered).toBe(true); + expect(receivedSignal).toBe(controller.signal); + }); + + it("allows only the run id to change when a harness session is attached", async () => { + const attachedRunIds: string[] = []; + class AttachableHarnessSession extends FakeHarnessSession { + async attachRun(input: { runId: string }) { + attachedRunIds.push(input.runId); + } + } + const backend = new HarnessDriverBackend({ + ...driver, + async openSession() { return new AttachableHarnessSession(); }, + }); + const originalIdentity = { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }; + const session = await backend.openSession({ + identity: originalIdentity, + workingDirectory: "/workspace", + }); + + for (const identity of [ + { ...originalIdentity, runId: "run-forged-company", companyId: "company-2" }, + { ...originalIdentity, runId: "run-forged-issue", issueId: "issue-2" }, + { ...originalIdentity, runId: "run-forged-agent", agentId: "agent-2" }, + { ...originalIdentity, runId: "run-forged-session", sessionId: "session-2" }, + ]) { + await expect(session.attachRun?.({ identity })).rejects.toThrow( + "native_session_attach_binding_mismatch", + ); + expect(session.identity()).toEqual(originalIdentity); + } + expect(attachedRunIds).toEqual([]); + + await expect(session.attachRun?.({ + identity: { ...originalIdentity, runId: "run-2" }, + })).resolves.toBeUndefined(); + expect(attachedRunIds).toEqual(["run-2"]); + expect(session.identity()).toEqual({ ...originalIdentity, runId: "run-2" }); + }); + + it("delegates native runtime-request resolutions to the harness session", async () => { + runtimeResolutions.length = 0; + const backend = new HarnessDriverBackend(driver); + const session = await backend.openSession({ + identity: { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + }); + + await session.resolveRuntimeRequest?.({ + requestId: "permission-1", + turnId: "turn-1", + resolution: { action: "accept_for_session" }, + }); + + expect(runtimeResolutions).toEqual([ + { + requestId: "permission-1", + turnId: "turn-1", + resolution: { action: "accept_for_session" }, + }, + ]); + }); + + it("emits one non-replayable input expiration and terminal wait after provider loss", async () => { + const questionSet = { + schema: "paperclip.question_set.v1" as const, + questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" as const }], + }; + class LostProviderSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: questionSet, + turnId: "turn-1", + itemId: "input-1", + } }); + throw new Error("provider transport lost"); + } + override async snapshot(): Promise { + return { driverKind: "fake", driverSessionId: "driver-1", lastSourceSequence: 1 }; + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new LostProviderSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ value: { eventType: "runtime_request.created" } }); + await expect(iterator.next()).resolves.toMatchObject({ value: { + eventType: "runtime_request.expired", + sourceSeq: 2, + payload: { + requestId: "input-1", + reason: "provider_process_lost", + replayAllowed: false, + request: { input: questionSet }, + }, + } }); + await expect(iterator.next()).resolves.toMatchObject({ value: { + eventType: "turn.interrupted", + sourceSeq: 3, + payload: { reason: "provider_process_lost" }, + } }); + await expect(iterator.next()).resolves.toMatchObject({ done: true }); + }); + + it("does not synthesize a fallback after the input was already resolved", async () => { + class ResolvedThenLostSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: { schema: "paperclip.question_set.v1", questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" }] }, + } }); + yield prpEvent(2, "runtime_request.resolved", { requestId: "input-1", action: "submit" }); + throw new Error("provider transport lost after resolution"); + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new ResolvedThenLostSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await iterator.next(); + await iterator.next(); + await expect(iterator.next()).rejects.toThrow("provider transport lost after resolution"); + }); + + it("does not synthesize a fallback after explicit run cancellation", async () => { + class CancelledProviderSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: { schema: "paperclip.question_set.v1", questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" }] }, + } }); + throw new Error("provider stopped after cancellation"); + } + async interrupt() {} + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new CancelledProviderSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ value: { eventType: "runtime_request.created" } }); + await session.cancel({ + reason: "operator cancelled the run", + signal: new AbortController().signal, + }).cleanup; + await expect(iterator.next()).rejects.toThrow("provider stopped after cancellation"); + }); +}); diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts new file mode 100644 index 0000000000..c66e69b3ae --- /dev/null +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -0,0 +1,455 @@ +import type { + HarnessDriver, + HarnessSession, + HarnessSessionRecoveryOptions, + PersistedHarnessSession, +} from "../contracts/harness-driver.js"; +import type { + NativeSession, + NativeSessionBackend, + NativeSessionBackendDescriptor, + NativeSessionRecoveryOptions, + OpenNativeSessionInput, + PersistedNativeSession, +} from "../contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpTerminalState, +} from "../protocol/replay-contract.js"; + +/** + * Package-owned adapter from the concrete harness driver contract to the + * normalized session boundary consumed by Paperclip. Provider mechanics stay + * behind HarnessDriver; the control plane sees only PRP events and results. + */ +export class HarnessDriverBackend implements NativeSessionBackend { + readonly #driver: HarnessDriver; + + constructor(driver: HarnessDriver) { + this.#driver = driver; + } + + async descriptor(): Promise { + const descriptor = await this.#driver.descriptor(); + return { + kind: "runner", + name: descriptor.kind, + version: descriptor.version, + capabilities: structuredClone(descriptor.capabilities), + runtimeContextCapabilities: descriptor.runtimeContextCapabilities === undefined ? undefined : structuredClone(descriptor.runtimeContextCapabilities), + }; + } + + async openSession(input: OpenNativeSessionInput): Promise { + const session = await this.#driver.openSession({ + runId: input.identity.runId, + normalizedSessionId: input.identity.sessionId, + workingDirectory: input.workingDirectory ?? process.cwd(), + ...(input.signal === undefined ? {} : { signal: input.signal }), + }); + try { + assertProviderSessionIdentity( + session, + (await this.#driver.descriptor()).kind, + "session.open", + ); + } catch (error) { + await session.close({ + reason: "provider session bootstrap returned an incomplete identity", + force: true, + }).catch(() => undefined); + throw error; + } + return new HarnessNativeSession(input, session); + } + + async recoverSession( + snapshot: PersistedNativeSession, + options: NativeSessionRecoveryOptions, + ) { + if (this.#driver.recoverSession === undefined) { + return { recovered: false, reason: "driver does not support recovery" }; + } + const persisted: PersistedHarnessSession = { + driverKind: snapshot.driverKind ?? snapshot.backendKind, + driverSessionId: snapshot.sessionId, + providerSessionId: snapshot.providerSessionId, + ...(snapshot.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(snapshot.providerIdentity) }), + ...(snapshot.providerRecoveryPolicy === undefined + ? {} + : { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }), + runId: snapshot.identity.runId, + normalizedSessionId: snapshot.identity.sessionId, + activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null, + lastSourceSequence: parseCursor(snapshot.cursor), + ...(snapshot.semanticResult === undefined || snapshot.semanticResult === null + ? {} + : { + semanticResult: { + result: snapshot.semanticResult, + fingerprint: canonicalJson(snapshot.semanticResult), + turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered", + }, + }), + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], + }; + const recoveryOptions: HarnessSessionRecoveryOptions = { + signal: options.signal, + }; + const recovered = await this.#driver.recoverSession( + persisted, + recoveryOptions, + ); + if (!recovered.recovered || recovered.session === undefined) { + return { recovered: false, reason: recovered.reason }; + } + try { + assertProviderSessionIdentity( + recovered.session, + (await this.#driver.descriptor()).kind, + "session.recover", + ); + } catch (error) { + await recovered.session.close({ + reason: "provider session recovery returned an incomplete identity", + force: true, + }).catch(() => undefined); + throw error; + } + return { + recovered: true, + session: new HarnessNativeSession( + { identity: snapshot.identity }, + recovered.session, + ), + }; + } +} + +function assertProviderSessionIdentity( + session: HarnessSession, + provider: string, + stage: "session.open" | "session.recover", +): void { + const ids = session.ids(); + if ( + typeof ids.driverSessionId !== "string" + || ids.driverSessionId.trim().length === 0 + || typeof ids.providerSessionId !== "string" + || ids.providerSessionId.trim().length === 0 + ) { + throw new Error( + `provider_initialize_protocol_error: provider=${provider} stage=${stage} missing durable provider session identity`, + ); + } +} + +class HarnessNativeSession implements NativeSession { + #input: OpenNativeSessionInput; + readonly #session: HarnessSession; + #terminal: PrpTerminalState | null = null; + #explicitlyCancelled = false; + + constructor(input: OpenNativeSessionInput, session: HarnessSession) { + this.#input = structuredClone(input); + this.#session = session; + } + + identity() { + return structuredClone(this.#input.identity); + } + + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: this.#session.steer !== undefined, + interruption: this.#session.interrupt !== undefined, + structuredResult: true, + read: this.#session.read !== undefined, + reconciliation: this.#session.reconcile !== undefined, + usage: this.#session.usage !== undefined, + runtimeRequestResolution: this.#session.resolveRuntimeRequest !== undefined, + runtimeRequestHandoff: this.#session.handoffRuntimeRequest !== undefined, + goals: this.#session.goal !== undefined, + threadLineage: this.#session.lineage !== undefined, + }; + } + + async attachRun(input: { identity: OpenNativeSessionInput["identity"] }): Promise { + const currentIdentity = this.#input.identity; + if ( + input.identity.sessionId !== currentIdentity.sessionId + || input.identity.companyId !== currentIdentity.companyId + || input.identity.issueId !== currentIdentity.issueId + || input.identity.agentId !== currentIdentity.agentId + ) { + throw new Error("native_session_attach_binding_mismatch"); + } + if (this.#session.attachRun === undefined) { + throw new Error("native_session_multi_run_unavailable"); + } + await this.#session.attachRun({ runId: input.identity.runId }); + this.#input = { ...this.#input, identity: structuredClone(input.identity) }; + this.#terminal = null; + this.#explicitlyCancelled = false; + } + + async *events(): AsyncIterable { + let sourceInstanceId: string | null = null; + let lastSourceSequence = 0; + let sawTerminal = false; + let synthesizedDurableWait = false; + let streamFailure: unknown = null; + const observedPendingInputs = new Map>(); + try { + for await (const event of this.#session.events()) { + sourceInstanceId = event.sourceInstanceId; + lastSourceSequence = Math.max(lastSourceSequence, event.sourceSeq); + if (event.eventType === "runtime_request.created") { + const request = plainRecord(event.payload.request); + if ( + request?.schema === "paperclip.runtime_request.v2" + && request.type === "input" + && typeof request.requestId === "string" + ) observedPendingInputs.set(request.requestId, structuredClone(request)); + } else if (["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(event.eventType)) { + const requestId = typeof event.payload.requestId === "string" ? event.payload.requestId : null; + if (requestId) observedPendingInputs.delete(requestId); + } + if (event.eventType === "run.terminal") { + sawTerminal = true; + this.#terminal = structuredClone(event.payload as PrpTerminalState); + } else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) { + sawTerminal = true; + const snapshot = await this.#session.snapshot(); + const disposition = snapshot.semanticResult?.result.reportedWorkDisposition ?? "yielded"; + this.#terminal = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: + event.eventType === "turn.completed" ? "completed" + : event.eventType === "turn.failed" ? "failed" + : event.eventType === "turn.interrupted" ? "interrupted" + : "cancelled", + runTerminalState: event.eventType === "turn.completed" ? "succeeded" : event.eventType === "turn.failed" ? "failed" : "cancelled", + reportedWorkDisposition: disposition, + }; + } + yield structuredClone(event); + } + } catch (error) { + streamFailure = error; + } + + // The provider can disappear while its native RPC is awaiting the user. + // Emit one canonical terminal fact and replace the stream failure with a + // governed wait; the control plane can materialize the continuation without + // ever trying to replay the dead provider request. + if (!sawTerminal && !this.#explicitlyCancelled && sourceInstanceId) { + const snapshot = await this.#session.snapshot().catch(() => null); + let governedWaitTurnId: string | undefined; + for (const request of observedPendingInputs.values()) { + const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + lastSourceSequence = sourceSeq; + const requestId = String(request.requestId); + const turnId = typeof request.turnId === "string" ? request.turnId : undefined; + governedWaitTurnId ??= turnId; + const itemId = typeof request.itemId === "string" ? request.itemId : requestId; + yield { + schema: "paperclip.prp.event.v1", + sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId, + sourceKind: "runner", + runId: this.#input.identity.runId, + normalizedSessionId: this.#input.identity.sessionId, + ...(turnId ? { turnId } : {}), + itemId, + eventType: "runtime_request.expired", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: { + requestId, + ...(turnId ? { turnId } : {}), + itemId, + requestKind: "runtime", + reason: "provider_process_lost", + replayAllowed: false, + requestType: "input", + ...(plainRecord(request.origin)?.adapter + ? { adapter: plainRecord(request.origin)?.adapter } + : {}), + request: structuredClone(request), + }, + }; + } + if (governedWaitTurnId) { + const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + lastSourceSequence = sourceSeq; + synthesizedDurableWait = true; + sawTerminal = true; + this.#terminal = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }; + yield { + schema: "paperclip.prp.event.v1", + sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId, + sourceKind: "runner", + runId: this.#input.identity.runId, + normalizedSessionId: this.#input.identity.sessionId, + turnId: governedWaitTurnId, + eventType: "turn.interrupted", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: { status: "interrupted", reason: "provider_process_lost" }, + }; + } + } + if (streamFailure && !synthesizedDurableWait) throw streamFailure; + } + + startTurn(input: Parameters[0]) { + return this.#session.startTurn(input); + } + + steer(input: { turnId: string; message: { role: "user"; text: string }; correlationId?: string }) { + if (this.#session.steer === undefined) throw new Error("steering is unavailable"); + return this.#session.steer(input); + } + + interrupt(input: { turnId?: string; reason?: string }) { + if (this.#session.interrupt === undefined) throw new Error("interruption is unavailable"); + return this.#session.interrupt(input); + } + + cancel(input: { reason: string; signal: AbortSignal }) { + if (input.signal.aborted) { + throw input.signal.reason ?? new Error("native session cancellation aborted"); + } + // This flag is the adapter's synchronous publication boundary. Provider + // interruption happens afterward as passive cleanup, so a slow or broken + // transport cannot synthesize or publish new accepted output for the turn. + this.#explicitlyCancelled = true; + const interrupt = this.#session.interrupt; + return { + cleanup: interrupt === undefined + ? Promise.resolve() + : Promise.resolve().then(() => interrupt.call(this.#session, { + reason: input.reason, + signal: input.signal, + })), + }; + } + + resolveRuntimeRequest(input: { + requestId: string; + turnId: string; + resolution: Parameters>[0]["resolution"]; + }) { + if (this.#session.resolveRuntimeRequest === undefined) { + throw new Error("native_runtime_request_resolution_unavailable"); + } + return this.#session.resolveRuntimeRequest(input); + } + + handoffRuntimeRequest(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + signal: AbortSignal; + }) { + if (this.#session.handoffRuntimeRequest === undefined) { + throw new Error("native_runtime_request_handoff_unavailable"); + } + return this.#session.handoffRuntimeRequest(input); + } + + async result() { + const snapshot = await this.#session.snapshot(); + if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + return null; + } + if (this.#terminal === null) { + return null; + } + return { + result: structuredClone(snapshot.semanticResult.result), + terminal: structuredClone(this.#terminal), + turnId: snapshot.semanticResult.turnId, + }; + } + + async snapshot(): Promise { + const snapshot = await this.#session.snapshot(); + return { + backendKind: "runner", + driverKind: snapshot.driverKind, + sessionId: snapshot.driverSessionId, + identity: structuredClone(this.#input.identity), + providerSessionId: snapshot.providerSessionId, + ...(snapshot.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(snapshot.providerIdentity) }), + ...(snapshot.providerRecoveryPolicy === undefined + ? {} + : { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }), + cursor: + snapshot.lastSourceSequence === undefined + ? null + : String(snapshot.lastSourceSequence), + semanticResult: snapshot.semanticResult?.result ?? null, + terminal: this.#terminal, + activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null, + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], + }; + } + + async usage(): Promise | null> { + return this.#session.usage?.() ?? null; + } + + close(input: { reason: string }) { + return this.#session.close(input); + } +} + +function plainRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : null; +} + +export function createHarnessDriverBackend(driver: HarnessDriver): NativeSessionBackend { + return new HarnessDriverBackend(driver); +} + +function parseCursor(cursor: string | null | undefined): number | undefined { + if (cursor === undefined || cursor === null || cursor === "") return undefined; + const parsed = Number(cursor); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/backends/runtime-context.test.ts b/packages/paperclip-runner/src/backends/runtime-context.test.ts new file mode 100644 index 0000000000..772b0def81 --- /dev/null +++ b/packages/paperclip-runner/src/backends/runtime-context.test.ts @@ -0,0 +1,57 @@ +import { + mkdtempSync, + mkdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import { nativeSystemInstructions } from "./runtime-context.js"; + +const temporaryRoots: string[] = []; + +function runtimeInput(rootPath: string, entryPath: string): NativeExecutionInput { + return { + runtimeContext: { + prompt: { text: "Paperclip runtime." }, + instructions: { bundle: { rootPath }, entryPath }, + }, + } as unknown as NativeExecutionInput; +} + +describe("native runtime context files", () => { + afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("reads an instruction entry contained by its bundle root", () => { + const temporaryRoot = mkdtempSync(join(tmpdir(), "paperclip-runtime-context-")); + temporaryRoots.push(temporaryRoot); + const bundleRoot = join(temporaryRoot, "bundle"); + mkdirSync(bundleRoot); + writeFileSync(join(bundleRoot, "AGENTS.md"), "Stay inside the bundle.\n"); + + expect(nativeSystemInstructions(runtimeInput(bundleRoot, "AGENTS.md"))) + .toContain("Stay inside the bundle."); + }); + + it("rejects traversal and symlink escapes from the bundle root", () => { + const temporaryRoot = mkdtempSync(join(tmpdir(), "paperclip-runtime-context-")); + temporaryRoots.push(temporaryRoot); + const bundleRoot = join(temporaryRoot, "bundle"); + mkdirSync(bundleRoot); + writeFileSync(join(temporaryRoot, "outside.md"), "outside"); + symlinkSync(join(temporaryRoot, "outside.md"), join(bundleRoot, "linked.md")); + + expect(() => nativeSystemInstructions(runtimeInput(bundleRoot, "../outside.md"))) + .toThrow("native_runtime_context_entry_outside_bundle"); + expect(() => nativeSystemInstructions(runtimeInput(bundleRoot, "linked.md"))) + .toThrow("native_runtime_context_entry_outside_bundle"); + }); +}); diff --git a/packages/paperclip-runner/src/backends/runtime-context.ts b/packages/paperclip-runner/src/backends/runtime-context.ts new file mode 100644 index 0000000000..1258e2374a --- /dev/null +++ b/packages/paperclip-runner/src/backends/runtime-context.ts @@ -0,0 +1,42 @@ +import { readFileSync, realpathSync } from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { CODEX_SKILLLESS_BASE_INSTRUCTIONS } from "../contracts/codex.js"; +import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import { composeNativeSystemInstructions } from "../contracts/runtime-context.js"; + +export function nativeSystemInstructions(input: NativeExecutionInput): string { + if (!("runtimeContext" in input)) return CODEX_SKILLLESS_BASE_INSTRUCTIONS; + const configuredRoot = resolve(input.runtimeContext.instructions.bundle.rootPath); + const bundleRoot = realpathSync(configuredRoot); + const entryPath = realpathSync(resolve( + configuredRoot, + input.runtimeContext.instructions.entryPath, + )); + const pathFromRoot = relative(bundleRoot, entryPath); + if ( + pathFromRoot === ".." + || pathFromRoot.startsWith(`..${sep}`) + || isAbsolute(pathFromRoot) + ) { + throw new Error("native_runtime_context_entry_outside_bundle"); + } + const entry = readFileSync(entryPath, "utf8"); + return composeNativeSystemInstructions(input.runtimeContext, entry); +} + +export function nativeTaskConstraints(input: NativeExecutionInput): string[] { + const finalResponseConstraint = + "Write the complete user-facing final response exactly once before invoking paperclip_finish or paperclip_block. Treat that semantic completion tool as the last action and do not emit a trailing acknowledgement."; + if (!("runtimeContext" in input)) { + return [ + "Do not discover or invoke skills.", + "Do not call a control-plane API.", + finalResponseConstraint, + ]; + } + return [ + "Use only the assigned skills and provider-native tools.", + "Use Paperclip semantic tools for coordination and finalization.", + finalResponseConstraint, + ]; +} diff --git a/packages/paperclip-runner/src/conformance/control-plane-port.ts b/packages/paperclip-runner/src/conformance/control-plane-port.ts new file mode 100644 index 0000000000..6364d184cf --- /dev/null +++ b/packages/paperclip-runner/src/conformance/control-plane-port.ts @@ -0,0 +1,188 @@ +import type { + CompleteControlPlaneRunInput, + ControlPlanePort, + OpenControlPlaneRunInput, +} from "../contracts/control-plane-port.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; + +export interface ControlPlanePortConformanceSnapshot { + eventCount: number; + highestContiguousSourceSeq: number; + duplicateDisposition: "duplicate"; + terminalReplayIdempotent: boolean; + openBindingRejected: boolean; + eventIdMutationRejected: boolean; + eventSequenceMutationRejected: boolean; + replayBindingRejected: boolean; + resultMutationRejected: boolean; +} + +export interface ControlPlanePortConformanceHarness { + port: ControlPlanePort; + start?(): Promise; + stop?(): Promise; +} + +export const CONTROL_PLANE_CONFORMANCE_OPEN: OpenControlPlaneRunInput = { + identity: { + runId: "00000000-0000-4000-8000-000000000006", + sessionId: "session-standalone-conformance", + companyId: "00000000-0000-4000-8000-000000000001", + issueId: "00000000-0000-4000-8000-000000000003", + agentId: "00000000-0000-4000-8000-000000000002", + }, + backendKind: "mock", + sourceInstanceId: "runner-standalone-conformance", +}; + +export const CONTROL_PLANE_CONFORMANCE_RESULT: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Standalone conformance task completed.", + completionClaim: { + contractRevision: "standalone-v1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: ["event:2"] }], + remainingWork: [], + }, + evidence: [{ kind: "event", ref: "event:2" }], + verification: [{ commandOrCheck: "standalone-conformance", status: "passed", artifactRef: "event:2" }], + attentionRequests: [], + artifacts: [], +}; + +export const CONTROL_PLANE_CONFORMANCE_TERMINAL: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", +}; + +function event(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-standalone-conformance:event:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-standalone-conformance", + sourceKind: "runner", + runId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.runId, + normalizedSessionId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.sessionId, + turnId: "turn-standalone-conformance", + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" ? 0 : 1, + emittedAt: `2026-08-09T00:00:0${sourceSeq}.000Z`, + payload, + }; +} + +export const CONTROL_PLANE_CONFORMANCE_EVENTS = [ + event(1, "session.started", { backend: "mock" }), + event(2, "run.result.proposed", CONTROL_PLANE_CONFORMANCE_RESULT), + event(3, "run.terminal", CONTROL_PLANE_CONFORMANCE_TERMINAL as unknown as Record), +] as const; + +export async function runControlPlanePortConformance( + harness: ControlPlanePortConformanceHarness, +): Promise { + await harness.start?.(); + try { + await harness.port.openRun(CONTROL_PLANE_CONFORMANCE_OPEN); + let openBindingRejected = false; + try { + await harness.port.openRun({ + ...CONTROL_PLANE_CONFORMANCE_OPEN, + identity: { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, issueId: "forged-issue" }, + }); + } catch { + openBindingRejected = true; + } + if (!openBindingRejected) throw new Error("mismatched run binding was accepted"); + const first = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[0]); + if (first.disposition !== "committed" || first.highestContiguousSourceSeq !== 1) { + throw new Error("first event did not commit at cursor 1"); + } + const gap = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[2]); + if (gap.highestContiguousSourceSeq !== 1) throw new Error("source gap was hidden"); + const recovered = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[1]); + if (recovered.highestContiguousSourceSeq !== 3) throw new Error("source gap did not recover"); + const duplicate = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[1]); + if (duplicate.disposition !== "duplicate") throw new Error("identical duplicate was not idempotent"); + let eventIdMutationRejected = false; + try { + await harness.port.appendEvent({ + ...CONTROL_PLANE_CONFORMANCE_EVENTS[1], + payload: { mutated: true }, + }); + } catch { + eventIdMutationRejected = true; + } + if (!eventIdMutationRejected) throw new Error("mutated event id replay was accepted"); + let eventSequenceMutationRejected = false; + try { + await harness.port.appendEvent({ + ...CONTROL_PLANE_CONFORMANCE_EVENTS[1], + sourceEventId: "runner-standalone-conformance:mutated-sequence-two", + payload: { mutated: true }, + }); + } catch { + eventSequenceMutationRejected = true; + } + if (!eventSequenceMutationRejected) throw new Error("mutated source sequence replay was accepted"); + + const replay = await harness.port.replayEvents({ + runId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.runId, + sourceInstanceId: "runner-standalone-conformance", + afterSourceSeq: 1, + limit: 10, + }); + if (replay.events.map((entry) => entry.sourceSeq).join(",") !== "2,3") { + throw new Error("exclusive replay cursor returned the wrong event set"); + } + let replayBindingRejected = false; + try { + await harness.port.replayEvents({ + runId: "forged-run", + sourceInstanceId: "runner-standalone-conformance", + afterSourceSeq: 0, + limit: 10, + }); + } catch { + replayBindingRejected = true; + } + if (!replayBindingRejected) throw new Error("mismatched replay binding was accepted"); + + const completion: CompleteControlPlaneRunInput = { + result: CONTROL_PLANE_CONFORMANCE_RESULT, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + turnId: "turn-standalone-conformance", + callerResultId: "result-standalone-conformance", + callerDedupeKey: "dedupe-standalone-conformance", + }; + await harness.port.completeRun(completion); + await harness.port.completeRun(completion); + let resultMutationRejected = false; + try { + await harness.port.completeRun({ + ...completion, + result: { ...completion.result, summary: "mutated result replay" }, + }); + } catch { + resultMutationRejected = true; + } + if (!resultMutationRejected) throw new Error("mutated result replay was accepted"); + return { + eventCount: replay.highestContiguousSourceSeq, + highestContiguousSourceSeq: replay.highestContiguousSourceSeq, + duplicateDisposition: duplicate.disposition, + terminalReplayIdempotent: true, + openBindingRejected, + eventIdMutationRejected, + eventSequenceMutationRejected, + replayBindingRejected, + resultMutationRejected, + }; + } finally { + await harness.stop?.(); + } +} diff --git a/packages/paperclip-runner/src/conformance/harness-driver.ts b/packages/paperclip-runner/src/conformance/harness-driver.ts new file mode 100644 index 0000000000..db08327bc9 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/harness-driver.ts @@ -0,0 +1,241 @@ +import { readFile } from "node:fs/promises"; + +import { + HARNESS_DRIVER_CONTRACT_VERSION, + type HarnessDriver, + type HarnessDriverDescriptor, +} from "../contracts/harness-driver.js"; +import { validatePrpEvent, type PrpEvent } from "../protocol/replay-contract.js"; + +export const HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA = + "paperclip-runner/harness-driver-conformance-fixture/v1" as const; +export const HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA = + "paperclip-runner/harness-driver-conformance-report/v1" as const; + +export const harnessDriverConformanceFixtureUrl = new URL( + "../../protocol/fixtures/evals/harness-driver-conformance.json", + import.meta.url, +); + +export interface HarnessDriverConformanceFixture { + schema: typeof HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA; + validConfig: Record; + invalidConfig: Record; + completionMessage: string; + interruptMessage: string; + requiredCapabilities: string[]; + unsupportedFeatures: string[]; +} + +export interface HarnessDriverConformanceReport { + schema: typeof HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA; + contractVersion: typeof HARNESS_DRIVER_CONTRACT_VERSION; + descriptor: HarnessDriverDescriptor; + checks: { + capabilityDescription: true; + configValidation: true; + sessionLifecycle: true; + sessionRecovery: true; + semanticTools: true; + eventValidation: true; + interruptAndCancel: true; + usage: true; + transcriptCompleteness: true; + unsupportedFeatures: true; + }; + eventCount: number; + semanticToolCallCount: number; + usage: Record; +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${path} must be a non-empty string`); + } + return value; +} + +function strings(value: unknown, path: string): string[] { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) { + throw new Error(`${path} must be an array of strings`); + } + return [...value] as string[]; +} + +export function parseHarnessDriverConformanceFixture( + value: unknown, +): HarnessDriverConformanceFixture { + const fixture = record(value, "fixture"); + if (fixture.schema !== HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA) { + throw new Error( + `fixture.schema must be ${HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA}`, + ); + } + return { + schema: HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA, + validConfig: structuredClone(record(fixture.validConfig, "fixture.validConfig")), + invalidConfig: structuredClone(record(fixture.invalidConfig, "fixture.invalidConfig")), + completionMessage: text(fixture.completionMessage, "fixture.completionMessage"), + interruptMessage: text(fixture.interruptMessage, "fixture.interruptMessage"), + requiredCapabilities: strings(fixture.requiredCapabilities, "fixture.requiredCapabilities"), + unsupportedFeatures: strings(fixture.unsupportedFeatures, "fixture.unsupportedFeatures"), + }; +} + +export async function loadHarnessDriverConformanceFixture( + url: URL = harnessDriverConformanceFixtureUrl, +): Promise { + return parseHarnessDriverConformanceFixture( + JSON.parse(await readFile(url, "utf8")) as unknown, + ); +} + +async function collectEvents(events: AsyncIterable): Promise { + const collected: PrpEvent[] = []; + for await (const event of events) { + const validation = validatePrpEvent(event); + if (!validation.ok) { + throw new Error( + `driver conformance received invalid event: ${validation.issues[0]?.message ?? "unknown issue"}`, + ); + } + collected.push(structuredClone(validation.event)); + } + return collected; +} + +/** Run the deterministic, provider-free harness-driver V1 contract. */ +export async function runHarnessDriverConformance(input: { + driver: HarnessDriver; + fixture?: HarnessDriverConformanceFixture; +}): Promise { + const fixture = input.fixture ?? await loadHarnessDriverConformanceFixture(); + const descriptor = await input.driver.descriptor(); + if (!descriptor.kind || !descriptor.displayName || !descriptor.version) { + throw new Error("driver descriptor is missing stable identity fields"); + } + for (const capability of fixture.requiredCapabilities) { + if (descriptor.capabilities[capability as keyof typeof descriptor.capabilities] !== true) { + throw new Error(`driver capability ${capability} is required by conformance V1`); + } + } + for (const unsupported of fixture.unsupportedFeatures) { + if (!descriptor.capabilities.unsupported?.includes(unsupported)) { + throw new Error(`driver must explicitly describe unsupported feature ${unsupported}`); + } + } + + if (input.driver.validateConfig === undefined) { + throw new Error("driver does not expose deterministic config validation"); + } + const valid = await input.driver.validateConfig(fixture.validConfig); + const invalid = await input.driver.validateConfig(fixture.invalidConfig); + if (!valid.ok || invalid.ok || invalid.issues.length === 0) { + throw new Error("driver config validation did not accept valid and reject invalid input"); + } + + const completeSession = await input.driver.openSession({ + runId: "run_driver_conformance_complete", + normalizedSessionId: "session_driver_conformance_complete", + workingDirectory: "/deterministic/conformance", + }); + await completeSession.startTurn({ message: { role: "user", text: fixture.completionMessage } }); + const completeEvents = await collectEvents(completeSession.events()); + const semanticInputs = completeEvents.filter((event) => event.eventType === "mcp_app.tool_input"); + const semanticResults = completeEvents.filter((event) => event.eventType === "mcp_app.tool_result"); + if (semanticInputs.length === 0 || semanticInputs.length !== semanticResults.length) { + throw new Error("driver conformance requires paired semantic tool input/result events"); + } + const snapshot = await completeSession.snapshot(); + if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + throw new Error("completed driver session did not persist a semantic result"); + } + const usage = await completeSession.usage?.(); + if (usage === undefined || usage === null) { + throw new Error("driver conformance requires a deterministic usage snapshot"); + } + const transcript = await completeSession.transcript?.(); + if ( + transcript === undefined + || !transcript.complete + || transcript.eventCount !== completeEvents.length + || transcript.events.length !== completeEvents.length + || transcript.omissionReason !== null + ) { + throw new Error("driver conformance transcript is incomplete or inconsistent"); + } + if (input.driver.recoverSession === undefined) { + throw new Error("driver declares resume but does not expose session recovery"); + } + const recovery = await input.driver.recoverSession(snapshot, { + signal: new AbortController().signal, + }); + if ( + !recovery.recovered + || recovery.session === undefined + || recovery.session.ids().driverSessionId !== snapshot.driverSessionId + ) { + throw new Error("driver did not recover the persisted conformance session identity"); + } + const recoveredSnapshot = await recovery.session.snapshot(); + if ( + recoveredSnapshot.runId !== snapshot.runId + || recoveredSnapshot.normalizedSessionId !== snapshot.normalizedSessionId + || recoveredSnapshot.semanticResult?.fingerprint !== snapshot.semanticResult?.fingerprint + ) { + throw new Error("driver recovery did not preserve the persisted conformance state"); + } + await recovery.session.close({ reason: "conformance_recovery_complete" }); + await completeSession.close({ reason: "conformance_complete" }); + + const interruptedSession = await input.driver.openSession({ + runId: "run_driver_conformance_interrupt", + normalizedSessionId: "session_driver_conformance_interrupt", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await interruptedSession.startTurn({ + message: { role: "user", text: fixture.interruptMessage }, + }); + if (interruptedSession.interrupt === undefined) { + throw new Error("driver conformance requires interrupt support"); + } + await interruptedSession.interrupt({ turnId, reason: "conformance_interrupt" }); + const interruptedEvents = await collectEvents(interruptedSession.events()); + if ( + !interruptedEvents.some((event) => event.eventType === "turn.interrupted") + || !interruptedEvents.some((event) => + event.eventType === "run.terminal" + && record(event.payload, "terminal payload").runTerminalState === "cancelled") + ) { + throw new Error("driver interrupt did not produce an explicit cancelled terminal"); + } + await interruptedSession.close({ reason: "conformance_cancel", force: true }); + + return { + schema: HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA, + contractVersion: HARNESS_DRIVER_CONTRACT_VERSION, + descriptor: structuredClone(descriptor), + checks: { + capabilityDescription: true, + configValidation: true, + sessionLifecycle: true, + sessionRecovery: true, + semanticTools: true, + eventValidation: true, + interruptAndCancel: true, + usage: true, + transcriptCompleteness: true, + unsupportedFeatures: true, + }, + eventCount: completeEvents.length, + semanticToolCallCount: semanticInputs.length, + usage: structuredClone(usage), + }; +} diff --git a/packages/paperclip-runner/src/contracts/control-plane-port.ts b/packages/paperclip-runner/src/contracts/control-plane-port.ts new file mode 100644 index 0000000000..d877316868 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/control-plane-port.ts @@ -0,0 +1,94 @@ +import type { + NativeRunEvent, + NativeRunIdentity, + NativeRunResult, +} from "./types.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; +import type { PersistedNativeSession } from "./native-session-backend.js"; + +export interface OpenControlPlaneRunInput { + identity: NativeRunIdentity; + backendKind: "runner" | "remote" | "mock"; + sourceInstanceId?: string; +} + +export interface AppendedEventReceipt { + cursor: number; + highestContiguousSourceSeq: number; + disposition: "committed" | "duplicate"; +} + +export interface AppendControlPlaneEventOptions { + /** + * Cancellation is a durability boundary: an aborted append must settle + * without committing before it rejects. + */ + signal: AbortSignal; +} + +export interface CheckpointControlPlaneSessionOptions { + /** + * Cancellation is a durability boundary: once aborted, checkpoint work + * must settle without committing a new snapshot. + */ + signal: AbortSignal; +} + +export interface ReplayControlPlaneEventsInput { + runId: string; + sourceInstanceId: string; + afterSourceSeq: number; + limit: number; +} + +export interface ReplayedControlPlaneEvents { + events: PrpEvent[]; + highestContiguousSourceSeq: number; +} + +export interface FinalizeControlPlaneOperationOptions { + /** + * Finalization cancellation is a durability boundary. Once aborted, a + * mutating operation must settle without committing new durable state. + */ + signal: AbortSignal; +} + +export interface CompleteControlPlaneRunInput { + result: PrpStructuredRunResult; + terminal: PrpTerminalState; + turnId?: string | null; + callerResultId?: string | null; + callerDedupeKey?: string | null; +} + +/** + * The only control-plane surface the standalone runner may call. + * + * Production Paperclip implements this port in a later integration phase. The + * runner package never imports the server, UI, or database that sits behind it. + */ +export interface ControlPlanePort { + openRun(input: OpenControlPlaneRunInput): Promise; + loadSessionCheckpoint?(): Promise; + checkpointSession?( + snapshot: PersistedNativeSession, + options?: CheckpointControlPlaneSessionOptions, + ): Promise; + appendEvent( + event: NativeRunEvent | PrpEvent, + options?: AppendControlPlaneEventOptions, + ): Promise; + replayEvents( + input: ReplayControlPlaneEventsInput, + options?: FinalizeControlPlaneOperationOptions, + ): Promise; + completeRun( + result: NativeRunResult | CompleteControlPlaneRunInput, + options?: FinalizeControlPlaneOperationOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts new file mode 100644 index 0000000000..16b8291986 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -0,0 +1,513 @@ +import type { + PrpEvent, + PrpStructuredRunResult, +} from "../protocol/replay-contract.js"; +import type { NativeSessionCapabilities, NativeUserMessage } from "./types.js"; +import { + PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2, + parsePaperclipQuestionResponse, + type PaperclipQuestionResponse, + type PaperclipQuestionSet, + type PaperclipRuntimeRequestOrigin, +} from "./question-set.js"; + +export * from "./question-set.js"; + +export const HARNESS_DRIVER_CONTRACT_VERSION = 1 as const; + +export interface HarnessDriverConfigIssue { + path: string; + code: string; + message: string; +} + +export type HarnessDriverConfigValidation = + | { ok: true; config: Record; issues: [] } + | { ok: false; config: null; issues: HarnessDriverConfigIssue[] }; + +export interface HarnessTranscriptSnapshot { + schema: "paperclip-runner/harness-transcript/v1"; + complete: boolean; + eventCount: number; + events: PrpEvent[]; + omissionReason: string | null; +} + +export interface HarnessDriverDescriptor { + kind: string; + displayName: string; + version: string; + protocolVersion?: string; + capabilities: NativeSessionCapabilities; + runtimeContextCapabilities?: NativeRuntimeContextCapabilities; +} + +export interface NativeRuntimeContextCapabilities { + instructions: "native" | "unsupported"; + skills: "native" | "unsupported"; + mcp: "native" | "unsupported"; +} + +export interface OpenHarnessSessionInput { + runId: string; + normalizedSessionId: string; + workingDirectory: string; + /** Abort provider bootstrap and release any not-yet-returned provider state. */ + signal?: AbortSignal; +} + +export interface HarnessSessionRecoveryOptions { + /** Abort provider recovery and release any not-yet-returned provider state. */ + signal: AbortSignal; +} + +export class HarnessCapabilityUnavailableError extends Error { + readonly operation: string; + + constructor(operation: string, detail: string) { + super(`${operation} is unavailable: ${detail}`); + this.name = "HarnessCapabilityUnavailableError"; + this.operation = operation; + } +} + +export class HarnessReconciliationError extends Error { + readonly recoverable = true; + + constructor(message: string) { + super(message); + this.name = "HarnessReconciliationError"; + } +} + +export class HarnessOperationAlreadyTerminalError extends Error { + readonly code = "already_terminal" as const; + + constructor(operation: string) { + super(`${operation} lost a race with the committed turn terminal`); + this.name = "HarnessOperationAlreadyTerminalError"; + } +} + +export class HarnessStaleTurnError extends Error { + readonly code = "stale_turn" as const; + + constructor(turnId: string) { + super(`turn ${turnId} is not the active turn`); + this.name = "HarnessStaleTurnError"; + } +} + +export type HarnessRuntimeRequestKind = + | "command_approval" + | "file_approval" + | "permission_approval" + | "user_input" + | "elicitation"; + +export interface HarnessRuntimeRequest { + requestId: string; + requestKind: HarnessRuntimeRequestKind; + method: string; + turnId: string; + itemId: string; + status: "pending"; + prompt: string; + details: Record; + /** Provider-neutral presentation model for structured input requests. */ + input?: PaperclipQuestionSet; + /** Diagnostic origin only; provider response shapes never enter PRP. */ + origin?: PaperclipRuntimeRequestOrigin; +} + +export type HarnessRuntimeRequestResolution = + | { action: "accept" | "accept_for_session" | "decline" | "cancel" } + | { + action: "submit"; + answers: Record; + } + | { + action: "submit"; + content: Record; + } + | { + action: "submit"; + response: PaperclipQuestionResponse; + }; + +export type HarnessRuntimeRequestAction = HarnessRuntimeRequestResolution["action"]; + +export type HarnessRuntimeRequestHandoffResult = "handed_off" | "already_settled"; + +/** + * A runtime-input handoff commits its durable state transition before the + * method returns. Provider interruption is cleanup only: it remains observed, + * but cannot acquire mutation authority or reverse an already committed turn. + */ +export interface HarnessRuntimeRequestHandoff { + result: HarnessRuntimeRequestHandoffResult; + cleanup: Promise; +} + +const RUNTIME_REQUEST_ACTIONS: readonly HarnessRuntimeRequestAction[] = [ + "accept", + "accept_for_session", + "decline", + "cancel", + "submit", +]; + +export class HarnessRuntimeRequestResolutionError extends Error { + readonly code = "invalid_resolution" as const; + readonly requestKind: HarnessRuntimeRequestKind; + + constructor(requestKind: HarnessRuntimeRequestKind, detail: string) { + super(`${requestKind} rejected its resolution: ${detail}`); + this.name = "HarnessRuntimeRequestResolutionError"; + this.requestKind = requestKind; + } +} + +function plainRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function parseAnswers(value: unknown): Record | null { + const fields = plainRecord(value); + if (fields === null || Object.keys(fields).length === 0) return null; + const parsed: Record = {}; + for (const [field, entry] of Object.entries(fields)) { + const answer = plainRecord(entry); + if (answer === null || !Array.isArray(answer.answers)) return null; + if (answer.answers.some((value) => typeof value !== "string")) return null; + parsed[field] = { answers: [...(answer.answers as string[])] }; + } + return parsed; +} + +/** + * Validates a resolution against the kind of request it answers, so a + * mismatched submit fails closed instead of degrading into an accepted empty + * response. Every driver runs this before it touches its provider, and the + * browser transport runs it again at its own untrusted edge. + */ +export function parseHarnessRuntimeRequestResolution( + requestKind: HarnessRuntimeRequestKind, + value: unknown, + questionSet?: PaperclipQuestionSet, +): HarnessRuntimeRequestResolution { + const candidate = plainRecord(value) ?? {}; + const rawAction = candidate.action; + if ( + typeof rawAction !== "string" || + !RUNTIME_REQUEST_ACTIONS.includes(rawAction as HarnessRuntimeRequestAction) + ) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + `unsupported action ${JSON.stringify(rawAction) ?? "undefined"}`, + ); + } + const action = rawAction as HarnessRuntimeRequestAction; + if (action !== "submit" && ("answers" in candidate || "content" in candidate || "response" in candidate)) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + `${action} does not carry submitted form data`, + ); + } + + if (action === "submit" && "response" in candidate) { + if (requestKind !== "user_input" && requestKind !== "elicitation") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "approval requests do not accept submitted question responses", + ); + } + if ("answers" in candidate || "content" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "canonical submissions cannot also carry provider-specific answers or content", + ); + } + if (questionSet === undefined) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "canonical submission requires the persisted question set", + ); + } + try { + return { + action, + response: parsePaperclipQuestionResponse(questionSet, candidate.response), + }; + } catch (error) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + error instanceof Error ? error.message : "invalid question response", + ); + } + } + + if (requestKind === "user_input") { + if (action === "accept" || action === "accept_for_session") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "user input requires submit, decline, or cancel", + ); + } + if (action !== "submit") return { action }; + if ("content" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "user input submissions carry answers, not content", + ); + } + const answers = parseAnswers(candidate.answers); + if (answers === null) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "submit requires answers shaped as { field: { answers: [string] } }", + ); + } + return { action, answers }; + } + + if (requestKind === "elicitation") { + if (action === "accept_for_session") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "elicitation does not support session acceptance", + ); + } + if (action !== "submit") return { action }; + if ("answers" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "elicitation submissions carry content, not answers", + ); + } + const content = plainRecord(candidate.content); + if (content === null || Object.keys(content).length === 0) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "submit requires a non-empty content object", + ); + } + return { action, content: structuredClone(content) }; + } + + if (action === "submit") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "approval requests do not accept submitted form data", + ); + } + return { action }; +} + +/** + * The single payload shape every driver emits for a terminal + * `runtime_request.*` fact. Request identity travels in the payload as well as + * the event binding, so a consumer that only reads payloads still settles the + * request it belongs to. + */ +export type HarnessRuntimeRequestOutcome = { + requestId: string; + requestKind: HarnessRuntimeRequestKind; + turnId: string; + itemId: string; + action?: HarnessRuntimeRequestAction; + reason?: string; + /** Enables content-free lifecycle counters partitioned by adapter. */ + adapter?: string; + requestType?: "input" | "permission"; + /** Canonical submitted answers retained for durable replay and audit UI. */ + response?: PaperclipQuestionResponse; +}; + +export function harnessRuntimeRequestOutcome( + request: Pick< + HarnessRuntimeRequest, + "requestId" | "requestKind" | "turnId" | "itemId" | "origin" | "input" + >, + outcome: { + action?: HarnessRuntimeRequestAction | null; + reason?: string | null; + response?: PaperclipQuestionResponse | null; + } = {}, +): HarnessRuntimeRequestOutcome { + return { + requestId: request.requestId, + requestKind: request.requestKind, + turnId: request.turnId, + itemId: request.itemId, + ...(outcome.action ? { action: outcome.action } : {}), + ...(outcome.reason ? { reason: outcome.reason } : {}), + ...(outcome.response ? { response: structuredClone(outcome.response) } : {}), + ...(request.input + ? { + ...(request.origin?.adapter ? { adapter: request.origin.adapter } : {}), + requestType: "input" as const, + } + : {}), + }; +} + +/** Canonical non-replayable expiration used when a live input moves to a durable wait. */ +export function harnessRuntimeInputExpiredOutcome( + request: HarnessRuntimeRequest, + reason: "durable_handoff" | "provider_process_lost", +): Omit & { + requestKind: "runtime"; + replayAllowed: false; + request: Record; +} { + return { + ...harnessRuntimeRequestOutcome(request, { reason }), + requestKind: "runtime", + replayAllowed: false, + request: { + schema: PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2, + requestKind: "runtime", + requestId: request.requestId, + type: "input", + status: request.status, + prompt: request.prompt, + input: structuredClone(request.input), + origin: structuredClone(request.origin), + turnId: request.turnId, + itemId: request.itemId, + }, + }; +} + +export interface HarnessThreadGoal { + threadId: string; + objective: string; + status: "active" | "paused" | "blocked" | "usageLimited" | "budgetLimited" | "complete"; + tokenBudget: number | null; + tokensUsed: number; + timeUsedSeconds: number; + createdAt: number; + updatedAt: number; +} + +export type HarnessGoalOperation = + | { action: "get" } + | { action: "set"; objective: string; tokenBudget?: number | null } + | { action: "pause" | "resume" | "clear" }; + +export interface HarnessThreadLineageEntry { + threadId: string; + providerSessionId: string | null; + parentThreadId: string | null; + depth: number; + nickname: string | null; + role: string | null; + status: string; +} + +export interface PersistedHarnessSemanticResult { + result: PrpStructuredRunResult; + fingerprint: string; + callId?: string | null; + turnId: string; +} + +export interface PersistedHarnessTurnTerminal { + turnId: string; + fingerprint: string; +} + +export interface AcpxSessionIdentity { + kind: "acpx"; + normalizedSessionId: string; + acpxRecordId: string; + backendSessionId: string; + agentSessionId: string; + profileDigest: string; + workspaceDigest: string; + requestedModel: string; + effectiveModel: string; + /** Missing on legacy snapshots; those used the historical approve-reads behavior. */ + permissionMode?: "approve-all" | "approve-reads" | "deny-all"; +} + +export type PersistedHarnessProviderIdentity = AcpxSessionIdentity; + +export interface PersistedHarnessSession { + driverKind: string; + driverSessionId: string; + providerSessionId?: string | null; + runId?: string; + normalizedSessionId?: string; + activeTurnId?: string | null; + semanticResult?: PersistedHarnessSemanticResult | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + pendingRuntimeRequests?: HarnessRuntimeRequest[]; + goal?: HarnessThreadGoal | null; + lineage?: HarnessThreadLineageEntry[]; + lastSourceSequence?: number; + /** Tagged provider identity used to reject cross-profile recovery. */ + providerIdentity?: PersistedHarnessProviderIdentity; + /** Narrow escape hatch for a durable response-wake when a provider cannot reload its prior native session. */ + providerRecoveryPolicy?: + | "same_session_only" + | "allow_replacement_after_governed_wait" + | "allow_replacement_after_resume_failure"; +} + +export interface HarnessSessionRecoveryResult { + recovered: boolean; + session?: HarnessSession; + reason?: string; +} + +export interface HarnessSession { + ids(): { + driverSessionId: string; + providerSessionId?: string | null; + displayId?: string | null; + }; + events(): AsyncIterable; + attachRun?(input: { runId: string }): Promise | void; + startTurn(input: { + message: NativeUserMessage; + requestedCollaborationMode?: "default" | "plan"; + }): Promise<{ turnId: string; effectiveCollaborationMode?: "default" | "plan" }>; + steer?(input: { turnId: string; message: NativeUserMessage; correlationId?: string }): Promise; + interrupt?(input: { turnId?: string; reason?: string; signal?: AbortSignal }): Promise; + pendingRuntimeRequests?(): HarnessRuntimeRequest[]; + resolveRuntimeRequest?(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise; + handoffRuntimeRequest?(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + /** Do not synchronously commit if runtime ownership is already revoked. */ + signal: AbortSignal; + }): HarnessRuntimeRequestHandoff; + goal?(input: HarnessGoalOperation): Promise; + lineage?(): HarnessThreadLineageEntry[]; + read?(): Promise>; + reconcile?(): Promise>; + usage?(): Promise | null>; + transcript?(): Promise; + snapshot(): Promise; + close(input: { reason: string; force?: boolean }): Promise; +} + +/** A local harness implementation hidden behind the runner daemon boundary. */ +export interface HarnessDriver { + descriptor(): Promise; + validateConfig?(config: unknown): Promise; + openSession(input: OpenHarnessSessionInput): Promise; + recoverSession?( + snapshot: PersistedHarnessSession, + options: HarnessSessionRecoveryOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts new file mode 100644 index 0000000000..b1898b2eb9 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -0,0 +1,148 @@ +import type { + NativeRunIdentity, + NativeSessionCapabilities, + NativeUserMessage, +} from "./types.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; +import type { + HarnessRuntimeRequest, + HarnessRuntimeRequestHandoff, + HarnessRuntimeRequestResolution, + HarnessThreadLineageEntry, + NativeRuntimeContextCapabilities, + PersistedHarnessProviderIdentity, + PersistedHarnessTurnTerminal, +} from "./harness-driver.js"; + +export interface NativeSessionBackendDescriptor { + kind: "runner" | "remote" | "mock"; + name: string; + version: string; + capabilities: NativeSessionCapabilities; + runtimeContextCapabilities?: NativeRuntimeContextCapabilities; +} + +export interface OpenNativeSessionInput { + identity: NativeRunIdentity; + workingDirectory?: string; + /** + * When present, aborts provider bootstrap if the caller's recovery deadline + * expires. Backends own cleanup for work that has not returned a session yet. + */ + signal?: AbortSignal; +} + +export interface NativeSessionRecoveryOptions { + /** Abort provider recovery and release any not-yet-returned provider state. */ + signal: AbortSignal; +} + +export interface PersistedNativeSession { + backendKind: NativeSessionBackendDescriptor["kind"]; + driverKind?: string | null; + sessionId: string; + identity: NativeRunIdentity; + providerSessionId?: string | null; + /** Tagged provider-owned identity required for safe driver recovery. */ + providerIdentity?: PersistedHarnessProviderIdentity; + providerRecoveryPolicy?: + | "same_session_only" + | "allow_replacement_after_governed_wait" + | "allow_replacement_after_resume_failure"; + cursor?: string | null; + semanticResult?: PrpStructuredRunResult | null; + terminal?: PrpTerminalState | null; + activeTurnId?: string | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + pendingRuntimeRequests?: HarnessRuntimeRequest[]; + lineage?: HarnessThreadLineageEntry[]; +} + +export interface NativeSessionRecoveryResult { + recovered: boolean; + session?: NativeSession; + reason?: string; +} + +/** + * Cancellation is a synchronous authority transition followed by passive + * provider cleanup. Once `cancel` returns, the provider session must no longer + * be able to publish accepted output or acquire new mutation authority for the + * cancelled turn. Cleanup may stop processes or transports, but it must not + * perform durable control-plane mutations. + */ +export interface NativeSessionCancellation { + cleanup: Promise; +} + +export interface NativeSessionSnapshotOptions { + /** Stop provider snapshot work that outlives the execution deadline. */ + signal: AbortSignal; +} + +export interface NativeSession { + identity(): NativeRunIdentity; + capabilities(): Promise; + attachRun?(input: { identity: NativeRunIdentity }): Promise; + events(input?: { afterCursor?: string | null }): AsyncIterable; + startTurn(input: { + message: NativeUserMessage; + requestedCollaborationMode?: "default" | "plan"; + }): Promise<{ turnId: string; effectiveCollaborationMode?: "default" | "plan" }>; + steer?(input: { turnId: string; message: NativeUserMessage; correlationId?: string }): Promise; + interrupt?(input: { turnId?: string; reason?: string }): Promise; + /** Commit cancellation synchronously; the returned promise owns cleanup only. */ + cancel?(input: { reason: string; signal: AbortSignal }): NativeSessionCancellation; + resolveRuntimeRequest?(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise; + handoffRuntimeRequest?(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + /** + * Revokes durable mutation authority when event consumption fails. The + * method must commit synchronously before returning; its returned promise + * owns provider cleanup only and must not mutate durable request state. + */ + signal: AbortSignal; + }): HarnessRuntimeRequestHandoff; + result(): Promise<{ + result: PrpStructuredRunResult; + terminal: PrpTerminalState; + turnId: string | null; + } | null>; + usage?(): Promise | null>; + snapshot(options?: NativeSessionSnapshotOptions): Promise; + /** + * Idempotently stop provider work and release every pending `events().next()` + * before this promise resolves. Implementations must settle every promise + * previously returned by the session (including interrupt, cancel, handoff, + * and iterator teardown). The runtime bounds its wait for a broken provider, + * revokes that session's mutation authority, removes it from reuse, and keeps + * observing late cleanup so a contract violation cannot defeat a run timeout + * or become an unhandled rejection. + */ + close(input: { reason: string }): Promise; +} + +/** Normalized control-plane boundary shared by runner and hosted backends. */ +export interface NativeSessionBackend { + descriptor(): Promise; + openSession(input: OpenNativeSessionInput): Promise; + /** Open a fresh provider session after an explicitly governed continuity break. */ + openReplacementSession?( + input: OpenNativeSessionInput, + previous: PersistedNativeSession, + ): Promise; + recoverSession?( + snapshot: PersistedNativeSession, + options: NativeSessionRecoveryOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 02210787d2..3c56edc441 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -1,12 +1,17 @@ export * from "./catalog/index.js"; +export * from "./contracts/control-plane-port.js"; export * from "./contracts/completion-result.js"; export * from "./contracts/codex.js"; export * from "./contracts/durable-recovery.js"; +export * from "./contracts/harness-driver.js"; export * from "./contracts/local-runner.js"; export * from "./contracts/native-execution.js"; +export * from "./contracts/native-session-backend.js"; export * from "./contracts/question-set.js"; export * from "./contracts/runtime-context.js"; export * from "./contracts/types.js"; +export * from "./backends/harness-driver-backend.js"; +export * from "./native-session-runtime.js"; export { DurablePrpControlPlane, type DurablePrpControlPlaneOptions, diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts new file mode 100644 index 0000000000..9ea07c4b6c --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -0,0 +1,3738 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ControlPlanePort } from "./contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "./contracts/native-execution.js"; +import type { NativeRunIdentity } from "./contracts/types.js"; +import type { + NativeSession, + NativeSessionBackend, + PersistedNativeSession, +} from "./contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "./protocol/replay-contract.js"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, +} from "./contracts/runtime-context.js"; +import { + executeNativeSession, + type ExecuteNativeSessionOptions, +} from "./native-session-runtime.js"; + +const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", +}; + +const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], +}; + +const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", +}; + +const yieldedResult: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], + remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [{ kind: "issue_thread_interaction", ref: "interaction:pending" }], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, +}; + +const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], +}; + +function controlEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, +): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "control-recovery", + sourceKind: "control_plane", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; +} + +function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, +): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; +} + +function highestContiguous(events: PrpEvent[]): number { + const sequences = new Set(events.map((event) => event.sourceSeq)); + let cursor = 0; + while (sequences.has(cursor + 1)) cursor += 1; + return cursor; +} + +describe("executeNativeSession recovery", () => { + it("keeps governed-wait discovery synchronous", () => { + type GovernedWaitResolver = NonNullable; + const resolver: GovernedWaitResolver = () => null; + // An async resolver could retain control-plane mutation authority after + // execution settles, so the public boundary rejects it at compile time. + // @ts-expect-error governed-wait discovery must not return a promise + const asynchronousResolver: GovernedWaitResolver = async () => null; + + expect(resolver).toBeTypeOf("function"); + void asynchronousResolver; + }); + + it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => { + const digest = "0".repeat(64); + const context = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath: "AGENTS.md", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: "/paperclip/context/instructions", + fileCount: 1, + totalBytes: 1, + }, + }, + skills: [], + mcp: { assignmentSetId: "none", digest, bindingId: null }, + } as const; + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "future-provider", + name: "future-provider", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input: { + ...input, + schema: "paperclip.native-execution-input.v3", + executionMode: "default", + planningContext: null, + runtimeContext: { + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("does not natively realize instructions, skills, mcp"); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("does not admit a fresh run when provider session initialization fails", async () => { + const providerFailure = new Error("provider initialization failed"); + const openSession = vi.fn(async () => { + throw providerFailure; + }); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "fresh-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toBe(providerFailure); + + expect(openSession).toHaveBeenCalledWith(expect.objectContaining({ + identity, + workingDirectory: input.workspace.cwd, + signal: expect.any(AbortSignal), + })); + expect(openRun).not.toHaveBeenCalled(); + }); + + it("bounds fresh session bootstrap and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + let resolveBootstrap = (_value: NativeSession) => {}; + const stalledBootstrap = new Promise((resolve) => { + resolveBootstrap = resolve; + }); + let markBootstrapStarted = () => {}; + const bootstrapStarted = new Promise((resolve) => { + markBootstrapStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late fresh session must not start"); }, + async result() { return null; }, + async snapshot() { throw new Error("late fresh session must not snapshot"); }, + close, + }; + let bootstrapSignal: AbortSignal | undefined; + const openSession = vi.fn((bootstrapInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + bootstrapSignal = bootstrapInput.signal; + bootstrapInput.signal?.addEventListener( + "abort", + () => resolveBootstrap(lateSession), + { once: true }, + ); + markBootstrapStarted(); + return stalledBootstrap; + }); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "fresh-stalled-backend", + version: "1", + capabilities: await lateSession.capabilities(), + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + }); + const rejection = expect(execution).rejects.toThrow( + "native session bootstrap timed out after 5ms", + ); + + await bootstrapStarted; + expect(bootstrapSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(bootstrapSignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session bootstrap timed out", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("closes the provider when owner quarantine notification throws", async () => { + const snapshotFailure = new Error("snapshot failed"); + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("unexpected turn"); }, + async result() { return null; }, + async snapshot() { throw snapshotFailure; }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "owner-notification-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + const retainedSessions: Array = []; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + keepSessionOpen: true, + onSession(current) { + retainedSessions.push(current); + if (current === null) throw new Error("owner notification failed"); + }, + })).rejects.toBe(snapshotFailure); + + expect(retainedSessions).toEqual([session, null]); + expect(close).toHaveBeenCalledOnce(); + }); + + it.each(["control-plane checkpoint", "owner checkpoint"] as const)( + "aborts consumption and closes the provider when the startup %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + let releaseStream = () => {}; + try { + const streamReleased = new Promise((resolve) => { + releaseStream = resolve; + }); + const never = new Promise(() => undefined); + let markCheckpointStalled = () => {}; + const checkpointStalled = new Promise((resolve) => { + markCheckpointStalled = resolve; + }); + let checkpointSignal: AbortSignal | undefined; + let controlPlaneCheckpointCount = 0; + const checkpointSession: NonNullable = async ( + _snapshot, + checkpointOptions, + ) => { + controlPlaneCheckpointCount += 1; + if ( + stalledBoundary === "control-plane checkpoint" + && controlPlaneCheckpointCount === 2 + ) { + checkpointSignal = checkpointOptions?.signal; + markCheckpointStalled(); + await never; + } + }; + let ownerCheckpointCount = 0; + const onCheckpoint: NonNullable = async ( + _snapshot, + checkpointOptions, + ) => { + ownerCheckpointCount += 1; + if ( + stalledBoundary === "owner checkpoint" + && ownerCheckpointCount === 2 + ) { + checkpointSignal = checkpointOptions?.signal; + markCheckpointStalled(); + await never; + } + }; + const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled" })); + const close = vi.fn(async () => { + releaseStream(); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + }, + async *events() { + await streamReleased; + }, + startTurn, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-checkpoint-stalled", + cursor: "0", + activeTurnId: "turn-checkpoint-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "checkpoint-stalled-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const openRun = vi.fn(async () => undefined); + const port: ControlPlanePort = { + openRun, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1_000, + checkpointTimeoutMs: 1, + keepSessionOpen: true, + onCheckpoint, + onSession: (current) => retainedSessions.push(current), + }); + const rejection = expect(execution).rejects.toThrow( + "native session checkpoint timed out after 1ms", + ); + await checkpointStalled; + expect(checkpointSignal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + await rejection; + + expect(checkpointSignal?.aborted).toBe(true); + expect(openRun).toHaveBeenCalledOnce(); + expect(startTurn).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + releaseStream(); + vi.useRealTimers(); + } + }, + ); + + it.each([ + "provider result", + "completion checkpoint", + "control-plane replay", + "final event append", + "run completion", + ] as const)( + "bounds post-terminal finalization when %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markFinalizationStalled = () => {}; + const finalizationStalled = new Promise((resolve) => { + markFinalizationStalled = resolve; + }); + let stalledSignal: AbortSignal | undefined; + let resultCalls = 0; + let resultResolved = false; + let completionCheckpointCalls = 0; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { + resultCalls += 1; + if (stalledBoundary === "provider result") { + markFinalizationStalled(); + return await never; + } + resultResolved = true; + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "finalization-timeout-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession(_snapshot, operationOptions) { + if (stalledBoundary === "completion checkpoint" && resultResolved) { + completionCheckpointCalls += 1; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + async appendEvent(event, operationOptions) { + if ( + stalledBoundary === "final event append" + && (event as PrpEvent).sourceKind === "control_plane" + ) { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + return { + cursor: 1, + highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, + disposition: "committed", + }; + }, + async replayEvents(_replay, operationOptions) { + if (stalledBoundary === "control-plane replay") { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun(_completion, operationOptions) { + if (stalledBoundary === "run completion") { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + const rejection = expect(execution).rejects.toThrow( + "native session finalization timed out after 10ms", + ); + await finalizationStalled; + if (stalledBoundary !== "provider result") { + expect(stalledSignal?.aborted).toBe(false); + } + + await vi.advanceTimersByTimeAsync(20); + await rejection; + + if (stalledBoundary !== "provider result") { + expect(stalledSignal?.aborted).toBe(true); + } + expect(resultCalls).toBe(1); + if (stalledBoundary === "completion checkpoint") { + expect(completionCheckpointCalls).toBe(1); + } + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + vi.useRealTimers(); + } + }, + ); + + it.each([ + "final event append", + "run completion", + ] as const)( + "confirms durable completion when %s commits before its acknowledgement stalls", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markFinalizationStalled = () => {}; + const finalizationStalled = new Promise((resolve) => { + markFinalizationStalled = resolve; + }); + let stalledOnce = false; + let stalledSignal: AbortSignal | undefined; + const events: PrpEvent[] = []; + let durableCompletion: unknown = null; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "durable-finalization-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event, operationOptions) { + const appended = structuredClone(event as PrpEvent); + const existing = events.find((candidate) => + candidate.sourceInstanceId === appended.sourceInstanceId + && candidate.sourceSeq === appended.sourceSeq + ); + if (existing === undefined) events.push(appended); + if ( + stalledBoundary === "final event append" + && appended.sourceKind === "control_plane" + && !stalledOnce + ) { + stalledOnce = true; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + const sourceEvents = events.filter( + (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId, + ); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: existing === undefined ? "committed" : "duplicate", + }; + }, + async replayEvents(replay) { + const sourceEvents = events.filter( + (event) => event.sourceInstanceId === replay.sourceInstanceId, + ); + return { + events: structuredClone(sourceEvents.filter( + (event) => event.sourceSeq > replay.afterSourceSeq, + )), + highestContiguousSourceSeq: highestContiguous(sourceEvents), + }; + }, + async completeRun(completion, operationOptions) { + if (durableCompletion === null) { + durableCompletion = structuredClone(completion); + } else { + expect(completion).toEqual(durableCompletion); + } + if (stalledBoundary === "run completion" && !stalledOnce) { + stalledOnce = true; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + }); + await finalizationStalled; + expect(stalledSignal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(10); + + await expect(execution).resolves.toMatchObject({ + result, + terminal, + nativeEventCount: 3, + }); + expect(stalledSignal?.aborted).toBe(true); + expect(durableCompletion).toMatchObject({ result, terminal }); + expect(events.filter((event) => event.sourceKind === "control_plane")) + .toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }, + ); + + it.each([ + "provider snapshot", + "post-completion checkpoint", + "provider usage", + ] as const)( + "preserves durable completion when %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markEnrichmentStalled = () => {}; + const enrichmentStalled = new Promise((resolve) => { + markEnrichmentStalled = resolve; + }); + let stalledSignal: AbortSignal | undefined; + let runCompleted = false; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot(snapshotOptions) { + if (stalledBoundary === "provider snapshot" && runCompleted) { + stalledSignal = snapshotOptions?.signal; + markEnrichmentStalled(); + return await never; + } + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async usage() { + if (stalledBoundary === "provider usage" && runCompleted) { + markEnrichmentStalled(); + return await never; + } + return { driverVersion: "2" }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "post-completion-enrichment-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const completeRun = vi.fn(async () => { + runCompleted = true; + }); + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession(_snapshot, operationOptions) { + if (stalledBoundary === "post-completion checkpoint" && runCompleted) { + stalledSignal = operationOptions?.signal; + markEnrichmentStalled(); + await never; + } + }, + async appendEvent(event) { + return { + cursor: 1, + highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + await enrichmentStalled; + if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(10); + + await expect(execution).resolves.toMatchObject({ + result, + terminal, + providerSessionId: "provider-recovery", + driverVersion: "1", + usage: null, + }); + if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + vi.useRealTimers(); + } + }, + ); + + it("contains a consumer rejection when starting the turn fails first", async () => { + let markAppendStarted = () => {}; + const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + let releaseAppend = () => {}; + const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + let appendCommitted = false; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.started"); }, + async startTurn() { + await appendStarted; + throw new Error("start turn failed"); + }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(_event, options) { + markAppendStarted(); + await Promise.race([ + appendReleased, + new Promise((_resolve, reject) => { + const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); + if (options?.signal.aborted) rejectAbort(); + else options?.signal.addEventListener("abort", rejectAbort, { once: true }); + }), + ]); + appendCommitted = true; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed" as const, + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + await appendStarted; + await expect(execution).rejects.toThrow("start turn failed"); + expect(close).toHaveBeenCalled(); + expect(appendCommitted).toBe(false); + releaseAppend(); + await new Promise((resolve) => setImmediate(resolve)); + expect(appendCommitted).toBe(false); + }); + + it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => { + let markAppendStarted = () => {}; + const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + let releaseAppend = () => {}; + const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + let releaseTeardown = () => {}; + const teardownReleased = new Promise((resolve) => { releaseTeardown = resolve; }); + const iteratorTeardown = vi.fn(); + let appendCommitted = false; + const appendEvent = vi.fn(async ( + _event: PrpEvent, + options?: { signal: AbortSignal }, + ) => { + markAppendStarted(); + await Promise.race([ + appendReleased, + new Promise((_resolve, reject) => { + const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); + if (options?.signal.aborted) rejectAbort(); + else options?.signal.addEventListener("abort", rejectAbort, { once: true }); + }), + ]); + appendCommitted = true; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed" as const, + }; + }); + const cancel = vi.fn(() => { + releaseTeardown(); + return { cleanup: Promise.resolve() }; + }); + const close = vi.fn(async () => { releaseTeardown(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + try { + yield runnerEvent(1, "turn.completed"); + } finally { + iteratorTeardown(); + await teardownReleased; + } + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + appendEvent, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }); + const rejection = expect(execution).rejects.toThrow("native session timed out"); + await appendStarted; + await vi.waitFor(() => expect(iteratorTeardown).toHaveBeenCalledOnce()); + await rejection; + expect(cancel).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalled(); + expect(appendEvent).toHaveBeenCalledOnce(); + expect(appendCommitted).toBe(false); + releaseAppend(); + await new Promise((resolve) => setImmediate(resolve)); + expect(appendCommitted).toBe(false); + }); + + it("closes a failed session while retaining an uncancellable event read", async () => { + let releaseStream = () => {}; + const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); + const close = vi.fn(async () => { releaseStream(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { + await streamReleased; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + })).rejects.toThrow("native session timed out"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("commits cancellation before bounding failed provider cleanup", async () => { + let releaseStream = () => {}; + const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); + let releaseCancellation = () => {}; + const cancellationReleased = new Promise((resolve) => { releaseCancellation = resolve; }); + const interrupt = vi.fn(() => cancellationReleased); + let cancellationCommitted = false; + const cancel = vi.fn(() => { + cancellationCommitted = true; + return { cleanup: cancellationReleased }; + }); + const close = vi.fn(async () => { releaseStream(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + await streamReleased; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + interrupt, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }); + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + expect(interrupt).not.toHaveBeenCalled(); + expect(cancel).toHaveBeenCalledOnce(); + await expect(execution).rejects.toThrow("native session timed out"); + expect(cancellationCommitted).toBe(true); + releaseCancellation(); + }); + + it("bounds failure when iterator teardown and provider close never settle", async () => { + const never = new Promise(() => undefined); + const close = vi.fn(() => never); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { + await never; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + })).rejects.toThrow("native session timed out"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("preserves durable success when provider close never settles", async () => { + const never = new Promise(() => undefined); + const close = vi.fn(() => never); + const completeRun = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ result, terminal }); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("closes after a synchronous governed-wait probe returns no result", async () => { + const resolveGovernedWait = vi.fn(() => null); + const lifecycle: string[] = []; + const close = vi.fn(async () => { lifecycle.push("closed"); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + yield runnerEvent(1, "item.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: "turn-recovery", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + resolveGovernedWait, + }); + await expect(execution).rejects.toThrow("before a turn terminal fact"); + expect(resolveGovernedWait).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalled(); + expect(lifecycle).toEqual(["closed"]); + }); + + it("commits a governed wait without waiting for abort-insensitive provider cleanup", async () => { + const lifecycle: string[] = []; + let cancellationSignal: AbortSignal | undefined; + const cancel = vi.fn(({ signal }: { signal: AbortSignal }) => { + cancellationSignal = signal; + lifecycle.push("cancelled"); + return { cleanup: new Promise(() => undefined) }; + }); + const close = vi.fn(async () => { lifecycle.push("closed"); }); + const events: PrpEvent[] = []; + const retainedSessions: Array = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + yield runnerEvent(1, "item.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: "turn-recovery", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: event.sourceSeq, + disposition: "committed", + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + resolveGovernedWait: () => yieldedResult, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + await expect(execution).resolves.toMatchObject({ result: yieldedResult }); + expect(cancel).toHaveBeenCalledOnce(); + expect(cancellationSignal?.aborted).toBe(true); + expect(close).toHaveBeenCalledOnce(); + expect(lifecycle).toEqual(["cancelled", "closed"]); + expect(retainedSessions.at(-1)).toBeNull(); + expect(events.map((event) => event.eventType)).toEqual([ + "item.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("rejects a mismatched checkpoint before it mutates control-plane state", async () => { + const openRun = vi.fn(async () => undefined); + const checkpointSession = vi.fn(async () => undefined); + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity: { ...identity, companyId: "other-company" }, + }; + }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("rejects a mismatched existing session before opening control-plane state", async () => { + const openRun = vi.fn(async () => undefined); + const attachRun = vi.fn(async () => undefined); + const existingSession: NativeSession = { + identity: () => ({ ...identity, companyId: "other-company" }), + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + async startTurn() { return { turnId: "unexpected" }; }, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + })).rejects.toThrow("native_session_attach_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(attachRun).not.toHaveBeenCalled(); + }); + + it("quarantines a retained session when attachment partially mutates then fails", async () => { + const attachmentFailure = new Error("provider attachment failed"); + const openRun = vi.fn(async () => undefined); + let retainedIdentity = { ...identity, runId: "run-previous" }; + const attachRun = vi.fn(async (input: { identity: NativeRunIdentity }) => { + retainedIdentity = structuredClone(input.identity); + throw attachmentFailure; + }); + const close = vi.fn(() => new Promise(() => {})); + const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); + const onSession = vi.fn(); + const existingSession: NativeSession = { + identity: () => structuredClone(retainedIdentity), + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + startTurn, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + })).rejects.toBe(attachmentFailure); + expect(attachRun).toHaveBeenCalledWith({ identity }); + expect(retainedIdentity).toEqual(identity); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native session attachment failed", + }); + expect(openRun).not.toHaveBeenCalled(); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it("quarantines an attached session when control-plane run admission fails", async () => { + const admissionFailure = new Error("control-plane admission failed"); + const openRun = vi.fn(async () => { + throw admissionFailure; + }); + const attachRun = vi.fn(async () => undefined); + const close = vi.fn(async () => undefined); + const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); + const onSession = vi.fn(); + const existingSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + startTurn, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + })).rejects.toBe(admissionFailure); + expect(attachRun).toHaveBeenCalledWith({ identity }); + expect(openRun).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native control-plane run admission failed", + }); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it("rejects checkpoint adoption when the requested session id is absent", async () => { + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { + return { + backendKind: "mock", + sessionId: "driver-other-session", + identity: { ...identity, sessionId: "other-session" }, + }; + }, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input: { + ...input, + session: { ...input.session, normalizedSessionId: null }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("proves required provider recovery before re-opening the durable run", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-unrecoverable", + identity, + providerSessionId: "provider-unrecoverable", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-unrecoverable", + pendingRuntimeRequests: [], + lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const completeRun = vi.fn(async () => undefined); + const recoverSession = vi.fn(async () => ({ + recovered: false as const, + reason: "provider session no longer exists", + })); + const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow( + "native_session_recovery_failed: provider session no longer exists", + ); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).not.toHaveBeenCalled(); + expect(completeRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { + vi.useFakeTimers(); + let rejectStalledReplay = (_error: Error) => {}; + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-replay-stalled", + identity, + providerSessionId: "provider-replay-stalled", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-replay-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + const stalledReplay = new Promise((_resolve, reject) => { + rejectStalledReplay = reject; + }); + let markSecondPageStarted = () => {}; + const secondPageStarted = new Promise((resolve) => { + markSecondPageStarted = resolve; + }); + const replaySignals: AbortSignal[] = []; + const replayEvents = vi.fn( + async (replay, operationOptions) => { + expect(operationOptions?.signal).toBeInstanceOf(AbortSignal); + replaySignals.push(operationOptions!.signal); + if (replay.afterSourceSeq === 0) { + return { + events: [runnerEvent(1, "item.completed", { kind: "progress" })], + highestContiguousSourceSeq: 1, + }; + } + markSecondPageStarted(); + return await stalledReplay; + }, + ); + const recoverSession = vi.fn(); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + replayEvents, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + }); + const rejection = expect(execution).rejects.toThrow( + "native session recovery replay timed out after 5ms", + ); + + await secondPageStarted; + expect(replayEvents).toHaveBeenCalledTimes(2); + expect(replaySignals).toHaveLength(2); + expect(replaySignals[1]).toBe(replaySignals[0]); + expect(replaySignals[0]?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(5); + await rejection; + + expect(replaySignals[0]?.aborted).toBe(true); + expect(recoverSession).not.toHaveBeenCalled(); + expect(openRun).not.toHaveBeenCalled(); + + // A broken adapter may ignore abort and reject later. The bounded helper + // keeps that losing operation observed after execution already rejected. + rejectStalledReplay(new Error("late recovery replay failure")); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds provider recovery and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery-stalled", + identity, + providerSessionId: "provider-recovery-stalled", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-recovery-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + let resolveRecovery = (_value: { recovered: true; session: NativeSession }) => {}; + const stalledRecovery = new Promise<{ recovered: true; session: NativeSession }>( + (resolve) => { resolveRecovery = resolve; }, + ); + let markRecoveryStarted = () => {}; + const recoveryStarted = new Promise((resolve) => { + markRecoveryStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late recovery session must not start"); }, + async result() { return null; }, + async snapshot() { return structuredClone(checkpoint); }, + close, + }; + let recoverySignal: AbortSignal | undefined; + const recoverSession = vi.fn( + (_checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }) => { + recoverySignal = recoveryOptions.signal; + recoveryOptions.signal.addEventListener( + "abort", + () => resolveRecovery({ recovered: true, session: lateSession }), + { once: true }, + ); + markRecoveryStarted(); + return stalledRecovery; + }, + ); + const openRun = vi.fn(async () => undefined); + const onSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + onSession, + }); + const rejection = expect(execution).rejects.toThrow( + "native session provider recovery timed out after 5ms", + ); + + await recoveryStarted; + expect(recoverySignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(recoverySignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + expect(onSession).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session provider recovery timed out", + }); + expect(onSession).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds replacement bootstrap and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-replacement-stalled", + identity, + providerSessionId: "provider-replacement-stalled", + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: "0", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + let resolveReplacement = (_value: NativeSession) => {}; + const stalledReplacement = new Promise((resolve) => { + resolveReplacement = resolve; + }); + let markReplacementStarted = () => {}; + const replacementStarted = new Promise((resolve) => { + markReplacementStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late replacement session must not start"); }, + async result() { return null; }, + async snapshot() { return structuredClone(checkpoint); }, + close, + }; + let replacementSignal: AbortSignal | undefined; + const openReplacementSession = vi.fn((replacementInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + replacementSignal = replacementInput.signal; + replacementInput.signal?.addEventListener( + "abort", + () => resolveReplacement(lateSession), + { once: true }, + ); + markReplacementStarted(); + return stalledReplacement; + }); + const openRun = vi.fn(async () => undefined); + const onSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "replacement-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + }; + }, + async openSession() { throw new Error("replacement seam must be used"); }, + async recoverSession() { + return { recovered: false, reason: "provider session is missing" }; + }, + openReplacementSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + onSession, + }); + const rejection = expect(execution).rejects.toThrow( + "native session replacement bootstrap timed out after 5ms", + ); + + await replacementStarted; + expect(replacementSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(replacementSignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + expect(onSession).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session replacement bootstrap timed out", + }); + expect(onSession).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("does not persist a reconciled recovery cursor when provider recovery rejects", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery-rejects", + identity, + providerSessionId: "provider-recovery-rejects", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-recovery-rejects", + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveryFailure = new Error("provider recovery rejected"); + const openRun = vi.fn(async () => undefined); + const checkpointSession = vi.fn(async () => undefined); + const onCheckpoint = vi.fn(async () => undefined); + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + throw recoveryFailure; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents(replay) { + return { + events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], + highestContiguousSourceSeq: 1, + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + })).rejects.toBe(recoveryFailure); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).not.toHaveBeenCalled(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(onCheckpoint).not.toHaveBeenCalled(); + }); + + it("does not persist a reconciled recovery cursor when run admission rejects", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-admission-rejects", + identity, + providerSessionId: "provider-admission-rejects", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-admission-rejects", + pendingRuntimeRequests: [], + lineage: [], + }; + const admissionFailure = new Error("control-plane admission rejected"); + const openRun = vi.fn(async () => { throw admissionFailure; }); + const checkpointSession = vi.fn(async () => undefined); + const onCheckpoint = vi.fn(async () => undefined); + const onSession = vi.fn(); + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() {}, + async startTurn() { throw new Error("unexpected turn"); }, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + return { recovered: true as const, session }; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents(replay) { + return { + events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], + highestContiguousSourceSeq: 1, + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + onSession, + })).rejects.toBe(admissionFailure); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).toHaveBeenCalledOnce(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(onCheckpoint).not.toHaveBeenCalled(); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native control-plane run admission failed", + }); + }); + + it("continues a provider-reported active turn without starting a duplicate turn", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "0", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const providerSnapshot: PersistedNativeSession = { + ...checkpoint, + cursor: "1", + activeTurnId: "turn-recovery", + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:1", + sourceSeq: 1, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload: {}, + }; + const bySource = new Map(); + const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" })); + const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + startTurn, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { return structuredClone(providerSnapshot); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + async recoverSession() { return { recovered: true, session }; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const list = bySource.get(replay.sourceInstanceId) ?? []; + return { + events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery" }); + expect(openSession).not.toHaveBeenCalled(); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it.each([ + { checkpointCursor: "12", expectedCursor: "41", terminalSequence: 42 }, + { checkpointCursor: "50", expectedCursor: "50", terminalSequence: 51 }, + ])( + "seeds recovery from the larger of checkpoint $checkpointCursor and the persisted source high-water mark", + async ({ checkpointCursor, expectedCursor, terminalSequence }) => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: checkpointCursor, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const runnerEvents = [ + runnerEvent(13, "item.completed", { kind: "progress" }), + runnerEvent(41, "item.completed", { kind: "progress" }), + ]; + const terminalEvent = runnerEvent(terminalSequence, "turn.completed"); + const controlEvents: PrpEvent[] = []; + const checkpoints: PersistedNativeSession[] = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null }; + }, + async close() {}, + }; + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe(expectedCursor); + return { recovered: true, session }; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + recoverSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async appendEvent(event) { + const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; + if (target.some((existing) => existing.sourceSeq === event.sourceSeq)) { + throw new Error(`native_event_replay_conflict:${event.sourceSeq}`); + } + target.push(structuredClone(event)); + return { + cursor: target.length, + highestContiguousSourceSeq: highestContiguous(target), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; + const events = source + .filter((event) => event.sourceSeq > replay.afterSourceSeq) + .sort((left, right) => left.sourceSeq - right.sourceSeq) + .slice(0, replay.limit); + return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source) }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-recovery" }); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(runnerEvents.some((event) => event.sourceSeq === terminalSequence)).toBe(true); + if (checkpointCursor === "12") { + expect(checkpoints[0]).toMatchObject({ cursor: "41" }); + } + }, + ); + + it("attempts exact recovery before opening an observable replacement session", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-old", + identity, + providerSessionId: "provider-old", + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const replacementSnapshot: PersistedNativeSession = { + ...checkpoint, + sessionId: "driver-new", + providerSessionId: "provider-new", + providerRecoveryPolicy: "same_session_only", + }; + const replacementSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-replacement" }; }, + async result() { return { result, terminal, turnId: "turn-replacement" }; }, + async snapshot() { return structuredClone(replacementSnapshot); }, + async close() {}, + }; + const recoverSession = vi.fn(async () => ({ + recovered: false as const, + reason: "provider reported the prior session missing", + })); + const openReplacementSession = vi.fn(async () => replacementSession); + const onContinuityBreak = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "replacement-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("replacement seam must be used"); }, + recoverSession, + openReplacementSession, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event)); + return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-replacement", + controlPlaneInstanceId: "control-replacement", + onContinuityBreak, + })).resolves.toMatchObject({ providerSessionId: "provider-new" }); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openReplacementSession).toHaveBeenCalledOnce(); + expect(onContinuityBreak).toHaveBeenCalledWith({ + reason: "provider reported the prior session missing", + previousDriverSessionId: "driver-old", + previousProviderSessionId: "provider-old", + replacementDriverSessionId: "driver-new", + replacementProviderSessionId: "provider-new", + }); + }); + + it("starts a continuation after the driver clears a stale active terminal turn", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: "turn-already-terminal", + terminalTurns: [{ turnId: "turn-already-terminal", fingerprint: "terminal-fingerprint" }], + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveredSnapshot: PersistedNativeSession = { + ...checkpoint, + activeTurnId: null, + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:2", + sourceSeq: 2, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-continuation", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:01.000Z", + payload: {}, + }; + const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + startTurn, + async result() { return { result, terminal, turnId: "turn-continuation" }; }, + async snapshot() { return structuredClone(recoveredSnapshot); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const bySource = new Map(); + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const list = bySource.get(replay.sourceInstanceId) ?? []; + return { + events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" }); + expect(startTurn).toHaveBeenCalledOnce(); + const recoveryEnvelope = JSON.parse( + startTurn.mock.calls[0]![0].message.text, + ) as { task: { prompt: string } }; + expect(recoveryEnvelope.task.prompt).toContain( + "semantic-result recovery for a prior completed provider turn", + ); + expect(recoveryEnvelope.task.prompt).toContain( + "Do not repeat implementation, tests, research, or the final answer", + ); + }); + + it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "4", + semanticResult: result, + terminal, + activeTurnId: "turn-recovery", + terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }], + pendingRuntimeRequests: [], + lineage: [], + }; + const events = [controlEvent(1, "run.result.accepted", { result })]; + const checkpoints: PersistedNativeSession[] = []; + const completeRun = vi.fn(async () => undefined); + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + const openSession = vi.fn(async () => { + throw new Error("a recovered run must not open a second provider session"); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() {}, + startTurn, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { return structuredClone(checkpoint); }, + async close() {}, + }; + const recoverSession = vi.fn(async () => ({ recovered: true, session })); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + expect(openSession).not.toHaveBeenCalled(); + expect(recoverSession).toHaveBeenCalledOnce(); + expect(startTurn).not.toHaveBeenCalled(); + expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]); + expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]); + expect(completeRun).toHaveBeenCalledOnce(); + expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 }); + expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal }); + }); + + it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => { + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:1", + sourceSeq: 1, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-waiting", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload: {}, + }; + const yielded: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], + remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, + }; + const events: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const resolveMissingResult = vi.fn(async () => yielded); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + async startTurn() { return { turnId: "turn-waiting" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "1", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "governed-wait-backend", + version: "1", + capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult, + }); + + expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent }); + expect(completed).toMatchObject({ + result: yielded, + terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" }, + turnId: "turn-waiting", + }); + expect(completeRun).toHaveBeenCalledWith( + expect.objectContaining({ result: yielded }), + { signal: expect.any(AbortSignal) }, + ); + expect(events.map((event) => event.eventType)).toEqual([ + "turn.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("parks a provider turn immediately after a durable governed wait appears", async () => { + const yielded: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [ + { + criterionId: "objective", + status: "unknown", + evidenceRefs: ["interaction:pending"], + }, + ], + remainingWork: [ + { description: "Resume after the response.", blocksCompletion: true }, + ], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, + }; + const itemCompleted: PrpEvent = { + ...controlEvent(1, "item.completed", { + kind: "dynamicToolCall", + item: { id: "ask-1", name: "ask_user_questions" }, + }), + sourceEventId: "provider-recovery:1", + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + turnId: "turn-waiting", + }; + const turnInterrupted: PrpEvent = { + ...controlEvent(2, "turn.interrupted", { reason: "governed_wait" }), + sourceEventId: "provider-recovery:2", + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + turnId: "turn-waiting", + }; + let releaseCancelled!: () => void; + const cancelled = new Promise((resolve) => { + releaseCancelled = resolve; + }); + const cancel = vi.fn(() => { + releaseCancelled(); + return { cleanup: Promise.resolve() }; + }); + const events: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield itemCompleted; + await cancelled; + yield turnInterrupted; + }, + async startTurn() { + return { turnId: "turn-waiting" }; + }, + cancel, + async result() { + return null; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "2", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "governed-wait-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + async openSession() { + return session; + }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, + ); + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(replayed), + }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveGovernedWait: ({ event }) => + event.eventType === "item.completed" ? yielded : null, + }); + + expect(cancel).toHaveBeenCalledOnce(); + expect(completed).toMatchObject({ + result: yielded, + terminal: { + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "yielded", + }, + }); + expect(events.map((event) => event.eventType)).toEqual([ + "item.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("hands a committed structured input to the durable wait after its live window", async () => { + vi.useFakeTimers(); + try { + const questionSet = { + schema: "paperclip.question_set.v1" as const, + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "single_select" as const, + options: [{ id: "us", label: "US" }, { id: "eu", label: "Europe" }], + }], + }; + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which region?", + input: questionSet, + origin: { adapter: "mock" }, + turnId: "turn-waiting", + itemId: "input-1", + }; + const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting" }; + const expired = { ...runnerEvent(2, "runtime_request.expired", { + requestId: "input-1", + requestKind: "runtime", + turnId: "turn-waiting", + itemId: "input-1", + reason: "durable_handoff", + replayAllowed: false, + requestType: "input", + request, + }), turnId: "turn-waiting" }; + const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting" }; + let releaseHandoff!: () => void; + const handedOff = new Promise((resolve) => { releaseHandoff = resolve; }); + let releaseCancelled!: () => void; + const cancelled = new Promise((resolve) => { releaseCancelled = resolve; }); + let releaseCreated!: () => void; + const createdCommitted = new Promise((resolve) => { releaseCreated = resolve; }); + const handoffRuntimeRequest = vi.fn(() => { + releaseHandoff(); + return { result: "handed_off" as const, cleanup: Promise.resolve() }; + }); + const cancel = vi.fn(() => { + releaseCancelled(); + return { cleanup: Promise.resolve() }; + }); + const events: PrpEvent[] = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield created; + await handedOff; + yield expired; + await cancelled; + yield interrupted; + }, + async startTurn() { return { turnId: "turn-waiting" }; }, + handoffRuntimeRequest, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "3", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "runtime-input-wait-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + if (event.eventType === "runtime_request.created") releaseCreated(); + const sourceEvents = events.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 120, + resolveGovernedWait: ({ event }) => + event.eventType === "runtime_request.expired" ? yieldedResult : null, + }); + await createdCommitted; + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(119); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + await expect(execution).resolves.toMatchObject({ result: yieldedResult }); + expect(handoffRuntimeRequest).toHaveBeenCalledWith({ + requestId: "input-1", + turnId: "turn-waiting", + reason: "durable_handoff", + signal: expect.any(AbortSignal), + }); + expect(cancel).toHaveBeenCalledOnce(); + expect(events.map((event) => event.eventType)).toContain("runtime_request.expired"); + } finally { + vi.useRealTimers(); + } + }); + + it("aborts and bounds a durable handoff that never settles", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-stalled", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-stalled", + itemId: "input-stalled", + }; + const created = { + ...runnerEvent(1, "runtime_request.created", { request }), + turnId: "turn-stalled", + }; + let releaseEvents = () => {}; + const eventsReleased = new Promise((resolve) => { + releaseEvents = resolve; + }); + let markHandoffStarted = () => {}; + const handoffStarted = new Promise((resolve) => { + markHandoffStarted = resolve; + }); + let handoffSignal: AbortSignal | undefined; + let releaseHandoff = () => {}; + const close = vi.fn(async () => releaseEvents()); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield created; + await eventsReleased; + }, + async startTurn() { + return { turnId: "turn-stalled" }; + }, + handoffRuntimeRequest(input) { + handoffSignal = input.signal; + markHandoffStarted(); + return { + result: "handed_off", + cleanup: new Promise((resolve) => { releaseHandoff = resolve; }), + }; + }, + cancel() { + releaseEvents(); + return { cleanup: Promise.resolve() }; + }, + async result() { + return null; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-stalled", + activeTurnId: "turn-stalled", + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "stalled-handoff-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 1, + timeoutMs: 25, + keepSessionOpen: true, + }); + await handoffStarted; + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + expect(handoffSignal?.aborted).toBe(true); + await expect(execution).rejects.toThrow("native session timed out"); + releaseHandoff(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("preserves terminal success while iterator teardown remains pending", async () => { + vi.useFakeTimers(); + try { + let releaseTeardown = () => {}; + const teardownStarted = vi.fn(); + const close = vi.fn(async () => undefined); + const readResult = vi.fn(async () => ({ + result, + terminal, + turnId: "turn-terminal", + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + try { + yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal" }; + } finally { + teardownStarted(); + await new Promise((resolve) => { + releaseTeardown = resolve; + }); + } + }, + async startTurn() { + return { turnId: "turn-terminal" }; + }, + result: readResult, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-terminal", + cursor: "1", + activeTurnId: null, + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "slow-teardown-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const completeRun = vi.fn(async () => undefined); + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + await vi.waitFor(() => expect(teardownStarted).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(100); + await expect(execution).resolves.toMatchObject({ result }); + expect(readResult).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + releaseTeardown(); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } + }); + + it("preserves terminal success while quarantining stalled handoff cleanup", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-terminal", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-terminal", + itemId: "input-terminal", + }; + let markHandoffStarted = () => {}; + const handoffStarted = new Promise((resolve) => { + markHandoffStarted = resolve; + }); + let releaseHandoff = () => {}; + let handoffSignal: AbortSignal | undefined; + const close = vi.fn(async () => undefined); + const onSession = vi.fn(); + const completeRun = vi.fn(async () => undefined); + const readResult = vi.fn(async () => ({ + result, + terminal, + turnId: "turn-terminal", + })); + const providerEvents = [ + { + ...runnerEvent(1, "runtime_request.created", { request }), + turnId: "turn-terminal", + }, + { ...runnerEvent(2, "turn.completed"), turnId: "turn-terminal" }, + ]; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield providerEvents[0]!; + await handoffStarted; + yield providerEvents[1]!; + }, + async startTurn() { + return { turnId: "turn-terminal" }; + }, + handoffRuntimeRequest(input) { + handoffSignal = input.signal; + markHandoffStarted(); + return { + result: "handed_off", + cleanup: new Promise((resolve) => { + releaseHandoff = resolve; + }), + }; + }, + result: readResult, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-terminal", + cursor: "2", + activeTurnId: null, + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "terminal-handoff-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const appended: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + appended.push(structuredClone(event as PrpEvent)); + return { + cursor: appended.length, + highestContiguousSourceSeq: highestContiguous(appended), + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 1, + keepSessionOpen: true, + onSession, + }); + await handoffStarted; + await vi.waitFor(() => expect(handoffSignal?.aborted).toBe(true)); + await expect(execution).resolves.toMatchObject({ result }); + expect(readResult).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenLastCalledWith(null); + releaseHandoff(); + await Promise.resolve(); + expect(close).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + }); + + it("keeps a settling structured input in the original turn while its append crosses expiry", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-live", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-live", + itemId: "input-live", + }; + const providerEvents = [ + { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live" }, + { + ...runnerEvent(2, "runtime_request.resolved", { + requestId: "input-live", + requestKind: "user_input", + turnId: "turn-live", + itemId: "input-live", + action: "submit", + requestType: "input", + }), + turnId: "turn-live", + }, + { ...runnerEvent(3, "turn.completed"), turnId: "turn-live" }, + ]; + const appended: PrpEvent[] = []; + let markSettlementAppendStarted!: () => void; + const settlementAppendStarted = new Promise((resolve) => { + markSettlementAppendStarted = resolve; + }); + let releaseSettlementAppend!: () => void; + const settlementAppendReleased = new Promise((resolve) => { + releaseSettlementAppend = resolve; + }); + const handoffRuntimeRequest = vi.fn(() => ({ + result: "handed_off" as const, + cleanup: Promise.resolve(), + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { yield* providerEvents; }, + async startTurn() { return { turnId: "turn-live" }; }, + handoffRuntimeRequest, + async result() { return { result, terminal, turnId: "turn-live" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-live", + cursor: "3", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "runtime-input-live-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + if (event.eventType === "runtime_request.resolved") { + markSettlementAppendStarted(); + await settlementAppendReleased; + } + appended.push(structuredClone(event as PrpEvent)); + const sourceEvents = appended.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + return { + cursor: appended.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = appended.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + async completeRun() {}, + }; + + const originalSetTimeout = globalThis.setTimeout; + let queuedHandoffCallback: (() => void) | null = null; + const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback, delay, ...args) => { + if (delay === 123_456) { + queuedHandoffCallback = () => callback(...args); + const handle = originalSetTimeout(() => undefined, 60_000); + handle.unref?.(); + return handle; + } + return originalSetTimeout(callback, delay, ...args); + }) as typeof setTimeout); + try { + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 123_456, + }); + await settlementAppendStarted; + expect(queuedHandoffCallback).not.toBeNull(); + queuedHandoffCallback?.(); + await Promise.resolve(); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + releaseSettlementAppend(); + await expect(execution).resolves.toMatchObject({ result }); + queuedHandoffCallback?.(); + await Promise.resolve(); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + expect(appended.some((event) => event.eventType === "runtime_request.expired")).toBe(false); + } finally { + releaseSettlementAppend(); + timeoutSpy.mockRestore(); + } + }); +}); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts new file mode 100644 index 0000000000..66e4696131 --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -0,0 +1,1205 @@ +import { randomUUID } from "node:crypto"; + +import type { + CheckpointControlPlaneSessionOptions, + ControlPlanePort, +} from "./contracts/control-plane-port.js"; +import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contracts/native-execution.js"; +import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js"; +import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js"; +import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js"; +import { parsePaperclipQuestionSet } from "./contracts/question-set.js"; + +export const DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS = 120_000; +const OPTIONAL_SESSION_CANCELLATION_GRACE_MS = 100; +const FAILED_OPERATION_SETTLEMENT_GRACE_MS = 100; +const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000; + +export interface ExecuteNativeSessionOptions { + input: NativeExecutionInput; + backend: NativeSessionBackend; + controlPlane: ControlPlanePort; + runnerInstanceId: string; + controlPlaneInstanceId: string; + timeoutMs?: number; + /** Internal test seam; production bounds checkpoint persistence to 30 seconds. */ + checkpointTimeoutMs?: number; + /** Internal test seam; production uses the fixed 120-second platform policy. */ + runtimeInputLiveWindowMs?: number; + onSession?: (session: NativeSession | null) => void; + existingSession?: NativeSession; + persistedSession?: PersistedNativeSession | null; + keepSessionOpen?: boolean; + onCheckpoint?: ( + snapshot: PersistedNativeSession, + options?: CheckpointControlPlaneSessionOptions, + ) => Promise | void; + /** Called when exact provider recovery failed and policy opened a new provider session. */ + onContinuityBreak?: (input: { + reason: string; + previousDriverSessionId: string; + previousProviderSessionId: string | null; + replacementDriverSessionId: string; + replacementProviderSessionId: string | null; + }) => Promise | void; + /** + * Control-plane policy seam for a provider turn that completed after + * durably creating a governed wait, but did not emit a semantic finish + * result. The runner package cannot inspect server-owned interactions, so + * it asks the embedding control plane whether that missing result is an + * intentional yield before treating it as provider failure. + */ + resolveMissingResult?: (input: { + turnId: string | null; + terminalEvent: PrpEvent; + }) => Promise; + /** + * Detect a durable server-owned wait as soon as its provider tool event is + * committed. Models are not trusted to stop or avoid polling after creating + * a question/review interaction; the control plane may park the turn here. + * This boundary is deliberately synchronous and observational: asynchronous + * mutation authority cannot be revoked safely after a failed execution. + */ + resolveGovernedWait?: (input: { + turnId: string | null; + event: PrpEvent; + }) => PrpStructuredRunResult | null; +} + +function isTurnTerminal(event: PrpEvent): boolean { + return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType); +} + +function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["reportedWorkDisposition"]): PrpTerminalState { + const states = event.eventType === "turn.completed" + ? { turnTerminalState: "completed" as const, runTerminalState: "succeeded" as const } + : event.eventType === "turn.failed" + ? { turnTerminalState: "failed" as const, runTerminalState: "failed" as const } + : event.eventType === "turn.interrupted" + ? { turnTerminalState: "interrupted" as const, runTerminalState: "cancelled" as const } + : { turnTerminalState: "cancelled" as const, runTerminalState: "cancelled" as const }; + return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition }; +} + +async function attemptOptionalSessionCancellation( + session: NativeSession, + reason: string, +): Promise<{ settlement: Promise[]> } | null> { + if (session.cancel === undefined) return null; + const cancellationAbort = new AbortController(); + let cleanup: Promise; + try { + // The session commits cancellation before returning. Only provider + // cleanup remains asynchronous, so bounded failure settlement cannot + // leave an operation with accepted-output or mutation authority. + cleanup = session.cancel({ + reason, + signal: cancellationAbort.signal, + }).cleanup; + } catch { + return null; + } + const attempts = Promise.allSettled([cleanup]); + if (await settlesWithin(attempts, OPTIONAL_SESSION_CANCELLATION_GRACE_MS)) { + return null; + } + cancellationAbort.abort(new Error("native session cancellation grace expired")); + return { settlement: attempts }; +} + +async function settlesWithin(operation: Promise, timeoutMs: number): Promise { + let graceTimer: ReturnType | undefined; + const settled = await Promise.race([ + operation.then(() => true), + new Promise((resolve) => { + graceTimer = setTimeout(() => resolve(false), timeoutMs); + graceTimer.unref?.(); + }), + ]); + if (graceTimer !== undefined) clearTimeout(graceTimer); + return settled; +} + +async function runAbortableOperationWithin(input: { + timeoutMs: number; + timeoutMessage: string; + timeoutError?: () => Error; + operation: (signal: AbortSignal) => Promise; + onLateResolution?: (value: T) => Promise | void; +}): Promise { + const operationAbort = new AbortController(); + let timer: ReturnType | undefined; + let timedOut = false; + let timeoutError: Error | undefined; + const operation = input.operation(operationAbort.signal); + // The timeout path deliberately stops awaiting an uncooperative adapter. + // Keep its eventual settlement observed after mutation authority is revoked. + void operation.catch(() => undefined); + if (input.onLateResolution !== undefined) { + void operation.then( + (value) => { + if (!timedOut) return; + void Promise.resolve() + .then(() => input.onLateResolution!(value)) + .catch(() => undefined); + }, + () => undefined, + ); + } + try { + const value = await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = input.timeoutError?.() ?? new Error(input.timeoutMessage); + timedOut = true; + timeoutError = error; + reject(error); + operationAbort.abort(error); + }, input.timeoutMs); + timer.unref?.(); + }), + ]); + // An abort listener can resolve synchronously before the timeout promise's + // rejection wins the race. The deadline still owns that boundary: the late + // result is disposed above and must never be admitted by the caller. + if (timedOut) throw timeoutError; + return value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +async function disposeUnadmittedSession( + session: NativeSession, + reason: string, +): Promise { + // A provider may ignore abort and return a session after its caller has + // already timed out. That session was never published through onSession, so + // close it without clearing ownership that a later execution may establish. + const closeSettlement = Promise.allSettled([ + Promise.resolve().then(() => session.close({ reason })), + ]); + await settlesWithin(closeSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); +} + +class NativeSessionFinalizationTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`native session finalization timed out after ${timeoutMs}ms`); + this.name = "NativeSessionFinalizationTimeoutError"; + } +} + +async function finalizeWithin(input: { + timeoutMs: number; + operation: (signal: AbortSignal) => Promise; +}): Promise { + return runAbortableOperationWithin({ + ...input, + timeoutMessage: `native session finalization timed out after ${input.timeoutMs}ms`, + timeoutError: () => new NativeSessionFinalizationTimeoutError(input.timeoutMs), + }); +} + +async function finalizeIdempotentControlPlaneWithin(input: { + timeoutMs: number; + operation: (signal: AbortSignal) => Promise; +}): Promise { + try { + return await finalizeWithin(input); + } catch (error) { + if (!(error instanceof NativeSessionFinalizationTimeoutError)) throw error; + // Only the deterministic control-plane transaction enters this retry. + // Its event ids and completion dedupe key make the second settlement an + // authoritative acknowledgement of any first-attempt commit. Provider + // result resolution and checkpointing are deliberately outside this + // boundary and are never started a second time. + return finalizeWithin(input); + } +} + +async function quarantineRetainedSession( + session: NativeSession, + onSession: ExecuteNativeSessionOptions["onSession"], + reason: string, +): Promise { + // Eviction and provider cleanup are independent obligations. Keep both + // observed so a throwing owner callback cannot prevent close from starting, + // and a broken provider cannot keep the failed execution pending forever. + const quarantineSettlement = Promise.allSettled([ + Promise.resolve().then(() => onSession?.(null)), + Promise.resolve().then(() => session.close({ reason })), + ]); + await settlesWithin( + quarantineSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); +} + +async function persistCheckpointWithin(input: { + snapshot: PersistedNativeSession; + controlPlane: ControlPlanePort; + onCheckpoint: ExecuteNativeSessionOptions["onCheckpoint"]; + timeoutMs: number; + externalSignal?: AbortSignal; +}): Promise { + const checkpointAbort = new AbortController(); + const externalSignal = input.externalSignal; + let timer: ReturnType | undefined; + let removeExternalAbort = () => {}; + const externalAbortFailure = externalSignal + ? new Promise((_resolve, reject) => { + const abort = () => { + const reason = externalSignal.reason + ?? new Error("native session checkpoint aborted"); + checkpointAbort.abort(reason); + reject(reason); + }; + if (externalSignal.aborted) { + abort(); + } else { + externalSignal.addEventListener("abort", abort, { once: true }); + removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + } + }) + : new Promise(() => undefined); + const checkpointing = (async () => { + const checkpointOptions = { signal: checkpointAbort.signal }; + await input.controlPlane.checkpointSession?.( + input.snapshot, + checkpointOptions, + ); + if (checkpointAbort.signal.aborted) { + throw checkpointAbort.signal.reason + ?? new Error("native session checkpoint aborted"); + } + await input.onCheckpoint?.(input.snapshot, checkpointOptions); + })(); + // The timeout path intentionally stops awaiting an uncooperative adapter. + // Keep its eventual rejection observed after execution has quarantined and + // closed the provider session. + void checkpointing.catch(() => undefined); + try { + await Promise.race([ + checkpointing, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new Error( + `native session checkpoint timed out after ${input.timeoutMs}ms`, + ); + checkpointAbort.abort(error); + reject(error); + }, input.timeoutMs); + }), + externalAbortFailure, + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + removeExternalAbort(); + } +} + +async function consumeTurn( + session: NativeSession, + controlPlane: ControlPlanePort, + timeoutMs: number, + runtimeInputLiveWindowMs: number, + closeFailedSession: () => Promise, + quarantineSession: () => void, + resolveGovernedWait?: ExecuteNativeSessionOptions["resolveGovernedWait"], + externalSignal?: AbortSignal, +) { + let timer: ReturnType | undefined; + const appendAbort = new AbortController(); + const governedCleanupOperations = new Set>(); + let governedCancellationCommitted = false; + let deferredGovernedCleanupSettlement: Promise | null = null; + let deferredSessionCancellationSettlement: Promise | null = null; + const inputTimers = new Map>(); + const handoffCleanupOperations = new Set>(); + const eventIterator = session.events()[Symbol.asyncIterator](); + let stopConsumer = false; + let rejectHandoff: ((error: unknown) => void) | null = null; + const handoffFailure = new Promise((_resolve, reject) => { + rejectHandoff = reject; + }); + let removeExternalAbort = () => {}; + const externalAbortFailure = externalSignal + ? new Promise((_resolve, reject) => { + const abort = () => { + const reason = externalSignal.reason ?? new Error("native event consumption aborted"); + stopConsumer = true; + appendAbort.abort(reason); + reject(reason); + }; + if (externalSignal.aborted) { + abort(); + } else { + externalSignal.addEventListener("abort", abort, { once: true }); + removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + } + }) + : new Promise(() => undefined); + const clearInputTimer = (requestId: string) => { + const inputTimer = inputTimers.get(requestId); + if (inputTimer !== undefined) clearTimeout(inputTimer); + inputTimers.delete(requestId); + }; + const consumer = (async () => { + let eventCount = 0; + let highestContiguousSourceSeq = 0; + let governedResult: PrpStructuredRunResult | null = null; + while (true) { + const next = await eventIterator.next(); + if (stopConsumer) throw new Error("native event consumer stopped"); + if (next.done) throw new Error("native event stream closed before a turn terminal fact"); + const event = next.value; + const payload = event.payload as Record; + const settlingRequestId = + ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"] + .includes(event.eventType) + && typeof payload.requestId === "string" + ? payload.requestId + : null; + // Beginning settlement revokes the expiry timer's handoff authority. + // appendEvent may remain pending across the live-window deadline; if + // the timer stayed live until the receipt returned, both settlement + // and a durable handoff could commit for the same request. + if (settlingRequestId !== null) clearInputTimer(settlingRequestId); + const receipt = await controlPlane.appendEvent(event, { + signal: appendAbort.signal, + }); + if (stopConsumer) throw new Error("native event consumer stopped"); + eventCount += receipt.disposition === "committed" ? 1 : 0; + highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); + const request = payload.request && typeof payload.request === "object" && !Array.isArray(payload.request) + ? payload.request as Record + : null; + if ( + receipt.disposition === "committed" + && event.eventType === "runtime_request.created" + && request?.schema === "paperclip.runtime_request.v2" + && request.type === "input" + && typeof request.requestId === "string" + && typeof request.turnId === "string" + ) { + try { + parsePaperclipQuestionSet(request.input); + const requestId = request.requestId; + const turnId = request.turnId; + clearInputTimer(requestId); + const inputTimer = setTimeout(() => { + // Clearing a timeout does not revoke a callback that is already + // queued. The map entry is the per-request authority token. + if (inputTimers.get(requestId) !== inputTimer) return; + inputTimers.delete(requestId); + // A timer callback can already be queued when teardown clears + // its handle. Re-check the live-turn authority inside the + // callback before starting or registering durable work. + if (stopConsumer || appendAbort.signal.aborted) return; + if (session.handoffRuntimeRequest === undefined) { + rejectHandoff?.(new Error("native_runtime_request_handoff_unavailable")); + return; + } + let handoffCleanup: Promise; + try { + const handoff = session.handoffRuntimeRequest({ + requestId, + turnId, + reason: "durable_handoff", + signal: appendAbort.signal, + }); + // Durable handoff mutation is synchronous. The returned + // promise owns provider interruption only, so it can remain + // observed without acquiring authority to delay or reverse + // a provider terminal fact. + handoffCleanup = handoff.cleanup; + } catch (error) { + rejectHandoff?.(error); + return; + } + handoffCleanupOperations.add(handoffCleanup); + void handoffCleanup + .catch((error) => { + if (!stopConsumer && !appendAbort.signal.aborted) { + rejectHandoff?.(error); + } + }) + .finally(() => handoffCleanupOperations.delete(handoffCleanup)); + }, runtimeInputLiveWindowMs); + inputTimer.unref?.(); + inputTimers.set(requestId, inputTimer); + } catch { + // Invalid structured inputs remain rejected by the driver and never become durable questions. + } + } + if (governedResult === null && resolveGovernedWait) { + if (appendAbort.signal.aborted) { + throw appendAbort.signal.reason ?? new Error("native event consumption aborted"); + } + governedResult = resolveGovernedWait({ + turnId: event.turnId ?? null, + event, + }); + if (governedResult !== null && !isTurnTerminal(event)) { + if (session.cancel === undefined) { + throw new Error("native_governed_wait_cancellation_unavailable"); + } + // A governed result is already durable. Commit cancellation + // synchronously, then stop consuming provider output now rather + // than waiting for an abort-insensitive cleanup or terminal event. + // The returned promise owns cleanup only and remains observed in + // finally, where its wait is bounded and the session quarantined. + const cancellation = session.cancel({ + reason: "Paperclip parked this turn on a durable governed interaction.", + signal: appendAbort.signal, + }); + governedCancellationCommitted = true; + const cleanup = cancellation.cleanup; + governedCleanupOperations.add(cleanup); + void cleanup + .catch(() => quarantineSession()) + .finally(() => governedCleanupOperations.delete(cleanup)); + return { event, eventCount, highestContiguousSourceSeq, governedResult }; + } + } + if (isTurnTerminal(event)) { + return { event, eventCount, highestContiguousSourceSeq, governedResult }; + } + } + })(); + // A timeout can win the race while an iterator is still waiting for data. + // Observe any later consumer rejection so it cannot become process-fatal. + void consumer.catch(() => undefined); + let consumptionFailed = false; + try { + return await Promise.race([ + consumer, + new Promise((_, reject) => { + timer = setTimeout(() => { + reject(new Error(`native session timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }), + handoffFailure, + externalAbortFailure, + ]); + } catch (error) { + consumptionFailed = true; + stopConsumer = true; + appendAbort.abort(error); + for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer); + inputTimers.clear(); + // Governed-wait discovery is synchronous and observational, and provider + // cancellation has already committed synchronously. Bound only the + // authority-free provider cleanup while keeping its outcome observed. + if (governedCleanupOperations.size > 0) { + const governedCleanupSettlement = Promise.allSettled([...governedCleanupOperations]); + if (!(await settlesWithin(governedCleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS))) { + deferredGovernedCleanupSettlement = governedCleanupSettlement; + } + } + if (!governedCancellationCommitted) { + const deferredCancellation = await attemptOptionalSessionCancellation( + session, + "Native session event consumption failed.", + ); + deferredSessionCancellationSettlement = deferredCancellation?.settlement ?? null; + } + throw error; + } finally { + stopConsumer = true; + for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer); + inputTimers.clear(); + const activeHandoffCleanupSettlement = handoffCleanupOperations.size > 0 + ? Promise.allSettled([...handoffCleanupOperations]) + : null; + const activeGovernedCleanupSettlement = deferredGovernedCleanupSettlement === null + && governedCleanupOperations.size > 0 + ? Promise.allSettled([...governedCleanupOperations]) + : null; + if (!consumptionFailed && !appendAbort.signal.aborted) { + // A provider terminal or synchronous governed cancellation revokes + // live-turn authority. Handoff state was already committed; abort only + // tells provider cleanup that it must not begin any new work. + appendAbort.abort(new Error("native turn reached a terminal state")); + } + // Do not let failure escape while the provider iterator still owns a live + // subscription. Cancellation above is responsible for releasing a blocked + // `next()`; awaiting `return()` then synchronizes the iterator's `finally` + // teardown before the session can be closed or reused. + const iteratorTeardown = eventIterator.return?.().catch(() => undefined); + // The consumer may already be past `next()` and awaiting a durable append. + // Abort is a control-plane durability boundary: appendEvent must settle + // without committing when its signal is aborted. Handoff and cancellation + // promises below own provider cleanup only; their durable transitions were + // synchronous, so a slow cleanup cannot reverse terminal completion. + const passiveTeardownSettlement = Promise.allSettled([ + iteratorTeardown, + consumer, + ...(activeHandoffCleanupSettlement ? [activeHandoffCleanupSettlement] : []), + ...(activeGovernedCleanupSettlement ? [activeGovernedCleanupSettlement] : []), + ...(deferredGovernedCleanupSettlement ? [deferredGovernedCleanupSettlement] : []), + ...(deferredSessionCancellationSettlement + ? [deferredSessionCancellationSettlement] + : []), + ]); + if (consumptionFailed) { + // Start provider close immediately so a cooperative implementation can + // release a blocked iterator or provider operation. Abort has already + // revoked every control-plane mutation capability and closeSession has + // removed this session from the caller, so an implementation that + // violates its cancellation contract is quarantined rather than allowed + // to defeat the execution deadline. Promise.allSettled keeps every late + // rejection observed after the bounded wait expires. + const cleanupSettlement = Promise.allSettled([ + passiveTeardownSettlement, + Promise.resolve().then(closeFailedSession), + ]); + await settlesWithin(cleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); + } else { + // Iterator and provider cleanup own no control-plane mutation authority. + // A slow subscription or cleanup remains observed and is released by the + // normal session close, but it cannot erase an already committed + // terminal fact or prevent result retrieval and durable finalization. + const teardownSettled = await settlesWithin( + passiveTeardownSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); + if (!teardownSettled) quarantineSession(); + } + if (timer !== undefined) clearTimeout(timer); + removeExternalAbort(); + } +} + +function checkpointCursor(cursor: string | null | undefined): number { + if (cursor === undefined || cursor === null || cursor === "") return 0; + const parsed = Number(cursor); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +async function reconcileRecoveryCursor(input: { + controlPlane: ControlPlanePort; + checkpoint: PersistedNativeSession; + runId: string; + sourceInstanceId: string; + signal: AbortSignal; +}): Promise { + const checkpointHighWater = checkpointCursor(input.checkpoint.cursor); + let afterSourceSeq = checkpointHighWater; + let persistedHighWater = checkpointHighWater; + while (true) { + const replay = await input.controlPlane.replayEvents( + { + runId: input.runId, + sourceInstanceId: input.sourceInstanceId, + afterSourceSeq, + limit: 1_000, + }, + { signal: input.signal }, + ); + input.signal.throwIfAborted(); + if (replay.events.length === 0) break; + const pageHighWater = replay.events.reduce( + (highest, event) => Math.max(highest, event.sourceSeq), + afterSourceSeq, + ); + if (pageHighWater <= afterSourceSeq) { + throw new Error("native_recovery_replay_did_not_advance"); + } + persistedHighWater = Math.max(persistedHighWater, pageHighWater); + afterSourceSeq = pageHighWater; + } + if (persistedHighWater === checkpointHighWater && input.checkpoint.cursor === String(checkpointHighWater)) { + return input.checkpoint; + } + return { ...input.checkpoint, cursor: String(persistedHighWater) }; +} + +/** + * Package-owned normalized session loop. Paperclip supplies persistence and + * authority through ControlPlanePort; provider/session behavior stays here. + */ +export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise { + const input = parseNativeExecutionInput(options.input); + const descriptor = await options.backend.descriptor(); + if ("runtimeContext" in input) { + const capabilities = descriptor.runtimeContextCapabilities; + const unsupported = (["instructions", "skills", "mcp"] as const).filter((key) => capabilities?.[key] !== "native"); + if (unsupported.length) throw new Error(`native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`); + } + let persistedSession = options.existingSession + ? null + : options.persistedSession ?? await options.controlPlane.loadSessionCheckpoint?.() ?? null; + if ( + persistedSession + && ( + persistedSession.identity.runId !== input.binding.runId + || persistedSession.identity.companyId !== input.binding.companyId + || persistedSession.identity.issueId !== input.binding.issueId + || persistedSession.identity.agentId !== input.binding.agentId + || input.session.normalizedSessionId === null + || persistedSession.identity.sessionId !== input.session.normalizedSessionId + ) + ) throw new Error("native_session_checkpoint_binding_mismatch"); + const existingIdentity = options.existingSession?.identity() ?? null; + if ( + existingIdentity + && ( + existingIdentity.companyId !== input.binding.companyId + || existingIdentity.issueId !== input.binding.issueId + || existingIdentity.agentId !== input.binding.agentId + || input.session.normalizedSessionId === null + || existingIdentity.sessionId !== input.session.normalizedSessionId + ) + ) throw new Error("native_session_attach_binding_mismatch"); + const normalizedSessionId = persistedSession?.identity.sessionId + ?? existingIdentity?.sessionId + ?? input.session.normalizedSessionId + ?? randomUUID(); + const identity = { + runId: input.binding.runId, + sessionId: normalizedSessionId, + companyId: input.binding.companyId, + issueId: input.binding.issueId, + agentId: input.binding.agentId, + }; + let recovered = false; + let session: NativeSession | null = null; + let continuityBreak: { + reason: string; + previousDriverSessionId: string; + previousProviderSessionId: string | null; + } | null = null; + let reconciledRecoveryCheckpoint: PersistedNativeSession | null = null; + if (options.existingSession) { + if (options.existingSession.attachRun === undefined) { + throw new Error("native_session_multi_run_unavailable"); + } + // Attaching can fail even after the retained session's identity passes the + // static binding check (for example, when the provider lost multi-run + // state). Prove the provider attachment before opening durable + // control-plane state because ControlPlanePort has no rollback operation. + try { + await options.existingSession.attachRun({ identity }); + } catch (error) { + // attachRun has no transactional guarantee: a provider may bind the new + // run before reporting a later failure. Conservatively quarantine the + // session so neither the old nor partially attached run can reuse it. + await quarantineRetainedSession( + options.existingSession, + options.onSession, + "native session attachment failed", + ); + throw error; + } + session = options.existingSession; + recovered = true; + } else if (persistedSession) { + // A persisted checkpoint proves that this is recovery of an existing + // durable run. Reconcile its cursor and prove provider continuity before + // re-opening that run in the control plane: ControlPlanePort has no + // rollback operation if same-session recovery fails. + const recoveryCheckpoint = persistedSession; + const recoveryTimeoutMs = options.timeoutMs ?? 900_000; + persistedSession = await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => reconcileRecoveryCursor({ + controlPlane: options.controlPlane, + checkpoint: recoveryCheckpoint, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + signal, + }), + }); + reconciledRecoveryCheckpoint = persistedSession; + const providerRecoveryCheckpoint = persistedSession; + + const replacementAllowed = + providerRecoveryCheckpoint.providerRecoveryPolicy === + "allow_replacement_after_resume_failure"; + const recovery = options.backend.recoverSession + ? await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => options.backend.recoverSession!( + providerRecoveryCheckpoint, + { signal }, + ), + onLateResolution: async (lateRecovery) => { + if (lateRecovery.session) { + await disposeUnadmittedSession( + lateRecovery.session, + "native session provider recovery timed out", + ); + } + }, + }) + : { recovered: false as const, reason: "driver does not support recovery" }; + if (!recovery.recovered || !recovery.session) { + if (!replacementAllowed) { + throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`); + } + continuityBreak = { + reason: recovery.reason ?? "provider session is no longer recoverable", + previousDriverSessionId: providerRecoveryCheckpoint.sessionId, + previousProviderSessionId: providerRecoveryCheckpoint.providerSessionId ?? null, + }; + const replacementInput = { + identity, + workingDirectory: input.workspace.cwd, + }; + session = await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session replacement bootstrap timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => { + const abortableReplacementInput = { ...replacementInput, signal }; + return options.backend.openReplacementSession + ? options.backend.openReplacementSession( + abortableReplacementInput, + providerRecoveryCheckpoint, + ) + : options.backend.openSession(abortableReplacementInput); + }, + onLateResolution: (lateSession) => disposeUnadmittedSession( + lateSession, + "native session replacement bootstrap timed out", + ), + }); + } else { + session = recovery.session; + recovered = true; + } + } + if (session === null) { + // A fresh provider session is part of admission. Prove that it exists + // before opening durable run state because ControlPlanePort intentionally + // exposes no rollback for an admitted run. + const bootstrapTimeoutMs = options.timeoutMs ?? 900_000; + const bootstrapInput = { + identity, + workingDirectory: input.workspace.cwd, + }; + session = await runAbortableOperationWithin({ + timeoutMs: bootstrapTimeoutMs, + timeoutMessage: `native session bootstrap timed out after ${bootstrapTimeoutMs}ms`, + operation: (signal) => options.backend.openSession({ + ...bootstrapInput, + signal, + }), + onLateResolution: (lateSession) => disposeUnadmittedSession( + lateSession, + "native session bootstrap timed out", + ), + }); + } + try { + await options.controlPlane.openRun({ + identity, + backendKind: descriptor.kind, + sourceInstanceId: options.runnerInstanceId, + }); + } catch (error) { + if (session) { + // Attachment and recovery both establish provider-side authority before + // durable admission. If admission then fails, the prepared session must + // not remain available for reuse. + await quarantineRetainedSession( + session, + options.onSession, + "native control-plane run admission failed", + ); + } + throw error; + } + let sessionClosePromise: Promise | null = null; + let sessionQuarantined = false; + const quarantineSession = () => { + if (sessionQuarantined) return; + sessionQuarantined = true; + try { + options.onSession?.(null); + } catch { + // Owner notification cannot prevent provider cleanup. + } + }; + const closeSession = () => { + if (sessionClosePromise === null) { + quarantineSession(); + sessionClosePromise = session.close({ + reason: "native session execution complete", + }); + } + return sessionClosePromise; + }; + let executionSucceeded = false; + try { + // Ownership publication is part of the execution-owned lifetime. If the + // callback fails, the finally block below still quarantines and closes the + // provider session. + options.onSession?.(session); + const checkpointTimeoutMs = options.checkpointTimeoutMs + ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS; + const persistCheckpoint = ( + snapshot: PersistedNativeSession, + externalSignal?: AbortSignal, + ) => + persistCheckpointWithin({ + snapshot, + controlPlane: options.controlPlane, + onCheckpoint: options.onCheckpoint, + timeoutMs: checkpointTimeoutMs, + externalSignal, + }); + // Cursor reconciliation is provisional until both provider recovery and + // durable run admission succeed. Persist it only after those boundaries; + // the execution-owned finally below quarantines the recovered session if + // checkpoint persistence itself fails. + if (reconciledRecoveryCheckpoint !== null) { + await persistCheckpoint(reconciledRecoveryCheckpoint); + } + const checkpoint = async (signal?: AbortSignal) => { + const snapshot = await session.snapshot(signal ? { signal } : undefined); + signal?.throwIfAborted(); + await persistCheckpoint(snapshot, signal); + }; + const recoveredSnapshot = await session.snapshot(); + if (continuityBreak) { + await options.onContinuityBreak?.({ + ...continuityBreak, + replacementDriverSessionId: recoveredSnapshot.sessionId, + replacementProviderSessionId: + recoveredSnapshot.providerSessionId ?? null, + }); + } + await persistCheckpoint(recoveredSnapshot); + + let consumed = { + event: null as PrpEvent | null, + eventCount: 0, + highestContiguousSourceSeq: 0, + governedResult: null as PrpStructuredRunResult | null, + }; + const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal + ? recoveredSnapshot + : persistedSession; + let completed = completionSnapshot?.semanticResult && completionSnapshot.terminal + ? { + result: completionSnapshot.semanticResult, + terminal: completionSnapshot.terminal, + turnId: completionSnapshot.activeTurnId ?? null, + } + : null; + if (!completed) { + const consumptionAbort = new AbortController(); + const consuming = consumeTurn( + session, + options.controlPlane, + options.timeoutMs ?? 900_000, + options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, + closeSession, + quarantineSession, + options.resolveGovernedWait, + consumptionAbort.signal, + ); + // Event consumption must begin before startTurn so an eager provider cannot + // outrun us. Observe its rejection immediately, though: if startTurn or + // checkpointing fails first, the outer finally closes the session and the + // abandoned consumer will reject when its stream closes. Without a handler + // that later rejection becomes process-fatal under Node's strict policy. + void consuming.catch(() => undefined); + // A recovered driver is authoritative about whether a provider turn is + // still active. In particular, drivers normalize the checkpoint race + // where a terminal fingerprint was persisted before activeTurnId was + // cleared. Falling back to the older control-plane checkpoint here + // resurrects that terminal turn and waits forever for an event that was + // already consumed. + const recoveredActiveTurnId = recovered + ? recoveredSnapshot.activeTurnId ?? null + : persistedSession?.activeTurnId ?? null; + try { + if (!recovered || !recoveredActiveTurnId) { + const modelEnvelope = buildNativeModelEnvelope(input); + const dispositionOnlyRecovery = Boolean( + recovered && + !recoveredSnapshot.semanticResult && + (persistedSession?.terminalTurns?.length ?? 0) > 0 && + !recoveredActiveTurnId + ); + if (dispositionOnlyRecovery) { + modelEnvelope.task.prompt = [ + "Paperclip semantic-result recovery for a prior completed provider turn.", + "The prior turn already performed the work and its user-facing final answer is recorded.", + "Do not repeat implementation, tests, research, or the final answer.", + "Use the existing session context to invoke exactly one paperclip_finish or paperclip_block with the accurate current disposition, then stop without additional user-facing prose.", + ].join("\n"); + } + await session.startTurn({ + message: { role: "user", text: JSON.stringify(modelEnvelope) }, + requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default", + }); + await checkpoint(); + } + } catch (error) { + // Consumption starts before provider launch so eager events cannot be + // lost. If launch or its checkpoint fails, abort any in-flight append + // before joining cleanup so the failed turn cannot commit late or + // strand execution on a never-settling durability call. + consumptionAbort.abort(error); + await consuming.catch(() => undefined); + throw error; + } + const terminalEvent = await consuming; + consumed = terminalEvent; + } + const finalizationTimeoutMs = options.timeoutMs ?? 900_000; + const preparedFinalization = await finalizeWithin({ + timeoutMs: finalizationTimeoutMs, + operation: async (signal) => { + let settledCompletion = completed; + if (settledCompletion === null) { + const terminalEvent = consumed.event; + if (terminalEvent === null) { + throw new Error("native_finalization_missing: session returned no terminal event"); + } + settledCompletion = consumed.governedResult === null + ? await session.result() + : { + result: consumed.governedResult, + terminal: { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: + consumed.governedResult.reportedWorkDisposition, + }, + turnId: terminalEvent.turnId ?? null, + }; + signal.throwIfAborted(); + if (settledCompletion === null && options.resolveMissingResult) { + const recoveredResult = await options.resolveMissingResult({ + turnId: terminalEvent.turnId ?? null, + terminalEvent, + }); + signal.throwIfAborted(); + if (recoveredResult !== null) { + settledCompletion = { + result: recoveredResult, + terminal: terminalFromEvent( + terminalEvent, + recoveredResult.reportedWorkDisposition, + ), + turnId: terminalEvent.turnId ?? null, + }; + } + } + // Do not publish the resolved result to the retryable phase until its + // checkpoint has settled. A timed-out checkpoint therefore cannot be + // skipped by a second attempt. + await checkpoint(signal); + signal.throwIfAborted(); + completed = settledCompletion; + } + if (settledCompletion === null) { + throw new Error("native_finalization_missing: session returned no semantic result"); + } + let terminal: PrpTerminalState; + if (consumed.governedResult !== null) { + terminal = settledCompletion.terminal; + } else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) { + terminal = completionSnapshot.terminal; + } else { + terminal = terminalFromEvent( + consumed.event!, + settledCompletion.result.reportedWorkDisposition, + ); + } + const eventTurnId = settledCompletion.turnId + ?? persistedSession?.activeTurnId + ?? persistedSession?.terminalTurns?.at(-1)?.turnId + ?? consumed.event?.turnId; + const controlEvent = ( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, + ): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: options.controlPlaneInstanceId, + sourceKind: "control_plane", + runId: input.binding.runId, + normalizedSessionId, + ...(eventTurnId ? { turnId: eventTurnId } : {}), + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload, + }); + return { + completed: settledCompletion, + terminal, + expectedControlEvents: [ + controlEvent(1, "run.result.accepted", { + result: settledCompletion.result, + }), + controlEvent( + 2, + "run.terminal", + terminal as unknown as Record, + ), + ], + }; + }, + }); + const baselineControlEventSequences = new Set(); + const accountedControlEventSequences = new Set(); + let baselineControlReplayCaptured = false; + const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({ + timeoutMs: finalizationTimeoutMs, + operation: async (signal) => { + const controlReplay = await options.controlPlane.replayEvents({ + runId: input.binding.runId, + sourceInstanceId: options.controlPlaneInstanceId, + afterSourceSeq: 0, + limit: 10, + }, { signal }); + signal.throwIfAborted(); + const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event])); + for (const existing of controlReplay.events) { + const expected = preparedFinalization.expectedControlEvents[existing.sourceSeq - 1]; + if ( + expected === undefined + || existing.eventType !== expected.eventType + || canonicalJson(existing.payload) !== canonicalJson(expected.payload) + ) { + throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`); + } + } + if (!baselineControlReplayCaptured) { + for (const existing of controlReplay.events) { + baselineControlEventSequences.add(existing.sourceSeq); + } + baselineControlReplayCaptured = true; + } else { + for (const existing of controlReplay.events) { + if ( + !baselineControlEventSequences.has(existing.sourceSeq) + && !accountedControlEventSequences.has(existing.sourceSeq) + ) { + accountedControlEventSequences.add(existing.sourceSeq); + consumed.eventCount += 1; + } + } + } + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + controlReplay.highestContiguousSourceSeq, + ); + for (const event of preparedFinalization.expectedControlEvents) { + if (replayBySequence.has(event.sourceSeq)) continue; + const receipt = await options.controlPlane.appendEvent(event, { signal }); + signal.throwIfAborted(); + if ( + receipt.disposition === "committed" + && !accountedControlEventSequences.has(event.sourceSeq) + ) { + accountedControlEventSequences.add(event.sourceSeq); + consumed.eventCount += 1; + } + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + receipt.highestContiguousSourceSeq, + ); + } + await options.controlPlane.completeRun({ + result: preparedFinalization.completed.result, + terminal: preparedFinalization.terminal, + turnId: preparedFinalization.completed.turnId, + callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`, + callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`, + }, { signal }); + return { + result: preparedFinalization.completed.result, + terminal: preparedFinalization.terminal, + turnId: preparedFinalization.completed.turnId, + normalizedSessionId, + driverKind: descriptor.name, + nativeEventCount: consumed.eventCount, + highestContiguousSourceSeq: consumed.highestContiguousSourceSeq, + }; + }, + }); + let enrichment: { + providerSessionId: string | null; + driverVersion: string; + usage: Record | null; + }; + try { + enrichment = await finalizeWithin({ + timeoutMs: options.timeoutMs ?? 900_000, + operation: async (signal) => { + const snapshot = await session.snapshot({ signal }); + signal.throwIfAborted(); + const completedSnapshot = { + ...snapshot, + semanticResult: durableExecutionResult.result, + terminal: durableExecutionResult.terminal, + }; + await persistCheckpoint(completedSnapshot, signal); + signal.throwIfAborted(); + const usage = await session.usage?.() ?? null; + signal.throwIfAborted(); + return { + providerSessionId: snapshot.providerSessionId ?? null, + driverVersion: typeof usage?.driverVersion === "string" + ? usage.driverVersion + : descriptor.version, + usage, + }; + }, + }); + } catch { + // completeRun is the durable commit boundary. Snapshot/checkpoint/usage + // enrichment cannot revoke that success, but a session whose final + // checkpoint is unknown must not remain available for reuse. + void closeSession().catch(() => undefined); + enrichment = { + providerSessionId: recoveredSnapshot.providerSessionId ?? null, + driverVersion: descriptor.version, + usage: null, + }; + } + executionSucceeded = true; + return { ...durableExecutionResult, ...enrichment }; + } finally { + if (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) { + // A provider that ignores close must not keep execution pending forever. + // closeSession removes it from the caller before invoking the backend; + // retain observation of the promise, but bound the final join. Provider + // cleanup cannot reverse a result the control plane already committed; + // after that durable boundary the session remains unavailable for reuse + // and late close rejection stays observed without contradicting success. + const closeSettlement = Promise.allSettled([closeSession()]); + await settlesWithin( + closeSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); + } + } +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index 68ab11bfac..cfc6a4bb0b 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -6,5 +6,7 @@ * accidental production dependency. */ export * from "./index.js"; +export * from "./conformance/control-plane-port.js"; +export * from "./conformance/harness-driver.js"; export * from "./conformance/semantic-conformance.js"; export * from "./protocol/replay-loader.js";