From d18e281c332ffc89693cbb2f529c19af40187bf9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:42:48 -0500 Subject: [PATCH] Add the native runner session runtime (#12352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The Paperclip Runner package needs one provider-neutral session loop > - Native execution contracts now define the accepted input and output > - Backends still need bounded execution, recovery, and control-plane ports > - This pull request adds that package-local session runtime > - It does not change server runtime selection or start the experimental runner > - The benefit is a tested execution core for later provider and server layers ## Linked Issues or Issue Description **Subsystem affected** packages/paperclip-runner **Problem or motivation** The runner contracts do not yet have a shared session loop. Each backend would otherwise duplicate event handoff, terminal handling, recovery cursors, timeouts, and checkpoint behavior. **Proposed solution** Add the native session runtime, backend interfaces, control-plane port, harness driver contract, deterministic conformance helpers, and bounded tests. **Alternatives considered** The server could own this loop. That would mix provider process behavior with server persistence and authority logic. **Roadmap alignment** This is part of the existing experimental Paperclip Runner rollout. It does not enable a production adapter. ## What Changed - Add a provider-neutral native session execution loop. - Add recovery cursor reconciliation and checkpoint hooks. - Add bounded timeout and governed-wait behavior. - Add harness driver and control-plane conformance helpers. - Add deterministic backend and session runtime tests. ## Verification - `pnpm --filter @paperclipai/paperclip-runner test:typescript` - `pnpm --filter @paperclipai/paperclip-runner typecheck:typescript` - `pnpm -r typecheck` - `pnpm build` - Protocol generation and manifest checks pass. - The branch changes 14 files relative to its declared base. ## Risks The main risk is a session that hangs, duplicates an event, or restores the wrong cursor after reconnect. Tests cover terminal events, timeouts, durable handoff, duplicate receipts, recovery, continuity breaks, and checkpoint updates. The change remains package-local. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, `gpt-5`, with agentic reasoning, tool use, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change and contains no internal Paperclip ticket id - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge --- .../evals/harness-driver-conformance.json | 29 + .../paperclip-runner/protocol/manifest.json | 6 + .../backends/harness-driver-backend.test.ts | 451 ++ .../src/backends/harness-driver-backend.ts | 455 ++ .../src/backends/runtime-context.test.ts | 57 + .../src/backends/runtime-context.ts | 42 + .../src/conformance/control-plane-port.ts | 188 + .../src/conformance/harness-driver.ts | 241 ++ .../src/contracts/control-plane-port.ts | 94 + .../src/contracts/harness-driver.ts | 513 +++ .../src/contracts/native-session-backend.ts | 148 + packages/paperclip-runner/src/index.ts | 5 + .../src/native-session-runtime.test.ts | 3738 +++++++++++++++++ .../src/native-session-runtime.ts | 1205 ++++++ packages/paperclip-runner/src/testing.ts | 2 + 15 files changed, 7174 insertions(+) create mode 100644 packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json create mode 100644 packages/paperclip-runner/src/backends/harness-driver-backend.test.ts create mode 100644 packages/paperclip-runner/src/backends/harness-driver-backend.ts create mode 100644 packages/paperclip-runner/src/backends/runtime-context.test.ts create mode 100644 packages/paperclip-runner/src/backends/runtime-context.ts create mode 100644 packages/paperclip-runner/src/conformance/control-plane-port.ts create mode 100644 packages/paperclip-runner/src/conformance/harness-driver.ts create mode 100644 packages/paperclip-runner/src/contracts/control-plane-port.ts create mode 100644 packages/paperclip-runner/src/contracts/harness-driver.ts create mode 100644 packages/paperclip-runner/src/contracts/native-session-backend.ts create mode 100644 packages/paperclip-runner/src/native-session-runtime.test.ts create mode 100644 packages/paperclip-runner/src/native-session-runtime.ts diff --git a/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json new file mode 100644 index 0000000000..670fc5b980 --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/evals/harness-driver-conformance.json @@ -0,0 +1,29 @@ +{ + "schema": "paperclip-runner/harness-driver-conformance-fixture/v1", + "validConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "semantic-tool-success" + }, + "invalidConfig": { + "schema": "paperclip-runner/deterministic-harness-driver-config/v1", + "scenario": "unknown" + }, + "completionMessage": "Run deterministic semantic-tool conformance.", + "interruptMessage": "[conformance:interrupt] Keep the turn active until interrupted.", + "requiredCapabilities": [ + "resume", + "typedEvents", + "interruption", + "structuredResult", + "read", + "reconciliation", + "usage", + "dynamicTools" + ], + "unsupportedFeatures": [ + "steering", + "runtimeRequestResolution", + "goals", + "threadLineage" + ] +} diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index 8575322f47..9400ab615a 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -121,6 +121,12 @@ "expectation": "accept", "compatibilityCase": "cross-language-input" }, + { + "path": "fixtures/evals/harness-driver-conformance.json", + "sha256": "ce84eb64faa6af78e17a156cde352a951db7c108c8819a20109b732b23a5bec5", + "expectation": "accept", + "compatibilityCase": "canonical" + }, { "path": "fixtures/local-runner/scripts/duplicate-terminal.json", "sha256": "56928a21318ea4b390c65fb34f0d3114760619d79301f4d535391a8566b693ab", diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts new file mode 100644 index 0000000000..689800e54e --- /dev/null +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -0,0 +1,451 @@ +import { describe, expect, it } from "vitest"; + +import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; +import type { PrpEvent, PrpStructuredRunResult } from "../protocol/replay-contract.js"; +import { HarnessDriverBackend } from "./harness-driver-backend.js"; + +const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Backend adapter completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "fake", status: "passed" }], + attentionRequests: [], + artifacts: [], +}; + +const providerIdentity = { + kind: "acpx" as const, + normalizedSessionId: "session-1", + acpxRecordId: "driver-1", + backendSessionId: "backend-1", + agentSessionId: "provider-1", + profileDigest: "sha256:profile", + workspaceDigest: "sha256:workspace", + requestedModel: "claude-sonnet-4-20250514", + effectiveModel: "claude-sonnet-4-20250514", +}; + +function prpEvent(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `fake:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "fake", + sourceKind: "runner", + runId: "run-1", + normalizedSessionId: "session-1", + turnId: "turn-1", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: `2026-08-09T00:00:0${sourceSeq}.000Z`, + payload, + }; +} + +const runtimeResolutions: unknown[] = []; + +class FakeHarnessSession implements HarnessSession { + ids() { return { driverSessionId: "driver-1", providerSessionId: "provider-1" }; } + async *events() { + yield prpEvent(1, "run.result.proposed", result); + yield prpEvent(2, "turn.completed", { status: "completed" }); + } + async startTurn() { return { turnId: "turn-1" }; } + async resolveRuntimeRequest(input: unknown) { + runtimeResolutions.push(structuredClone(input)); + } + async snapshot(): Promise { + return { + driverKind: "fake", + driverSessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + semanticResult: { result, fingerprint: "fingerprint", turnId: "turn-1" }, + lastSourceSequence: 2, + }; + } + async close() {} +} + +const driver: HarnessDriver = { + async descriptor() { + return { + kind: "fake", + displayName: "Fake harness", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + async openSession() { return new FakeHarnessSession(); }, +}; + +describe("HarnessDriverBackend", () => { + it("rejects and closes a provider session without a durable provider identity", async () => { + let closed = false; + class MissingProviderIdentitySession extends FakeHarnessSession { + override ids() { + return { + driverSessionId: "driver-missing-provider", + providerSessionId: null, + }; + } + + override async close() { + closed = true; + } + } + const incompleteDriver: HarnessDriver = { + ...driver, + async openSession() { + return new MissingProviderIdentitySession(); + }, + }; + const backend = new HarnessDriverBackend(incompleteDriver); + + await expect(backend.openSession({ + identity: { + runId: "run-incomplete", + sessionId: "session-incomplete", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + })).rejects.toThrow( + "provider_initialize_protocol_error: provider=fake stage=session.open missing durable provider session identity", + ); + expect(closed).toBe(true); + }); + + it("rejects and closes a recovered session without a durable provider identity", async () => { + let closed = false; + class MissingRecoveredIdentitySession extends FakeHarnessSession { + override ids() { + return { + driverSessionId: "driver-missing-recovered-provider", + providerSessionId: null, + }; + } + + override async close() { + closed = true; + } + } + const incompleteDriver: HarnessDriver = { + ...driver, + async recoverSession() { + return { + recovered: true, + session: new MissingRecoveredIdentitySession(), + }; + }, + }; + const backend = new HarnessDriverBackend(incompleteDriver); + + await expect(backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + identity: { + runId: "run-recover-incomplete", + sessionId: "session-recover-incomplete", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + cursor: "0", + }, { + signal: new AbortController().signal, + })).rejects.toThrow( + "provider_initialize_protocol_error: provider=fake stage=session.recover missing durable provider session identity", + ); + expect(closed).toBe(true); + }); + + it("normalizes harness events, result, terminal, and snapshot", async () => { + const backend = new HarnessDriverBackend(driver); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + expect(events).toHaveLength(2); + await expect(session.result()).resolves.toMatchObject({ result, turnId: "turn-1", terminal: { runTerminalState: "succeeded" } }); + await expect(session.snapshot()).resolves.toMatchObject({ + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + }); + }); + + it("passes the persisted harness driver kind through recovery", async () => { + let recoveredDriverKind: string | null = null; + let recoveredProviderIdentity: PersistedHarnessSession["providerIdentity"]; + const recoveryDriver: HarnessDriver = { + ...driver, + async recoverSession(snapshot) { + recoveredDriverKind = snapshot.driverKind; + recoveredProviderIdentity = snapshot.providerIdentity; + return { recovered: true, session: new FakeHarnessSession() }; + }, + }; + const backend = new HarnessDriverBackend(recoveryDriver); + const recovery = await backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + providerRecoveryPolicy: "allow_replacement_after_governed_wait", + identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { + signal: new AbortController().signal, + }); + expect(recovery.recovered).toBe(true); + expect(recoveredDriverKind).toBe("fake"); + expect(recoveredProviderIdentity).toEqual(providerIdentity); + }); + + it("forwards the native bootstrap signal to the harness driver", async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const backend = new HarnessDriverBackend({ + ...driver, + async openSession(input) { + receivedSignal = input.signal; + return new FakeHarnessSession(); + }, + }); + + await backend.openSession({ + identity: { runId: "run-signal", sessionId: "session-signal", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + signal: controller.signal, + }); + + expect(receivedSignal).toBe(controller.signal); + }); + + it("forwards the native recovery signal to the harness driver", async () => { + const controller = new AbortController(); + let receivedSignal: AbortSignal | undefined; + const backend = new HarnessDriverBackend({ + ...driver, + async recoverSession(_snapshot, options) { + receivedSignal = options.signal; + return { recovered: true, session: new FakeHarnessSession() }; + }, + }); + + const recovery = await backend.recoverSession({ + backendKind: "runner", + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + providerIdentity, + identity: { runId: "run-recover-signal", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + }, { + signal: controller.signal, + }); + + expect(recovery.recovered).toBe(true); + expect(receivedSignal).toBe(controller.signal); + }); + + it("allows only the run id to change when a harness session is attached", async () => { + const attachedRunIds: string[] = []; + class AttachableHarnessSession extends FakeHarnessSession { + async attachRun(input: { runId: string }) { + attachedRunIds.push(input.runId); + } + } + const backend = new HarnessDriverBackend({ + ...driver, + async openSession() { return new AttachableHarnessSession(); }, + }); + const originalIdentity = { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }; + const session = await backend.openSession({ + identity: originalIdentity, + workingDirectory: "/workspace", + }); + + for (const identity of [ + { ...originalIdentity, runId: "run-forged-company", companyId: "company-2" }, + { ...originalIdentity, runId: "run-forged-issue", issueId: "issue-2" }, + { ...originalIdentity, runId: "run-forged-agent", agentId: "agent-2" }, + { ...originalIdentity, runId: "run-forged-session", sessionId: "session-2" }, + ]) { + await expect(session.attachRun?.({ identity })).rejects.toThrow( + "native_session_attach_binding_mismatch", + ); + expect(session.identity()).toEqual(originalIdentity); + } + expect(attachedRunIds).toEqual([]); + + await expect(session.attachRun?.({ + identity: { ...originalIdentity, runId: "run-2" }, + })).resolves.toBeUndefined(); + expect(attachedRunIds).toEqual(["run-2"]); + expect(session.identity()).toEqual({ ...originalIdentity, runId: "run-2" }); + }); + + it("delegates native runtime-request resolutions to the harness session", async () => { + runtimeResolutions.length = 0; + const backend = new HarnessDriverBackend(driver); + const session = await backend.openSession({ + identity: { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + workingDirectory: "/workspace", + }); + + await session.resolveRuntimeRequest?.({ + requestId: "permission-1", + turnId: "turn-1", + resolution: { action: "accept_for_session" }, + }); + + expect(runtimeResolutions).toEqual([ + { + requestId: "permission-1", + turnId: "turn-1", + resolution: { action: "accept_for_session" }, + }, + ]); + }); + + it("emits one non-replayable input expiration and terminal wait after provider loss", async () => { + const questionSet = { + schema: "paperclip.question_set.v1" as const, + questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" as const }], + }; + class LostProviderSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: questionSet, + turnId: "turn-1", + itemId: "input-1", + } }); + throw new Error("provider transport lost"); + } + override async snapshot(): Promise { + return { driverKind: "fake", driverSessionId: "driver-1", lastSourceSequence: 1 }; + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new LostProviderSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ value: { eventType: "runtime_request.created" } }); + await expect(iterator.next()).resolves.toMatchObject({ value: { + eventType: "runtime_request.expired", + sourceSeq: 2, + payload: { + requestId: "input-1", + reason: "provider_process_lost", + replayAllowed: false, + request: { input: questionSet }, + }, + } }); + await expect(iterator.next()).resolves.toMatchObject({ value: { + eventType: "turn.interrupted", + sourceSeq: 3, + payload: { reason: "provider_process_lost" }, + } }); + await expect(iterator.next()).resolves.toMatchObject({ done: true }); + }); + + it("does not synthesize a fallback after the input was already resolved", async () => { + class ResolvedThenLostSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: { schema: "paperclip.question_set.v1", questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" }] }, + } }); + yield prpEvent(2, "runtime_request.resolved", { requestId: "input-1", action: "submit" }); + throw new Error("provider transport lost after resolution"); + } + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new ResolvedThenLostSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await iterator.next(); + await iterator.next(); + await expect(iterator.next()).rejects.toThrow("provider transport lost after resolution"); + }); + + it("does not synthesize a fallback after explicit run cancellation", async () => { + class CancelledProviderSession extends FakeHarnessSession { + override async *events() { + yield prpEvent(1, "runtime_request.created", { request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which target?", + input: { schema: "paperclip.question_set.v1", questions: [{ id: "target", prompt: "Which target?", required: true, answerMode: "text" }] }, + } }); + throw new Error("provider stopped after cancellation"); + } + async interrupt() {} + } + const backend = new HarnessDriverBackend({ ...driver, async openSession() { return new CancelledProviderSession(); } }); + const session = await backend.openSession({ + identity: { runId: "run-1", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, + workingDirectory: "/workspace", + }); + const iterator = session.events()[Symbol.asyncIterator](); + await expect(iterator.next()).resolves.toMatchObject({ value: { eventType: "runtime_request.created" } }); + await session.cancel({ + reason: "operator cancelled the run", + signal: new AbortController().signal, + }).cleanup; + await expect(iterator.next()).rejects.toThrow("provider stopped after cancellation"); + }); +}); diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts new file mode 100644 index 0000000000..c66e69b3ae --- /dev/null +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -0,0 +1,455 @@ +import type { + HarnessDriver, + HarnessSession, + HarnessSessionRecoveryOptions, + PersistedHarnessSession, +} from "../contracts/harness-driver.js"; +import type { + NativeSession, + NativeSessionBackend, + NativeSessionBackendDescriptor, + NativeSessionRecoveryOptions, + OpenNativeSessionInput, + PersistedNativeSession, +} from "../contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpTerminalState, +} from "../protocol/replay-contract.js"; + +/** + * Package-owned adapter from the concrete harness driver contract to the + * normalized session boundary consumed by Paperclip. Provider mechanics stay + * behind HarnessDriver; the control plane sees only PRP events and results. + */ +export class HarnessDriverBackend implements NativeSessionBackend { + readonly #driver: HarnessDriver; + + constructor(driver: HarnessDriver) { + this.#driver = driver; + } + + async descriptor(): Promise { + const descriptor = await this.#driver.descriptor(); + return { + kind: "runner", + name: descriptor.kind, + version: descriptor.version, + capabilities: structuredClone(descriptor.capabilities), + runtimeContextCapabilities: descriptor.runtimeContextCapabilities === undefined ? undefined : structuredClone(descriptor.runtimeContextCapabilities), + }; + } + + async openSession(input: OpenNativeSessionInput): Promise { + const session = await this.#driver.openSession({ + runId: input.identity.runId, + normalizedSessionId: input.identity.sessionId, + workingDirectory: input.workingDirectory ?? process.cwd(), + ...(input.signal === undefined ? {} : { signal: input.signal }), + }); + try { + assertProviderSessionIdentity( + session, + (await this.#driver.descriptor()).kind, + "session.open", + ); + } catch (error) { + await session.close({ + reason: "provider session bootstrap returned an incomplete identity", + force: true, + }).catch(() => undefined); + throw error; + } + return new HarnessNativeSession(input, session); + } + + async recoverSession( + snapshot: PersistedNativeSession, + options: NativeSessionRecoveryOptions, + ) { + if (this.#driver.recoverSession === undefined) { + return { recovered: false, reason: "driver does not support recovery" }; + } + const persisted: PersistedHarnessSession = { + driverKind: snapshot.driverKind ?? snapshot.backendKind, + driverSessionId: snapshot.sessionId, + providerSessionId: snapshot.providerSessionId, + ...(snapshot.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(snapshot.providerIdentity) }), + ...(snapshot.providerRecoveryPolicy === undefined + ? {} + : { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }), + runId: snapshot.identity.runId, + normalizedSessionId: snapshot.identity.sessionId, + activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null, + lastSourceSequence: parseCursor(snapshot.cursor), + ...(snapshot.semanticResult === undefined || snapshot.semanticResult === null + ? {} + : { + semanticResult: { + result: snapshot.semanticResult, + fingerprint: canonicalJson(snapshot.semanticResult), + turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered", + }, + }), + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], + }; + const recoveryOptions: HarnessSessionRecoveryOptions = { + signal: options.signal, + }; + const recovered = await this.#driver.recoverSession( + persisted, + recoveryOptions, + ); + if (!recovered.recovered || recovered.session === undefined) { + return { recovered: false, reason: recovered.reason }; + } + try { + assertProviderSessionIdentity( + recovered.session, + (await this.#driver.descriptor()).kind, + "session.recover", + ); + } catch (error) { + await recovered.session.close({ + reason: "provider session recovery returned an incomplete identity", + force: true, + }).catch(() => undefined); + throw error; + } + return { + recovered: true, + session: new HarnessNativeSession( + { identity: snapshot.identity }, + recovered.session, + ), + }; + } +} + +function assertProviderSessionIdentity( + session: HarnessSession, + provider: string, + stage: "session.open" | "session.recover", +): void { + const ids = session.ids(); + if ( + typeof ids.driverSessionId !== "string" + || ids.driverSessionId.trim().length === 0 + || typeof ids.providerSessionId !== "string" + || ids.providerSessionId.trim().length === 0 + ) { + throw new Error( + `provider_initialize_protocol_error: provider=${provider} stage=${stage} missing durable provider session identity`, + ); + } +} + +class HarnessNativeSession implements NativeSession { + #input: OpenNativeSessionInput; + readonly #session: HarnessSession; + #terminal: PrpTerminalState | null = null; + #explicitlyCancelled = false; + + constructor(input: OpenNativeSessionInput, session: HarnessSession) { + this.#input = structuredClone(input); + this.#session = session; + } + + identity() { + return structuredClone(this.#input.identity); + } + + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: this.#session.steer !== undefined, + interruption: this.#session.interrupt !== undefined, + structuredResult: true, + read: this.#session.read !== undefined, + reconciliation: this.#session.reconcile !== undefined, + usage: this.#session.usage !== undefined, + runtimeRequestResolution: this.#session.resolveRuntimeRequest !== undefined, + runtimeRequestHandoff: this.#session.handoffRuntimeRequest !== undefined, + goals: this.#session.goal !== undefined, + threadLineage: this.#session.lineage !== undefined, + }; + } + + async attachRun(input: { identity: OpenNativeSessionInput["identity"] }): Promise { + const currentIdentity = this.#input.identity; + if ( + input.identity.sessionId !== currentIdentity.sessionId + || input.identity.companyId !== currentIdentity.companyId + || input.identity.issueId !== currentIdentity.issueId + || input.identity.agentId !== currentIdentity.agentId + ) { + throw new Error("native_session_attach_binding_mismatch"); + } + if (this.#session.attachRun === undefined) { + throw new Error("native_session_multi_run_unavailable"); + } + await this.#session.attachRun({ runId: input.identity.runId }); + this.#input = { ...this.#input, identity: structuredClone(input.identity) }; + this.#terminal = null; + this.#explicitlyCancelled = false; + } + + async *events(): AsyncIterable { + let sourceInstanceId: string | null = null; + let lastSourceSequence = 0; + let sawTerminal = false; + let synthesizedDurableWait = false; + let streamFailure: unknown = null; + const observedPendingInputs = new Map>(); + try { + for await (const event of this.#session.events()) { + sourceInstanceId = event.sourceInstanceId; + lastSourceSequence = Math.max(lastSourceSequence, event.sourceSeq); + if (event.eventType === "runtime_request.created") { + const request = plainRecord(event.payload.request); + if ( + request?.schema === "paperclip.runtime_request.v2" + && request.type === "input" + && typeof request.requestId === "string" + ) observedPendingInputs.set(request.requestId, structuredClone(request)); + } else if (["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(event.eventType)) { + const requestId = typeof event.payload.requestId === "string" ? event.payload.requestId : null; + if (requestId) observedPendingInputs.delete(requestId); + } + if (event.eventType === "run.terminal") { + sawTerminal = true; + this.#terminal = structuredClone(event.payload as PrpTerminalState); + } else if (["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType)) { + sawTerminal = true; + const snapshot = await this.#session.snapshot(); + const disposition = snapshot.semanticResult?.result.reportedWorkDisposition ?? "yielded"; + this.#terminal = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: + event.eventType === "turn.completed" ? "completed" + : event.eventType === "turn.failed" ? "failed" + : event.eventType === "turn.interrupted" ? "interrupted" + : "cancelled", + runTerminalState: event.eventType === "turn.completed" ? "succeeded" : event.eventType === "turn.failed" ? "failed" : "cancelled", + reportedWorkDisposition: disposition, + }; + } + yield structuredClone(event); + } + } catch (error) { + streamFailure = error; + } + + // The provider can disappear while its native RPC is awaiting the user. + // Emit one canonical terminal fact and replace the stream failure with a + // governed wait; the control plane can materialize the continuation without + // ever trying to replay the dead provider request. + if (!sawTerminal && !this.#explicitlyCancelled && sourceInstanceId) { + const snapshot = await this.#session.snapshot().catch(() => null); + let governedWaitTurnId: string | undefined; + for (const request of observedPendingInputs.values()) { + const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + lastSourceSequence = sourceSeq; + const requestId = String(request.requestId); + const turnId = typeof request.turnId === "string" ? request.turnId : undefined; + governedWaitTurnId ??= turnId; + const itemId = typeof request.itemId === "string" ? request.itemId : requestId; + yield { + schema: "paperclip.prp.event.v1", + sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId, + sourceKind: "runner", + runId: this.#input.identity.runId, + normalizedSessionId: this.#input.identity.sessionId, + ...(turnId ? { turnId } : {}), + itemId, + eventType: "runtime_request.expired", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: { + requestId, + ...(turnId ? { turnId } : {}), + itemId, + requestKind: "runtime", + reason: "provider_process_lost", + replayAllowed: false, + requestType: "input", + ...(plainRecord(request.origin)?.adapter + ? { adapter: plainRecord(request.origin)?.adapter } + : {}), + request: structuredClone(request), + }, + }; + } + if (governedWaitTurnId) { + const sourceSeq = Math.max(lastSourceSequence, snapshot?.lastSourceSequence ?? 0) + 1; + lastSourceSequence = sourceSeq; + synthesizedDurableWait = true; + sawTerminal = true; + this.#terminal = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }; + yield { + schema: "paperclip.prp.event.v1", + sourceEventId: `${sourceInstanceId}:${this.#input.identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId, + sourceKind: "runner", + runId: this.#input.identity.runId, + normalizedSessionId: this.#input.identity.sessionId, + turnId: governedWaitTurnId, + eventType: "turn.interrupted", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: { status: "interrupted", reason: "provider_process_lost" }, + }; + } + } + if (streamFailure && !synthesizedDurableWait) throw streamFailure; + } + + startTurn(input: Parameters[0]) { + return this.#session.startTurn(input); + } + + steer(input: { turnId: string; message: { role: "user"; text: string }; correlationId?: string }) { + if (this.#session.steer === undefined) throw new Error("steering is unavailable"); + return this.#session.steer(input); + } + + interrupt(input: { turnId?: string; reason?: string }) { + if (this.#session.interrupt === undefined) throw new Error("interruption is unavailable"); + return this.#session.interrupt(input); + } + + cancel(input: { reason: string; signal: AbortSignal }) { + if (input.signal.aborted) { + throw input.signal.reason ?? new Error("native session cancellation aborted"); + } + // This flag is the adapter's synchronous publication boundary. Provider + // interruption happens afterward as passive cleanup, so a slow or broken + // transport cannot synthesize or publish new accepted output for the turn. + this.#explicitlyCancelled = true; + const interrupt = this.#session.interrupt; + return { + cleanup: interrupt === undefined + ? Promise.resolve() + : Promise.resolve().then(() => interrupt.call(this.#session, { + reason: input.reason, + signal: input.signal, + })), + }; + } + + resolveRuntimeRequest(input: { + requestId: string; + turnId: string; + resolution: Parameters>[0]["resolution"]; + }) { + if (this.#session.resolveRuntimeRequest === undefined) { + throw new Error("native_runtime_request_resolution_unavailable"); + } + return this.#session.resolveRuntimeRequest(input); + } + + handoffRuntimeRequest(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + signal: AbortSignal; + }) { + if (this.#session.handoffRuntimeRequest === undefined) { + throw new Error("native_runtime_request_handoff_unavailable"); + } + return this.#session.handoffRuntimeRequest(input); + } + + async result() { + const snapshot = await this.#session.snapshot(); + if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + return null; + } + if (this.#terminal === null) { + return null; + } + return { + result: structuredClone(snapshot.semanticResult.result), + terminal: structuredClone(this.#terminal), + turnId: snapshot.semanticResult.turnId, + }; + } + + async snapshot(): Promise { + const snapshot = await this.#session.snapshot(); + return { + backendKind: "runner", + driverKind: snapshot.driverKind, + sessionId: snapshot.driverSessionId, + identity: structuredClone(this.#input.identity), + providerSessionId: snapshot.providerSessionId, + ...(snapshot.providerIdentity === undefined + ? {} + : { providerIdentity: structuredClone(snapshot.providerIdentity) }), + ...(snapshot.providerRecoveryPolicy === undefined + ? {} + : { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }), + cursor: + snapshot.lastSourceSequence === undefined + ? null + : String(snapshot.lastSourceSequence), + semanticResult: snapshot.semanticResult?.result ?? null, + terminal: this.#terminal, + activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null, + terminalTurns: snapshot.terminalTurns ?? [], + pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage ?? [], + }; + } + + async usage(): Promise | null> { + return this.#session.usage?.() ?? null; + } + + close(input: { reason: string }) { + return this.#session.close(input); + } +} + +function plainRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : null; +} + +export function createHarnessDriverBackend(driver: HarnessDriver): NativeSessionBackend { + return new HarnessDriverBackend(driver); +} + +function parseCursor(cursor: string | null | undefined): number | undefined { + if (cursor === undefined || cursor === null || cursor === "") return undefined; + const parsed = Number(cursor); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : undefined; +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/backends/runtime-context.test.ts b/packages/paperclip-runner/src/backends/runtime-context.test.ts new file mode 100644 index 0000000000..772b0def81 --- /dev/null +++ b/packages/paperclip-runner/src/backends/runtime-context.test.ts @@ -0,0 +1,57 @@ +import { + mkdtempSync, + mkdirSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; + +import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import { nativeSystemInstructions } from "./runtime-context.js"; + +const temporaryRoots: string[] = []; + +function runtimeInput(rootPath: string, entryPath: string): NativeExecutionInput { + return { + runtimeContext: { + prompt: { text: "Paperclip runtime." }, + instructions: { bundle: { rootPath }, entryPath }, + }, + } as unknown as NativeExecutionInput; +} + +describe("native runtime context files", () => { + afterEach(() => { + for (const root of temporaryRoots.splice(0)) { + rmSync(root, { recursive: true, force: true }); + } + }); + + it("reads an instruction entry contained by its bundle root", () => { + const temporaryRoot = mkdtempSync(join(tmpdir(), "paperclip-runtime-context-")); + temporaryRoots.push(temporaryRoot); + const bundleRoot = join(temporaryRoot, "bundle"); + mkdirSync(bundleRoot); + writeFileSync(join(bundleRoot, "AGENTS.md"), "Stay inside the bundle.\n"); + + expect(nativeSystemInstructions(runtimeInput(bundleRoot, "AGENTS.md"))) + .toContain("Stay inside the bundle."); + }); + + it("rejects traversal and symlink escapes from the bundle root", () => { + const temporaryRoot = mkdtempSync(join(tmpdir(), "paperclip-runtime-context-")); + temporaryRoots.push(temporaryRoot); + const bundleRoot = join(temporaryRoot, "bundle"); + mkdirSync(bundleRoot); + writeFileSync(join(temporaryRoot, "outside.md"), "outside"); + symlinkSync(join(temporaryRoot, "outside.md"), join(bundleRoot, "linked.md")); + + expect(() => nativeSystemInstructions(runtimeInput(bundleRoot, "../outside.md"))) + .toThrow("native_runtime_context_entry_outside_bundle"); + expect(() => nativeSystemInstructions(runtimeInput(bundleRoot, "linked.md"))) + .toThrow("native_runtime_context_entry_outside_bundle"); + }); +}); diff --git a/packages/paperclip-runner/src/backends/runtime-context.ts b/packages/paperclip-runner/src/backends/runtime-context.ts new file mode 100644 index 0000000000..1258e2374a --- /dev/null +++ b/packages/paperclip-runner/src/backends/runtime-context.ts @@ -0,0 +1,42 @@ +import { readFileSync, realpathSync } from "node:fs"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +import { CODEX_SKILLLESS_BASE_INSTRUCTIONS } from "../contracts/codex.js"; +import type { NativeExecutionInput } from "../contracts/native-execution.js"; +import { composeNativeSystemInstructions } from "../contracts/runtime-context.js"; + +export function nativeSystemInstructions(input: NativeExecutionInput): string { + if (!("runtimeContext" in input)) return CODEX_SKILLLESS_BASE_INSTRUCTIONS; + const configuredRoot = resolve(input.runtimeContext.instructions.bundle.rootPath); + const bundleRoot = realpathSync(configuredRoot); + const entryPath = realpathSync(resolve( + configuredRoot, + input.runtimeContext.instructions.entryPath, + )); + const pathFromRoot = relative(bundleRoot, entryPath); + if ( + pathFromRoot === ".." + || pathFromRoot.startsWith(`..${sep}`) + || isAbsolute(pathFromRoot) + ) { + throw new Error("native_runtime_context_entry_outside_bundle"); + } + const entry = readFileSync(entryPath, "utf8"); + return composeNativeSystemInstructions(input.runtimeContext, entry); +} + +export function nativeTaskConstraints(input: NativeExecutionInput): string[] { + const finalResponseConstraint = + "Write the complete user-facing final response exactly once before invoking paperclip_finish or paperclip_block. Treat that semantic completion tool as the last action and do not emit a trailing acknowledgement."; + if (!("runtimeContext" in input)) { + return [ + "Do not discover or invoke skills.", + "Do not call a control-plane API.", + finalResponseConstraint, + ]; + } + return [ + "Use only the assigned skills and provider-native tools.", + "Use Paperclip semantic tools for coordination and finalization.", + finalResponseConstraint, + ]; +} diff --git a/packages/paperclip-runner/src/conformance/control-plane-port.ts b/packages/paperclip-runner/src/conformance/control-plane-port.ts new file mode 100644 index 0000000000..6364d184cf --- /dev/null +++ b/packages/paperclip-runner/src/conformance/control-plane-port.ts @@ -0,0 +1,188 @@ +import type { + CompleteControlPlaneRunInput, + ControlPlanePort, + OpenControlPlaneRunInput, +} from "../contracts/control-plane-port.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; + +export interface ControlPlanePortConformanceSnapshot { + eventCount: number; + highestContiguousSourceSeq: number; + duplicateDisposition: "duplicate"; + terminalReplayIdempotent: boolean; + openBindingRejected: boolean; + eventIdMutationRejected: boolean; + eventSequenceMutationRejected: boolean; + replayBindingRejected: boolean; + resultMutationRejected: boolean; +} + +export interface ControlPlanePortConformanceHarness { + port: ControlPlanePort; + start?(): Promise; + stop?(): Promise; +} + +export const CONTROL_PLANE_CONFORMANCE_OPEN: OpenControlPlaneRunInput = { + identity: { + runId: "00000000-0000-4000-8000-000000000006", + sessionId: "session-standalone-conformance", + companyId: "00000000-0000-4000-8000-000000000001", + issueId: "00000000-0000-4000-8000-000000000003", + agentId: "00000000-0000-4000-8000-000000000002", + }, + backendKind: "mock", + sourceInstanceId: "runner-standalone-conformance", +}; + +export const CONTROL_PLANE_CONFORMANCE_RESULT: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Standalone conformance task completed.", + completionClaim: { + contractRevision: "standalone-v1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: ["event:2"] }], + remainingWork: [], + }, + evidence: [{ kind: "event", ref: "event:2" }], + verification: [{ commandOrCheck: "standalone-conformance", status: "passed", artifactRef: "event:2" }], + attentionRequests: [], + artifacts: [], +}; + +export const CONTROL_PLANE_CONFORMANCE_TERMINAL: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", +}; + +function event(sourceSeq: number, eventType: PrpEvent["eventType"], payload: Record): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-standalone-conformance:event:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-standalone-conformance", + sourceKind: "runner", + runId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.runId, + normalizedSessionId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.sessionId, + turnId: "turn-standalone-conformance", + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" ? 0 : 1, + emittedAt: `2026-08-09T00:00:0${sourceSeq}.000Z`, + payload, + }; +} + +export const CONTROL_PLANE_CONFORMANCE_EVENTS = [ + event(1, "session.started", { backend: "mock" }), + event(2, "run.result.proposed", CONTROL_PLANE_CONFORMANCE_RESULT), + event(3, "run.terminal", CONTROL_PLANE_CONFORMANCE_TERMINAL as unknown as Record), +] as const; + +export async function runControlPlanePortConformance( + harness: ControlPlanePortConformanceHarness, +): Promise { + await harness.start?.(); + try { + await harness.port.openRun(CONTROL_PLANE_CONFORMANCE_OPEN); + let openBindingRejected = false; + try { + await harness.port.openRun({ + ...CONTROL_PLANE_CONFORMANCE_OPEN, + identity: { ...CONTROL_PLANE_CONFORMANCE_OPEN.identity, issueId: "forged-issue" }, + }); + } catch { + openBindingRejected = true; + } + if (!openBindingRejected) throw new Error("mismatched run binding was accepted"); + const first = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[0]); + if (first.disposition !== "committed" || first.highestContiguousSourceSeq !== 1) { + throw new Error("first event did not commit at cursor 1"); + } + const gap = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[2]); + if (gap.highestContiguousSourceSeq !== 1) throw new Error("source gap was hidden"); + const recovered = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[1]); + if (recovered.highestContiguousSourceSeq !== 3) throw new Error("source gap did not recover"); + const duplicate = await harness.port.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[1]); + if (duplicate.disposition !== "duplicate") throw new Error("identical duplicate was not idempotent"); + let eventIdMutationRejected = false; + try { + await harness.port.appendEvent({ + ...CONTROL_PLANE_CONFORMANCE_EVENTS[1], + payload: { mutated: true }, + }); + } catch { + eventIdMutationRejected = true; + } + if (!eventIdMutationRejected) throw new Error("mutated event id replay was accepted"); + let eventSequenceMutationRejected = false; + try { + await harness.port.appendEvent({ + ...CONTROL_PLANE_CONFORMANCE_EVENTS[1], + sourceEventId: "runner-standalone-conformance:mutated-sequence-two", + payload: { mutated: true }, + }); + } catch { + eventSequenceMutationRejected = true; + } + if (!eventSequenceMutationRejected) throw new Error("mutated source sequence replay was accepted"); + + const replay = await harness.port.replayEvents({ + runId: CONTROL_PLANE_CONFORMANCE_OPEN.identity.runId, + sourceInstanceId: "runner-standalone-conformance", + afterSourceSeq: 1, + limit: 10, + }); + if (replay.events.map((entry) => entry.sourceSeq).join(",") !== "2,3") { + throw new Error("exclusive replay cursor returned the wrong event set"); + } + let replayBindingRejected = false; + try { + await harness.port.replayEvents({ + runId: "forged-run", + sourceInstanceId: "runner-standalone-conformance", + afterSourceSeq: 0, + limit: 10, + }); + } catch { + replayBindingRejected = true; + } + if (!replayBindingRejected) throw new Error("mismatched replay binding was accepted"); + + const completion: CompleteControlPlaneRunInput = { + result: CONTROL_PLANE_CONFORMANCE_RESULT, + terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL, + turnId: "turn-standalone-conformance", + callerResultId: "result-standalone-conformance", + callerDedupeKey: "dedupe-standalone-conformance", + }; + await harness.port.completeRun(completion); + await harness.port.completeRun(completion); + let resultMutationRejected = false; + try { + await harness.port.completeRun({ + ...completion, + result: { ...completion.result, summary: "mutated result replay" }, + }); + } catch { + resultMutationRejected = true; + } + if (!resultMutationRejected) throw new Error("mutated result replay was accepted"); + return { + eventCount: replay.highestContiguousSourceSeq, + highestContiguousSourceSeq: replay.highestContiguousSourceSeq, + duplicateDisposition: duplicate.disposition, + terminalReplayIdempotent: true, + openBindingRejected, + eventIdMutationRejected, + eventSequenceMutationRejected, + replayBindingRejected, + resultMutationRejected, + }; + } finally { + await harness.stop?.(); + } +} diff --git a/packages/paperclip-runner/src/conformance/harness-driver.ts b/packages/paperclip-runner/src/conformance/harness-driver.ts new file mode 100644 index 0000000000..db08327bc9 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/harness-driver.ts @@ -0,0 +1,241 @@ +import { readFile } from "node:fs/promises"; + +import { + HARNESS_DRIVER_CONTRACT_VERSION, + type HarnessDriver, + type HarnessDriverDescriptor, +} from "../contracts/harness-driver.js"; +import { validatePrpEvent, type PrpEvent } from "../protocol/replay-contract.js"; + +export const HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA = + "paperclip-runner/harness-driver-conformance-fixture/v1" as const; +export const HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA = + "paperclip-runner/harness-driver-conformance-report/v1" as const; + +export const harnessDriverConformanceFixtureUrl = new URL( + "../../protocol/fixtures/evals/harness-driver-conformance.json", + import.meta.url, +); + +export interface HarnessDriverConformanceFixture { + schema: typeof HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA; + validConfig: Record; + invalidConfig: Record; + completionMessage: string; + interruptMessage: string; + requiredCapabilities: string[]; + unsupportedFeatures: string[]; +} + +export interface HarnessDriverConformanceReport { + schema: typeof HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA; + contractVersion: typeof HARNESS_DRIVER_CONTRACT_VERSION; + descriptor: HarnessDriverDescriptor; + checks: { + capabilityDescription: true; + configValidation: true; + sessionLifecycle: true; + sessionRecovery: true; + semanticTools: true; + eventValidation: true; + interruptAndCancel: true; + usage: true; + transcriptCompleteness: true; + unsupportedFeatures: true; + }; + eventCount: number; + semanticToolCallCount: number; + usage: Record; +} + +function record(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + return value as Record; +} + +function text(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${path} must be a non-empty string`); + } + return value; +} + +function strings(value: unknown, path: string): string[] { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== "string")) { + throw new Error(`${path} must be an array of strings`); + } + return [...value] as string[]; +} + +export function parseHarnessDriverConformanceFixture( + value: unknown, +): HarnessDriverConformanceFixture { + const fixture = record(value, "fixture"); + if (fixture.schema !== HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA) { + throw new Error( + `fixture.schema must be ${HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA}`, + ); + } + return { + schema: HARNESS_DRIVER_CONFORMANCE_FIXTURE_SCHEMA, + validConfig: structuredClone(record(fixture.validConfig, "fixture.validConfig")), + invalidConfig: structuredClone(record(fixture.invalidConfig, "fixture.invalidConfig")), + completionMessage: text(fixture.completionMessage, "fixture.completionMessage"), + interruptMessage: text(fixture.interruptMessage, "fixture.interruptMessage"), + requiredCapabilities: strings(fixture.requiredCapabilities, "fixture.requiredCapabilities"), + unsupportedFeatures: strings(fixture.unsupportedFeatures, "fixture.unsupportedFeatures"), + }; +} + +export async function loadHarnessDriverConformanceFixture( + url: URL = harnessDriverConformanceFixtureUrl, +): Promise { + return parseHarnessDriverConformanceFixture( + JSON.parse(await readFile(url, "utf8")) as unknown, + ); +} + +async function collectEvents(events: AsyncIterable): Promise { + const collected: PrpEvent[] = []; + for await (const event of events) { + const validation = validatePrpEvent(event); + if (!validation.ok) { + throw new Error( + `driver conformance received invalid event: ${validation.issues[0]?.message ?? "unknown issue"}`, + ); + } + collected.push(structuredClone(validation.event)); + } + return collected; +} + +/** Run the deterministic, provider-free harness-driver V1 contract. */ +export async function runHarnessDriverConformance(input: { + driver: HarnessDriver; + fixture?: HarnessDriverConformanceFixture; +}): Promise { + const fixture = input.fixture ?? await loadHarnessDriverConformanceFixture(); + const descriptor = await input.driver.descriptor(); + if (!descriptor.kind || !descriptor.displayName || !descriptor.version) { + throw new Error("driver descriptor is missing stable identity fields"); + } + for (const capability of fixture.requiredCapabilities) { + if (descriptor.capabilities[capability as keyof typeof descriptor.capabilities] !== true) { + throw new Error(`driver capability ${capability} is required by conformance V1`); + } + } + for (const unsupported of fixture.unsupportedFeatures) { + if (!descriptor.capabilities.unsupported?.includes(unsupported)) { + throw new Error(`driver must explicitly describe unsupported feature ${unsupported}`); + } + } + + if (input.driver.validateConfig === undefined) { + throw new Error("driver does not expose deterministic config validation"); + } + const valid = await input.driver.validateConfig(fixture.validConfig); + const invalid = await input.driver.validateConfig(fixture.invalidConfig); + if (!valid.ok || invalid.ok || invalid.issues.length === 0) { + throw new Error("driver config validation did not accept valid and reject invalid input"); + } + + const completeSession = await input.driver.openSession({ + runId: "run_driver_conformance_complete", + normalizedSessionId: "session_driver_conformance_complete", + workingDirectory: "/deterministic/conformance", + }); + await completeSession.startTurn({ message: { role: "user", text: fixture.completionMessage } }); + const completeEvents = await collectEvents(completeSession.events()); + const semanticInputs = completeEvents.filter((event) => event.eventType === "mcp_app.tool_input"); + const semanticResults = completeEvents.filter((event) => event.eventType === "mcp_app.tool_result"); + if (semanticInputs.length === 0 || semanticInputs.length !== semanticResults.length) { + throw new Error("driver conformance requires paired semantic tool input/result events"); + } + const snapshot = await completeSession.snapshot(); + if (snapshot.semanticResult === undefined || snapshot.semanticResult === null) { + throw new Error("completed driver session did not persist a semantic result"); + } + const usage = await completeSession.usage?.(); + if (usage === undefined || usage === null) { + throw new Error("driver conformance requires a deterministic usage snapshot"); + } + const transcript = await completeSession.transcript?.(); + if ( + transcript === undefined + || !transcript.complete + || transcript.eventCount !== completeEvents.length + || transcript.events.length !== completeEvents.length + || transcript.omissionReason !== null + ) { + throw new Error("driver conformance transcript is incomplete or inconsistent"); + } + if (input.driver.recoverSession === undefined) { + throw new Error("driver declares resume but does not expose session recovery"); + } + const recovery = await input.driver.recoverSession(snapshot, { + signal: new AbortController().signal, + }); + if ( + !recovery.recovered + || recovery.session === undefined + || recovery.session.ids().driverSessionId !== snapshot.driverSessionId + ) { + throw new Error("driver did not recover the persisted conformance session identity"); + } + const recoveredSnapshot = await recovery.session.snapshot(); + if ( + recoveredSnapshot.runId !== snapshot.runId + || recoveredSnapshot.normalizedSessionId !== snapshot.normalizedSessionId + || recoveredSnapshot.semanticResult?.fingerprint !== snapshot.semanticResult?.fingerprint + ) { + throw new Error("driver recovery did not preserve the persisted conformance state"); + } + await recovery.session.close({ reason: "conformance_recovery_complete" }); + await completeSession.close({ reason: "conformance_complete" }); + + const interruptedSession = await input.driver.openSession({ + runId: "run_driver_conformance_interrupt", + normalizedSessionId: "session_driver_conformance_interrupt", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await interruptedSession.startTurn({ + message: { role: "user", text: fixture.interruptMessage }, + }); + if (interruptedSession.interrupt === undefined) { + throw new Error("driver conformance requires interrupt support"); + } + await interruptedSession.interrupt({ turnId, reason: "conformance_interrupt" }); + const interruptedEvents = await collectEvents(interruptedSession.events()); + if ( + !interruptedEvents.some((event) => event.eventType === "turn.interrupted") + || !interruptedEvents.some((event) => + event.eventType === "run.terminal" + && record(event.payload, "terminal payload").runTerminalState === "cancelled") + ) { + throw new Error("driver interrupt did not produce an explicit cancelled terminal"); + } + await interruptedSession.close({ reason: "conformance_cancel", force: true }); + + return { + schema: HARNESS_DRIVER_CONFORMANCE_REPORT_SCHEMA, + contractVersion: HARNESS_DRIVER_CONTRACT_VERSION, + descriptor: structuredClone(descriptor), + checks: { + capabilityDescription: true, + configValidation: true, + sessionLifecycle: true, + sessionRecovery: true, + semanticTools: true, + eventValidation: true, + interruptAndCancel: true, + usage: true, + transcriptCompleteness: true, + unsupportedFeatures: true, + }, + eventCount: completeEvents.length, + semanticToolCallCount: semanticInputs.length, + usage: structuredClone(usage), + }; +} diff --git a/packages/paperclip-runner/src/contracts/control-plane-port.ts b/packages/paperclip-runner/src/contracts/control-plane-port.ts new file mode 100644 index 0000000000..d877316868 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/control-plane-port.ts @@ -0,0 +1,94 @@ +import type { + NativeRunEvent, + NativeRunIdentity, + NativeRunResult, +} from "./types.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; +import type { PersistedNativeSession } from "./native-session-backend.js"; + +export interface OpenControlPlaneRunInput { + identity: NativeRunIdentity; + backendKind: "runner" | "remote" | "mock"; + sourceInstanceId?: string; +} + +export interface AppendedEventReceipt { + cursor: number; + highestContiguousSourceSeq: number; + disposition: "committed" | "duplicate"; +} + +export interface AppendControlPlaneEventOptions { + /** + * Cancellation is a durability boundary: an aborted append must settle + * without committing before it rejects. + */ + signal: AbortSignal; +} + +export interface CheckpointControlPlaneSessionOptions { + /** + * Cancellation is a durability boundary: once aborted, checkpoint work + * must settle without committing a new snapshot. + */ + signal: AbortSignal; +} + +export interface ReplayControlPlaneEventsInput { + runId: string; + sourceInstanceId: string; + afterSourceSeq: number; + limit: number; +} + +export interface ReplayedControlPlaneEvents { + events: PrpEvent[]; + highestContiguousSourceSeq: number; +} + +export interface FinalizeControlPlaneOperationOptions { + /** + * Finalization cancellation is a durability boundary. Once aborted, a + * mutating operation must settle without committing new durable state. + */ + signal: AbortSignal; +} + +export interface CompleteControlPlaneRunInput { + result: PrpStructuredRunResult; + terminal: PrpTerminalState; + turnId?: string | null; + callerResultId?: string | null; + callerDedupeKey?: string | null; +} + +/** + * The only control-plane surface the standalone runner may call. + * + * Production Paperclip implements this port in a later integration phase. The + * runner package never imports the server, UI, or database that sits behind it. + */ +export interface ControlPlanePort { + openRun(input: OpenControlPlaneRunInput): Promise; + loadSessionCheckpoint?(): Promise; + checkpointSession?( + snapshot: PersistedNativeSession, + options?: CheckpointControlPlaneSessionOptions, + ): Promise; + appendEvent( + event: NativeRunEvent | PrpEvent, + options?: AppendControlPlaneEventOptions, + ): Promise; + replayEvents( + input: ReplayControlPlaneEventsInput, + options?: FinalizeControlPlaneOperationOptions, + ): Promise; + completeRun( + result: NativeRunResult | CompleteControlPlaneRunInput, + options?: FinalizeControlPlaneOperationOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts new file mode 100644 index 0000000000..16b8291986 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -0,0 +1,513 @@ +import type { + PrpEvent, + PrpStructuredRunResult, +} from "../protocol/replay-contract.js"; +import type { NativeSessionCapabilities, NativeUserMessage } from "./types.js"; +import { + PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2, + parsePaperclipQuestionResponse, + type PaperclipQuestionResponse, + type PaperclipQuestionSet, + type PaperclipRuntimeRequestOrigin, +} from "./question-set.js"; + +export * from "./question-set.js"; + +export const HARNESS_DRIVER_CONTRACT_VERSION = 1 as const; + +export interface HarnessDriverConfigIssue { + path: string; + code: string; + message: string; +} + +export type HarnessDriverConfigValidation = + | { ok: true; config: Record; issues: [] } + | { ok: false; config: null; issues: HarnessDriverConfigIssue[] }; + +export interface HarnessTranscriptSnapshot { + schema: "paperclip-runner/harness-transcript/v1"; + complete: boolean; + eventCount: number; + events: PrpEvent[]; + omissionReason: string | null; +} + +export interface HarnessDriverDescriptor { + kind: string; + displayName: string; + version: string; + protocolVersion?: string; + capabilities: NativeSessionCapabilities; + runtimeContextCapabilities?: NativeRuntimeContextCapabilities; +} + +export interface NativeRuntimeContextCapabilities { + instructions: "native" | "unsupported"; + skills: "native" | "unsupported"; + mcp: "native" | "unsupported"; +} + +export interface OpenHarnessSessionInput { + runId: string; + normalizedSessionId: string; + workingDirectory: string; + /** Abort provider bootstrap and release any not-yet-returned provider state. */ + signal?: AbortSignal; +} + +export interface HarnessSessionRecoveryOptions { + /** Abort provider recovery and release any not-yet-returned provider state. */ + signal: AbortSignal; +} + +export class HarnessCapabilityUnavailableError extends Error { + readonly operation: string; + + constructor(operation: string, detail: string) { + super(`${operation} is unavailable: ${detail}`); + this.name = "HarnessCapabilityUnavailableError"; + this.operation = operation; + } +} + +export class HarnessReconciliationError extends Error { + readonly recoverable = true; + + constructor(message: string) { + super(message); + this.name = "HarnessReconciliationError"; + } +} + +export class HarnessOperationAlreadyTerminalError extends Error { + readonly code = "already_terminal" as const; + + constructor(operation: string) { + super(`${operation} lost a race with the committed turn terminal`); + this.name = "HarnessOperationAlreadyTerminalError"; + } +} + +export class HarnessStaleTurnError extends Error { + readonly code = "stale_turn" as const; + + constructor(turnId: string) { + super(`turn ${turnId} is not the active turn`); + this.name = "HarnessStaleTurnError"; + } +} + +export type HarnessRuntimeRequestKind = + | "command_approval" + | "file_approval" + | "permission_approval" + | "user_input" + | "elicitation"; + +export interface HarnessRuntimeRequest { + requestId: string; + requestKind: HarnessRuntimeRequestKind; + method: string; + turnId: string; + itemId: string; + status: "pending"; + prompt: string; + details: Record; + /** Provider-neutral presentation model for structured input requests. */ + input?: PaperclipQuestionSet; + /** Diagnostic origin only; provider response shapes never enter PRP. */ + origin?: PaperclipRuntimeRequestOrigin; +} + +export type HarnessRuntimeRequestResolution = + | { action: "accept" | "accept_for_session" | "decline" | "cancel" } + | { + action: "submit"; + answers: Record; + } + | { + action: "submit"; + content: Record; + } + | { + action: "submit"; + response: PaperclipQuestionResponse; + }; + +export type HarnessRuntimeRequestAction = HarnessRuntimeRequestResolution["action"]; + +export type HarnessRuntimeRequestHandoffResult = "handed_off" | "already_settled"; + +/** + * A runtime-input handoff commits its durable state transition before the + * method returns. Provider interruption is cleanup only: it remains observed, + * but cannot acquire mutation authority or reverse an already committed turn. + */ +export interface HarnessRuntimeRequestHandoff { + result: HarnessRuntimeRequestHandoffResult; + cleanup: Promise; +} + +const RUNTIME_REQUEST_ACTIONS: readonly HarnessRuntimeRequestAction[] = [ + "accept", + "accept_for_session", + "decline", + "cancel", + "submit", +]; + +export class HarnessRuntimeRequestResolutionError extends Error { + readonly code = "invalid_resolution" as const; + readonly requestKind: HarnessRuntimeRequestKind; + + constructor(requestKind: HarnessRuntimeRequestKind, detail: string) { + super(`${requestKind} rejected its resolution: ${detail}`); + this.name = "HarnessRuntimeRequestResolutionError"; + this.requestKind = requestKind; + } +} + +function plainRecord(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function parseAnswers(value: unknown): Record | null { + const fields = plainRecord(value); + if (fields === null || Object.keys(fields).length === 0) return null; + const parsed: Record = {}; + for (const [field, entry] of Object.entries(fields)) { + const answer = plainRecord(entry); + if (answer === null || !Array.isArray(answer.answers)) return null; + if (answer.answers.some((value) => typeof value !== "string")) return null; + parsed[field] = { answers: [...(answer.answers as string[])] }; + } + return parsed; +} + +/** + * Validates a resolution against the kind of request it answers, so a + * mismatched submit fails closed instead of degrading into an accepted empty + * response. Every driver runs this before it touches its provider, and the + * browser transport runs it again at its own untrusted edge. + */ +export function parseHarnessRuntimeRequestResolution( + requestKind: HarnessRuntimeRequestKind, + value: unknown, + questionSet?: PaperclipQuestionSet, +): HarnessRuntimeRequestResolution { + const candidate = plainRecord(value) ?? {}; + const rawAction = candidate.action; + if ( + typeof rawAction !== "string" || + !RUNTIME_REQUEST_ACTIONS.includes(rawAction as HarnessRuntimeRequestAction) + ) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + `unsupported action ${JSON.stringify(rawAction) ?? "undefined"}`, + ); + } + const action = rawAction as HarnessRuntimeRequestAction; + if (action !== "submit" && ("answers" in candidate || "content" in candidate || "response" in candidate)) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + `${action} does not carry submitted form data`, + ); + } + + if (action === "submit" && "response" in candidate) { + if (requestKind !== "user_input" && requestKind !== "elicitation") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "approval requests do not accept submitted question responses", + ); + } + if ("answers" in candidate || "content" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "canonical submissions cannot also carry provider-specific answers or content", + ); + } + if (questionSet === undefined) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "canonical submission requires the persisted question set", + ); + } + try { + return { + action, + response: parsePaperclipQuestionResponse(questionSet, candidate.response), + }; + } catch (error) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + error instanceof Error ? error.message : "invalid question response", + ); + } + } + + if (requestKind === "user_input") { + if (action === "accept" || action === "accept_for_session") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "user input requires submit, decline, or cancel", + ); + } + if (action !== "submit") return { action }; + if ("content" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "user input submissions carry answers, not content", + ); + } + const answers = parseAnswers(candidate.answers); + if (answers === null) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "submit requires answers shaped as { field: { answers: [string] } }", + ); + } + return { action, answers }; + } + + if (requestKind === "elicitation") { + if (action === "accept_for_session") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "elicitation does not support session acceptance", + ); + } + if (action !== "submit") return { action }; + if ("answers" in candidate) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "elicitation submissions carry content, not answers", + ); + } + const content = plainRecord(candidate.content); + if (content === null || Object.keys(content).length === 0) { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "submit requires a non-empty content object", + ); + } + return { action, content: structuredClone(content) }; + } + + if (action === "submit") { + throw new HarnessRuntimeRequestResolutionError( + requestKind, + "approval requests do not accept submitted form data", + ); + } + return { action }; +} + +/** + * The single payload shape every driver emits for a terminal + * `runtime_request.*` fact. Request identity travels in the payload as well as + * the event binding, so a consumer that only reads payloads still settles the + * request it belongs to. + */ +export type HarnessRuntimeRequestOutcome = { + requestId: string; + requestKind: HarnessRuntimeRequestKind; + turnId: string; + itemId: string; + action?: HarnessRuntimeRequestAction; + reason?: string; + /** Enables content-free lifecycle counters partitioned by adapter. */ + adapter?: string; + requestType?: "input" | "permission"; + /** Canonical submitted answers retained for durable replay and audit UI. */ + response?: PaperclipQuestionResponse; +}; + +export function harnessRuntimeRequestOutcome( + request: Pick< + HarnessRuntimeRequest, + "requestId" | "requestKind" | "turnId" | "itemId" | "origin" | "input" + >, + outcome: { + action?: HarnessRuntimeRequestAction | null; + reason?: string | null; + response?: PaperclipQuestionResponse | null; + } = {}, +): HarnessRuntimeRequestOutcome { + return { + requestId: request.requestId, + requestKind: request.requestKind, + turnId: request.turnId, + itemId: request.itemId, + ...(outcome.action ? { action: outcome.action } : {}), + ...(outcome.reason ? { reason: outcome.reason } : {}), + ...(outcome.response ? { response: structuredClone(outcome.response) } : {}), + ...(request.input + ? { + ...(request.origin?.adapter ? { adapter: request.origin.adapter } : {}), + requestType: "input" as const, + } + : {}), + }; +} + +/** Canonical non-replayable expiration used when a live input moves to a durable wait. */ +export function harnessRuntimeInputExpiredOutcome( + request: HarnessRuntimeRequest, + reason: "durable_handoff" | "provider_process_lost", +): Omit & { + requestKind: "runtime"; + replayAllowed: false; + request: Record; +} { + return { + ...harnessRuntimeRequestOutcome(request, { reason }), + requestKind: "runtime", + replayAllowed: false, + request: { + schema: PAPERCLIP_RUNTIME_REQUEST_SCHEMA_V2, + requestKind: "runtime", + requestId: request.requestId, + type: "input", + status: request.status, + prompt: request.prompt, + input: structuredClone(request.input), + origin: structuredClone(request.origin), + turnId: request.turnId, + itemId: request.itemId, + }, + }; +} + +export interface HarnessThreadGoal { + threadId: string; + objective: string; + status: "active" | "paused" | "blocked" | "usageLimited" | "budgetLimited" | "complete"; + tokenBudget: number | null; + tokensUsed: number; + timeUsedSeconds: number; + createdAt: number; + updatedAt: number; +} + +export type HarnessGoalOperation = + | { action: "get" } + | { action: "set"; objective: string; tokenBudget?: number | null } + | { action: "pause" | "resume" | "clear" }; + +export interface HarnessThreadLineageEntry { + threadId: string; + providerSessionId: string | null; + parentThreadId: string | null; + depth: number; + nickname: string | null; + role: string | null; + status: string; +} + +export interface PersistedHarnessSemanticResult { + result: PrpStructuredRunResult; + fingerprint: string; + callId?: string | null; + turnId: string; +} + +export interface PersistedHarnessTurnTerminal { + turnId: string; + fingerprint: string; +} + +export interface AcpxSessionIdentity { + kind: "acpx"; + normalizedSessionId: string; + acpxRecordId: string; + backendSessionId: string; + agentSessionId: string; + profileDigest: string; + workspaceDigest: string; + requestedModel: string; + effectiveModel: string; + /** Missing on legacy snapshots; those used the historical approve-reads behavior. */ + permissionMode?: "approve-all" | "approve-reads" | "deny-all"; +} + +export type PersistedHarnessProviderIdentity = AcpxSessionIdentity; + +export interface PersistedHarnessSession { + driverKind: string; + driverSessionId: string; + providerSessionId?: string | null; + runId?: string; + normalizedSessionId?: string; + activeTurnId?: string | null; + semanticResult?: PersistedHarnessSemanticResult | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + pendingRuntimeRequests?: HarnessRuntimeRequest[]; + goal?: HarnessThreadGoal | null; + lineage?: HarnessThreadLineageEntry[]; + lastSourceSequence?: number; + /** Tagged provider identity used to reject cross-profile recovery. */ + providerIdentity?: PersistedHarnessProviderIdentity; + /** Narrow escape hatch for a durable response-wake when a provider cannot reload its prior native session. */ + providerRecoveryPolicy?: + | "same_session_only" + | "allow_replacement_after_governed_wait" + | "allow_replacement_after_resume_failure"; +} + +export interface HarnessSessionRecoveryResult { + recovered: boolean; + session?: HarnessSession; + reason?: string; +} + +export interface HarnessSession { + ids(): { + driverSessionId: string; + providerSessionId?: string | null; + displayId?: string | null; + }; + events(): AsyncIterable; + attachRun?(input: { runId: string }): Promise | void; + startTurn(input: { + message: NativeUserMessage; + requestedCollaborationMode?: "default" | "plan"; + }): Promise<{ turnId: string; effectiveCollaborationMode?: "default" | "plan" }>; + steer?(input: { turnId: string; message: NativeUserMessage; correlationId?: string }): Promise; + interrupt?(input: { turnId?: string; reason?: string; signal?: AbortSignal }): Promise; + pendingRuntimeRequests?(): HarnessRuntimeRequest[]; + resolveRuntimeRequest?(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise; + handoffRuntimeRequest?(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + /** Do not synchronously commit if runtime ownership is already revoked. */ + signal: AbortSignal; + }): HarnessRuntimeRequestHandoff; + goal?(input: HarnessGoalOperation): Promise; + lineage?(): HarnessThreadLineageEntry[]; + read?(): Promise>; + reconcile?(): Promise>; + usage?(): Promise | null>; + transcript?(): Promise; + snapshot(): Promise; + close(input: { reason: string; force?: boolean }): Promise; +} + +/** A local harness implementation hidden behind the runner daemon boundary. */ +export interface HarnessDriver { + descriptor(): Promise; + validateConfig?(config: unknown): Promise; + openSession(input: OpenHarnessSessionInput): Promise; + recoverSession?( + snapshot: PersistedHarnessSession, + options: HarnessSessionRecoveryOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts new file mode 100644 index 0000000000..b1898b2eb9 --- /dev/null +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -0,0 +1,148 @@ +import type { + NativeRunIdentity, + NativeSessionCapabilities, + NativeUserMessage, +} from "./types.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; +import type { + HarnessRuntimeRequest, + HarnessRuntimeRequestHandoff, + HarnessRuntimeRequestResolution, + HarnessThreadLineageEntry, + NativeRuntimeContextCapabilities, + PersistedHarnessProviderIdentity, + PersistedHarnessTurnTerminal, +} from "./harness-driver.js"; + +export interface NativeSessionBackendDescriptor { + kind: "runner" | "remote" | "mock"; + name: string; + version: string; + capabilities: NativeSessionCapabilities; + runtimeContextCapabilities?: NativeRuntimeContextCapabilities; +} + +export interface OpenNativeSessionInput { + identity: NativeRunIdentity; + workingDirectory?: string; + /** + * When present, aborts provider bootstrap if the caller's recovery deadline + * expires. Backends own cleanup for work that has not returned a session yet. + */ + signal?: AbortSignal; +} + +export interface NativeSessionRecoveryOptions { + /** Abort provider recovery and release any not-yet-returned provider state. */ + signal: AbortSignal; +} + +export interface PersistedNativeSession { + backendKind: NativeSessionBackendDescriptor["kind"]; + driverKind?: string | null; + sessionId: string; + identity: NativeRunIdentity; + providerSessionId?: string | null; + /** Tagged provider-owned identity required for safe driver recovery. */ + providerIdentity?: PersistedHarnessProviderIdentity; + providerRecoveryPolicy?: + | "same_session_only" + | "allow_replacement_after_governed_wait" + | "allow_replacement_after_resume_failure"; + cursor?: string | null; + semanticResult?: PrpStructuredRunResult | null; + terminal?: PrpTerminalState | null; + activeTurnId?: string | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + pendingRuntimeRequests?: HarnessRuntimeRequest[]; + lineage?: HarnessThreadLineageEntry[]; +} + +export interface NativeSessionRecoveryResult { + recovered: boolean; + session?: NativeSession; + reason?: string; +} + +/** + * Cancellation is a synchronous authority transition followed by passive + * provider cleanup. Once `cancel` returns, the provider session must no longer + * be able to publish accepted output or acquire new mutation authority for the + * cancelled turn. Cleanup may stop processes or transports, but it must not + * perform durable control-plane mutations. + */ +export interface NativeSessionCancellation { + cleanup: Promise; +} + +export interface NativeSessionSnapshotOptions { + /** Stop provider snapshot work that outlives the execution deadline. */ + signal: AbortSignal; +} + +export interface NativeSession { + identity(): NativeRunIdentity; + capabilities(): Promise; + attachRun?(input: { identity: NativeRunIdentity }): Promise; + events(input?: { afterCursor?: string | null }): AsyncIterable; + startTurn(input: { + message: NativeUserMessage; + requestedCollaborationMode?: "default" | "plan"; + }): Promise<{ turnId: string; effectiveCollaborationMode?: "default" | "plan" }>; + steer?(input: { turnId: string; message: NativeUserMessage; correlationId?: string }): Promise; + interrupt?(input: { turnId?: string; reason?: string }): Promise; + /** Commit cancellation synchronously; the returned promise owns cleanup only. */ + cancel?(input: { reason: string; signal: AbortSignal }): NativeSessionCancellation; + resolveRuntimeRequest?(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise; + handoffRuntimeRequest?(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + /** + * Revokes durable mutation authority when event consumption fails. The + * method must commit synchronously before returning; its returned promise + * owns provider cleanup only and must not mutate durable request state. + */ + signal: AbortSignal; + }): HarnessRuntimeRequestHandoff; + result(): Promise<{ + result: PrpStructuredRunResult; + terminal: PrpTerminalState; + turnId: string | null; + } | null>; + usage?(): Promise | null>; + snapshot(options?: NativeSessionSnapshotOptions): Promise; + /** + * Idempotently stop provider work and release every pending `events().next()` + * before this promise resolves. Implementations must settle every promise + * previously returned by the session (including interrupt, cancel, handoff, + * and iterator teardown). The runtime bounds its wait for a broken provider, + * revokes that session's mutation authority, removes it from reuse, and keeps + * observing late cleanup so a contract violation cannot defeat a run timeout + * or become an unhandled rejection. + */ + close(input: { reason: string }): Promise; +} + +/** Normalized control-plane boundary shared by runner and hosted backends. */ +export interface NativeSessionBackend { + descriptor(): Promise; + openSession(input: OpenNativeSessionInput): Promise; + /** Open a fresh provider session after an explicitly governed continuity break. */ + openReplacementSession?( + input: OpenNativeSessionInput, + previous: PersistedNativeSession, + ): Promise; + recoverSession?( + snapshot: PersistedNativeSession, + options: NativeSessionRecoveryOptions, + ): Promise; +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 02210787d2..3c56edc441 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -1,12 +1,17 @@ export * from "./catalog/index.js"; +export * from "./contracts/control-plane-port.js"; export * from "./contracts/completion-result.js"; export * from "./contracts/codex.js"; export * from "./contracts/durable-recovery.js"; +export * from "./contracts/harness-driver.js"; export * from "./contracts/local-runner.js"; export * from "./contracts/native-execution.js"; +export * from "./contracts/native-session-backend.js"; export * from "./contracts/question-set.js"; export * from "./contracts/runtime-context.js"; export * from "./contracts/types.js"; +export * from "./backends/harness-driver-backend.js"; +export * from "./native-session-runtime.js"; export { DurablePrpControlPlane, type DurablePrpControlPlaneOptions, diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts new file mode 100644 index 0000000000..9ea07c4b6c --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -0,0 +1,3738 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ControlPlanePort } from "./contracts/control-plane-port.js"; +import type { NativeExecutionInputV1 } from "./contracts/native-execution.js"; +import type { NativeRunIdentity } from "./contracts/types.js"; +import type { + NativeSession, + NativeSessionBackend, + PersistedNativeSession, +} from "./contracts/native-session-backend.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "./protocol/replay-contract.js"; +import { + NATIVE_RUNTIME_ASSET_SCHEMA, + PAPERCLIP_EXECUTION_PROMPT, + PAPERCLIP_EXECUTION_PROMPT_REVISION, + canonicalNativeRuntimeContextDigest, + nativeRuntimePromptDigest, +} from "./contracts/runtime-context.js"; +import { + executeNativeSession, + type ExecuteNativeSessionOptions, +} from "./native-session-runtime.js"; + +const identity = { + runId: "run-recovery", + sessionId: "session-recovery", + companyId: "company-recovery", + issueId: "issue-recovery", + agentId: "agent-recovery", +}; + +const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Recovered native work completed.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: true, + criteria: [{ criterionId: "objective", status: "satisfied", evidenceRefs: [] }], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "recovery", status: "passed" }], + attentionRequests: [], + artifacts: [], +}; + +const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", +}; + +const yieldedResult: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], + remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [{ kind: "issue_thread_interaction", ref: "interaction:pending" }], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, +}; + +const input: NativeExecutionInputV1 = { + schema: "paperclip.native-execution-input.v1", + binding: { + companyId: identity.companyId, + runId: identity.runId, + issueId: identity.issueId, + agentId: identity.agentId, + executionWorkspaceId: "workspace-recovery", + }, + task: { + identifier: "PAP-RECOVERY", + title: "Recover native work", + description: null, + prompt: "# PAP-RECOVERY: Recover native work", + workMode: "standard", + }, + workspace: { cwd: "/workspace", repoUrl: null, repoRef: null, branchName: null }, + session: { normalizedSessionId: identity.sessionId, driverKind: "codex_app_server", protocolVersion: 1 }, + provider: { kind: "codex", model: null }, + completionContract: { + id: "contract-recovery", + sha256: "contract-recovery-sha", + schemaVersion: "paperclip.completion-contract.v1", + contract: { + revision: "1", + objective: "Recover native work", + criteria: [{ id: "objective", requirement: "Complete after recovery" }], + }, + }, + interactionResponses: [], + credentialBindings: [], +}; + +function controlEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, +): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `control-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "control-recovery", + sourceKind: "control_plane", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; +} + +function runnerEvent( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record = {}, +): PrpEvent { + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `runner-recovery:${identity.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "runner-recovery", + sourceKind: "runner", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload, + }; +} + +function highestContiguous(events: PrpEvent[]): number { + const sequences = new Set(events.map((event) => event.sourceSeq)); + let cursor = 0; + while (sequences.has(cursor + 1)) cursor += 1; + return cursor; +} + +describe("executeNativeSession recovery", () => { + it("keeps governed-wait discovery synchronous", () => { + type GovernedWaitResolver = NonNullable; + const resolver: GovernedWaitResolver = () => null; + // An async resolver could retain control-plane mutation authority after + // execution settles, so the public boundary rejects it at compile time. + // @ts-expect-error governed-wait discovery must not return a promise + const asynchronousResolver: GovernedWaitResolver = async () => null; + + expect(resolver).toBeTypeOf("function"); + void asynchronousResolver; + }); + + it("fails closed before launch when a v3 driver does not declare complete native context realization", async () => { + const digest = "0".repeat(64); + const context = { + prompt: { + revision: PAPERCLIP_EXECUTION_PROMPT_REVISION, + text: PAPERCLIP_EXECUTION_PROMPT, + digest: nativeRuntimePromptDigest(), + }, + instructions: { + entryPath: "AGENTS.md", + bundle: { + schema: NATIVE_RUNTIME_ASSET_SCHEMA, + digest, + manifestDigest: digest, + rootPath: "/paperclip/context/instructions", + fileCount: 1, + totalBytes: 1, + }, + }, + skills: [], + mcp: { assignmentSetId: "none", digest, bindingId: null }, + } as const; + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "future-provider", + name: "future-provider", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input: { + ...input, + schema: "paperclip.native-execution-input.v3", + executionMode: "default", + planningContext: null, + runtimeContext: { + ...context, + aggregateDigest: canonicalNativeRuntimeContextDigest(context), + }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("does not natively realize instructions, skills, mcp"); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("does not admit a fresh run when provider session initialization fails", async () => { + const providerFailure = new Error("provider initialization failed"); + const openSession = vi.fn(async () => { + throw providerFailure; + }); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "fresh-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toBe(providerFailure); + + expect(openSession).toHaveBeenCalledWith(expect.objectContaining({ + identity, + workingDirectory: input.workspace.cwd, + signal: expect.any(AbortSignal), + })); + expect(openRun).not.toHaveBeenCalled(); + }); + + it("bounds fresh session bootstrap and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + let resolveBootstrap = (_value: NativeSession) => {}; + const stalledBootstrap = new Promise((resolve) => { + resolveBootstrap = resolve; + }); + let markBootstrapStarted = () => {}; + const bootstrapStarted = new Promise((resolve) => { + markBootstrapStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late fresh session must not start"); }, + async result() { return null; }, + async snapshot() { throw new Error("late fresh session must not snapshot"); }, + close, + }; + let bootstrapSignal: AbortSignal | undefined; + const openSession = vi.fn((bootstrapInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + bootstrapSignal = bootstrapInput.signal; + bootstrapInput.signal?.addEventListener( + "abort", + () => resolveBootstrap(lateSession), + { once: true }, + ); + markBootstrapStarted(); + return stalledBootstrap; + }); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "fresh-stalled-backend", + version: "1", + capabilities: await lateSession.capabilities(), + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + }); + const rejection = expect(execution).rejects.toThrow( + "native session bootstrap timed out after 5ms", + ); + + await bootstrapStarted; + expect(bootstrapSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(bootstrapSignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session bootstrap timed out", + }); + } finally { + vi.useRealTimers(); + } + }); + + it("closes the provider when owner quarantine notification throws", async () => { + const snapshotFailure = new Error("snapshot failed"); + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("unexpected turn"); }, + async result() { return null; }, + async snapshot() { throw snapshotFailure; }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "owner-notification-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + const retainedSessions: Array = []; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + keepSessionOpen: true, + onSession(current) { + retainedSessions.push(current); + if (current === null) throw new Error("owner notification failed"); + }, + })).rejects.toBe(snapshotFailure); + + expect(retainedSessions).toEqual([session, null]); + expect(close).toHaveBeenCalledOnce(); + }); + + it.each(["control-plane checkpoint", "owner checkpoint"] as const)( + "aborts consumption and closes the provider when the startup %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + let releaseStream = () => {}; + try { + const streamReleased = new Promise((resolve) => { + releaseStream = resolve; + }); + const never = new Promise(() => undefined); + let markCheckpointStalled = () => {}; + const checkpointStalled = new Promise((resolve) => { + markCheckpointStalled = resolve; + }); + let checkpointSignal: AbortSignal | undefined; + let controlPlaneCheckpointCount = 0; + const checkpointSession: NonNullable = async ( + _snapshot, + checkpointOptions, + ) => { + controlPlaneCheckpointCount += 1; + if ( + stalledBoundary === "control-plane checkpoint" + && controlPlaneCheckpointCount === 2 + ) { + checkpointSignal = checkpointOptions?.signal; + markCheckpointStalled(); + await never; + } + }; + let ownerCheckpointCount = 0; + const onCheckpoint: NonNullable = async ( + _snapshot, + checkpointOptions, + ) => { + ownerCheckpointCount += 1; + if ( + stalledBoundary === "owner checkpoint" + && ownerCheckpointCount === 2 + ) { + checkpointSignal = checkpointOptions?.signal; + markCheckpointStalled(); + await never; + } + }; + const startTurn = vi.fn(async () => ({ turnId: "turn-checkpoint-stalled" })); + const close = vi.fn(async () => { + releaseStream(); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + }; + }, + async *events() { + await streamReleased; + }, + startTurn, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-checkpoint-stalled", + cursor: "0", + activeTurnId: "turn-checkpoint-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "checkpoint-stalled-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const openRun = vi.fn(async () => undefined); + const port: ControlPlanePort = { + openRun, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1_000, + checkpointTimeoutMs: 1, + keepSessionOpen: true, + onCheckpoint, + onSession: (current) => retainedSessions.push(current), + }); + const rejection = expect(execution).rejects.toThrow( + "native session checkpoint timed out after 1ms", + ); + await checkpointStalled; + expect(checkpointSignal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + await rejection; + + expect(checkpointSignal?.aborted).toBe(true); + expect(openRun).toHaveBeenCalledOnce(); + expect(startTurn).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + releaseStream(); + vi.useRealTimers(); + } + }, + ); + + it.each([ + "provider result", + "completion checkpoint", + "control-plane replay", + "final event append", + "run completion", + ] as const)( + "bounds post-terminal finalization when %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markFinalizationStalled = () => {}; + const finalizationStalled = new Promise((resolve) => { + markFinalizationStalled = resolve; + }); + let stalledSignal: AbortSignal | undefined; + let resultCalls = 0; + let resultResolved = false; + let completionCheckpointCalls = 0; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { + resultCalls += 1; + if (stalledBoundary === "provider result") { + markFinalizationStalled(); + return await never; + } + resultResolved = true; + return { result, terminal, turnId: "turn-recovery" }; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "finalization-timeout-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession(_snapshot, operationOptions) { + if (stalledBoundary === "completion checkpoint" && resultResolved) { + completionCheckpointCalls += 1; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + async appendEvent(event, operationOptions) { + if ( + stalledBoundary === "final event append" + && (event as PrpEvent).sourceKind === "control_plane" + ) { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + return { + cursor: 1, + highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, + disposition: "committed", + }; + }, + async replayEvents(_replay, operationOptions) { + if (stalledBoundary === "control-plane replay") { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun(_completion, operationOptions) { + if (stalledBoundary === "run completion") { + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + const rejection = expect(execution).rejects.toThrow( + "native session finalization timed out after 10ms", + ); + await finalizationStalled; + if (stalledBoundary !== "provider result") { + expect(stalledSignal?.aborted).toBe(false); + } + + await vi.advanceTimersByTimeAsync(20); + await rejection; + + if (stalledBoundary !== "provider result") { + expect(stalledSignal?.aborted).toBe(true); + } + expect(resultCalls).toBe(1); + if (stalledBoundary === "completion checkpoint") { + expect(completionCheckpointCalls).toBe(1); + } + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + vi.useRealTimers(); + } + }, + ); + + it.each([ + "final event append", + "run completion", + ] as const)( + "confirms durable completion when %s commits before its acknowledgement stalls", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markFinalizationStalled = () => {}; + const finalizationStalled = new Promise((resolve) => { + markFinalizationStalled = resolve; + }); + let stalledOnce = false; + let stalledSignal: AbortSignal | undefined; + const events: PrpEvent[] = []; + let durableCompletion: unknown = null; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "durable-finalization-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event, operationOptions) { + const appended = structuredClone(event as PrpEvent); + const existing = events.find((candidate) => + candidate.sourceInstanceId === appended.sourceInstanceId + && candidate.sourceSeq === appended.sourceSeq + ); + if (existing === undefined) events.push(appended); + if ( + stalledBoundary === "final event append" + && appended.sourceKind === "control_plane" + && !stalledOnce + ) { + stalledOnce = true; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + return await never; + } + const sourceEvents = events.filter( + (candidate) => candidate.sourceInstanceId === appended.sourceInstanceId, + ); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: existing === undefined ? "committed" : "duplicate", + }; + }, + async replayEvents(replay) { + const sourceEvents = events.filter( + (event) => event.sourceInstanceId === replay.sourceInstanceId, + ); + return { + events: structuredClone(sourceEvents.filter( + (event) => event.sourceSeq > replay.afterSourceSeq, + )), + highestContiguousSourceSeq: highestContiguous(sourceEvents), + }; + }, + async completeRun(completion, operationOptions) { + if (durableCompletion === null) { + durableCompletion = structuredClone(completion); + } else { + expect(completion).toEqual(durableCompletion); + } + if (stalledBoundary === "run completion" && !stalledOnce) { + stalledOnce = true; + stalledSignal = operationOptions?.signal; + markFinalizationStalled(); + await never; + } + }, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + }); + await finalizationStalled; + expect(stalledSignal?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(10); + + await expect(execution).resolves.toMatchObject({ + result, + terminal, + nativeEventCount: 3, + }); + expect(stalledSignal?.aborted).toBe(true); + expect(durableCompletion).toMatchObject({ result, terminal }); + expect(events.filter((event) => event.sourceKind === "control_plane")) + .toHaveLength(2); + } finally { + vi.useRealTimers(); + } + }, + ); + + it.each([ + "provider snapshot", + "post-completion checkpoint", + "provider usage", + ] as const)( + "preserves durable completion when %s never settles", + async (stalledBoundary) => { + vi.useFakeTimers(); + try { + const never = new Promise(() => undefined); + let markEnrichmentStalled = () => {}; + const enrichmentStalled = new Promise((resolve) => { + markEnrichmentStalled = resolve; + }); + let stalledSignal: AbortSignal | undefined; + let runCompleted = false; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot(snapshotOptions) { + if (stalledBoundary === "provider snapshot" && runCompleted) { + stalledSignal = snapshotOptions?.signal; + markEnrichmentStalled(); + return await never; + } + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async usage() { + if (stalledBoundary === "provider usage" && runCompleted) { + markEnrichmentStalled(); + return await never; + } + return { driverVersion: "2" }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "post-completion-enrichment-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { return session; }, + }; + const completeRun = vi.fn(async () => { + runCompleted = true; + }); + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession(_snapshot, operationOptions) { + if (stalledBoundary === "post-completion checkpoint" && runCompleted) { + stalledSignal = operationOptions?.signal; + markEnrichmentStalled(); + await never; + } + }, + async appendEvent(event) { + return { + cursor: 1, + highestContiguousSourceSeq: (event as PrpEvent).sourceSeq, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + const retainedSessions: Array = []; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 10, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + await enrichmentStalled; + if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(10); + + await expect(execution).resolves.toMatchObject({ + result, + terminal, + providerSessionId: "provider-recovery", + driverVersion: "1", + usage: null, + }); + if (stalledSignal !== undefined) expect(stalledSignal.aborted).toBe(true); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(retainedSessions).toEqual([session, null]); + } finally { + vi.useRealTimers(); + } + }, + ); + + it("contains a consumer rejection when starting the turn fails first", async () => { + let markAppendStarted = () => {}; + const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + let releaseAppend = () => {}; + const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + let appendCommitted = false; + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.started"); }, + async startTurn() { + await appendStarted; + throw new Error("start turn failed"); + }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(_event, options) { + markAppendStarted(); + await Promise.race([ + appendReleased, + new Promise((_resolve, reject) => { + const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); + if (options?.signal.aborted) rejectAbort(); + else options?.signal.addEventListener("abort", rejectAbort, { once: true }); + }), + ]); + appendCommitted = true; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed" as const, + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + await appendStarted; + await expect(execution).rejects.toThrow("start turn failed"); + expect(close).toHaveBeenCalled(); + expect(appendCommitted).toBe(false); + releaseAppend(); + await new Promise((resolve) => setImmediate(resolve)); + expect(appendCommitted).toBe(false); + }); + + it("stops and closes a timed-out consumer even when the caller requested a warm session", async () => { + let markAppendStarted = () => {}; + const appendStarted = new Promise((resolve) => { markAppendStarted = resolve; }); + let releaseAppend = () => {}; + const appendReleased = new Promise((resolve) => { releaseAppend = resolve; }); + let releaseTeardown = () => {}; + const teardownReleased = new Promise((resolve) => { releaseTeardown = resolve; }); + const iteratorTeardown = vi.fn(); + let appendCommitted = false; + const appendEvent = vi.fn(async ( + _event: PrpEvent, + options?: { signal: AbortSignal }, + ) => { + markAppendStarted(); + await Promise.race([ + appendReleased, + new Promise((_resolve, reject) => { + const rejectAbort = () => reject(options?.signal.reason ?? new Error("append aborted")); + if (options?.signal.aborted) rejectAbort(); + else options?.signal.addEventListener("abort", rejectAbort, { once: true }); + }), + ]); + appendCommitted = true; + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed" as const, + }; + }); + const cancel = vi.fn(() => { + releaseTeardown(); + return { cleanup: Promise.resolve() }; + }); + const close = vi.fn(async () => { releaseTeardown(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + try { + yield runnerEvent(1, "turn.completed"); + } finally { + iteratorTeardown(); + await teardownReleased; + } + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + appendEvent, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }); + const rejection = expect(execution).rejects.toThrow("native session timed out"); + await appendStarted; + await vi.waitFor(() => expect(iteratorTeardown).toHaveBeenCalledOnce()); + await rejection; + expect(cancel).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalled(); + expect(appendEvent).toHaveBeenCalledOnce(); + expect(appendCommitted).toBe(false); + releaseAppend(); + await new Promise((resolve) => setImmediate(resolve)); + expect(appendCommitted).toBe(false); + }); + + it("closes a failed session while retaining an uncancellable event read", async () => { + let releaseStream = () => {}; + const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); + const close = vi.fn(async () => { releaseStream(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { + await streamReleased; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + })).rejects.toThrow("native session timed out"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("commits cancellation before bounding failed provider cleanup", async () => { + let releaseStream = () => {}; + const streamReleased = new Promise((resolve) => { releaseStream = resolve; }); + let releaseCancellation = () => {}; + const cancellationReleased = new Promise((resolve) => { releaseCancellation = resolve; }); + const interrupt = vi.fn(() => cancellationReleased); + let cancellationCommitted = false; + const cancel = vi.fn(() => { + cancellationCommitted = true; + return { cleanup: cancellationReleased }; + }); + const close = vi.fn(async () => { releaseStream(); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + await streamReleased; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + interrupt, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + }); + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + expect(interrupt).not.toHaveBeenCalled(); + expect(cancel).toHaveBeenCalledOnce(); + await expect(execution).rejects.toThrow("native session timed out"); + expect(cancellationCommitted).toBe(true); + releaseCancellation(); + }); + + it("bounds failure when iterator teardown and provider close never settle", async () => { + const never = new Promise(() => undefined); + const close = vi.fn(() => never); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { + await never; + yield runnerEvent(1, "turn.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 1, + keepSessionOpen: true, + })).rejects.toThrow("native session timed out"); + expect(close).toHaveBeenCalledOnce(); + }); + + it("preserves durable success when provider close never settles", async () => { + const never = new Promise(() => undefined); + const close = vi.fn(() => never); + const completeRun = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ result, terminal }); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("closes after a synchronous governed-wait probe returns no result", async () => { + const resolveGovernedWait = vi.fn(() => null); + const lifecycle: string[] = []; + const close = vi.fn(async () => { lifecycle.push("closed"); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + yield runnerEvent(1, "item.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: "turn-recovery", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { cursor: 1, highestContiguousSourceSeq: 1, disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + resolveGovernedWait, + }); + await expect(execution).rejects.toThrow("before a turn terminal fact"); + expect(resolveGovernedWait).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalled(); + expect(lifecycle).toEqual(["closed"]); + }); + + it("commits a governed wait without waiting for abort-insensitive provider cleanup", async () => { + const lifecycle: string[] = []; + let cancellationSignal: AbortSignal | undefined; + const cancel = vi.fn(({ signal }: { signal: AbortSignal }) => { + cancellationSignal = signal; + lifecycle.push("cancelled"); + return { cleanup: new Promise(() => undefined) }; + }); + const close = vi.fn(async () => { lifecycle.push("closed"); }); + const events: PrpEvent[] = []; + const retainedSessions: Array = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { + yield runnerEvent(1, "item.completed"); + }, + async startTurn() { return { turnId: "turn-recovery" }; }, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: null, + activeTurnId: "turn-recovery", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: event.sourceSeq, + disposition: "committed", + }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + timeoutMs: 5, + resolveGovernedWait: () => yieldedResult, + keepSessionOpen: true, + onSession: (current) => retainedSessions.push(current), + }); + await expect(execution).resolves.toMatchObject({ result: yieldedResult }); + expect(cancel).toHaveBeenCalledOnce(); + expect(cancellationSignal?.aborted).toBe(true); + expect(close).toHaveBeenCalledOnce(); + expect(lifecycle).toEqual(["cancelled", "closed"]); + expect(retainedSessions.at(-1)).toBeNull(); + expect(events.map((event) => event.eventType)).toEqual([ + "item.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("rejects a mismatched checkpoint before it mutates control-plane state", async () => { + const openRun = vi.fn(async () => undefined); + const checkpointSession = vi.fn(async () => undefined); + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { + return { + backendKind: "mock", + sessionId: "driver-recovery", + identity: { ...identity, companyId: "other-company" }, + }; + }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("rejects a mismatched existing session before opening control-plane state", async () => { + const openRun = vi.fn(async () => undefined); + const attachRun = vi.fn(async () => undefined); + const existingSession: NativeSession = { + identity: () => ({ ...identity, companyId: "other-company" }), + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + async startTurn() { return { turnId: "unexpected" }; }, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + })).rejects.toThrow("native_session_attach_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(attachRun).not.toHaveBeenCalled(); + }); + + it("quarantines a retained session when attachment partially mutates then fails", async () => { + const attachmentFailure = new Error("provider attachment failed"); + const openRun = vi.fn(async () => undefined); + let retainedIdentity = { ...identity, runId: "run-previous" }; + const attachRun = vi.fn(async (input: { identity: NativeRunIdentity }) => { + retainedIdentity = structuredClone(input.identity); + throw attachmentFailure; + }); + const close = vi.fn(() => new Promise(() => {})); + const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); + const onSession = vi.fn(); + const existingSession: NativeSession = { + identity: () => structuredClone(retainedIdentity), + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + startTurn, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + })).rejects.toBe(attachmentFailure); + expect(attachRun).toHaveBeenCalledWith({ identity }); + expect(retainedIdentity).toEqual(identity); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native session attachment failed", + }); + expect(openRun).not.toHaveBeenCalled(); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it("quarantines an attached session when control-plane run admission fails", async () => { + const admissionFailure = new Error("control-plane admission failed"); + const openRun = vi.fn(async () => { + throw admissionFailure; + }); + const attachRun = vi.fn(async () => undefined); + const close = vi.fn(async () => undefined); + const startTurn = vi.fn(async () => ({ turnId: "unexpected" })); + const onSession = vi.fn(); + const existingSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + attachRun, + async *events() {}, + startTurn, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "existing-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected open"); }, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + existingSession, + onSession, + })).rejects.toBe(admissionFailure); + expect(attachRun).toHaveBeenCalledWith({ identity }); + expect(openRun).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native control-plane run admission failed", + }); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it("rejects checkpoint adoption when the requested session id is absent", async () => { + const openRun = vi.fn(async () => undefined); + const openSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { + return { + backendKind: "mock", + sessionId: "driver-other-session", + identity: { ...identity, sessionId: "other-session" }, + }; + }, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input: { + ...input, + session: { ...input.session, normalizedSessionId: null }, + }, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow("native_session_checkpoint_binding_mismatch"); + expect(openRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("proves required provider recovery before re-opening the durable run", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-unrecoverable", + identity, + providerSessionId: "provider-unrecoverable", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-unrecoverable", + pendingRuntimeRequests: [], + lineage: [], + }; + const openRun = vi.fn(async () => undefined); + const completeRun = vi.fn(async () => undefined); + const recoverSession = vi.fn(async () => ({ + recovered: false as const, + reason: "provider session no longer exists", + })); + const openSession = vi.fn(async () => { throw new Error("replacement is forbidden"); }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + completeRun, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).rejects.toThrow( + "native_session_recovery_failed: provider session no longer exists", + ); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).not.toHaveBeenCalled(); + expect(completeRun).not.toHaveBeenCalled(); + expect(openSession).not.toHaveBeenCalled(); + }); + + it("bounds paginated recovery replay with one signal and observes a late rejection", async () => { + vi.useFakeTimers(); + let rejectStalledReplay = (_error: Error) => {}; + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-replay-stalled", + identity, + providerSessionId: "provider-replay-stalled", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-replay-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + const stalledReplay = new Promise((_resolve, reject) => { + rejectStalledReplay = reject; + }); + let markSecondPageStarted = () => {}; + const secondPageStarted = new Promise((resolve) => { + markSecondPageStarted = resolve; + }); + const replaySignals: AbortSignal[] = []; + const replayEvents = vi.fn( + async (replay, operationOptions) => { + expect(operationOptions?.signal).toBeInstanceOf(AbortSignal); + replaySignals.push(operationOptions!.signal); + if (replay.afterSourceSeq === 0) { + return { + events: [runnerEvent(1, "item.completed", { kind: "progress" })], + highestContiguousSourceSeq: 1, + }; + } + markSecondPageStarted(); + return await stalledReplay; + }, + ); + const recoverSession = vi.fn(); + const openRun = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + replayEvents, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + }); + const rejection = expect(execution).rejects.toThrow( + "native session recovery replay timed out after 5ms", + ); + + await secondPageStarted; + expect(replayEvents).toHaveBeenCalledTimes(2); + expect(replaySignals).toHaveLength(2); + expect(replaySignals[1]).toBe(replaySignals[0]); + expect(replaySignals[0]?.aborted).toBe(false); + + await vi.advanceTimersByTimeAsync(5); + await rejection; + + expect(replaySignals[0]?.aborted).toBe(true); + expect(recoverSession).not.toHaveBeenCalled(); + expect(openRun).not.toHaveBeenCalled(); + + // A broken adapter may ignore abort and reject later. The bounded helper + // keeps that losing operation observed after execution already rejected. + rejectStalledReplay(new Error("late recovery replay failure")); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds provider recovery and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery-stalled", + identity, + providerSessionId: "provider-recovery-stalled", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-recovery-stalled", + pendingRuntimeRequests: [], + lineage: [], + }; + let resolveRecovery = (_value: { recovered: true; session: NativeSession }) => {}; + const stalledRecovery = new Promise<{ recovered: true; session: NativeSession }>( + (resolve) => { resolveRecovery = resolve; }, + ); + let markRecoveryStarted = () => {}; + const recoveryStarted = new Promise((resolve) => { + markRecoveryStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late recovery session must not start"); }, + async result() { return null; }, + async snapshot() { return structuredClone(checkpoint); }, + close, + }; + let recoverySignal: AbortSignal | undefined; + const recoverSession = vi.fn( + (_checkpoint: PersistedNativeSession, recoveryOptions: { signal: AbortSignal }) => { + recoverySignal = recoveryOptions.signal; + recoveryOptions.signal.addEventListener( + "abort", + () => resolveRecovery({ recovered: true, session: lateSession }), + { once: true }, + ); + markRecoveryStarted(); + return stalledRecovery; + }, + ); + const openRun = vi.fn(async () => undefined); + const onSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + onSession, + }); + const rejection = expect(execution).rejects.toThrow( + "native session provider recovery timed out after 5ms", + ); + + await recoveryStarted; + expect(recoverySignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(recoverySignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + expect(onSession).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session provider recovery timed out", + }); + expect(onSession).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("bounds replacement bootstrap and closes a session returned after timeout", async () => { + vi.useFakeTimers(); + try { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-replacement-stalled", + identity, + providerSessionId: "provider-replacement-stalled", + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: "0", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + let resolveReplacement = (_value: NativeSession) => {}; + const stalledReplacement = new Promise((resolve) => { + resolveReplacement = resolve; + }); + let markReplacementStarted = () => {}; + const replacementStarted = new Promise((resolve) => { + markReplacementStarted = resolve; + }); + let markCloseStarted = () => {}; + const closeStarted = new Promise((resolve) => { + markCloseStarted = resolve; + }); + const close = vi.fn(async () => { markCloseStarted(); }); + const lateSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true }; + }, + async *events() {}, + async startTurn() { throw new Error("late replacement session must not start"); }, + async result() { return null; }, + async snapshot() { return structuredClone(checkpoint); }, + close, + }; + let replacementSignal: AbortSignal | undefined; + const openReplacementSession = vi.fn((replacementInput: { + identity: NativeRunIdentity; + workingDirectory?: string; + signal?: AbortSignal; + }) => { + replacementSignal = replacementInput.signal; + replacementInput.signal?.addEventListener( + "abort", + () => resolveReplacement(lateSession), + { once: true }, + ); + markReplacementStarted(); + return stalledReplacement; + }); + const openRun = vi.fn(async () => undefined); + const onSession = vi.fn(); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "replacement-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true }, + }; + }, + async openSession() { throw new Error("replacement seam must be used"); }, + async recoverSession() { + return { recovered: false, reason: "provider session is missing" }; + }, + openReplacementSession, + }; + const port: ControlPlanePort = { + openRun, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + persistedSession: structuredClone(checkpoint), + timeoutMs: 5, + onSession, + }); + const rejection = expect(execution).rejects.toThrow( + "native session replacement bootstrap timed out after 5ms", + ); + + await replacementStarted; + expect(replacementSignal?.aborted).toBe(false); + await vi.advanceTimersByTimeAsync(5); + await rejection; + expect(replacementSignal?.aborted).toBe(true); + expect(openRun).not.toHaveBeenCalled(); + expect(onSession).not.toHaveBeenCalled(); + + await closeStarted; + expect(close).toHaveBeenCalledWith({ + reason: "native session replacement bootstrap timed out", + }); + expect(onSession).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + } + }); + + it("does not persist a reconciled recovery cursor when provider recovery rejects", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery-rejects", + identity, + providerSessionId: "provider-recovery-rejects", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-recovery-rejects", + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveryFailure = new Error("provider recovery rejected"); + const openRun = vi.fn(async () => undefined); + const checkpointSession = vi.fn(async () => undefined); + const onCheckpoint = vi.fn(async () => undefined); + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + throw recoveryFailure; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents(replay) { + return { + events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], + highestContiguousSourceSeq: 1, + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + })).rejects.toBe(recoveryFailure); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).not.toHaveBeenCalled(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(onCheckpoint).not.toHaveBeenCalled(); + }); + + it("does not persist a reconciled recovery cursor when run admission rejects", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-admission-rejects", + identity, + providerSessionId: "provider-admission-rejects", + providerRecoveryPolicy: "same_session_only", + cursor: "0", + activeTurnId: "turn-admission-rejects", + pendingRuntimeRequests: [], + lineage: [], + }; + const admissionFailure = new Error("control-plane admission rejected"); + const openRun = vi.fn(async () => { throw admissionFailure; }); + const checkpointSession = vi.fn(async () => undefined); + const onCheckpoint = vi.fn(async () => undefined); + const onSession = vi.fn(); + const close = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() {}, + async startTurn() { throw new Error("unexpected turn"); }, + async result() { return null; }, + async snapshot() { throw new Error("unexpected snapshot"); }, + close, + }; + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe("1"); + return { recovered: true as const, session }; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("unexpected replacement"); }, + recoverSession, + }; + const port: ControlPlanePort = { + openRun, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + checkpointSession, + async appendEvent() { throw new Error("unexpected event"); }, + async replayEvents(replay) { + return { + events: replay.afterSourceSeq === 0 ? [runnerEvent(1)] : [], + highestContiguousSourceSeq: 1, + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + onCheckpoint, + onSession, + })).rejects.toBe(admissionFailure); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openRun).toHaveBeenCalledOnce(); + expect(checkpointSession).not.toHaveBeenCalled(); + expect(onCheckpoint).not.toHaveBeenCalled(); + expect(onSession).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenCalledWith(null); + expect(close).toHaveBeenCalledWith({ + reason: "native control-plane run admission failed", + }); + }); + + it("continues a provider-reported active turn without starting a duplicate turn", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "0", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const providerSnapshot: PersistedNativeSession = { + ...checkpoint, + cursor: "1", + activeTurnId: "turn-recovery", + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:1", + sourceSeq: 1, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-recovery", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload: {}, + }; + const bySource = new Map(); + const startTurn = vi.fn(async () => ({ turnId: "duplicate-turn" })); + const openSession = vi.fn(async () => { throw new Error("must recover the provider session"); }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + startTurn, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { return structuredClone(providerSnapshot); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + async recoverSession() { return { recovered: true, session }; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const list = bySource.get(replay.sourceInstanceId) ?? []; + return { + events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-recovery", providerSessionId: "provider-recovery" }); + expect(openSession).not.toHaveBeenCalled(); + expect(startTurn).not.toHaveBeenCalled(); + }); + + it.each([ + { checkpointCursor: "12", expectedCursor: "41", terminalSequence: 42 }, + { checkpointCursor: "50", expectedCursor: "50", terminalSequence: 51 }, + ])( + "seeds recovery from the larger of checkpoint $checkpointCursor and the persisted source high-water mark", + async ({ checkpointCursor, expectedCursor, terminalSequence }) => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: checkpointCursor, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const runnerEvents = [ + runnerEvent(13, "item.completed", { kind: "progress" }), + runnerEvent(41, "item.completed", { kind: "progress" }), + ]; + const terminalEvent = runnerEvent(terminalSequence, "turn.completed"); + const controlEvents: PrpEvent[] = []; + const checkpoints: PersistedNativeSession[] = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + async startTurn() { return { turnId: "turn-recovery" }; }, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { + return { ...checkpoint, cursor: String(terminalSequence), activeTurnId: null }; + }, + async close() {}, + }; + const recoverSession = vi.fn(async (recoveryCheckpoint: PersistedNativeSession) => { + expect(recoveryCheckpoint.cursor).toBe(expectedCursor); + return { recovered: true, session }; + }); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + recoverSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async appendEvent(event) { + const target = event.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; + if (target.some((existing) => existing.sourceSeq === event.sourceSeq)) { + throw new Error(`native_event_replay_conflict:${event.sourceSeq}`); + } + target.push(structuredClone(event)); + return { + cursor: target.length, + highestContiguousSourceSeq: highestContiguous(target), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const source = replay.sourceInstanceId === "runner-recovery" ? runnerEvents : controlEvents; + const events = source + .filter((event) => event.sourceSeq > replay.afterSourceSeq) + .sort((left, right) => left.sourceSeq - right.sourceSeq) + .slice(0, replay.limit); + return { events: structuredClone(events), highestContiguousSourceSeq: highestContiguous(source) }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-recovery" }); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(runnerEvents.some((event) => event.sourceSeq === terminalSequence)).toBe(true); + if (checkpointCursor === "12") { + expect(checkpoints[0]).toMatchObject({ cursor: "41" }); + } + }, + ); + + it("attempts exact recovery before opening an observable replacement session", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-old", + identity, + providerSessionId: "provider-old", + providerRecoveryPolicy: "allow_replacement_after_resume_failure", + cursor: null, + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + const replacementSnapshot: PersistedNativeSession = { + ...checkpoint, + sessionId: "driver-new", + providerSessionId: "provider-new", + providerRecoveryPolicy: "same_session_only", + }; + const replacementSession: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield runnerEvent(1, "turn.completed"); }, + async startTurn() { return { turnId: "turn-replacement" }; }, + async result() { return { result, terminal, turnId: "turn-replacement" }; }, + async snapshot() { return structuredClone(replacementSnapshot); }, + async close() {}, + }; + const recoverSession = vi.fn(async () => ({ + recovered: false as const, + reason: "provider reported the prior session missing", + })); + const openReplacementSession = vi.fn(async () => replacementSession); + const onContinuityBreak = vi.fn(async () => undefined); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "replacement-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("replacement seam must be used"); }, + recoverSession, + openReplacementSession, + }; + const events: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event)); + return { cursor: events.length, highestContiguousSourceSeq: highestContiguous(events), disposition: "committed" }; + }, + async replayEvents() { return { events: [], highestContiguousSourceSeq: 0 }; }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-replacement", + controlPlaneInstanceId: "control-replacement", + onContinuityBreak, + })).resolves.toMatchObject({ providerSessionId: "provider-new" }); + + expect(recoverSession).toHaveBeenCalledOnce(); + expect(openReplacementSession).toHaveBeenCalledOnce(); + expect(onContinuityBreak).toHaveBeenCalledWith({ + reason: "provider reported the prior session missing", + previousDriverSessionId: "driver-old", + previousProviderSessionId: "provider-old", + replacementDriverSessionId: "driver-new", + replacementProviderSessionId: "provider-new", + }); + }); + + it("starts a continuation after the driver clears a stale active terminal turn", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: "turn-already-terminal", + terminalTurns: [{ turnId: "turn-already-terminal", fingerprint: "terminal-fingerprint" }], + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveredSnapshot: PersistedNativeSession = { + ...checkpoint, + activeTurnId: null, + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:2", + sourceSeq: 2, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-continuation", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:01.000Z", + payload: {}, + }; + const startTurn = vi.fn(async () => ({ turnId: "turn-continuation" })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + startTurn, + async result() { return { result, terminal, turnId: "turn-continuation" }; }, + async snapshot() { return structuredClone(recoveredSnapshot); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const bySource = new Map(); + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const list = bySource.get(replay.sourceInstanceId) ?? []; + return { + events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" }); + expect(startTurn).toHaveBeenCalledOnce(); + const recoveryEnvelope = JSON.parse( + startTurn.mock.calls[0]![0].message.text, + ) as { task: { prompt: string } }; + expect(recoveryEnvelope.task.prompt).toContain( + "semantic-result recovery for a prior completed provider turn", + ); + expect(recoveryEnvelope.task.prompt).toContain( + "Do not repeat implementation, tests, research, or the final answer", + ); + }); + + it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "4", + semanticResult: result, + terminal, + activeTurnId: "turn-recovery", + terminalTurns: [{ turnId: "turn-recovery", fingerprint: "terminal-fingerprint" }], + pendingRuntimeRequests: [], + lineage: [], + }; + const events = [controlEvent(1, "run.result.accepted", { result })]; + const checkpoints: PersistedNativeSession[] = []; + const completeRun = vi.fn(async () => undefined); + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + const openSession = vi.fn(async () => { + throw new Error("a recovered run must not open a second provider session"); + }); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() {}, + startTurn, + async result() { return { result, terminal, turnId: "turn-recovery" }; }, + async snapshot() { return structuredClone(checkpoint); }, + async close() {}, + }; + const recoverSession = vi.fn(async () => ({ recovered: true, session })); + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + openSession, + recoverSession, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { checkpoints.push(structuredClone(snapshot)); }, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => event.sourceSeq > replay.afterSourceSeq); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(events) }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + + expect(openSession).not.toHaveBeenCalled(); + expect(recoverSession).toHaveBeenCalledOnce(); + expect(startTurn).not.toHaveBeenCalled(); + expect(events.map((event) => event.eventType)).toEqual(["run.result.accepted", "run.terminal"]); + expect(events.map((event) => event.sourceSeq)).toEqual([1, 2]); + expect(completeRun).toHaveBeenCalledOnce(); + expect(completed).toMatchObject({ nativeEventCount: 1, highestContiguousSourceSeq: 2 }); + expect(checkpoints.at(-1)).toMatchObject({ semanticResult: result, terminal }); + }); + + it("accepts a control-plane governed wait when a completed turn omitted its semantic result", async () => { + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:1", + sourceSeq: 1, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-waiting", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:00.000Z", + payload: {}, + }; + const yielded: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [{ criterionId: "objective", status: "unknown", evidenceRefs: ["interaction:pending"] }], + remainingWork: [{ description: "Resume after the response.", blocksCompletion: true }], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, + }; + const events: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const resolveMissingResult = vi.fn(async () => yielded); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + async startTurn() { return { turnId: "turn-waiting" }; }, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "1", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "governed-wait-backend", + version: "1", + capabilities: { resume: false, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult, + }); + + expect(resolveMissingResult).toHaveBeenCalledWith({ turnId: "turn-waiting", terminalEvent }); + expect(completed).toMatchObject({ + result: yielded, + terminal: { runTerminalState: "succeeded", reportedWorkDisposition: "yielded" }, + turnId: "turn-waiting", + }); + expect(completeRun).toHaveBeenCalledWith( + expect.objectContaining({ result: yielded }), + { signal: expect.any(AbortSignal) }, + ); + expect(events.map((event) => event.eventType)).toEqual([ + "turn.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("parks a provider turn immediately after a durable governed wait appears", async () => { + const yielded: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary: "Waiting for the requested response.", + completionClaim: { + contractRevision: "1", + objectiveSatisfied: false, + criteria: [ + { + criterionId: "objective", + status: "unknown", + evidenceRefs: ["interaction:pending"], + }, + ], + remainingWork: [ + { description: "Resume after the response.", blocksCompletion: true }, + ], + }, + evidence: [{ ref: "interaction:pending" }], + verification: [], + attentionRequests: [], + artifacts: [], + continuation: { + kind: "response_wake", + summary: "Resume from the answer.", + idempotencyKey: "interaction-response:pending", + }, + }; + const itemCompleted: PrpEvent = { + ...controlEvent(1, "item.completed", { + kind: "dynamicToolCall", + item: { id: "ask-1", name: "ask_user_questions" }, + }), + sourceEventId: "provider-recovery:1", + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + turnId: "turn-waiting", + }; + const turnInterrupted: PrpEvent = { + ...controlEvent(2, "turn.interrupted", { reason: "governed_wait" }), + sourceEventId: "provider-recovery:2", + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + turnId: "turn-waiting", + }; + let releaseCancelled!: () => void; + const cancelled = new Promise((resolve) => { + releaseCancelled = resolve; + }); + const cancel = vi.fn(() => { + releaseCancelled(); + return { cleanup: Promise.resolve() }; + }); + const events: PrpEvent[] = []; + const completeRun = vi.fn(async () => undefined); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + yield itemCompleted; + await cancelled; + yield turnInterrupted; + }, + async startTurn() { + return { turnId: "turn-waiting" }; + }, + cancel, + async result() { + return null; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "2", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "governed-wait-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }, + }; + }, + async openSession() { + return session; + }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter( + (event) => + event.sourceInstanceId === replay.sourceInstanceId && + event.sourceSeq > replay.afterSourceSeq, + ); + return { + events: structuredClone(replayed), + highestContiguousSourceSeq: highestContiguous(replayed), + }; + }, + completeRun, + }; + + const completed = await executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveGovernedWait: ({ event }) => + event.eventType === "item.completed" ? yielded : null, + }); + + expect(cancel).toHaveBeenCalledOnce(); + expect(completed).toMatchObject({ + result: yielded, + terminal: { + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "yielded", + }, + }); + expect(events.map((event) => event.eventType)).toEqual([ + "item.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("hands a committed structured input to the durable wait after its live window", async () => { + vi.useFakeTimers(); + try { + const questionSet = { + schema: "paperclip.question_set.v1" as const, + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "single_select" as const, + options: [{ id: "us", label: "US" }, { id: "eu", label: "Europe" }], + }], + }; + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-1", + type: "input", + status: "pending", + prompt: "Which region?", + input: questionSet, + origin: { adapter: "mock" }, + turnId: "turn-waiting", + itemId: "input-1", + }; + const created = { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-waiting" }; + const expired = { ...runnerEvent(2, "runtime_request.expired", { + requestId: "input-1", + requestKind: "runtime", + turnId: "turn-waiting", + itemId: "input-1", + reason: "durable_handoff", + replayAllowed: false, + requestType: "input", + request, + }), turnId: "turn-waiting" }; + const interrupted = { ...runnerEvent(3, "turn.interrupted", { reason: "governed_wait" }), turnId: "turn-waiting" }; + let releaseHandoff!: () => void; + const handedOff = new Promise((resolve) => { releaseHandoff = resolve; }); + let releaseCancelled!: () => void; + const cancelled = new Promise((resolve) => { releaseCancelled = resolve; }); + let releaseCreated!: () => void; + const createdCommitted = new Promise((resolve) => { releaseCreated = resolve; }); + const handoffRuntimeRequest = vi.fn(() => { + releaseHandoff(); + return { result: "handed_off" as const, cleanup: Promise.resolve() }; + }); + const cancel = vi.fn(() => { + releaseCancelled(); + return { cleanup: Promise.resolve() }; + }); + const events: PrpEvent[] = []; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield created; + await handedOff; + yield expired; + await cancelled; + yield interrupted; + }, + async startTurn() { return { turnId: "turn-waiting" }; }, + handoffRuntimeRequest, + cancel, + async result() { return null; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-waiting", + cursor: "3", + activeTurnId: "turn-waiting", + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "runtime-input-wait-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + events.push(structuredClone(event as PrpEvent)); + if (event.eventType === "runtime_request.created") releaseCreated(); + const sourceEvents = events.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = events.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 120, + resolveGovernedWait: ({ event }) => + event.eventType === "runtime_request.expired" ? yieldedResult : null, + }); + await createdCommitted; + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(119); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + + await expect(execution).resolves.toMatchObject({ result: yieldedResult }); + expect(handoffRuntimeRequest).toHaveBeenCalledWith({ + requestId: "input-1", + turnId: "turn-waiting", + reason: "durable_handoff", + signal: expect.any(AbortSignal), + }); + expect(cancel).toHaveBeenCalledOnce(); + expect(events.map((event) => event.eventType)).toContain("runtime_request.expired"); + } finally { + vi.useRealTimers(); + } + }); + + it("aborts and bounds a durable handoff that never settles", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-stalled", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-stalled", + itemId: "input-stalled", + }; + const created = { + ...runnerEvent(1, "runtime_request.created", { request }), + turnId: "turn-stalled", + }; + let releaseEvents = () => {}; + const eventsReleased = new Promise((resolve) => { + releaseEvents = resolve; + }); + let markHandoffStarted = () => {}; + const handoffStarted = new Promise((resolve) => { + markHandoffStarted = resolve; + }); + let handoffSignal: AbortSignal | undefined; + let releaseHandoff = () => {}; + const close = vi.fn(async () => releaseEvents()); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield created; + await eventsReleased; + }, + async startTurn() { + return { turnId: "turn-stalled" }; + }, + handoffRuntimeRequest(input) { + handoffSignal = input.signal; + markHandoffStarted(); + return { + result: "handed_off", + cleanup: new Promise((resolve) => { releaseHandoff = resolve; }), + }; + }, + cancel() { + releaseEvents(); + return { cleanup: Promise.resolve() }; + }, + async result() { + return null; + }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-stalled", + activeTurnId: "turn-stalled", + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "stalled-handoff-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + async completeRun() {}, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 1, + timeoutMs: 25, + keepSessionOpen: true, + }); + await handoffStarted; + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce()); + expect(handoffSignal?.aborted).toBe(true); + await expect(execution).rejects.toThrow("native session timed out"); + releaseHandoff(); + expect(close).toHaveBeenCalledOnce(); + }); + + it("preserves terminal success while iterator teardown remains pending", async () => { + vi.useFakeTimers(); + try { + let releaseTeardown = () => {}; + const teardownStarted = vi.fn(); + const close = vi.fn(async () => undefined); + const readResult = vi.fn(async () => ({ + result, + terminal, + turnId: "turn-terminal", + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + }; + }, + async *events() { + try { + yield { ...runnerEvent(1, "turn.completed"), turnId: "turn-terminal" }; + } finally { + teardownStarted(); + await new Promise((resolve) => { + releaseTeardown = resolve; + }); + } + }, + async startTurn() { + return { turnId: "turn-terminal" }; + }, + result: readResult, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-terminal", + cursor: "1", + activeTurnId: null, + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "slow-teardown-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const completeRun = vi.fn(async () => undefined); + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent() { + return { + cursor: 1, + highestContiguousSourceSeq: 1, + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + }); + await vi.waitFor(() => expect(teardownStarted).toHaveBeenCalledOnce()); + await vi.advanceTimersByTimeAsync(100); + await expect(execution).resolves.toMatchObject({ result }); + expect(readResult).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + releaseTeardown(); + await Promise.resolve(); + } finally { + vi.useRealTimers(); + } + }); + + it("preserves terminal success while quarantining stalled handoff cleanup", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-terminal", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-terminal", + itemId: "input-terminal", + }; + let markHandoffStarted = () => {}; + const handoffStarted = new Promise((resolve) => { + markHandoffStarted = resolve; + }); + let releaseHandoff = () => {}; + let handoffSignal: AbortSignal | undefined; + const close = vi.fn(async () => undefined); + const onSession = vi.fn(); + const completeRun = vi.fn(async () => undefined); + const readResult = vi.fn(async () => ({ + result, + terminal, + turnId: "turn-terminal", + })); + const providerEvents = [ + { + ...runnerEvent(1, "runtime_request.created", { request }), + turnId: "turn-terminal", + }, + { ...runnerEvent(2, "turn.completed"), turnId: "turn-terminal" }, + ]; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { + yield providerEvents[0]!; + await handoffStarted; + yield providerEvents[1]!; + }, + async startTurn() { + return { turnId: "turn-terminal" }; + }, + handoffRuntimeRequest(input) { + handoffSignal = input.signal; + markHandoffStarted(); + return { + result: "handed_off", + cleanup: new Promise((resolve) => { + releaseHandoff = resolve; + }), + }; + }, + result: readResult, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-terminal", + cursor: "2", + activeTurnId: null, + }; + }, + close, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "terminal-handoff-backend", + version: "1", + capabilities: await session.capabilities(), + }; + }, + async openSession() { + return session; + }, + }; + const appended: PrpEvent[] = []; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + appended.push(structuredClone(event as PrpEvent)); + return { + cursor: appended.length, + highestContiguousSourceSeq: highestContiguous(appended), + disposition: "committed", + }; + }, + async replayEvents() { + return { events: [], highestContiguousSourceSeq: 0 }; + }, + completeRun, + }; + + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 1, + keepSessionOpen: true, + onSession, + }); + await handoffStarted; + await vi.waitFor(() => expect(handoffSignal?.aborted).toBe(true)); + await expect(execution).resolves.toMatchObject({ result }); + expect(readResult).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + expect(close).toHaveBeenCalledOnce(); + expect(onSession).toHaveBeenLastCalledWith(null); + releaseHandoff(); + await Promise.resolve(); + expect(close).toHaveBeenCalledOnce(); + expect(completeRun).toHaveBeenCalledOnce(); + }); + + it("keeps a settling structured input in the original turn while its append crosses expiry", async () => { + const request = { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "input-live", + type: "input", + status: "pending", + prompt: "Which region?", + input: { + schema: "paperclip.question_set.v1", + questions: [{ + id: "region", + prompt: "Which region?", + required: true, + answerMode: "text", + }], + }, + origin: { adapter: "mock" }, + turnId: "turn-live", + itemId: "input-live", + }; + const providerEvents = [ + { ...runnerEvent(1, "runtime_request.created", { request }), turnId: "turn-live" }, + { + ...runnerEvent(2, "runtime_request.resolved", { + requestId: "input-live", + requestKind: "user_input", + turnId: "turn-live", + itemId: "input-live", + action: "submit", + requestType: "input", + }), + turnId: "turn-live", + }, + { ...runnerEvent(3, "turn.completed"), turnId: "turn-live" }, + ]; + const appended: PrpEvent[] = []; + let markSettlementAppendStarted!: () => void; + const settlementAppendStarted = new Promise((resolve) => { + markSettlementAppendStarted = resolve; + }); + let releaseSettlementAppend!: () => void; + const settlementAppendReleased = new Promise((resolve) => { + releaseSettlementAppend = resolve; + }); + const handoffRuntimeRequest = vi.fn(() => ({ + result: "handed_off" as const, + cleanup: Promise.resolve(), + })); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }; + }, + async *events() { yield* providerEvents; }, + async startTurn() { return { turnId: "turn-live" }; }, + handoffRuntimeRequest, + async result() { return { result, terminal, turnId: "turn-live" }; }, + async snapshot() { + return { + backendKind: "mock", + sessionId: identity.sessionId, + identity, + providerSessionId: "provider-live", + cursor: "3", + activeTurnId: null, + pendingRuntimeRequests: [], + lineage: [], + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "runtime-input-live-backend", + version: "1", + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + runtimeRequestHandoff: true, + }, + }; + }, + async openSession() { return session; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async checkpointSession() {}, + async appendEvent(event) { + if (event.eventType === "runtime_request.resolved") { + markSettlementAppendStarted(); + await settlementAppendReleased; + } + appended.push(structuredClone(event as PrpEvent)); + const sourceEvents = appended.filter((candidate) => candidate.sourceInstanceId === event.sourceInstanceId); + return { + cursor: appended.length, + highestContiguousSourceSeq: highestContiguous(sourceEvents), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const replayed = appended.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId && event.sourceSeq > replay.afterSourceSeq + ); + return { events: structuredClone(replayed), highestContiguousSourceSeq: highestContiguous(replayed) }; + }, + async completeRun() {}, + }; + + const originalSetTimeout = globalThis.setTimeout; + let queuedHandoffCallback: (() => void) | null = null; + const timeoutSpy = vi.spyOn(globalThis, "setTimeout").mockImplementation(((callback, delay, ...args) => { + if (delay === 123_456) { + queuedHandoffCallback = () => callback(...args); + const handle = originalSetTimeout(() => undefined, 60_000); + handle.unref?.(); + return handle; + } + return originalSetTimeout(callback, delay, ...args); + }) as typeof setTimeout); + try { + const execution = executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + runtimeInputLiveWindowMs: 123_456, + }); + await settlementAppendStarted; + expect(queuedHandoffCallback).not.toBeNull(); + queuedHandoffCallback?.(); + await Promise.resolve(); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + releaseSettlementAppend(); + await expect(execution).resolves.toMatchObject({ result }); + queuedHandoffCallback?.(); + await Promise.resolve(); + expect(handoffRuntimeRequest).not.toHaveBeenCalled(); + expect(appended.some((event) => event.eventType === "runtime_request.expired")).toBe(false); + } finally { + releaseSettlementAppend(); + timeoutSpy.mockRestore(); + } + }); +}); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts new file mode 100644 index 0000000000..66e4696131 --- /dev/null +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -0,0 +1,1205 @@ +import { randomUUID } from "node:crypto"; + +import type { + CheckpointControlPlaneSessionOptions, + ControlPlanePort, +} from "./contracts/control-plane-port.js"; +import type { NativeExecutionInput, NativeSessionExecutionResult } from "./contracts/native-execution.js"; +import { buildNativeModelEnvelope, parseNativeExecutionInput } from "./contracts/native-execution.js"; +import type { NativeSession, NativeSessionBackend } from "./contracts/native-session-backend.js"; +import type { PersistedNativeSession } from "./contracts/native-session-backend.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "./protocol/replay-contract.js"; +import { parsePaperclipQuestionSet } from "./contracts/question-set.js"; + +export const DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS = 120_000; +const OPTIONAL_SESSION_CANCELLATION_GRACE_MS = 100; +const FAILED_OPERATION_SETTLEMENT_GRACE_MS = 100; +const DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS = 30_000; + +export interface ExecuteNativeSessionOptions { + input: NativeExecutionInput; + backend: NativeSessionBackend; + controlPlane: ControlPlanePort; + runnerInstanceId: string; + controlPlaneInstanceId: string; + timeoutMs?: number; + /** Internal test seam; production bounds checkpoint persistence to 30 seconds. */ + checkpointTimeoutMs?: number; + /** Internal test seam; production uses the fixed 120-second platform policy. */ + runtimeInputLiveWindowMs?: number; + onSession?: (session: NativeSession | null) => void; + existingSession?: NativeSession; + persistedSession?: PersistedNativeSession | null; + keepSessionOpen?: boolean; + onCheckpoint?: ( + snapshot: PersistedNativeSession, + options?: CheckpointControlPlaneSessionOptions, + ) => Promise | void; + /** Called when exact provider recovery failed and policy opened a new provider session. */ + onContinuityBreak?: (input: { + reason: string; + previousDriverSessionId: string; + previousProviderSessionId: string | null; + replacementDriverSessionId: string; + replacementProviderSessionId: string | null; + }) => Promise | void; + /** + * Control-plane policy seam for a provider turn that completed after + * durably creating a governed wait, but did not emit a semantic finish + * result. The runner package cannot inspect server-owned interactions, so + * it asks the embedding control plane whether that missing result is an + * intentional yield before treating it as provider failure. + */ + resolveMissingResult?: (input: { + turnId: string | null; + terminalEvent: PrpEvent; + }) => Promise; + /** + * Detect a durable server-owned wait as soon as its provider tool event is + * committed. Models are not trusted to stop or avoid polling after creating + * a question/review interaction; the control plane may park the turn here. + * This boundary is deliberately synchronous and observational: asynchronous + * mutation authority cannot be revoked safely after a failed execution. + */ + resolveGovernedWait?: (input: { + turnId: string | null; + event: PrpEvent; + }) => PrpStructuredRunResult | null; +} + +function isTurnTerminal(event: PrpEvent): boolean { + return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes(event.eventType); +} + +function terminalFromEvent(event: PrpEvent, disposition: PrpTerminalState["reportedWorkDisposition"]): PrpTerminalState { + const states = event.eventType === "turn.completed" + ? { turnTerminalState: "completed" as const, runTerminalState: "succeeded" as const } + : event.eventType === "turn.failed" + ? { turnTerminalState: "failed" as const, runTerminalState: "failed" as const } + : event.eventType === "turn.interrupted" + ? { turnTerminalState: "interrupted" as const, runTerminalState: "cancelled" as const } + : { turnTerminalState: "cancelled" as const, runTerminalState: "cancelled" as const }; + return { schema: "paperclip.prp.terminal.v1", ...states, reportedWorkDisposition: disposition }; +} + +async function attemptOptionalSessionCancellation( + session: NativeSession, + reason: string, +): Promise<{ settlement: Promise[]> } | null> { + if (session.cancel === undefined) return null; + const cancellationAbort = new AbortController(); + let cleanup: Promise; + try { + // The session commits cancellation before returning. Only provider + // cleanup remains asynchronous, so bounded failure settlement cannot + // leave an operation with accepted-output or mutation authority. + cleanup = session.cancel({ + reason, + signal: cancellationAbort.signal, + }).cleanup; + } catch { + return null; + } + const attempts = Promise.allSettled([cleanup]); + if (await settlesWithin(attempts, OPTIONAL_SESSION_CANCELLATION_GRACE_MS)) { + return null; + } + cancellationAbort.abort(new Error("native session cancellation grace expired")); + return { settlement: attempts }; +} + +async function settlesWithin(operation: Promise, timeoutMs: number): Promise { + let graceTimer: ReturnType | undefined; + const settled = await Promise.race([ + operation.then(() => true), + new Promise((resolve) => { + graceTimer = setTimeout(() => resolve(false), timeoutMs); + graceTimer.unref?.(); + }), + ]); + if (graceTimer !== undefined) clearTimeout(graceTimer); + return settled; +} + +async function runAbortableOperationWithin(input: { + timeoutMs: number; + timeoutMessage: string; + timeoutError?: () => Error; + operation: (signal: AbortSignal) => Promise; + onLateResolution?: (value: T) => Promise | void; +}): Promise { + const operationAbort = new AbortController(); + let timer: ReturnType | undefined; + let timedOut = false; + let timeoutError: Error | undefined; + const operation = input.operation(operationAbort.signal); + // The timeout path deliberately stops awaiting an uncooperative adapter. + // Keep its eventual settlement observed after mutation authority is revoked. + void operation.catch(() => undefined); + if (input.onLateResolution !== undefined) { + void operation.then( + (value) => { + if (!timedOut) return; + void Promise.resolve() + .then(() => input.onLateResolution!(value)) + .catch(() => undefined); + }, + () => undefined, + ); + } + try { + const value = await Promise.race([ + operation, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = input.timeoutError?.() ?? new Error(input.timeoutMessage); + timedOut = true; + timeoutError = error; + reject(error); + operationAbort.abort(error); + }, input.timeoutMs); + timer.unref?.(); + }), + ]); + // An abort listener can resolve synchronously before the timeout promise's + // rejection wins the race. The deadline still owns that boundary: the late + // result is disposed above and must never be admitted by the caller. + if (timedOut) throw timeoutError; + return value; + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +async function disposeUnadmittedSession( + session: NativeSession, + reason: string, +): Promise { + // A provider may ignore abort and return a session after its caller has + // already timed out. That session was never published through onSession, so + // close it without clearing ownership that a later execution may establish. + const closeSettlement = Promise.allSettled([ + Promise.resolve().then(() => session.close({ reason })), + ]); + await settlesWithin(closeSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); +} + +class NativeSessionFinalizationTimeoutError extends Error { + constructor(timeoutMs: number) { + super(`native session finalization timed out after ${timeoutMs}ms`); + this.name = "NativeSessionFinalizationTimeoutError"; + } +} + +async function finalizeWithin(input: { + timeoutMs: number; + operation: (signal: AbortSignal) => Promise; +}): Promise { + return runAbortableOperationWithin({ + ...input, + timeoutMessage: `native session finalization timed out after ${input.timeoutMs}ms`, + timeoutError: () => new NativeSessionFinalizationTimeoutError(input.timeoutMs), + }); +} + +async function finalizeIdempotentControlPlaneWithin(input: { + timeoutMs: number; + operation: (signal: AbortSignal) => Promise; +}): Promise { + try { + return await finalizeWithin(input); + } catch (error) { + if (!(error instanceof NativeSessionFinalizationTimeoutError)) throw error; + // Only the deterministic control-plane transaction enters this retry. + // Its event ids and completion dedupe key make the second settlement an + // authoritative acknowledgement of any first-attempt commit. Provider + // result resolution and checkpointing are deliberately outside this + // boundary and are never started a second time. + return finalizeWithin(input); + } +} + +async function quarantineRetainedSession( + session: NativeSession, + onSession: ExecuteNativeSessionOptions["onSession"], + reason: string, +): Promise { + // Eviction and provider cleanup are independent obligations. Keep both + // observed so a throwing owner callback cannot prevent close from starting, + // and a broken provider cannot keep the failed execution pending forever. + const quarantineSettlement = Promise.allSettled([ + Promise.resolve().then(() => onSession?.(null)), + Promise.resolve().then(() => session.close({ reason })), + ]); + await settlesWithin( + quarantineSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); +} + +async function persistCheckpointWithin(input: { + snapshot: PersistedNativeSession; + controlPlane: ControlPlanePort; + onCheckpoint: ExecuteNativeSessionOptions["onCheckpoint"]; + timeoutMs: number; + externalSignal?: AbortSignal; +}): Promise { + const checkpointAbort = new AbortController(); + const externalSignal = input.externalSignal; + let timer: ReturnType | undefined; + let removeExternalAbort = () => {}; + const externalAbortFailure = externalSignal + ? new Promise((_resolve, reject) => { + const abort = () => { + const reason = externalSignal.reason + ?? new Error("native session checkpoint aborted"); + checkpointAbort.abort(reason); + reject(reason); + }; + if (externalSignal.aborted) { + abort(); + } else { + externalSignal.addEventListener("abort", abort, { once: true }); + removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + } + }) + : new Promise(() => undefined); + const checkpointing = (async () => { + const checkpointOptions = { signal: checkpointAbort.signal }; + await input.controlPlane.checkpointSession?.( + input.snapshot, + checkpointOptions, + ); + if (checkpointAbort.signal.aborted) { + throw checkpointAbort.signal.reason + ?? new Error("native session checkpoint aborted"); + } + await input.onCheckpoint?.(input.snapshot, checkpointOptions); + })(); + // The timeout path intentionally stops awaiting an uncooperative adapter. + // Keep its eventual rejection observed after execution has quarantined and + // closed the provider session. + void checkpointing.catch(() => undefined); + try { + await Promise.race([ + checkpointing, + new Promise((_resolve, reject) => { + timer = setTimeout(() => { + const error = new Error( + `native session checkpoint timed out after ${input.timeoutMs}ms`, + ); + checkpointAbort.abort(error); + reject(error); + }, input.timeoutMs); + }), + externalAbortFailure, + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + removeExternalAbort(); + } +} + +async function consumeTurn( + session: NativeSession, + controlPlane: ControlPlanePort, + timeoutMs: number, + runtimeInputLiveWindowMs: number, + closeFailedSession: () => Promise, + quarantineSession: () => void, + resolveGovernedWait?: ExecuteNativeSessionOptions["resolveGovernedWait"], + externalSignal?: AbortSignal, +) { + let timer: ReturnType | undefined; + const appendAbort = new AbortController(); + const governedCleanupOperations = new Set>(); + let governedCancellationCommitted = false; + let deferredGovernedCleanupSettlement: Promise | null = null; + let deferredSessionCancellationSettlement: Promise | null = null; + const inputTimers = new Map>(); + const handoffCleanupOperations = new Set>(); + const eventIterator = session.events()[Symbol.asyncIterator](); + let stopConsumer = false; + let rejectHandoff: ((error: unknown) => void) | null = null; + const handoffFailure = new Promise((_resolve, reject) => { + rejectHandoff = reject; + }); + let removeExternalAbort = () => {}; + const externalAbortFailure = externalSignal + ? new Promise((_resolve, reject) => { + const abort = () => { + const reason = externalSignal.reason ?? new Error("native event consumption aborted"); + stopConsumer = true; + appendAbort.abort(reason); + reject(reason); + }; + if (externalSignal.aborted) { + abort(); + } else { + externalSignal.addEventListener("abort", abort, { once: true }); + removeExternalAbort = () => externalSignal.removeEventListener("abort", abort); + } + }) + : new Promise(() => undefined); + const clearInputTimer = (requestId: string) => { + const inputTimer = inputTimers.get(requestId); + if (inputTimer !== undefined) clearTimeout(inputTimer); + inputTimers.delete(requestId); + }; + const consumer = (async () => { + let eventCount = 0; + let highestContiguousSourceSeq = 0; + let governedResult: PrpStructuredRunResult | null = null; + while (true) { + const next = await eventIterator.next(); + if (stopConsumer) throw new Error("native event consumer stopped"); + if (next.done) throw new Error("native event stream closed before a turn terminal fact"); + const event = next.value; + const payload = event.payload as Record; + const settlingRequestId = + ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"] + .includes(event.eventType) + && typeof payload.requestId === "string" + ? payload.requestId + : null; + // Beginning settlement revokes the expiry timer's handoff authority. + // appendEvent may remain pending across the live-window deadline; if + // the timer stayed live until the receipt returned, both settlement + // and a durable handoff could commit for the same request. + if (settlingRequestId !== null) clearInputTimer(settlingRequestId); + const receipt = await controlPlane.appendEvent(event, { + signal: appendAbort.signal, + }); + if (stopConsumer) throw new Error("native event consumer stopped"); + eventCount += receipt.disposition === "committed" ? 1 : 0; + highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); + const request = payload.request && typeof payload.request === "object" && !Array.isArray(payload.request) + ? payload.request as Record + : null; + if ( + receipt.disposition === "committed" + && event.eventType === "runtime_request.created" + && request?.schema === "paperclip.runtime_request.v2" + && request.type === "input" + && typeof request.requestId === "string" + && typeof request.turnId === "string" + ) { + try { + parsePaperclipQuestionSet(request.input); + const requestId = request.requestId; + const turnId = request.turnId; + clearInputTimer(requestId); + const inputTimer = setTimeout(() => { + // Clearing a timeout does not revoke a callback that is already + // queued. The map entry is the per-request authority token. + if (inputTimers.get(requestId) !== inputTimer) return; + inputTimers.delete(requestId); + // A timer callback can already be queued when teardown clears + // its handle. Re-check the live-turn authority inside the + // callback before starting or registering durable work. + if (stopConsumer || appendAbort.signal.aborted) return; + if (session.handoffRuntimeRequest === undefined) { + rejectHandoff?.(new Error("native_runtime_request_handoff_unavailable")); + return; + } + let handoffCleanup: Promise; + try { + const handoff = session.handoffRuntimeRequest({ + requestId, + turnId, + reason: "durable_handoff", + signal: appendAbort.signal, + }); + // Durable handoff mutation is synchronous. The returned + // promise owns provider interruption only, so it can remain + // observed without acquiring authority to delay or reverse + // a provider terminal fact. + handoffCleanup = handoff.cleanup; + } catch (error) { + rejectHandoff?.(error); + return; + } + handoffCleanupOperations.add(handoffCleanup); + void handoffCleanup + .catch((error) => { + if (!stopConsumer && !appendAbort.signal.aborted) { + rejectHandoff?.(error); + } + }) + .finally(() => handoffCleanupOperations.delete(handoffCleanup)); + }, runtimeInputLiveWindowMs); + inputTimer.unref?.(); + inputTimers.set(requestId, inputTimer); + } catch { + // Invalid structured inputs remain rejected by the driver and never become durable questions. + } + } + if (governedResult === null && resolveGovernedWait) { + if (appendAbort.signal.aborted) { + throw appendAbort.signal.reason ?? new Error("native event consumption aborted"); + } + governedResult = resolveGovernedWait({ + turnId: event.turnId ?? null, + event, + }); + if (governedResult !== null && !isTurnTerminal(event)) { + if (session.cancel === undefined) { + throw new Error("native_governed_wait_cancellation_unavailable"); + } + // A governed result is already durable. Commit cancellation + // synchronously, then stop consuming provider output now rather + // than waiting for an abort-insensitive cleanup or terminal event. + // The returned promise owns cleanup only and remains observed in + // finally, where its wait is bounded and the session quarantined. + const cancellation = session.cancel({ + reason: "Paperclip parked this turn on a durable governed interaction.", + signal: appendAbort.signal, + }); + governedCancellationCommitted = true; + const cleanup = cancellation.cleanup; + governedCleanupOperations.add(cleanup); + void cleanup + .catch(() => quarantineSession()) + .finally(() => governedCleanupOperations.delete(cleanup)); + return { event, eventCount, highestContiguousSourceSeq, governedResult }; + } + } + if (isTurnTerminal(event)) { + return { event, eventCount, highestContiguousSourceSeq, governedResult }; + } + } + })(); + // A timeout can win the race while an iterator is still waiting for data. + // Observe any later consumer rejection so it cannot become process-fatal. + void consumer.catch(() => undefined); + let consumptionFailed = false; + try { + return await Promise.race([ + consumer, + new Promise((_, reject) => { + timer = setTimeout(() => { + reject(new Error(`native session timed out after ${timeoutMs}ms`)); + }, timeoutMs); + }), + handoffFailure, + externalAbortFailure, + ]); + } catch (error) { + consumptionFailed = true; + stopConsumer = true; + appendAbort.abort(error); + for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer); + inputTimers.clear(); + // Governed-wait discovery is synchronous and observational, and provider + // cancellation has already committed synchronously. Bound only the + // authority-free provider cleanup while keeping its outcome observed. + if (governedCleanupOperations.size > 0) { + const governedCleanupSettlement = Promise.allSettled([...governedCleanupOperations]); + if (!(await settlesWithin(governedCleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS))) { + deferredGovernedCleanupSettlement = governedCleanupSettlement; + } + } + if (!governedCancellationCommitted) { + const deferredCancellation = await attemptOptionalSessionCancellation( + session, + "Native session event consumption failed.", + ); + deferredSessionCancellationSettlement = deferredCancellation?.settlement ?? null; + } + throw error; + } finally { + stopConsumer = true; + for (const inputTimer of inputTimers.values()) clearTimeout(inputTimer); + inputTimers.clear(); + const activeHandoffCleanupSettlement = handoffCleanupOperations.size > 0 + ? Promise.allSettled([...handoffCleanupOperations]) + : null; + const activeGovernedCleanupSettlement = deferredGovernedCleanupSettlement === null + && governedCleanupOperations.size > 0 + ? Promise.allSettled([...governedCleanupOperations]) + : null; + if (!consumptionFailed && !appendAbort.signal.aborted) { + // A provider terminal or synchronous governed cancellation revokes + // live-turn authority. Handoff state was already committed; abort only + // tells provider cleanup that it must not begin any new work. + appendAbort.abort(new Error("native turn reached a terminal state")); + } + // Do not let failure escape while the provider iterator still owns a live + // subscription. Cancellation above is responsible for releasing a blocked + // `next()`; awaiting `return()` then synchronizes the iterator's `finally` + // teardown before the session can be closed or reused. + const iteratorTeardown = eventIterator.return?.().catch(() => undefined); + // The consumer may already be past `next()` and awaiting a durable append. + // Abort is a control-plane durability boundary: appendEvent must settle + // without committing when its signal is aborted. Handoff and cancellation + // promises below own provider cleanup only; their durable transitions were + // synchronous, so a slow cleanup cannot reverse terminal completion. + const passiveTeardownSettlement = Promise.allSettled([ + iteratorTeardown, + consumer, + ...(activeHandoffCleanupSettlement ? [activeHandoffCleanupSettlement] : []), + ...(activeGovernedCleanupSettlement ? [activeGovernedCleanupSettlement] : []), + ...(deferredGovernedCleanupSettlement ? [deferredGovernedCleanupSettlement] : []), + ...(deferredSessionCancellationSettlement + ? [deferredSessionCancellationSettlement] + : []), + ]); + if (consumptionFailed) { + // Start provider close immediately so a cooperative implementation can + // release a blocked iterator or provider operation. Abort has already + // revoked every control-plane mutation capability and closeSession has + // removed this session from the caller, so an implementation that + // violates its cancellation contract is quarantined rather than allowed + // to defeat the execution deadline. Promise.allSettled keeps every late + // rejection observed after the bounded wait expires. + const cleanupSettlement = Promise.allSettled([ + passiveTeardownSettlement, + Promise.resolve().then(closeFailedSession), + ]); + await settlesWithin(cleanupSettlement, FAILED_OPERATION_SETTLEMENT_GRACE_MS); + } else { + // Iterator and provider cleanup own no control-plane mutation authority. + // A slow subscription or cleanup remains observed and is released by the + // normal session close, but it cannot erase an already committed + // terminal fact or prevent result retrieval and durable finalization. + const teardownSettled = await settlesWithin( + passiveTeardownSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); + if (!teardownSettled) quarantineSession(); + } + if (timer !== undefined) clearTimeout(timer); + removeExternalAbort(); + } +} + +function checkpointCursor(cursor: string | null | undefined): number { + if (cursor === undefined || cursor === null || cursor === "") return 0; + const parsed = Number(cursor); + return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : 0; +} + +async function reconcileRecoveryCursor(input: { + controlPlane: ControlPlanePort; + checkpoint: PersistedNativeSession; + runId: string; + sourceInstanceId: string; + signal: AbortSignal; +}): Promise { + const checkpointHighWater = checkpointCursor(input.checkpoint.cursor); + let afterSourceSeq = checkpointHighWater; + let persistedHighWater = checkpointHighWater; + while (true) { + const replay = await input.controlPlane.replayEvents( + { + runId: input.runId, + sourceInstanceId: input.sourceInstanceId, + afterSourceSeq, + limit: 1_000, + }, + { signal: input.signal }, + ); + input.signal.throwIfAborted(); + if (replay.events.length === 0) break; + const pageHighWater = replay.events.reduce( + (highest, event) => Math.max(highest, event.sourceSeq), + afterSourceSeq, + ); + if (pageHighWater <= afterSourceSeq) { + throw new Error("native_recovery_replay_did_not_advance"); + } + persistedHighWater = Math.max(persistedHighWater, pageHighWater); + afterSourceSeq = pageHighWater; + } + if (persistedHighWater === checkpointHighWater && input.checkpoint.cursor === String(checkpointHighWater)) { + return input.checkpoint; + } + return { ...input.checkpoint, cursor: String(persistedHighWater) }; +} + +/** + * Package-owned normalized session loop. Paperclip supplies persistence and + * authority through ControlPlanePort; provider/session behavior stays here. + */ +export async function executeNativeSession(options: ExecuteNativeSessionOptions): Promise { + const input = parseNativeExecutionInput(options.input); + const descriptor = await options.backend.descriptor(); + if ("runtimeContext" in input) { + const capabilities = descriptor.runtimeContextCapabilities; + const unsupported = (["instructions", "skills", "mcp"] as const).filter((key) => capabilities?.[key] !== "native"); + if (unsupported.length) throw new Error(`native_runtime_context_unsupported: ${descriptor.name} does not natively realize ${unsupported.join(", ")}`); + } + let persistedSession = options.existingSession + ? null + : options.persistedSession ?? await options.controlPlane.loadSessionCheckpoint?.() ?? null; + if ( + persistedSession + && ( + persistedSession.identity.runId !== input.binding.runId + || persistedSession.identity.companyId !== input.binding.companyId + || persistedSession.identity.issueId !== input.binding.issueId + || persistedSession.identity.agentId !== input.binding.agentId + || input.session.normalizedSessionId === null + || persistedSession.identity.sessionId !== input.session.normalizedSessionId + ) + ) throw new Error("native_session_checkpoint_binding_mismatch"); + const existingIdentity = options.existingSession?.identity() ?? null; + if ( + existingIdentity + && ( + existingIdentity.companyId !== input.binding.companyId + || existingIdentity.issueId !== input.binding.issueId + || existingIdentity.agentId !== input.binding.agentId + || input.session.normalizedSessionId === null + || existingIdentity.sessionId !== input.session.normalizedSessionId + ) + ) throw new Error("native_session_attach_binding_mismatch"); + const normalizedSessionId = persistedSession?.identity.sessionId + ?? existingIdentity?.sessionId + ?? input.session.normalizedSessionId + ?? randomUUID(); + const identity = { + runId: input.binding.runId, + sessionId: normalizedSessionId, + companyId: input.binding.companyId, + issueId: input.binding.issueId, + agentId: input.binding.agentId, + }; + let recovered = false; + let session: NativeSession | null = null; + let continuityBreak: { + reason: string; + previousDriverSessionId: string; + previousProviderSessionId: string | null; + } | null = null; + let reconciledRecoveryCheckpoint: PersistedNativeSession | null = null; + if (options.existingSession) { + if (options.existingSession.attachRun === undefined) { + throw new Error("native_session_multi_run_unavailable"); + } + // Attaching can fail even after the retained session's identity passes the + // static binding check (for example, when the provider lost multi-run + // state). Prove the provider attachment before opening durable + // control-plane state because ControlPlanePort has no rollback operation. + try { + await options.existingSession.attachRun({ identity }); + } catch (error) { + // attachRun has no transactional guarantee: a provider may bind the new + // run before reporting a later failure. Conservatively quarantine the + // session so neither the old nor partially attached run can reuse it. + await quarantineRetainedSession( + options.existingSession, + options.onSession, + "native session attachment failed", + ); + throw error; + } + session = options.existingSession; + recovered = true; + } else if (persistedSession) { + // A persisted checkpoint proves that this is recovery of an existing + // durable run. Reconcile its cursor and prove provider continuity before + // re-opening that run in the control plane: ControlPlanePort has no + // rollback operation if same-session recovery fails. + const recoveryCheckpoint = persistedSession; + const recoveryTimeoutMs = options.timeoutMs ?? 900_000; + persistedSession = await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session recovery replay timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => reconcileRecoveryCursor({ + controlPlane: options.controlPlane, + checkpoint: recoveryCheckpoint, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + signal, + }), + }); + reconciledRecoveryCheckpoint = persistedSession; + const providerRecoveryCheckpoint = persistedSession; + + const replacementAllowed = + providerRecoveryCheckpoint.providerRecoveryPolicy === + "allow_replacement_after_resume_failure"; + const recovery = options.backend.recoverSession + ? await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => options.backend.recoverSession!( + providerRecoveryCheckpoint, + { signal }, + ), + onLateResolution: async (lateRecovery) => { + if (lateRecovery.session) { + await disposeUnadmittedSession( + lateRecovery.session, + "native session provider recovery timed out", + ); + } + }, + }) + : { recovered: false as const, reason: "driver does not support recovery" }; + if (!recovery.recovered || !recovery.session) { + if (!replacementAllowed) { + throw new Error(`native_session_recovery_failed: ${recovery.reason ?? "unknown"}`); + } + continuityBreak = { + reason: recovery.reason ?? "provider session is no longer recoverable", + previousDriverSessionId: providerRecoveryCheckpoint.sessionId, + previousProviderSessionId: providerRecoveryCheckpoint.providerSessionId ?? null, + }; + const replacementInput = { + identity, + workingDirectory: input.workspace.cwd, + }; + session = await runAbortableOperationWithin({ + timeoutMs: recoveryTimeoutMs, + timeoutMessage: `native session replacement bootstrap timed out after ${recoveryTimeoutMs}ms`, + operation: (signal) => { + const abortableReplacementInput = { ...replacementInput, signal }; + return options.backend.openReplacementSession + ? options.backend.openReplacementSession( + abortableReplacementInput, + providerRecoveryCheckpoint, + ) + : options.backend.openSession(abortableReplacementInput); + }, + onLateResolution: (lateSession) => disposeUnadmittedSession( + lateSession, + "native session replacement bootstrap timed out", + ), + }); + } else { + session = recovery.session; + recovered = true; + } + } + if (session === null) { + // A fresh provider session is part of admission. Prove that it exists + // before opening durable run state because ControlPlanePort intentionally + // exposes no rollback for an admitted run. + const bootstrapTimeoutMs = options.timeoutMs ?? 900_000; + const bootstrapInput = { + identity, + workingDirectory: input.workspace.cwd, + }; + session = await runAbortableOperationWithin({ + timeoutMs: bootstrapTimeoutMs, + timeoutMessage: `native session bootstrap timed out after ${bootstrapTimeoutMs}ms`, + operation: (signal) => options.backend.openSession({ + ...bootstrapInput, + signal, + }), + onLateResolution: (lateSession) => disposeUnadmittedSession( + lateSession, + "native session bootstrap timed out", + ), + }); + } + try { + await options.controlPlane.openRun({ + identity, + backendKind: descriptor.kind, + sourceInstanceId: options.runnerInstanceId, + }); + } catch (error) { + if (session) { + // Attachment and recovery both establish provider-side authority before + // durable admission. If admission then fails, the prepared session must + // not remain available for reuse. + await quarantineRetainedSession( + session, + options.onSession, + "native control-plane run admission failed", + ); + } + throw error; + } + let sessionClosePromise: Promise | null = null; + let sessionQuarantined = false; + const quarantineSession = () => { + if (sessionQuarantined) return; + sessionQuarantined = true; + try { + options.onSession?.(null); + } catch { + // Owner notification cannot prevent provider cleanup. + } + }; + const closeSession = () => { + if (sessionClosePromise === null) { + quarantineSession(); + sessionClosePromise = session.close({ + reason: "native session execution complete", + }); + } + return sessionClosePromise; + }; + let executionSucceeded = false; + try { + // Ownership publication is part of the execution-owned lifetime. If the + // callback fails, the finally block below still quarantines and closes the + // provider session. + options.onSession?.(session); + const checkpointTimeoutMs = options.checkpointTimeoutMs + ?? DEFAULT_NATIVE_CHECKPOINT_TIMEOUT_MS; + const persistCheckpoint = ( + snapshot: PersistedNativeSession, + externalSignal?: AbortSignal, + ) => + persistCheckpointWithin({ + snapshot, + controlPlane: options.controlPlane, + onCheckpoint: options.onCheckpoint, + timeoutMs: checkpointTimeoutMs, + externalSignal, + }); + // Cursor reconciliation is provisional until both provider recovery and + // durable run admission succeed. Persist it only after those boundaries; + // the execution-owned finally below quarantines the recovered session if + // checkpoint persistence itself fails. + if (reconciledRecoveryCheckpoint !== null) { + await persistCheckpoint(reconciledRecoveryCheckpoint); + } + const checkpoint = async (signal?: AbortSignal) => { + const snapshot = await session.snapshot(signal ? { signal } : undefined); + signal?.throwIfAborted(); + await persistCheckpoint(snapshot, signal); + }; + const recoveredSnapshot = await session.snapshot(); + if (continuityBreak) { + await options.onContinuityBreak?.({ + ...continuityBreak, + replacementDriverSessionId: recoveredSnapshot.sessionId, + replacementProviderSessionId: + recoveredSnapshot.providerSessionId ?? null, + }); + } + await persistCheckpoint(recoveredSnapshot); + + let consumed = { + event: null as PrpEvent | null, + eventCount: 0, + highestContiguousSourceSeq: 0, + governedResult: null as PrpStructuredRunResult | null, + }; + const completionSnapshot = recoveredSnapshot.semanticResult && recoveredSnapshot.terminal + ? recoveredSnapshot + : persistedSession; + let completed = completionSnapshot?.semanticResult && completionSnapshot.terminal + ? { + result: completionSnapshot.semanticResult, + terminal: completionSnapshot.terminal, + turnId: completionSnapshot.activeTurnId ?? null, + } + : null; + if (!completed) { + const consumptionAbort = new AbortController(); + const consuming = consumeTurn( + session, + options.controlPlane, + options.timeoutMs ?? 900_000, + options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, + closeSession, + quarantineSession, + options.resolveGovernedWait, + consumptionAbort.signal, + ); + // Event consumption must begin before startTurn so an eager provider cannot + // outrun us. Observe its rejection immediately, though: if startTurn or + // checkpointing fails first, the outer finally closes the session and the + // abandoned consumer will reject when its stream closes. Without a handler + // that later rejection becomes process-fatal under Node's strict policy. + void consuming.catch(() => undefined); + // A recovered driver is authoritative about whether a provider turn is + // still active. In particular, drivers normalize the checkpoint race + // where a terminal fingerprint was persisted before activeTurnId was + // cleared. Falling back to the older control-plane checkpoint here + // resurrects that terminal turn and waits forever for an event that was + // already consumed. + const recoveredActiveTurnId = recovered + ? recoveredSnapshot.activeTurnId ?? null + : persistedSession?.activeTurnId ?? null; + try { + if (!recovered || !recoveredActiveTurnId) { + const modelEnvelope = buildNativeModelEnvelope(input); + const dispositionOnlyRecovery = Boolean( + recovered && + !recoveredSnapshot.semanticResult && + (persistedSession?.terminalTurns?.length ?? 0) > 0 && + !recoveredActiveTurnId + ); + if (dispositionOnlyRecovery) { + modelEnvelope.task.prompt = [ + "Paperclip semantic-result recovery for a prior completed provider turn.", + "The prior turn already performed the work and its user-facing final answer is recorded.", + "Do not repeat implementation, tests, research, or the final answer.", + "Use the existing session context to invoke exactly one paperclip_finish or paperclip_block with the accurate current disposition, then stop without additional user-facing prose.", + ].join("\n"); + } + await session.startTurn({ + message: { role: "user", text: JSON.stringify(modelEnvelope) }, + requestedCollaborationMode: "executionMode" in input ? input.executionMode : "default", + }); + await checkpoint(); + } + } catch (error) { + // Consumption starts before provider launch so eager events cannot be + // lost. If launch or its checkpoint fails, abort any in-flight append + // before joining cleanup so the failed turn cannot commit late or + // strand execution on a never-settling durability call. + consumptionAbort.abort(error); + await consuming.catch(() => undefined); + throw error; + } + const terminalEvent = await consuming; + consumed = terminalEvent; + } + const finalizationTimeoutMs = options.timeoutMs ?? 900_000; + const preparedFinalization = await finalizeWithin({ + timeoutMs: finalizationTimeoutMs, + operation: async (signal) => { + let settledCompletion = completed; + if (settledCompletion === null) { + const terminalEvent = consumed.event; + if (terminalEvent === null) { + throw new Error("native_finalization_missing: session returned no terminal event"); + } + settledCompletion = consumed.governedResult === null + ? await session.result() + : { + result: consumed.governedResult, + terminal: { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: + consumed.governedResult.reportedWorkDisposition, + }, + turnId: terminalEvent.turnId ?? null, + }; + signal.throwIfAborted(); + if (settledCompletion === null && options.resolveMissingResult) { + const recoveredResult = await options.resolveMissingResult({ + turnId: terminalEvent.turnId ?? null, + terminalEvent, + }); + signal.throwIfAborted(); + if (recoveredResult !== null) { + settledCompletion = { + result: recoveredResult, + terminal: terminalFromEvent( + terminalEvent, + recoveredResult.reportedWorkDisposition, + ), + turnId: terminalEvent.turnId ?? null, + }; + } + } + // Do not publish the resolved result to the retryable phase until its + // checkpoint has settled. A timed-out checkpoint therefore cannot be + // skipped by a second attempt. + await checkpoint(signal); + signal.throwIfAborted(); + completed = settledCompletion; + } + if (settledCompletion === null) { + throw new Error("native_finalization_missing: session returned no semantic result"); + } + let terminal: PrpTerminalState; + if (consumed.governedResult !== null) { + terminal = settledCompletion.terminal; + } else if (completionSnapshot?.semanticResult && completionSnapshot.terminal) { + terminal = completionSnapshot.terminal; + } else { + terminal = terminalFromEvent( + consumed.event!, + settledCompletion.result.reportedWorkDisposition, + ); + } + const eventTurnId = settledCompletion.turnId + ?? persistedSession?.activeTurnId + ?? persistedSession?.terminalTurns?.at(-1)?.turnId + ?? consumed.event?.turnId; + const controlEvent = ( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, + ): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${options.controlPlaneInstanceId}:${input.binding.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: options.controlPlaneInstanceId, + sourceKind: "control_plane", + runId: input.binding.runId, + normalizedSessionId, + ...(eventTurnId ? { turnId: eventTurnId } : {}), + eventType, + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload, + }); + return { + completed: settledCompletion, + terminal, + expectedControlEvents: [ + controlEvent(1, "run.result.accepted", { + result: settledCompletion.result, + }), + controlEvent( + 2, + "run.terminal", + terminal as unknown as Record, + ), + ], + }; + }, + }); + const baselineControlEventSequences = new Set(); + const accountedControlEventSequences = new Set(); + let baselineControlReplayCaptured = false; + const durableExecutionResult = await finalizeIdempotentControlPlaneWithin({ + timeoutMs: finalizationTimeoutMs, + operation: async (signal) => { + const controlReplay = await options.controlPlane.replayEvents({ + runId: input.binding.runId, + sourceInstanceId: options.controlPlaneInstanceId, + afterSourceSeq: 0, + limit: 10, + }, { signal }); + signal.throwIfAborted(); + const replayBySequence = new Map(controlReplay.events.map((event) => [event.sourceSeq, event])); + for (const existing of controlReplay.events) { + const expected = preparedFinalization.expectedControlEvents[existing.sourceSeq - 1]; + if ( + expected === undefined + || existing.eventType !== expected.eventType + || canonicalJson(existing.payload) !== canonicalJson(expected.payload) + ) { + throw new Error(`native_control_event_replay_conflict:${existing.sourceSeq}`); + } + } + if (!baselineControlReplayCaptured) { + for (const existing of controlReplay.events) { + baselineControlEventSequences.add(existing.sourceSeq); + } + baselineControlReplayCaptured = true; + } else { + for (const existing of controlReplay.events) { + if ( + !baselineControlEventSequences.has(existing.sourceSeq) + && !accountedControlEventSequences.has(existing.sourceSeq) + ) { + accountedControlEventSequences.add(existing.sourceSeq); + consumed.eventCount += 1; + } + } + } + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + controlReplay.highestContiguousSourceSeq, + ); + for (const event of preparedFinalization.expectedControlEvents) { + if (replayBySequence.has(event.sourceSeq)) continue; + const receipt = await options.controlPlane.appendEvent(event, { signal }); + signal.throwIfAborted(); + if ( + receipt.disposition === "committed" + && !accountedControlEventSequences.has(event.sourceSeq) + ) { + accountedControlEventSequences.add(event.sourceSeq); + consumed.eventCount += 1; + } + consumed.highestContiguousSourceSeq = Math.max( + consumed.highestContiguousSourceSeq, + receipt.highestContiguousSourceSeq, + ); + } + await options.controlPlane.completeRun({ + result: preparedFinalization.completed.result, + terminal: preparedFinalization.terminal, + turnId: preparedFinalization.completed.turnId, + callerResultId: `${options.runnerInstanceId}:${input.binding.runId}:result`, + callerDedupeKey: `${input.binding.runId}:${input.completionContract.sha256}`, + }, { signal }); + return { + result: preparedFinalization.completed.result, + terminal: preparedFinalization.terminal, + turnId: preparedFinalization.completed.turnId, + normalizedSessionId, + driverKind: descriptor.name, + nativeEventCount: consumed.eventCount, + highestContiguousSourceSeq: consumed.highestContiguousSourceSeq, + }; + }, + }); + let enrichment: { + providerSessionId: string | null; + driverVersion: string; + usage: Record | null; + }; + try { + enrichment = await finalizeWithin({ + timeoutMs: options.timeoutMs ?? 900_000, + operation: async (signal) => { + const snapshot = await session.snapshot({ signal }); + signal.throwIfAborted(); + const completedSnapshot = { + ...snapshot, + semanticResult: durableExecutionResult.result, + terminal: durableExecutionResult.terminal, + }; + await persistCheckpoint(completedSnapshot, signal); + signal.throwIfAborted(); + const usage = await session.usage?.() ?? null; + signal.throwIfAborted(); + return { + providerSessionId: snapshot.providerSessionId ?? null, + driverVersion: typeof usage?.driverVersion === "string" + ? usage.driverVersion + : descriptor.version, + usage, + }; + }, + }); + } catch { + // completeRun is the durable commit boundary. Snapshot/checkpoint/usage + // enrichment cannot revoke that success, but a session whose final + // checkpoint is unknown must not remain available for reuse. + void closeSession().catch(() => undefined); + enrichment = { + providerSessionId: recoveredSnapshot.providerSessionId ?? null, + driverVersion: descriptor.version, + usage: null, + }; + } + executionSucceeded = true; + return { ...durableExecutionResult, ...enrichment }; + } finally { + if (!options.keepSessionOpen || !executionSucceeded || sessionQuarantined) { + // A provider that ignores close must not keep execution pending forever. + // closeSession removes it from the caller before invoking the backend; + // retain observation of the promise, but bound the final join. Provider + // cleanup cannot reverse a result the control plane already committed; + // after that durable boundary the session remains unavailable for reuse + // and late close rejection stays observed without contradicting success. + const closeSettlement = Promise.allSettled([closeSession()]); + await settlesWithin( + closeSettlement, + FAILED_OPERATION_SETTLEMENT_GRACE_MS, + ); + } + } +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index 68ab11bfac..cfc6a4bb0b 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -6,5 +6,7 @@ * accidental production dependency. */ export * from "./index.js"; +export * from "./conformance/control-plane-port.js"; +export * from "./conformance/harness-driver.js"; export * from "./conformance/semantic-conformance.js"; export * from "./protocol/replay-loader.js";