diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index c439227f03..abcf8b8638 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -200,11 +200,14 @@ describe("HarnessDriverBackend", () => { it("passes the persisted harness driver kind through recovery", async () => { let recoveredDriverKind: string | null = null; let recoveredProviderIdentity: PersistedHarnessSession["providerIdentity"]; + let recoveredDispositionAllowance: boolean | undefined; const recoveryDriver: HarnessDriver = { ...driver, async recoverSession(snapshot) { recoveredDriverKind = snapshot.driverKind; recoveredProviderIdentity = snapshot.providerIdentity; + recoveredDispositionAllowance = + snapshot.dispositionOnlyRecoveryConsumed; return { recovered: true, session: new FakeHarnessSession() }; }, }; @@ -216,6 +219,7 @@ describe("HarnessDriverBackend", () => { providerSessionId: "provider-1", providerIdentity, providerRecoveryPolicy: "allow_replacement_after_governed_wait", + dispositionOnlyRecoveryConsumed: true, identity: { runId: "run-2", sessionId: "session-1", companyId: "company-1", issueId: "issue-1", agentId: "agent-1" }, }, { signal: new AbortController().signal, @@ -223,6 +227,37 @@ describe("HarnessDriverBackend", () => { expect(recovery.recovered).toBe(true); expect(recoveredDriverKind).toBe("fake"); expect(recoveredProviderIdentity).toEqual(providerIdentity); + expect(recoveredDispositionAllowance).toBe(true); + }); + + it("preserves a consumed disposition allowance in native snapshots", async () => { + class ConsumedRecoverySession extends FakeHarnessSession { + override async snapshot(): Promise { + return { + ...(await super.snapshot()), + dispositionOnlyRecoveryConsumed: true, + }; + } + } + const backend = new HarnessDriverBackend({ + ...driver, + async openSession() { + return new ConsumedRecoverySession(); + }, + }); + const session = await backend.openSession({ + identity: { + runId: "run-consumed-recovery", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + }); + + await expect(session.snapshot()).resolves.toMatchObject({ + dispositionOnlyRecoveryConsumed: true, + }); }); it("forwards the native bootstrap signal to the harness driver", async () => { @@ -329,11 +364,9 @@ describe("HarnessDriverBackend", () => { terminalTurns: [{ turnId: "turn-1", fingerprint: "terminal-1" }], }; - const recoveryOptions = { signal: new AbortController().signal }; - const firstRecovery = await backend.recoverSession( - settledSnapshot, - recoveryOptions, - ); + const firstRecovery = await backend.recoverSession(settledSnapshot, { + signal: new AbortController().signal, + }); expect(firstRecovery.recovered).toBe(true); const firstSession = firstRecovery.session!; const firstRecoveredSnapshot = await firstSession.snapshot(); @@ -348,10 +381,9 @@ describe("HarnessDriverBackend", () => { turnId: "turn-1", }); - const secondRecovery = await backend.recoverSession( - firstRecoveredSnapshot, - recoveryOptions, - ); + const secondRecovery = await backend.recoverSession(firstRecoveredSnapshot, { + signal: new AbortController().signal, + }); expect(secondRecovery.recovered).toBe(true); const secondSession = secondRecovery.session!; await expect(secondSession.snapshot()).resolves.toMatchObject({ diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index 9c39f8ccb4..a5164d6298 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -100,6 +100,10 @@ export class HarnessDriverBackend implements NativeSessionBackend { }, }), terminalTurns: snapshot.terminalTurns ?? [], + dispositionOnlyRecoveryConsumed: + snapshot.dispositionOnlyRecoveryConsumed ?? false, + dispositionOnlyRecoveryTurnId: + snapshot.dispositionOnlyRecoveryTurnId ?? null, pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], lineage: snapshot.lineage ?? [], }; @@ -429,6 +433,10 @@ class HarnessNativeSession implements NativeSession { ? snapshot.semanticResult?.turnId ?? null : snapshot.activeTurnId, terminalTurns: snapshot.terminalTurns ?? [], + dispositionOnlyRecoveryConsumed: + snapshot.dispositionOnlyRecoveryConsumed ?? false, + dispositionOnlyRecoveryTurnId: + snapshot.dispositionOnlyRecoveryTurnId ?? null, pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], lineage: snapshot.lineage ?? [], }; diff --git a/packages/paperclip-runner/src/contracts/harness-driver.ts b/packages/paperclip-runner/src/contracts/harness-driver.ts index 16b8291986..da3017e853 100644 --- a/packages/paperclip-runner/src/contracts/harness-driver.ts +++ b/packages/paperclip-runner/src/contracts/harness-driver.ts @@ -445,6 +445,10 @@ export interface PersistedHarnessSession { activeTurnId?: string | null; semanticResult?: PersistedHarnessSemanticResult | null; terminalTurns?: PersistedHarnessTurnTerminal[]; + /** A result-less terminal task may spend this fail-closed one-shot recovery allowance. */ + dispositionOnlyRecoveryConsumed?: boolean; + /** Exact accepted provider turn that spent the disposition-only allowance. */ + dispositionOnlyRecoveryTurnId?: string | null; pendingRuntimeRequests?: HarnessRuntimeRequest[]; goal?: HarnessThreadGoal | null; lineage?: HarnessThreadLineageEntry[]; diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index b1898b2eb9..dfa4cd597b 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -58,6 +58,9 @@ export interface PersistedNativeSession { terminal?: PrpTerminalState | null; activeTurnId?: string | null; terminalTurns?: PersistedHarnessTurnTerminal[]; + /** Durable at-most-once marker for a resultless terminal recovery turn. */ + dispositionOnlyRecoveryConsumed?: boolean; + dispositionOnlyRecoveryTurnId?: string | null; pendingRuntimeRequests?: HarnessRuntimeRequest[]; lineage?: HarnessThreadLineageEntry[]; } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts new file mode 100644 index 0000000000..950cd02090 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -0,0 +1,768 @@ +import { resolve } from "node:path"; + +import type { + HarnessDriver, + HarnessDriverDescriptor, + HarnessRuntimeRequest, + HarnessSession, + HarnessSessionRecoveryOptions, + HarnessSessionRecoveryResult, + HarnessThreadGoal, + HarnessThreadLineageEntry, + OpenHarnessSessionInput, + PersistedHarnessSemanticResult, + PersistedHarnessSession, + PersistedHarnessTurnTerminal, +} from "../../contracts/harness-driver.js"; +import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import { + CODEX_CODEX_PROTOCOL_VERSION, + CODEX_SEMANTIC_TOOL_NAMES, + CODEX_SKILLLESS_BASE_INSTRUCTIONS, +} from "../../contracts/codex.js"; +import { providerFamilyCapabilities } from "../../provider-events.js"; +import { + ProcessCodexAppServerTransport, + createSanitizedCodexEnvironment, + isCodexMethodUnavailable, + redactCodexDiagnostic, + type CodexAppServerTransport, + type CodexTransportProcessInfo, +} from "./app-server-transport.js"; +import { + boundedCodexValue, + validateCodexWorkingDirectory as validateWorkingDirectory, +} from "./codex-boundaries.js"; +import { + CODEX_PLANNING_PERMISSION_PROFILE as PLANNING_PERMISSION_PROFILE, + CODEX_SKILLLESS_PERMISSION_PROFILE as SKILLLESS_PERMISSION_PROFILE, + codexCommandEnvironment, + createIsolatedCodexAppServerArgs, + createSecuredCodexThreadParams, + createSkilllessCodexThreadConfig, +} from "./codex-security-config.js"; +import { + codexThreadLineage as lineageFromThread, + codexThreadStatus as threadStatus, + parseCodexThreadGoal as parseThreadGoal, +} from "./codex-thread-normalization.js"; +import { CodexHarnessSession } from "./codex-harness-session.js"; +import type { + CodexAppServerDriverOptions, + CodexCapabilities, + OpenedCodexThread, +} from "./codex-driver-types.js"; +import { + boundedText, + codexSemanticToolSpecs, + differingJsonPaths, + record, + text, +} from "./codex-driver-values.js"; + +const DRIVER_KIND = "codex_app_server"; +const DRIVER_VERSION = "codex-v2"; + +interface BootstrapCancellation { + close(): Promise; + detach(): void; + wait(operation: Promise): Promise; +} + +function bootstrapCancellation( + transport: CodexAppServerTransport, + signal: AbortSignal | undefined, +): BootstrapCancellation { + let closePromise: Promise | null = null; + const close = (): Promise => { + closePromise ??= transport.close(); + return closePromise; + }; + let rejectAborted!: (reason: unknown) => void; + const aborted = new Promise((_resolve, reject) => { + rejectAborted = reject; + }); + let abortStarted = false; + const onAbort = (): void => { + if (abortStarted) return; + abortStarted = true; + // Provider bootstrap owns a local process. Do not report cancellation + // until its pending RPCs have been rejected and TERM/KILL cleanup has + // completed. The cancellation reason remains authoritative if cleanup + // itself reports a secondary failure. + void close().then( + () => rejectAborted(signal?.reason ?? new Error("Codex bootstrap aborted")), + () => rejectAborted(signal?.reason ?? new Error("Codex bootstrap aborted")), + ); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + // Close the construction/listener race after the caller's preflight abort + // check without creating a provider for an already-aborted operation. + if (signal?.aborted) onAbort(); + return { + close, + detach(): void { + signal?.removeEventListener("abort", onAbort); + }, + async wait(operation: Promise): Promise { + if (signal === undefined) return await operation; + const result = await Promise.race([operation, aborted]); + // An operation and abort cleanup may settle in the same microtask turn. + // Cancellation wins until session admission, and still awaits cleanup. + if (signal.aborted) return await aborted; + return result; + }, + }; +} + +export class CodexAppServerDriver implements HarnessDriver { + readonly #options: CodexAppServerDriverOptions; + readonly #caps: CodexCapabilities; + + constructor(options: CodexAppServerDriverOptions) { + this.#options = options; + this.#caps = { + resume: true, + read: true, + steering: true, + interruption: true, + usage: true, + reconciliation: true, + dynamicTools: true, + runtimeRequestResolution: true, + goals: true, + threadLineage: true, + ...options.capabilities, + }; + if (!this.#caps.read) this.#caps.reconciliation = false; + } + + #direct(): boolean { + return this.#options.conversationMode === "direct"; + } + + #baseInstructions(): string { + return this.#options.baseInstructions ?? CODEX_SKILLLESS_BASE_INSTRUCTIONS; + } + + async descriptor(): Promise { + const unsupported = Object.entries(this.#caps) + .filter(([, supported]) => !supported) + .map(([operation]) => operation); + return { + kind: this.#options.driverIdentity?.kind ?? DRIVER_KIND, + displayName: + this.#options.driverIdentity?.displayName ?? "Codex app-server", + version: this.#options.driverIdentity?.version ?? DRIVER_VERSION, + protocolVersion: CODEX_CODEX_PROTOCOL_VERSION, + runtimeContextCapabilities: { instructions: "native", skills: "native", mcp: "native" }, + capabilities: { + resume: this.#caps.resume, + typedEvents: true, + typedEventFamilies: providerFamilyCapabilities({ + plan: "available", + tool_execution: "available", + research: "available", + delegation: "available", + model_identity: "available", + context: "available", + artifact: "policy_disabled", + review: "available", + hook: "available", + memory: "available", + safety: "available", + terminal: "available", + wait: "available", + provider_notice: "available", + }), + steering: this.#caps.steering, + interruption: this.#caps.interruption, + structuredResult: true, + read: this.#caps.read, + reconciliation: this.#caps.reconciliation, + usage: this.#caps.usage, + dynamicTools: this.#caps.dynamicTools, + runtimeRequestResolution: this.#caps.runtimeRequestResolution, + runtimeRequestHandoff: this.#caps.runtimeRequestResolution, + goals: this.#caps.goals, + threadLineage: this.#caps.threadLineage, + collaborationModes: [ + ...(this.#options.collaborationModes ?? ["default", "plan"]), + ], + unsupported, + }, + }; + } + + async openSession(input: OpenHarnessSessionInput): Promise { + input.signal?.throwIfAborted(); + const workingDirectory = validateWorkingDirectory( + input.workingDirectory, + this.#options.environment, + ); + const transport = this.#transport(); + const cancellation = bootstrapCancellation(transport, input.signal); + try { + await cancellation.wait(this.#persistProcessOwnership(transport)); + const initialize = await cancellation.wait(this.#initialize(transport)); + const requestedMode = + this.#options.requestedCollaborationMode ?? "default"; + const response = await cancellation.wait(transport.request("thread/start", { + ...createSecuredCodexThreadParams( + workingDirectory, + requestedMode, + this.#options.includeCollaborationModeInstructions ?? true, + this.#options.includeSkillInstructions ?? false, + ), + approvalPolicy: this.#options.approvalPolicy ?? "never", + ...(this.#options.model ? { model: this.#options.model } : {}), + ...(this.#direct() + ? {} + : { + baseInstructions: this.#baseInstructions(), + completionContract: { + revision: + this.#options.taskEnvelope.completionContract.revision, + criterionIds: + this.#options.taskEnvelope.completionContract.criteria.map( + (criterion) => criterion.id, + ), + }, + }), + dynamicTools: this.#direct() + ? [] + : this.#caps.dynamicTools + ? [ + ...(this.#options.dynamicTools ?? []), + ...codexSemanticToolSpecs(), + ] + : [], + experimentalRawEvents: false, + persistExtendedHistory: false, + })); + const collaborationMode = await cancellation.wait(this.#negotiateCollaborationMode( + transport, + response, + requestedMode, + )); + const opened = this.#openedThread( + response, + initialize, + workingDirectory, + collaborationMode, + ); + const goal = await cancellation.wait( + this.#discoverGoal(transport, opened.threadId), + ); + if (opened.context.liveConsole) + opened.context.liveConsole.goals = this.#caps.goals; + return this.#session({ + transport, + runId: input.runId, + normalizedSessionId: input.normalizedSessionId, + opened, + goal, + resumed: false, + sourceSequence: 0, + }); + } catch (error) { + // Cleanup must never replace the provider/bootstrap failure that caused + // the session open to abort. Remote transports may perform checkpoint + // work during close; when no durable provider identity exists that + // cleanup can fail independently. + await cancellation.close().catch(() => {}); + if (input.signal?.aborted) input.signal.throwIfAborted(); + throw error; + } finally { + cancellation.detach(); + } + } + + async recoverSession( + snapshot: PersistedHarnessSession, + options: HarnessSessionRecoveryOptions = { + signal: new AbortController().signal, + }, + ): Promise { + options.signal.throwIfAborted(); + if (!this.#caps.resume) { + return { recovered: false, reason: "resume capability is unavailable" }; + } + if (!this.#caps.read) { + return { + recovered: false, + reason: "read capability is required for safe resume", + }; + } + if ( + !snapshot.runId || + !snapshot.normalizedSessionId || + !snapshot.driverSessionId + ) { + return { + recovered: false, + reason: "persisted session identity is incomplete", + }; + } + const transport = this.#transport({ + providerRecoveryPolicy: snapshot.providerRecoveryPolicy, + }); + const cancellation = bootstrapCancellation(transport, options.signal); + try { + await cancellation.wait(this.#persistProcessOwnership(transport)); + const initialize = await cancellation.wait(this.#initialize(transport)); + const existing = await cancellation.wait(transport.request("thread/read", { + threadId: snapshot.driverSessionId, + includeTurns: true, + })); + const existingThread = record(existing.thread); + if (text(existingThread.id) !== snapshot.driverSessionId) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider read a different session", + }; + } + const workingDirectory = validateWorkingDirectory( + text(existingThread.cwd), + this.#options.environment, + ); + const response = await cancellation.wait(transport.request("thread/resume", { + threadId: snapshot.driverSessionId, + ...createSecuredCodexThreadParams( + workingDirectory, + this.#options.requestedCollaborationMode ?? "default", + this.#options.includeCollaborationModeInstructions ?? true, + this.#options.includeSkillInstructions ?? false, + ), + baseInstructions: this.#direct() + ? "" + : this.#baseInstructions(), + approvalPolicy: this.#options.approvalPolicy ?? "never", + ...(this.#options.model ? { model: this.#options.model } : {}), + persistExtendedHistory: false, + })); + const collaborationMode = await cancellation.wait(this.#negotiateCollaborationMode( + transport, + response, + this.#options.requestedCollaborationMode ?? "default", + )); + const opened = this.#openedThread( + response, + initialize, + workingDirectory, + collaborationMode, + ); + if (opened.threadId !== snapshot.driverSessionId) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider resumed a different session", + }; + } + if ( + snapshot.providerSessionId && + opened.providerSessionId !== snapshot.providerSessionId + ) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider resumed a different provider session", + }; + } + const checkpointedActiveTurnId = snapshot.activeTurnId ?? null; + // A terminal fingerprint is the durable provider fact. A crash can + // persist it before the following active-turn clear reaches the same + // checkpoint, so never resurrect that already-terminal turn as active. + const checkpointedActiveTurnIsTerminal = + checkpointedActiveTurnId !== null && + (snapshot.terminalTurns ?? []).some( + (terminal) => terminal.turnId === checkpointedActiveTurnId, + ); + let recoveredActiveTurnId = checkpointedActiveTurnIsTerminal + ? null + : checkpointedActiveTurnId; + let dispositionOnlyRecoveryConsumed = + snapshot.dispositionOnlyRecoveryConsumed ?? false; + let dispositionOnlyRecoveryTurnId = + snapshot.dispositionOnlyRecoveryTurnId ?? null; + let reconcileUncheckpointedDispositionTurn = false; + let providerTurnIds: Set | null = null; + if ( + !this.#direct() + && snapshot.semanticResult == null + && recoveredActiveTurnId === null + && (snapshot.terminalTurns?.length ?? 0) > 0 + ) { + const providerHistory = existingThread.turns; + const providerHistoryIsArray = Array.isArray(providerHistory); + const turns = providerHistoryIsArray + ? providerHistory.map(record) + : []; + const terminalIds = new Set( + (snapshot.terminalTurns ?? []).map((turn) => turn.turnId), + ); + let lastKnownTerminalIndex = -1; + turns.forEach((turn, index) => { + if (terminalIds.has(text(turn.id))) lastKnownTerminalIndex = index; + }); + // Releasing a consumed marker requires both an actual history array + // and a checkpointed terminal that anchors its ordering. An array that + // omits every durable terminal may be truncated or inconsistent, so + // absence from it is not proof that the provider rejected the turn. + if (providerHistoryIsArray && lastKnownTerminalIndex >= 0) { + providerTurnIds = new Set( + turns + .map((turn) => text(turn.id)) + .filter((turnId) => turnId.length > 0), + ); + } + const laterTurns = lastKnownTerminalIndex < 0 + ? [] + : turns.slice(lastKnownTerminalIndex + 1); + if (laterTurns.length > 1) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider exposed multiple uncheckpointed disposition recovery turns", + }; + } + const uncheckpointedTurnId = text(laterTurns[0]?.id); + if (laterTurns.length === 1 && uncheckpointedTurnId.length === 0) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider exposed an unidentifiable disposition recovery turn", + }; + } + if (uncheckpointedTurnId.length > 0) { + if ( + dispositionOnlyRecoveryTurnId !== null + && dispositionOnlyRecoveryTurnId !== uncheckpointedTurnId + ) { + await cancellation.wait(cancellation.close()); + return { + recovered: false, + reason: "provider changed the bound disposition recovery turn", + }; + } + // A previous process reached the provider but crashed before it + // could checkpoint the accepted turn or durably append its terminal. + // Inspect provider history even when the one-shot submitted marker + // was checkpointed, and adopt the exact turn instead of waiting on a + // reconstructed terminal session or submitting a duplicate. + recoveredActiveTurnId = uncheckpointedTurnId; + dispositionOnlyRecoveryConsumed = true; + dispositionOnlyRecoveryTurnId = uncheckpointedTurnId; + reconcileUncheckpointedDispositionTurn = true; + } + } + if ( + dispositionOnlyRecoveryConsumed + && recoveredActiveTurnId === null + && !reconcileUncheckpointedDispositionTurn + && providerTurnIds !== null + && ( + dispositionOnlyRecoveryTurnId === null + || ( + !providerTurnIds.has(dispositionOnlyRecoveryTurnId) + && !(snapshot.terminalTurns ?? []).some( + (terminal) => terminal.turnId === dispositionOnlyRecoveryTurnId, + ) + ) + ) + ) { + // Older or crash-raced checkpoints could persist the pre-request + // one-shot marker, including a requested turn id, without an accepted + // provider turn. With no matching provider history or checkpointed + // terminal, the marker alone is not acceptance evidence. The native + // runtime still checks durable replay before allowing a retry, and the + // reconstructed session must retain disposition-only mode so that a + // safe retry cannot repeat the original task envelope. + dispositionOnlyRecoveryConsumed = false; + dispositionOnlyRecoveryTurnId = null; + } + const goal = await cancellation.wait( + this.#discoverGoal(transport, opened.threadId), + ); + if (opened.context.liveConsole) + opened.context.liveConsole.goals = this.#caps.goals; + const session = this.#session({ + transport, + runId: snapshot.runId, + normalizedSessionId: snapshot.normalizedSessionId, + opened, + goal, + resumed: true, + activeTurnId: recoveredActiveTurnId, + semanticResult: snapshot.semanticResult ?? null, + terminalTurns: snapshot.terminalTurns ?? [], + dispositionOnlyRecoveryConsumed, + dispositionOnlyRecoveryTurnId, + stalePendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], + lineage: snapshot.lineage, + sourceSequence: snapshot.lastSourceSequence ?? 0, + }); + if (reconcileUncheckpointedDispositionTurn) { + await cancellation.wait(session.reconcile?.() ?? Promise.resolve({})); + } + return { + recovered: true, + session, + }; + } catch (error) { + await cancellation.close().catch(() => {}); + if (options.signal.aborted) options.signal.throwIfAborted(); + return { recovered: false, reason: redactCodexDiagnostic(String(error)) }; + } finally { + cancellation.detach(); + } + } + + #transport(context?: { + providerRecoveryPolicy?: PersistedHarnessSession["providerRecoveryPolicy"]; + }): CodexAppServerTransport { + return ( + this.#options.transportFactory?.(context) ?? + new ProcessCodexAppServerTransport({ + args: createIsolatedCodexAppServerArgs(this.#options.environment), + environment: createSanitizedCodexEnvironment(this.#options.environment), + onDiagnostic: this.#options.onDiagnostic, + processGroup: true, + }) + ); + } + + async #negotiateCollaborationMode( + transport: CodexAppServerTransport, + threadResponse: Record, + requested: "default" | "plan", + ): Promise | null> { + if (requested !== "plan") return null; + try { + const response = await transport.request("collaborationMode/list", {}); + const preset = Array.isArray(response.data) + ? response.data + .map(record) + .find((candidate) => text(candidate.mode) === "plan") + : undefined; + if (!preset) throw new Error("plan preset is absent"); + const model = text( + preset.model, + text(threadResponse.model, text(record(threadResponse.thread).model)), + ); + if (model.length === 0) + throw new Error("plan preset did not resolve a model"); + return { + mode: "plan", + settings: { + model, + reasoning_effort: preset.reasoning_effort ?? null, + developer_instructions: null, + }, + }; + } catch (cause) { + const error = new Error( + `planning_mode_unsupported: installed Codex app-server did not expose a usable native plan collaboration mode (${redactCodexDiagnostic(String(cause))})`, + ); + error.name = "PlanningModeUnsupportedError"; + throw error; + } + } + + async #persistProcessOwnership( + transport: CodexAppServerTransport, + ): Promise { + if (!this.#options.onSpawn) return; + const processInfo: CodexTransportProcessInfo | undefined = + transport.processInfo?.(); + if (!processInfo || processInfo.exited || processInfo.pid === null) return; + await this.#options.onSpawn({ + pid: processInfo.pid, + processGroupId: processInfo.processGroupId, + startedAt: processInfo.startedAt, + }); + } + + async #initialize( + transport: CodexAppServerTransport, + ): Promise> { + const initialized = await transport.request("initialize", { + clientInfo: { + name: "paperclip-runner", + title: "Paperclip Runner", + version: DRIVER_VERSION, + }, + capabilities: { experimentalApi: true, requestAttestation: false }, + }); + transport.notify("initialized"); + return initialized; + } + + async #discoverGoal( + transport: CodexAppServerTransport, + threadId: string, + ): Promise { + if (!this.#caps.goals) return undefined; + try { + const response = await transport.request("thread/goal/get", { threadId }); + return parseThreadGoal(response.goal); + } catch (error) { + if (isCodexMethodUnavailable(error)) { + // The provider answered, and its answer is that this build has no goal + // API. That is the only evidence that retires the capability. + this.#caps.goals = false; + this.#options.onDiagnostic?.( + redactCodexDiagnostic(`thread goals unavailable: ${String(error)}`), + ); + return undefined; + } + // A transport or protocol failure says nothing about what the provider + // supports, so the capability stays as advertised and the goal is merely + // unknown until the next call. + this.#options.onDiagnostic?.( + redactCodexDiagnostic(`thread goal probe failed: ${String(error)}`), + ); + return undefined; + } + } + + #openedThread( + response: Record, + initialize: Record, + workingDirectory: string, + collaborationMode: Record | null, + ): OpenedCodexThread { + const thread = record(response.thread); + const threadId = text(thread.id); + if (threadId.length === 0) + throw new Error("Codex thread response omitted thread.id"); + const providerSessionId = text(thread.sessionId) || null; + if (this.#options.requireProviderSessionIdentity && providerSessionId === null) { + throw new Error( + `provider_initialize_protocol_error: provider=${this.#options.driverIdentity?.kind ?? "codex"} stage=session.open omitted provider session identity`, + ); + } + const activePermissionProfile = record(thread.activePermissionProfile); + const permissionProfileId = text(activePermissionProfile.id); + const requestedMode = this.#options.requestedCollaborationMode ?? "default"; + const requiredPermissionProfile = + requestedMode === "plan" + ? PLANNING_PERMISSION_PROFILE + : SKILLLESS_PERMISSION_PROFILE; + if ( + permissionProfileId.length > 0 && + permissionProfileId !== requiredPermissionProfile + ) { + throw new Error( + "Codex thread did not activate the required filesystem permission profile", + ); + } + const configuredPermissionProfile = { + ...activePermissionProfile, + id: requiredPermissionProfile, + }; + const returnedWorkingDirectory = text(response.cwd, workingDirectory); + if (resolve(returnedWorkingDirectory) !== workingDirectory) { + throw new Error( + "Codex thread response changed the assigned working directory", + ); + } + return { + threadId, + providerSessionId, + collaborationMode, + context: { + protocolVersion: CODEX_CODEX_PROTOCOL_VERSION, + codexVersion: boundedText(initialize.userAgent), + clientInfo: { + name: "paperclip-runner", + title: "Paperclip Runner", + version: DRIVER_VERSION, + }, + model: boundedText(response.model), + modelProvider: boundedText( + response.modelProvider, + boundedText(thread.modelProvider), + ), + workingDirectory, + collaborationMode: collaborationMode === null ? "default" : "plan", + sandbox: { + permissionProfile: boundedCodexValue(configuredPermissionProfile), + legacyPolicy: boundedCodexValue(response.sandbox ?? null), + rootAccess: "none", + minimalRuntimeAccess: "read", + workspaceAccess: requestedMode === "plan" ? "read" : "write", + networkAccess: false, + }, + approvalPolicy: boundedCodexValue( + response.approvalPolicy ?? this.#options.approvalPolicy ?? "never", + ), + baseInstructions: this.#baseInstructions(), + instructionSources: Array.isArray(response.instructionSources) + ? response.instructionSources + .filter((value): value is string => typeof value === "string") + .slice(0, 32) + .map((value) => boundedText(value)) + : [], + instructionPolicy: { + skillInstructions: this.#options.includeSkillInstructions ?? false, + appInstructions: false, + collaborationInstructions: + this.#options.includeCollaborationModeInstructions ?? true, + }, + environmentKeys: Object.keys( + codexCommandEnvironment(this.#options.environment), + ).sort(), + dynamicToolNames: this.#direct() + ? [] + : this.#caps.dynamicTools + ? [ + ...(this.#options.dynamicTools ?? []).map((tool) => + text(tool.name), + ), + ...CODEX_SEMANTIC_TOOL_NAMES, + ] + : [], + modelInputKinds: ["text"], + liveConsole: { + conversationMode: this.#direct() ? "direct" : "task", + runtimeRequestResolution: this.#caps.runtimeRequestResolution, + goals: this.#caps.goals, + threadLineage: this.#caps.threadLineage, + }, + envelope: structuredClone(this.#options.taskEnvelope), + }, + lineage: lineageFromThread(thread), + }; + } + + #session(input: { + transport: CodexAppServerTransport; + runId: string; + normalizedSessionId: string; + opened: OpenedCodexThread; + goal?: HarnessThreadGoal | null; + resumed: boolean; + activeTurnId?: string | null; + semanticResult?: PersistedHarnessSemanticResult | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + dispositionOnlyRecoveryConsumed?: boolean; + dispositionOnlyRecoveryTurnId?: string | null; + stalePendingRuntimeRequests?: HarnessRuntimeRequest[]; + lineage?: HarnessThreadLineageEntry[]; + sourceSequence: number; + }): CodexHarnessSession { + return new CodexHarnessSession({ + ...input, + taskEnvelope: this.#options.taskEnvelope, + conversationMode: this.#direct() ? "direct" : "task", + now: this.#options.now ?? (() => new Date()), + runnerInstanceId: this.#options.runnerInstanceId ?? "runner-codex", + driverKind: this.#options.driverIdentity?.kind ?? DRIVER_KIND, + capabilities: this.#caps, + dynamicTools: this.#options.dynamicTools ?? [], + dynamicToolHandler: this.#options.dynamicToolHandler, + }); + } +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.conformance.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.conformance.test.ts new file mode 100644 index 0000000000..a9369e76b6 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.conformance.test.ts @@ -0,0 +1,329 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("degrades unsupported operations with explicit redacted diagnostics", async () => { + const transport = new FakeCodexTransport(); + transport.rejectMethods.set( + "turn/steer", + new Error("method not found Bearer super-secret api_key=also-secret"), + ); + const session = await makeDriver([transport]).openSession({ + runId: "run-degrade", + normalizedSessionId: "normalized-degrade", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Start." }, + }); + await expect( + session.steer?.({ turnId, message: { role: "user", text: "Steer." } }), + ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); + const iterator = session.events()[Symbol.asyncIterator](); + const events: PrpEvent[] = []; + for (let index = 0; index < 6; index += 1) { + const next = await iterator.next(); + if (next.value) events.push(next.value); + } + const diagnostic = events.find( + (event) => event.eventType === "harness.diagnostic", + ); + expect(diagnostic?.payload).toMatchObject({ + code: "unsupported_operation", + operation: "steering", + }); + expect(JSON.stringify(diagnostic)).not.toContain("super-secret"); + expect(JSON.stringify(diagnostic)).not.toContain("also-secret"); + }); + + it("rejects proposals that do not match the exact task envelope", () => { + const wrongRevision = structuredClone(result); + wrongRevision.completionClaim.contractRevision = "codex-demo-v0"; + expect(validateCodexResultProposal(wrongRevision, envelope)).toMatchObject({ + status: "rejected", + issues: [{ code: "contract_revision_mismatch" }], + }); + + const unknownCriterion = structuredClone(result); + unknownCriterion.completionClaim.criteria = [ + { + criterionId: "not-in-envelope", + status: "satisfied", + evidenceRefs: [], + }, + ]; + const criterionDecision = validateCodexResultProposal( + unknownCriterion, + envelope, + ); + expect(criterionDecision).toMatchObject({ status: "rejected" }); + if (criterionDecision.status === "rejected") { + expect(criterionDecision.issues.map((issue) => issue.code)).toEqual( + expect.arrayContaining(["unknown_criterion", "missing_criterion"]), + ); + } + + const invalidDone = structuredClone(result); + invalidDone.completionClaim.objectiveSatisfied = false; + expect(validateCodexResultProposal(invalidDone, envelope)).toMatchObject({ + status: "rejected", + issues: [{ code: "invalid_disposition" }], + }); + }); + + it.each(["needs_review", "blocked"] as const)( + "keeps a completed runtime successful for an accepted %s advisory proposal", + async (disposition) => { + const proposal: PrpStructuredRunResult = + disposition === "needs_review" + ? { + ...structuredClone(result), + reportedWorkDisposition: "needs_review", + attentionRequests: [ + { kind: "review", summary: "Review the completed artifact." }, + ], + } + : { + ...structuredClone(result), + reportedWorkDisposition: "blocked", + summary: "Waiting on fixture input.", + completionClaim: { + ...structuredClone(result.completionClaim), + objectiveSatisfied: false, + criteria: [ + { + criterionId: "file", + status: "not_satisfied", + evidenceRefs: [], + }, + ], + remainingWork: [ + { + description: "Provide fixture input.", + blocksCompletion: true, + }, + ], + }, + blocker: { + reasonCode: "fixture_input_missing", + owner: { kind: "external", name: "fixture owner" }, + unblockAction: "Provide fixture input.", + scope: "task_wide", + }, + artifacts: [], + }; + const { trace } = await traceCompletedProposal(proposal); + expect(trace.resultDecision.status).toBe("accepted"); + expect( + trace.events.find((event) => event.eventType === "run.terminal") + ?.payload, + ).toMatchObject({ + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: disposition, + }); + }, + ); + + it("records rejected and missing proposals as recovery evidence instead of review status", async () => { + const wrongRevision = structuredClone(result); + wrongRevision.completionClaim.contractRevision = "wrong-revision"; + const rejected = await traceCompletedProposal(wrongRevision); + expect(rejected.trace.result).toBeNull(); + expect(rejected.trace.proposedResult).toMatchObject({ + reportedWorkDisposition: "done", + }); + expect( + rejected.trace.events.find( + (event) => event.eventType === "run.result.rejected", + )?.payload, + ).toMatchObject({ recovery: { required: true, recoverable: true } }); + expect( + rejected.trace.events.find((event) => event.eventType === "run.terminal") + ?.payload, + ).toMatchObject({ + runTerminalState: "failed", + reportedWorkDisposition: "yielded", + }); + + const missing = await traceCompletedProposal(null); + expect(missing.trace.resultDecision).toMatchObject({ status: "rejected" }); + expect( + missing.trace.events.find((event) => event.eventType === "run.terminal") + ?.payload, + ).toMatchObject({ + runTerminalState: "failed", + reportedWorkDisposition: "yielded", + }); + expect( + missing.trace.events.some( + (event) => + event.eventType === "run.result.proposed" && + event.payload.reportedWorkDisposition === "needs_review", + ), + ).toBe(false); + }); + + it("preserves four independent stable identities under controller ownership", async () => { + const { trace } = await traceCompletedProposal(result, { + runId: "runtime-run-identity", + normalizedSessionId: "controller-session-identity", + }); + expect(trace.assertions.stableIdentity).toBe(true); + expect(trace.metadata.identity).toMatchObject({ + runId: "runtime-run-identity", + normalizedSessionId: "controller-session-identity", + driverSessionId: "thread-1", + providerSessionId: "provider-session-1", + }); + expect( + trace.events.every( + (event) => + event.runId === "runtime-run-identity" && + event.normalizedSessionId === "controller-session-identity", + ), + ).toBe(true); + }); + + it("validates persisted identity, uniqueness, terminals, and line bounds before replay", async () => { + const { trace } = await traceCompletedProposal(result, { + runId: "runtime-run-persisted", + normalizedSessionId: "controller-session-persisted", + }); + const serialize = (events: PrpEvent[]) => + `${events.map((event) => JSON.stringify(event)).join("\n")}\n`; + expect( + replayPersistedCodexEvents(serialize(trace.events), trace.metadata), + ).toEqual(trace.replaySnapshot); + + const mismatched = structuredClone(trace.events); + mismatched[0]!.normalizedSessionId = "another-session"; + expect(() => + replayPersistedCodexEvents(serialize(mismatched), trace.metadata), + ).toThrow("identity did not match"); + + const terminalIndex = trace.events.findIndex( + (event) => event.eventType === "run.terminal", + ); + const duplicated = trace.events.toSpliced( + terminalIndex, + 0, + structuredClone(trace.events[0]!), + ); + expect(() => + replayPersistedCodexEvents(serialize(duplicated), trace.metadata), + ).toThrow("event id was duplicated"); + + const terminal = structuredClone(trace.events[terminalIndex]!); + terminal.sourceEventId = `${terminal.sourceEventId}:conflict`; + terminal.sourceSeq += 1; + expect(() => + replayPersistedCodexEvents( + serialize([...trace.events, terminal]), + trace.metadata, + ), + ).toThrow("after the run terminal"); + expect(() => + replayPersistedCodexEvents( + `{\"payload\":\"${"x".repeat(300 * 1024)}\"}\n`, + trace.metadata, + ), + ).toThrow("line was empty or oversized"); + expect(() => + replayPersistedCodexEvents("{not-json}\n", trace.metadata), + ).toThrow("malformed JSON"); + }); + + it("degrades declared unsupported capabilities without Codex-specific core branches", async () => { + const capabilities = { + resume: false, + read: false, + steering: false, + interruption: false, + usage: false, + reconciliation: false, + dynamicTools: false, + }; + const { trace, transport } = await traceCompletedProposal(result, { + capabilities, + steer: "Do not call this unsupported path.", + interrupt: true, + }); + expect(trace.resultDecision.status).toBe("accepted"); + expect(trace.assertions.contextIsSkillless).toBe(true); + expect(trace.diagnostics).toEqual( + expect.arrayContaining([ + expect.stringContaining("steering is unavailable"), + expect.stringContaining("interruption is unavailable"), + ]), + ); + expect(transport.calls.map((call) => call.method)).not.toEqual( + expect.arrayContaining(["turn/steer", "turn/interrupt", "thread/read"]), + ); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params + .dynamicTools, + ).toEqual([]); + + const directTransport = new FakeCodexTransport(); + const driver = makeDriver([directTransport], { capabilities }); + const session = await driver.openSession({ + runId: "run-no-capabilities", + normalizedSessionId: "normalized-no-capabilities", + workingDirectory: WORKSPACE, + }); + await expect(session.read?.()).rejects.toBeInstanceOf( + HarnessCapabilityUnavailableError, + ); + await expect(session.usage?.()).rejects.toBeInstanceOf( + HarnessCapabilityUnavailableError, + ); + const snapshot = await session.snapshot(); + expect(await driver.recoverSession?.(snapshot)).toMatchObject({ + recovered: false, + reason: "resume capability is unavailable", + }); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.context-control.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.context-control.test.ts new file mode 100644 index 0000000000..c52117f7f9 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.context-control.test.ts @@ -0,0 +1,639 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("captures an exact skillless model/environment snapshot with credentials absent", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-context", + normalizedSessionId: "normalized-context", + workingDirectory: WORKSPACE, + }); + const iterator = session.events()[Symbol.asyncIterator](); + const first = await iterator.next(); + const context = first.value?.payload.context as Parameters< + typeof isSkilllessCodexContext + >[0]; + expect(context).toMatchObject({ + codexVersion: "codex-cli/0.132.0", + model: "gpt-test", + modelProvider: "openai", + workingDirectory: WORKSPACE, + approvalPolicy: "never", + instructionSources: [], + instructionPolicy: { + skillInstructions: false, + appInstructions: false, + collaborationInstructions: true, + }, + environmentKeys: ["LANG", "PATH"], + envelope, + }); + expect(isSkilllessCodexContext(context)).toBe(true); + expect(JSON.stringify(context)).not.toContain("must-not-pass"); + expect(JSON.stringify(transport.calls)).not.toContain("RANDOM_SKILL_PATH"); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ + approvalPolicy: "never", + config: { + "skills.include_instructions": false, + include_apps_instructions: false, + include_collaboration_mode_instructions: true, + "features.image_generation": false, + }, + permissions: "paperclip-runner-workspace-only", + runtimeWorkspaceRoots: [WORKSPACE], + dynamicTools: [{ name: "paperclip_finish" }, { name: "paperclip_block" }], + }); + const commandPolicy = transport.calls.find( + (call) => call.method === "thread/start", + )?.params.config as Record; + expect(JSON.stringify(commandPolicy)).not.toContain("/isolated/home"); + expect(JSON.stringify(commandPolicy)).not.toContain("/isolated/codex"); + expect(JSON.stringify(commandPolicy)).not.toContain("CODEX_HOME"); + const appServerArgs = createIsolatedCodexAppServerArgs({ + PATH: "/bin", + LANG: "C.UTF-8", + HOME: "/isolated/home", + CODEX_HOME: "/isolated/codex", + }); + expect(appServerArgs).toEqual( + expect.arrayContaining([ + expect.stringContaining( + 'default_permissions="paperclip-runner-workspace-only"', + ), + expect.stringContaining( + "permissions.paperclip-runner-workspace-only.filesystem=", + ), + "permissions.paperclip-runner-workspace-only.network.enabled=false", + 'shell_environment_policy.inherit="none"', + expect.stringContaining( + 'shell_environment_policy.set={PATH="/bin",LANG="C.UTF-8"}', + ), + "--disable", + "image_generation", + ]), + ); + expect( + appServerArgs.find((arg) => arg.startsWith("permissions.")), + ).toContain('"/isolated/home"="none"'); + expect( + appServerArgs.find((arg) => arg.startsWith("permissions.")), + ).toContain('"/isolated/codex"="none"'); + expect(JSON.stringify(appServerArgs)).not.toContain("HOME="); + expect(CODEX_RESULT_OUTPUT_SCHEMA.properties.schema).toEqual({ + type: "string", + const: "paperclip.run_result.v1", + }); + expect( + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA.properties.reportedWorkDisposition, + ).toEqual({ + type: "string", + const: "blocked", + }); + }); + + it.each(["never", "on-request", "untrusted"] as const)( + "pins approval policy %s for both thread start and resume", + async (approvalPolicy) => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second], { approvalPolicy }); + const original = await driver.openSession({ + runId: `run-policy-${approvalPolicy}`, + normalizedSessionId: `normalized-policy-${approvalPolicy}`, + workingDirectory: WORKSPACE, + }); + const started = await original.events()[Symbol.asyncIterator]().next(); + const snapshot = await original.snapshot(); + await original.close({ reason: "policy recovery test" }); + const recovered = await driver.recoverSession?.(snapshot); + expect(recovered).toMatchObject({ recovered: true }); + const resumed = await recovered!.session!.events()[Symbol.asyncIterator]().next(); + expect(first.calls.find((call) => call.method === "thread/start")?.params) + .toMatchObject({ approvalPolicy }); + expect(second.calls.find((call) => call.method === "thread/resume")?.params) + .toMatchObject({ approvalPolicy }); + expect(started.value).toMatchObject({ eventType: "session.started", payload: { context: { approvalPolicy } } }); + expect(resumed.value).toMatchObject({ eventType: "session.resumed", payload: { context: { approvalPolicy } } }); + await recovered?.session?.close({ reason: "policy recovery complete" }); + }, + ); + + it("allows eval fixtures to opt out of Codex collaboration instructions", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport], { + includeCollaborationModeInstructions: false, + }).openSession({ + runId: "run-no-collaboration-instructions", + normalizedSessionId: "session-no-collaboration-instructions", + workingDirectory: WORKSPACE, + }); + const first = await session.events()[Symbol.asyncIterator]().next(); + + expect(first.value?.payload.context).toMatchObject({ + instructionPolicy: { collaborationInstructions: false }, + }); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ + config: { include_collaboration_mode_instructions: false }, + }); + }); + + it("negotiates genuine plan mode with collaboration instructions and read-only workspace access", async () => { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport], { + requestedCollaborationMode: "plan", + }); + const session = await driver.openSession({ + runId: "run-plan", + normalizedSessionId: "session-plan", + workingDirectory: WORKSPACE, + }); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ + permissions: "paperclip-runner-workspace-read-only", + config: { include_collaboration_mode_instructions: true }, + }); + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).not.toHaveProperty("collaborationMode"); + expect( + transport.calls.some((call) => call.method === "collaborationMode/list"), + ).toBe(true); + await expect( + session.startTurn({ + message: { role: "user", text: "Author a plan." }, + requestedCollaborationMode: "plan", + }), + ).resolves.toMatchObject({ effectiveCollaborationMode: "plan" }); + expect( + transport.calls.find((call) => call.method === "turn/start")?.params, + ).toMatchObject({ + collaborationMode: { + mode: "plan", + settings: { + model: "gpt-test", + reasoning_effort: "high", + developer_instructions: null, + }, + }, + permissions: "paperclip-runner-workspace-read-only", + }); + }); + + it("fails closed when the installed app-server does not confirm plan mode", async () => { + const transport = new FakeCodexTransport(); + transport.confirmCollaborationMode = false; + await expect( + makeDriver([transport], { + requestedCollaborationMode: "plan", + }).openSession({ + runId: "run-plan-unsupported", + normalizedSessionId: "session-plan-unsupported", + workingDirectory: WORKSPACE, + }), + ).rejects.toThrow("planning_mode_unsupported"); + }); + + it("refuses to turn host credential roots into model-writable workspaces", async () => { + await expect( + makeDriver([], { + environment: { + HOME: "/isolated/home", + CODEX_HOME: WORKSPACE, + }, + }).openSession({ + runId: "run-codex-home", + normalizedSessionId: "normalized-codex-home", + workingDirectory: WORKSPACE, + }), + ).rejects.toThrow("cannot overlap host CODEX_HOME"); + await expect( + makeDriver([], { + environment: { + HOME: `${WORKSPACE}/host-home`, + CODEX_HOME: "/isolated/codex", + }, + }).openSession({ + runId: "run-host-home", + normalizedSessionId: "normalized-host-home", + workingDirectory: WORKSPACE, + }), + ).rejects.toThrow("cannot contain the host HOME"); + await expect( + makeDriver([]).openSession({ + runId: "run-root", + normalizedSessionId: "normalized-root", + workingDirectory: "/", + }), + ).rejects.toThrow("cannot be a filesystem root"); + }); + + it("steers and interrupts an active turn without replacing the session", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-controls", + normalizedSessionId: "normalized-controls", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Start." }, + }); + await session.steer?.({ + turnId, + message: { role: "user", text: "Use a shorter answer." }, + }); + await session.interrupt?.({ turnId, reason: "operator requested" }); + expect(transport.calls.map((call) => call.method)).toEqual([ + "initialize", + "thread/start", + "thread/goal/get", + "turn/start", + "turn/steer", + "turn/interrupt", + ]); + expect(session.ids()).toEqual({ + driverSessionId: "thread-1", + providerSessionId: "provider-session-1", + displayId: "thread-1", + }); + }); + + it("loads the deterministic Live console wire fixture", async () => { + const fixture = await loadLiveConsoleConformanceFixture( + liveConsoleFixturePath, + ); + expect( + fixture.runtimeRequests.map(({ requestKind }) => requestKind), + ).toEqual([ + "command_approval", + "file_approval", + "permission_approval", + "user_input", + "elicitation", + ]); + expect(fixture.goals.map(({ action }) => action)).toEqual([ + "get", + "set", + "pause", + "resume", + "clear", + ]); + }); + + it("holds browser-resolved upstream requests and returns the exact fixture responses", async () => { + const fixture = await loadLiveConsoleConformanceFixture( + liveConsoleFixturePath, + ); + for (const scenario of fixture.runtimeRequests) { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: `run-${scenario.id}`, + normalizedSessionId: `normalized-${scenario.id}`, + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Exercise request." }, + }); + const response = transport.invoke({ + id: scenario.id, + method: scenario.method, + params: { + threadId: "thread-1", + turnId, + itemId: `item-${scenario.id}`, + reason: `fixture ${scenario.id}`, + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(session.pendingRuntimeRequests?.()).toEqual([ + expect.objectContaining({ + requestId: scenario.id, + requestKind: scenario.requestKind, + method: scenario.method, + turnId, + }), + ]); + await expect( + session.resolveRuntimeRequest?.({ + requestId: scenario.id, + turnId, + resolution: { + action: "forged", + } as unknown as HarnessRuntimeRequestResolution, + }), + ).rejects.toThrow("unsupported action"); + expect(session.pendingRuntimeRequests?.()).toHaveLength(1); + await session.resolveRuntimeRequest?.({ + requestId: scenario.id, + turnId, + resolution: scenario.resolution as HarnessRuntimeRequestResolution, + }); + expect(await response).toEqual(scenario.expectedResponse); + expect(session.pendingRuntimeRequests?.()).toEqual([]); + await expect( + session.resolveRuntimeRequest?.({ + requestId: scenario.id, + turnId, + resolution: { action: "cancel" }, + }), + ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); + await session.close({ reason: "fixture complete" }); + } + }); + + it("acknowledges same-turn steering and rejects stale or child steering", async () => { + const fixture = await loadLiveConsoleConformanceFixture( + liveConsoleFixturePath, + ); + const transport = new FakeCodexTransport("thread-root"); + const session = await makeDriver([transport]).openSession({ + runId: "run-live-console-controls", + normalizedSessionId: "normalized-live-console-controls", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Start." }, + }); + await session.steer?.({ + turnId, + message: { role: "user", text: "Stay concise." }, + correlationId: "queued-comment-1", + }); + transport.push("item/completed", { + kind: "steering_acknowledgement", + status: "acknowledged", + commandId: "runnerd-steer-command", + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await session.steer?.({ + turnId, + message: { role: "user", text: "Stay concise." }, + correlationId: "queued-comment-1", + }); + expect( + transport.calls.find((call) => call.method === "turn/steer")?.params, + ).toMatchObject({ correlationId: "queued-comment-1" }); + expect( + transport.calls.filter((call) => call.method === "turn/steer"), + ).toHaveLength(1); + transport.rejectMethods.set( + "turn/steer", + new Error("provider rejected steering"), + ); + const rejectedSteering = await session + .steer?.({ + turnId, + message: { role: "user", text: "Try this again later." }, + correlationId: "queued-comment-2", + }) + .then( + () => null, + (error: unknown) => error, + ); + expect(rejectedSteering).toBeInstanceOf(Error); + expect(rejectedSteering).not.toBeInstanceOf( + HarnessCapabilityUnavailableError, + ); + expect(String(rejectedSteering)).toContain("provider rejected steering"); + transport.rejectMethods.delete("turn/steer"); + await expect( + session.steer?.({ + turnId: fixture.controls.staleTurnSteer.turnId!, + message: { role: "user", text: "Stale." }, + }), + ).rejects.toBeInstanceOf(HarnessStaleTurnError); + transport.push("thread/started", { thread: fixture.lineage.childThread }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(session.lineage?.()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + threadId: "thread-root", + parentThreadId: null, + depth: 0, + }), + expect.objectContaining({ + threadId: "thread-child", + parentThreadId: "thread-root", + depth: 1, + nickname: "Scout", + role: "researcher", + }), + ]), + ); + await expect( + session.steer?.({ + turnId: "thread-child", + message: { role: "user", text: "Do not emulate child steering." }, + }), + ).rejects.toBeInstanceOf(HarnessStaleTurnError); + const events = session.events()[Symbol.asyncIterator](); + const observed: PrpEvent[] = []; + for (let index = 0; index < 9; index += 1) { + const next = await events.next(); + if (next.done) break; + observed.push(next.value); + } + expect(observed).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + eventType: "item.completed", + itemId: `${turnId}:steer:queued-comment-1`, + payload: expect.objectContaining({ + kind: "steering_acknowledgement", + status: "acknowledged", + }), + }), + expect.objectContaining({ + eventType: "harness.diagnostic", + payload: expect.objectContaining({ code: "stale_turn_rejected" }), + }), + expect.objectContaining({ + eventType: "item.started", + payload: expect.objectContaining({ kind: "thread_lineage" }), + }), + ]), + ); + expect(observed.some((event) => event.eventType === "session.failed")) + .toBe(false); + await session.close({ reason: "fixture complete" }); + }); + + it("queues an interrupt before turn identity and reports a terminal race precisely", async () => { + const transport = new FakeCodexTransport(); + let releaseStart!: (value: Record) => void; + transport.turnStartResponse = new Promise((resolve) => { + releaseStart = resolve; + }); + const session = await makeDriver([transport]).openSession({ + runId: "run-interrupt-races", + normalizedSessionId: "normalized-interrupt-races", + workingDirectory: WORKSPACE, + }); + const starting = session.startTurn({ + message: { role: "user", text: "Start slowly." }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await session.interrupt?.({ reason: "operator" }); + releaseStart({ turn: { id: "turn-1", status: "inProgress", items: [] } }); + await expect(starting).resolves.toEqual({ + turnId: "turn-1", + effectiveCollaborationMode: "default", + }); + expect(transport.calls.map(({ method }) => method)).toEqual( + expect.arrayContaining(["turn/start", "turn/interrupt"]), + ); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "interrupted", items: [] }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await expect( + session.interrupt?.({ turnId: "turn-1" }), + ).rejects.toBeInstanceOf(HarnessOperationAlreadyTerminalError); + await session.close({ reason: "fixture complete" }); + }); + + it("maps all goal operations and advertises an exact unsupported state", async () => { + const fixture = await loadLiveConsoleConformanceFixture( + liveConsoleFixturePath, + ); + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-goals", + normalizedSessionId: "normalized-goals", + workingDirectory: WORKSPACE, + }); + await session.goal?.({ action: "get" }); + await session.goal?.({ + action: "set", + objective: "Ship the Live console tracer", + tokenBudget: 4096, + }); + await session.goal?.({ action: "pause" }); + await session.goal?.({ action: "resume" }); + await session.goal?.({ action: "clear" }); + const goalCalls = transport.calls.filter(({ method }) => + method.startsWith("thread/goal/"), + ); + expect( + goalCalls.slice(1).map(({ method, params }) => ({ + method, + params: Object.fromEntries( + Object.entries(params).filter(([key]) => key !== "threadId"), + ), + })), + ).toEqual(fixture.goals.map(({ method, params }) => ({ method, params }))); + + expect( + (await makeDriver([new FakeCodexTransport()]).descriptor()).capabilities, + ).toMatchObject({ goals: true }); + + // Both denials a real app-server sends: the method is absent, and the + // build has the feature switched off. + for (const denial of [ + new CodexRpcError( + '{"code":-32601,"message":"method not found"}', + CODEX_METHOD_NOT_FOUND, + ), + new CodexRpcError( + '{"code":-32600,"message":"goals feature is disabled"}', + CODEX_INVALID_REQUEST, + ), + ]) { + const unsupportedTransport = new FakeCodexTransport(); + unsupportedTransport.rejectMethods.set("thread/goal/get", denial); + const unsupportedDriver = makeDriver([unsupportedTransport]); + const unsupported = await unsupportedDriver.openSession({ + runId: "run-goals-unsupported", + normalizedSessionId: "normalized-goals-unsupported", + workingDirectory: WORKSPACE, + }); + expect((await unsupportedDriver.descriptor()).capabilities).toMatchObject( + { goals: false }, + ); + await expect( + unsupported.goal?.({ action: "get" }), + ).rejects.toBeInstanceOf(HarnessCapabilityUnavailableError); + await unsupported.close({ reason: "fixture complete" }); + } + await session.close({ reason: "fixture complete" }); + }); + + it("keeps goal support advertised when the probe fails transiently", async () => { + // A transport or protocol failure is evidence about this call, not about + // what the provider implements, so it must not retire the capability. + for (const failure of [ + new Error("codex app-server transport closed"), + new CodexRpcError('{"code":-32603,"message":"internal error"}', -32_603), + ]) { + const transport = new FakeCodexTransport(); + transport.rejectMethods.set("thread/goal/get", failure); + const driver = makeDriver([transport]); + const session = await driver.openSession({ + runId: "run-goals-transient", + normalizedSessionId: "normalized-goals-transient", + workingDirectory: WORKSPACE, + }); + + expect((await driver.descriptor()).capabilities).toMatchObject({ + goals: true, + }); + // The capability survives, so the operation is still offered and the + // next call reaches the provider instead of failing closed locally. + transport.rejectMethods.delete("thread/goal/get"); + await expect(session.goal?.({ action: "get" })).resolves.toBeNull(); + expect( + transport.calls.filter(({ method }) => method === "thread/goal/get"), + ).toHaveLength(2); + await session.close({ reason: "fixture complete" }); + } + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts new file mode 100644 index 0000000000..9f26c42ed2 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts @@ -0,0 +1,466 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("admits a strictly bound semantic result from the durable runner", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-durable-result", + normalizedSessionId: "normalized-durable-result", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Finish through runnerd." }, + }); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + transport.push("paperclip/runResult", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "semantic-result-1", + result, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.find((event) => event.eventType === "run.result.proposed"), + ).toMatchObject({ turnId: turn.turnId, itemId: "semantic-result-1" }); + }); + + it("correlates rehydrated notifications with the final canonical PRP event ids", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-traced", + normalizedSessionId: "normalized-traced", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Trace the response." }, + }); + + transport.pushTraced( + "turn/started", + { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }, + "event_runner_000001", + "turn.started", + ); + transport.pushTraced( + "item/started", + { + threadId: "thread-1", + turnId: turn.turnId, + item: { id: "answer-1", type: "agentMessage", phase: "final_answer" }, + }, + "event_runner_000002", + "item.started", + ); + transport.pushTraced( + "turn/completed", + { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }, + "event_runner_000003", + "turn.completed", + ); + + const events = await collectUntilTerminal(session.events()); + const itemMapping = transport.traceInterpretations.find( + (entry) => entry.providerMethod === "item/started", + ); + expect(itemMapping).toMatchObject({ + sourceEventId: "event_runner_000002", + sourceEventType: "item.started", + disposition: "mapped", + }); + expect(itemMapping?.emittedEventIds.length).toBeGreaterThan(0); + expect(events.map((event) => event.sourceEventId)).toEqual( + expect.arrayContaining(itemMapping?.emittedEventIds ?? []), + ); + expect(JSON.stringify(events)).not.toContain("paperclipTrace"); + }); + + it("maps canonical workspace snapshots with monotonic revisions and one terminal diff", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-canonical-workspace", + normalizedSessionId: "normalized-canonical-workspace", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Change the workspace." }, + }); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + const first = { + schema: "paperclip.workspace.diff.v1", + changeSetId: `${turn.turnId}:workspace`, + revision: 1, + source: "harness_reported", + complete: false, + files: [{ + path: "src/index.ts", + operation: "modify", + previousPath: null, + additions: 1, + deletions: 0, + binary: false, + diff: "+first\n", + }], + totals: { files: 1, additions: 1, deletions: 0 }, + patchArtifactRef: null, + }; + const second = { + ...first, + revision: 1, + files: [ + first.files[0], + { + path: "src/new-name.ts", + operation: "rename", + previousPath: "src/old-name.ts", + additions: 0, + deletions: 0, + binary: false, + diff: "rename from src/old-name.ts\nrename to src/new-name.ts\n", + }, + { + path: "public/image.png", + operation: "modify", + previousPath: null, + additions: null, + deletions: null, + binary: true, + diff: null, + }, + ], + totals: { files: 3, additions: null, deletions: null }, + }; + transport.pushTraced( + "paperclip/workspaceChange/updated", + { threadId: "thread-1", turnId: turn.turnId, workspaceChange: first }, + "event_workspace_1", + "workspace.change.updated", + ); + transport.pushTraced( + "paperclip/workspaceChange/updated", + { threadId: "thread-1", turnId: turn.turnId, workspaceChange: second }, + "event_workspace_2", + "workspace.change.updated", + ); + transport.push("paperclip/workspaceChange/updated", { + threadId: "thread-1", + turnId: turn.turnId, + workspaceChange: second, + }); + transport.push("paperclip/workspaceChange/updated", { + threadId: "thread-1", + turnId: turn.turnId, + workspaceChange: { + ...second, + files: [{ ...second.files[0], path: "/absolute/not-allowed.ts" }], + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + const updates = events.filter( + (event) => event.eventType === "workspace.change.updated", + ); + expect( + updates.map( + (event) => (event.payload as Record).revision, + ), + ).toEqual([1, 2]); + expect( + (updates[1]?.payload as Record).files, + ).toHaveLength(3); + expect(events.filter((event) => event.eventType === "workspace.diff.recorded")) + .toHaveLength(1); + expect( + events.find((event) => event.eventType === "workspace.diff.recorded")?.payload, + ).toMatchObject({ + revision: 2, + source: "runner_verified", + complete: true, + totals: { files: 3, additions: null, deletions: null }, + }); + expect( + transport.traceInterpretations.filter((entry) => + entry.providerMethod === "paperclip/workspaceChange/updated" + ).map((entry) => entry.disposition), + ).toEqual(["mapped", "mapped"]); + }); + + it("finalizes an authoritative empty workspace snapshot when a turn is interrupted", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-empty-interrupted-workspace", + normalizedSessionId: "normalized-empty-interrupted-workspace", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Inspect without changing files." }, + }); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + transport.push("paperclip/workspaceChange/updated", { + threadId: "thread-1", + turnId: turn.turnId, + workspaceChange: { + schema: "paperclip.workspace.diff.v1", + changeSetId: `${turn.turnId}:workspace`, + revision: 1, + source: "harness_reported", + complete: false, + files: [], + totals: { files: 0, additions: 0, deletions: 0 }, + patchArtifactRef: null, + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "interrupted", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect(events.find((event) => event.eventType === "workspace.change.updated")?.payload) + .toMatchObject({ revision: 1, complete: false, totals: { files: 0 } }); + expect(events.filter((event) => event.eventType === "workspace.diff.recorded")) + .toHaveLength(1); + expect(events.find((event) => event.eventType === "workspace.diff.recorded")?.payload) + .toMatchObject({ revision: 1, source: "runner_verified", complete: true, totals: { files: 0 } }); + expect(events.some((event) => event.eventType === "turn.interrupted")).toBe(true); + }); + + it("accepts a thread usage snapshot replayed before a resumed turn starts", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-usage-replay", + normalizedSessionId: "normalized-usage-replay", + workingDirectory: WORKSPACE, + }); + + transport.push("thread/tokenUsage/updated", { + threadId: "thread-1", + turnId: "prior-turn", + tokenUsage: { total: { inputTokens: 10, outputTokens: 4 } }, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Continue." }, + }); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect(events.some((event) => event.eventType === "session.failed")).toBe( + false, + ); + expect(await session.usage()).toMatchObject({ + total: { inputTokens: 10, outputTokens: 4 }, + }); + }); + + it("normalizes provider message and reasoning phases onto every streamed item event", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-stream-channels", + normalizedSessionId: "normalized-stream-channels", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Stream progress." }, + }); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + transport.push("item/started", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "progress-1", + type: "agentMessage", + phase: "commentary", + text: "", + }, + }); + transport.push("item/agentMessage/delta", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "progress-1", + delta: "Running it now.", + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "progress-1", + type: "agentMessage", + phase: "commentary", + text: "Running it now.", + }, + }); + transport.push("item/started", { + threadId: "thread-1", + turnId: turn.turnId, + item: { id: "reason-1", type: "reasoning", summary: [] }, + }); + transport.push("item/reasoning/summaryTextDelta", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "reason-1", + delta: "Summary", + }); + transport.push("item/reasoning/textDelta", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "reason-1", + delta: "Detail", + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "reason-1", + type: "reasoning", + summary: [{ text: "Summary" }], + }, + }); + transport.push("item/started", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "final-1", + type: "agentMessage", + phase: "final_answer", + text: "", + }, + }); + transport.push("item/agentMessage/delta", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "final-1", + delta: JSON.stringify(result), + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "final-1", + type: "agentMessage", + phase: "final_answer", + text: JSON.stringify(result), + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + const pick = (kind: string, channel: string) => + events.find( + (event) => + event.payload.kind === kind && event.payload.channel === channel, + ); + expect(pick("agentMessage", "progress")?.payload).toMatchObject({ + providerPhase: "commentary", + }); + expect( + events.find( + (event) => + event.eventType === "item.delta" && + event.payload.channel === "progress", + )?.payload, + ).toMatchObject({ + providerMethod: "item/agentMessage/delta", + text: "Running it now.", + }); + expect( + events.find( + (event) => + event.eventType === "item.delta" && + event.payload.channel === "summary", + )?.payload, + ).toMatchObject({ providerMethod: "item/reasoning/summaryTextDelta" }); + expect( + events.find( + (event) => + event.eventType === "item.delta" && + event.payload.channel === "detail", + )?.payload, + ).toMatchObject({ providerMethod: "item/reasoning/textDelta" }); + expect(pick("agentMessage", "final")?.payload).toMatchObject({ + providerPhase: "final_answer", + }); + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts new file mode 100644 index 0000000000..7ce0c83f3c --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts @@ -0,0 +1,555 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +class BlockingBootstrapTransport extends FakeCodexTransport { + readonly blocked: Promise; + readonly closeStarted: Promise; + closeCalls = 0; + #resolveBlocked!: () => void; + #resolveCloseStarted!: () => void; + #resolveClose!: () => void; + #rejectBlocked: ((reason: Error) => void) | null = null; + readonly #closeReleased: Promise; + + constructor(readonly blockedMethod: "thread/start" | "thread/read") { + super(); + this.blocked = new Promise((resolve) => { + this.#resolveBlocked = resolve; + }); + this.closeStarted = new Promise((resolve) => { + this.#resolveCloseStarted = resolve; + }); + this.#closeReleased = new Promise((resolve) => { + this.#resolveClose = resolve; + }); + } + + override request( + method: string, + params: Record, + ): Promise> { + if (method !== this.blockedMethod) return super.request(method, params); + this.calls.push({ method, params: structuredClone(params) }); + this.#resolveBlocked(); + return new Promise>((_resolve, reject) => { + this.#rejectBlocked = reject; + }); + } + + releaseClose(): void { + this.#resolveClose(); + } + + override async close(): Promise { + this.closeCalls += 1; + this.#rejectBlocked?.(new Error("blocked bootstrap transport closed")); + this.#rejectBlocked = null; + this.#resolveCloseStarted(); + await this.#closeReleased; + await super.close(); + } +} + +describe("Codex app-server Codex driver", () => { + it("does not create a transport for a pre-aborted session open", async () => { + const transportFactory = vi.fn(() => new FakeCodexTransport()); + const driver = makeDriver([], { transportFactory }); + const controller = new AbortController(); + const cancelled = new Error("session open cancelled before admission"); + controller.abort(cancelled); + + await expect(driver.openSession({ + runId: "run-pre-aborted", + normalizedSessionId: "normalized-pre-aborted", + workingDirectory: WORKSPACE, + signal: controller.signal, + })).rejects.toBe(cancelled); + expect(transportFactory).not.toHaveBeenCalled(); + }); + + it("does not create a transport for pre-aborted session recovery", async () => { + const initialDriver = makeDriver([new FakeCodexTransport()]); + const original = await initialDriver.openSession({ + runId: "run-pre-aborted-recovery", + normalizedSessionId: "normalized-pre-aborted-recovery", + workingDirectory: WORKSPACE, + }); + const snapshot = await original.snapshot(); + await original.close({ reason: "prepare recovery checkpoint" }); + const transportFactory = vi.fn(() => new FakeCodexTransport()); + const recoveryDriver = makeDriver([], { transportFactory }); + const controller = new AbortController(); + const cancelled = new Error("session recovery cancelled before admission"); + controller.abort(cancelled); + + await expect(recoveryDriver.recoverSession(snapshot, { + signal: controller.signal, + })).rejects.toBe(cancelled); + expect(transportFactory).not.toHaveBeenCalled(); + }); + + it("closes and awaits a blocked session-open transport on abort", async () => { + const transport = new BlockingBootstrapTransport("thread/start"); + const driver = makeDriver([transport]); + const controller = new AbortController(); + const cancelled = new Error("session open cancelled while blocked"); + let settled = false; + + const opening = driver.openSession({ + runId: "run-blocked-open", + normalizedSessionId: "normalized-blocked-open", + workingDirectory: WORKSPACE, + signal: controller.signal, + }).then( + () => ({ error: null }), + (error: unknown) => ({ error }), + ).finally(() => { + settled = true; + }); + await transport.blocked; + controller.abort(cancelled); + await transport.closeStarted; + await Promise.resolve(); + + expect(settled).toBe(false); + expect(transport.closeCalls).toBe(1); + transport.releaseClose(); + await expect(opening).resolves.toEqual({ error: cancelled }); + expect(transport.calls.map((call) => call.method)).toEqual([ + "initialize", + "thread/start", + ]); + }); + + it("closes and awaits a blocked recovery transport on abort", async () => { + const initial = new FakeCodexTransport(); + const recoveryTransport = new BlockingBootstrapTransport("thread/read"); + const driver = makeDriver([initial, recoveryTransport]); + const original = await driver.openSession({ + runId: "run-blocked-recovery", + normalizedSessionId: "normalized-blocked-recovery", + workingDirectory: WORKSPACE, + }); + const snapshot = await original.snapshot(); + await original.close({ reason: "simulate transport loss" }); + const controller = new AbortController(); + const cancelled = new Error("session recovery cancelled while blocked"); + let settled = false; + + const recovering = driver.recoverSession(snapshot, { + signal: controller.signal, + }).then( + (value) => ({ value, error: null }), + (error: unknown) => ({ value: null, error }), + ).finally(() => { + settled = true; + }); + await recoveryTransport.blocked; + controller.abort(cancelled); + await recoveryTransport.closeStarted; + await Promise.resolve(); + + expect(settled).toBe(false); + expect(recoveryTransport.closeCalls).toBe(1); + recoveryTransport.releaseClose(); + await expect(recovering).resolves.toEqual({ + value: null, + error: cancelled, + }); + expect(recoveryTransport.calls.map((call) => call.method)).toEqual([ + "initialize", + "thread/read", + ]); + }); + + it("persists process ownership before sending provider requests", async () => { + const transport = new FakeCodexTransport(); + Object.assign(transport, { + processInfo: () => ({ + pid: 71_001, + processGroupId: 71_001, + startedAt: "2026-08-18T18:00:00.000Z", + exited: false, + exitCode: null, + signal: null, + }), + }); + let releaseOwnership!: () => void; + const ownershipPersisted = new Promise((resolve) => { + releaseOwnership = resolve; + }); + const onSpawn = vi.fn(() => ownershipPersisted); + const driver = makeDriver([transport], { onSpawn }); + + const opening = driver.openSession({ + runId: "run-owned", + normalizedSessionId: "normalized-owned", + workingDirectory: WORKSPACE, + }); + await vi.waitFor(() => expect(onSpawn).toHaveBeenCalledOnce()); + expect(transport.calls).toEqual([]); + + releaseOwnership(); + await opening; + expect(onSpawn).toHaveBeenCalledWith({ + pid: 71_001, + processGroupId: 71_001, + startedAt: "2026-08-18T18:00:00.000Z", + }); + expect(transport.calls[0]?.method).toBe("initialize"); + }); + + it("sends direct chat as plain text and permits a follow-up turn", async () => { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport], { conversationMode: "direct" }); + const session = await driver.openSession({ + runId: "run-chat", + normalizedSessionId: "normalized-chat", + workingDirectory: WORKSPACE, + }); + + const threadStart = transport.calls.find( + (call) => call.method === "thread/start", + ); + expect(threadStart?.params).not.toHaveProperty("baseInstructions"); + expect(threadStart?.params.dynamicTools).toEqual([]); + + const first = await session.startTurn({ + message: { role: "user", text: "Hello Codex" }, + }); + const firstStart = transport.calls.find( + (call) => call.method === "turn/start", + ); + expect(firstStart?.params).not.toHaveProperty("outputSchema"); + expect(firstStart?.params.input).toEqual([ + { type: "text", text: "Hello Codex", text_elements: [] }, + ]); + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: first.turnId, status: "inProgress" }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: first.turnId, status: "completed", items: [] }, + }); + await collectUntilTerminal(session.events()); + + transport.turnStartResponse = Promise.resolve({ + turn: { id: "turn-2", status: "inProgress", items: [] }, + }); + await expect( + session.startTurn({ message: { role: "user", text: "And a follow-up" } }), + ).resolves.toEqual({ + turnId: "turn-2", + effectiveCollaborationMode: "default", + }); + }); + + it("forwards the persisted native model to the runner transport", async () => { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport], { model: "qualified-provider-model" }); + + await driver.openSession({ + runId: "run-qualified-model", + normalizedSessionId: "normalized-qualified-model", + workingDirectory: WORKSPACE, + }); + + expect( + transport.calls.find((call) => call.method === "thread/start")?.params, + ).toMatchObject({ + model: "qualified-provider-model", + completionContract: { + revision: "codex-demo-v1", + criterionIds: ["file"], + }, + }); + }); + + it("places Paperclip runtime instructions in Codex's system channel and enables only selected skill instructions", async () => { + const transport = new FakeCodexTransport(); + const baseInstructions = [ + "You are running as a Paperclip agent.", + "Follow the attached AGENTS.md instructions.", + "Read-only instruction sibling root: /paperclip/context/instructions", + ].join("\n\n"); + const driver = makeDriver([transport], { + baseInstructions, + includeSkillInstructions: true, + }); + + await driver.openSession({ + runId: "run-runtime-context", + normalizedSessionId: "normalized-runtime-context", + workingDirectory: WORKSPACE, + }); + + const threadStart = transport.calls.find((call) => call.method === "thread/start"); + expect(threadStart?.params).toMatchObject({ + baseInstructions, + config: { + "skills.include_instructions": true, + include_apps_instructions: false, + }, + }); + expect(JSON.stringify(threadStart?.params.input ?? null)).not.toContain(baseInstructions); + }); + + it("passes the common typed-event contract and reports one provider turn terminal", async () => { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport]); + const descriptor = await driver.descriptor(); + const session = await driver.openSession({ + runId: "run-1", + normalizedSessionId: "normalized-1", + workingDirectory: WORKSPACE, + }); + const turn = await session.startTurn({ + message: { role: "user", text: "Do the safe task." }, + }); + + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "inProgress" }, + }); + transport.push("turn/plan/updated", { + threadId: "thread-1", + turnId: turn.turnId, + revision: 1, + plan: [ + { step: "Inspect", status: "completed" }, + { step: "Implement", status: "inProgress" }, + ], + }); + transport.push("turn/diff/updated", { + threadId: "thread-1", + turnId: turn.turnId, + revision: 1, + diff: [ + "diff --git a/hello.txt b/hello.txt", + "--- a/hello.txt", + "+++ b/hello.txt", + "@@ -1 +1,2 @@", + "-hello", + "+hello world", + "+again", + ].join("\n"), + }); + transport.push("turn/plan/updated", { + threadId: "thread-1", + turnId: turn.turnId, + revision: 2, + plan: [ + { step: "Inspect", status: "completed" }, + { step: "Implement", status: "completed" }, + ], + }); + transport.push("item/started", { + threadId: "thread-1", + turnId: turn.turnId, + item: { id: "cmd-1", type: "commandExecution", command: "printf hello" }, + }); + transport.push("item/commandExecution/outputDelta", { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "cmd-1", + delta: "hello", + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "file-1", + type: "fileChange", + changes: [{ path: "hello.txt" }], + }, + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "reference-1", + type: "agentMessage", + text: "Open [hello.txt](hello.txt).", + }, + }); + const requestResolution = transport.invoke({ + id: "request-1", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId: turn.turnId, + itemId: "question-1", + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(session.pendingRuntimeRequests?.()).toHaveLength(1); + await session.resolveRuntimeRequest?.({ + requestId: "request-1", + turnId: turn.turnId, + resolution: { action: "cancel" }, + }); + expect(await requestResolution).toEqual({ answers: {} }); + transport.push("thread/tokenUsage/updated", { + threadId: "thread-1", + turnId: turn.turnId, + tokenUsage: { + total: { inputTokens: 10, outputTokens: 4 }, + modelContextWindow: 128000, + }, + }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: turn.turnId, + item: { + id: "answer-1", + type: "agentMessage", + text: JSON.stringify(result), + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: turn.turnId, status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect(events.every((event) => validatePrpEvent(event).ok)).toBe(true); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + expect( + events.filter((event) => event.eventType === "run.terminal"), + ).toHaveLength(0); + const workspaceEvents = events.filter((event) => event.eventType === "workspace.change.updated"); + expect(workspaceEvents[0]?.payload).toMatchObject({ + schema: "paperclip.workspace.diff.v1", + changeSetId: `${turn.turnId}:workspace`, + complete: false, + totals: { files: 1, additions: 2, deletions: 1 }, + }); + expect(workspaceEvents.at(-1)?.payload).toMatchObject({ + schema: "paperclip.workspace.diff.v1", + complete: true, + totals: { files: 1 }, + }); + const planEvents = events.filter((event) => event.eventType === "plan.updated"); + expect(planEvents).toHaveLength(2); + expect(new Set(planEvents.map((event) => event.itemId))).toEqual(new Set([turn.turnId])); + expect(planEvents.at(-1)?.payload).toMatchObject({ + planId: turn.turnId, + revision: 2, + complete: true, + syncStatus: "not_applicable", + }); + expect( + events.find((event) => event.eventType === "workspace.diff.recorded") + ?.payload, + ).toMatchObject({ schema: "paperclip.workspace.diff.v1", complete: true }); + expect( + events.find((event) => event.eventType === "workspace.file.referenced") + ?.payload, + ).toMatchObject({ + schema: "paperclip.workspace.file_reference.v1", + path: "hello.txt", + }); + expect(events.map((event) => event.eventType)).toEqual( + expect.arrayContaining([ + "session.started", + "turn.started", + "item.started", + "item.delta", + "item.completed", + "run.result.proposed", + "turn.completed", + "runtime_request.created", + "runtime_request.resolved", + "workspace.change.updated", + "workspace.diff.recorded", + ]), + ); + + const capabilities: PrpCapabilities = { + schema: "paperclip.prp.capabilities.v1", + sessionReusePolicy: "reuse_per_issue", + driver: { kind: descriptor.kind, version: descriptor.version }, + steer: true, + interrupt: true, + resume: true, + runtimeRequests: true, + structuredResult: true, + typedEvents: true, + }; + const metadata = { + fixtureName: "codex-conformance", + identity: { + schema: "paperclip.prp.identity.v1" as const, + companyId: "company-1", + issueId: "issue-1", + runId: "run-1", + environmentLeaseId: "lease-1", + runnerInstanceId: "runner-codex", + normalizedSessionId: "normalized-1", + }, + capabilities, + }; + const live = events.reduce( + applyPrpEvent, + createSessionSnapshotFromMetadata(metadata), + ); + const replay = events.reduce( + applyPrpEvent, + createSessionSnapshotFromMetadata(metadata), + ); + expect(live).toEqual(replay); + expect(live.integrity).toBe("complete"); + expect(await session.usage?.()).toMatchObject({ + modelContextWindow: 128000, + }); + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts new file mode 100644 index 0000000000..9737a80903 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts @@ -0,0 +1,1032 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("resumes and reconciles the exact provider thread after transport loss", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-recover", + normalizedSessionId: "normalized-recover", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Work." } }); + const snapshot = await original.snapshot(); + await original.close({ reason: "transport lost" }); + const recovery = await driver.recoverSession?.(snapshot); + expect(recovery).toMatchObject({ recovered: true }); + expect(recovery?.session?.ids()).toEqual(original.ids()); + expect(await recovery?.session?.reconcile?.()).toMatchObject({ + thread: { id: "thread-1" }, + }); + expect(second.calls.map((call) => call.method)).toEqual([ + "initialize", + "thread/read", + "thread/resume", + "thread/goal/get", + "thread/read", + ]); + expect((await recovery?.session?.snapshot())?.activeTurnId).toBe("turn-1"); + }); + + it("ignores an old terminal turn when the persisted active turn is still running", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { id: "turn-old", status: "completed", items: [] }, + { id: "turn-1", status: "inProgress", items: [] }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-history", + normalizedSessionId: "normalized-history", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ + message: { role: "user", text: "Keep working." }, + }); + const snapshot = await original.snapshot(); + await original.close({ reason: "transport lost" }); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + await recovered!.reconcile?.(); + expect(await recovered!.snapshot()).toMatchObject({ + activeTurnId: "turn-1", + }); + await recovered!.close({ reason: "test complete" }); + const events: PrpEvent[] = []; + for await (const event of recovered!.events()) events.push(event); + expect(events.some((event) => event.eventType === "turn.completed")).toBe( + false, + ); + }); + + it("preserves proposal idempotency and rejects a changed result after transport loss", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-proposal-loss", + normalizedSessionId: "normalized-proposal-loss", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await first.invoke({ + id: "call-before-loss", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-before-loss", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + const snapshot = await original.snapshot(); + expect(snapshot.semanticResult).toMatchObject({ + callId: "call-before-loss", + turnId: "turn-1", + result, + }); + await original.close({ reason: "transport lost after proposal" }); + const originalEvents: PrpEvent[] = []; + for await (const event of original.events()) originalEvents.push(event); + expect( + originalEvents.filter( + (event) => event.eventType === "run.result.proposed", + ), + ).toHaveLength(1); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + const collecting = collectUntilTerminal(recovered!.events()); + await recovered!.reconcile?.(); + expect( + await second.invoke({ + id: "call-after-loss", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-after-loss", + tool: "paperclip_finish", + arguments: reverseObjectKeys(result), + }, + }), + ).toMatchObject({ success: true }); + const changed = structuredClone(result); + changed.summary = "Changed after transport loss."; + expect( + await second.invoke({ + id: "changed-after-loss", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "changed-after-loss", + tool: "paperclip_finish", + arguments: changed, + }, + }), + ).toMatchObject({ success: false }); + second.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + const events = await collecting; + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(0); + expect( + events.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + }); + + it("rejects changed terminal result content discovered during recovery", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const changed = structuredClone(result); + changed.summary = "Conflicting recovered terminal result."; + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { + id: "turn-1", + status: "completed", + items: [ + { + id: "terminal-result", + type: "agentMessage", + text: JSON.stringify(changed), + }, + ], + }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-recovered-result-conflict", + normalizedSessionId: "normalized-recovered-result-conflict", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await first.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const snapshot = await original.snapshot(); + expect(snapshot.terminalTurns).toHaveLength(1); + await original.close({ reason: "transport lost after terminal" }); + + const recovery = await driver.recoverSession?.(snapshot); + expect(recovery).toMatchObject({ recovered: true }); + await expect( + recovery!.session!.reconcile!(), + ).rejects.toMatchObject({ + name: "HarnessReconciliationError", + recoverable: true, + message: expect.stringContaining( + "contains a conflicting semantic result", + ), + }); + expect( + (await recovery!.session!.snapshot()).semanticResult?.result, + ).toEqual(result); + await recovery!.session!.close({ reason: "test complete" }); + }); + + it("does not re-emit an already observed terminal after recovery", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-terminal-loss", + normalizedSessionId: "normalized-terminal-loss", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await first.invoke({ + id: "terminal-call", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "terminal-call", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const snapshot = await original.snapshot(); + expect(snapshot.terminalTurns).toHaveLength(1); + await original.close({ reason: "transport lost after terminal" }); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + await recovered!.reconcile?.(); + await recovered!.close({ reason: "test complete" }); + const events: PrpEvent[] = []; + for await (const event of recovered!.events()) events.push(event); + expect(events.some((event) => event.eventType.startsWith("turn."))).toBe( + false, + ); + expect( + events.some((event) => event.eventType === "run.result.proposed"), + ).toBe(false); + }); + + it("clears a checkpointed active turn that already has a durable terminal", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }], + }, + }; + second.turnStartResponse = Promise.resolve({ + turn: { id: "turn-2", status: "inProgress", items: [] }, + }); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-terminal-checkpoint-race", + normalizedSessionId: "normalized-terminal-checkpoint-race", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ + message: { role: "user", text: "Complete without a result." }, + }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const snapshot = await original.snapshot(); + expect(snapshot).toMatchObject({ + activeTurnId: null, + semanticResult: null, + terminalTurns: [{ turnId: "turn-1" }], + }); + // Model the crash window after the terminal fingerprint was checkpointed + // but before the subsequent active-turn clear was durable. + snapshot.activeTurnId = "turn-1"; + await original.close({ reason: "transport lost during terminal checkpoint" }); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + await expect(recovered!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: null, + terminalTurns: [{ turnId: "turn-1" }], + }); + await expect(recovered!.startTurn({ + message: { + role: "user", + text: "Recover only the missing disposition.", + }, + })).resolves.toMatchObject({ turnId: "turn-2" }); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(1); + await recovered!.close({ reason: "test complete" }); + }); + + it("permits one recovery turn for a result-less terminal task", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const third = new FakeCodexTransport(); + const fourth = new FakeCodexTransport(); + third.turnStartResponse = Promise.reject( + new CodexRpcError('{"code":-32602,"message":"invalid params"}', -32_602), + ); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }], + }, + }; + third.readResponse = structuredClone(second.readResponse); + fourth.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { id: "turn-1", status: "completed", items: [] }, + { id: "turn-2", status: "completed", items: [] }, + ], + }, + }; + fourth.turnStartResponse = Promise.resolve({ + turn: { id: "turn-3", status: "inProgress", items: [] }, + }); + const driver = makeDriver([first, second, third, fourth]); + const original = await driver.openSession({ + runId: "run-result-less-terminal", + normalizedSessionId: "normalized-result-less-terminal", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const snapshot = await original.snapshot(); + expect(snapshot).toMatchObject({ + activeTurnId: null, + semanticResult: null, + terminalTurns: [{ turnId: "turn-1" }], + }); + await original.close({ reason: "transport lost after terminal" }); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + await recovered!.reconcile?.(); + await expect(recovered!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + terminalTurns: [{ turnId: "turn-1" }], + }); + const preStartSnapshot = await recovered!.snapshot(); + await recovered!.close({ reason: "crash before disposition turn submission" }); + + const retry = await driver.recoverSession?.(preStartSnapshot); + const retried = retry?.session; + expect(retried).toBeDefined(); + await retried!.reconcile?.(); + await expect(retried!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + }); + const recoveryMessage = "Do not execute twice; report disposition only."; + const recoveryTerminal = collectUntilTerminal(retried!.events()); + await expect(retried!.startTurn({ + message: { role: "user", text: recoveryMessage }, + })).rejects.toThrow("invalid params"); + await expect(retried!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + }); + third.turnStartResponse = Promise.resolve({ + turn: { id: "turn-2", status: "inProgress", items: [] }, + }); + await expect(retried!.startTurn({ + message: { role: "user", text: recoveryMessage }, + })).resolves.toMatchObject({ turnId: "turn-2" }); + await expect(retried!.snapshot()).resolves.toMatchObject({ + activeTurnId: "turn-2", + dispositionOnlyRecoveryConsumed: true, + }); + await expect(retried!.startTurn({ + message: { role: "user", text: "Do not start concurrently." }, + })).rejects.toThrow("session cannot start another turn"); + const recoveryStarts = third.calls.filter( + (call) => call.method === "turn/start", + ); + expect(recoveryStarts).toHaveLength(2); + expect(recoveryStarts[0]!.params.input).toEqual([ + { type: "text", text: recoveryMessage, text_elements: [] }, + ]); + expect(JSON.stringify(recoveryStarts[0]!.params.input)).not.toContain( + "Complete.", + ); + third.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-2", status: "completed", items: [] }, + }); + await recoveryTerminal; + const spentSnapshot = await retried!.snapshot(); + expect(spentSnapshot).toMatchObject({ + activeTurnId: null, + semanticResult: null, + dispositionOnlyRecoveryConsumed: true, + terminalTurns: [{ turnId: "turn-1" }, { turnId: "turn-2" }], + }); + await retried!.close({ reason: "simulate another restart" }); + + const repeatedRecovery = await driver.recoverSession?.(spentSnapshot); + expect(repeatedRecovery?.session).toBeDefined(); + await repeatedRecovery!.session!.reconcile?.(); + await expect(repeatedRecovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: null, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-2", + terminalTurns: [{ turnId: "turn-1" }, { turnId: "turn-2" }], + }); + await expect(repeatedRecovery!.session!.startTurn({ + message: { role: "user", text: "Do not retry the completed disposition." }, + })).rejects.toThrow("session cannot start another turn"); + expect( + fourth.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(0); + await repeatedRecovery!.session!.close({ reason: "test complete" }); + }); + + it("adopts a checkpointed disposition turn before its terminal is durable", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { id: "turn-1", status: "completed", items: [] }, + { id: "turn-2", status: "inProgress", items: [] }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-uncheckpointed-disposition", + normalizedSessionId: "normalized-uncheckpointed-disposition", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const preDispositionSnapshot = await original.snapshot(); + await original.close({ reason: "simulate crash after provider acceptance" }); + + const recovery = await driver.recoverSession?.({ + ...preDispositionSnapshot, + dispositionOnlyRecoveryConsumed: true, + }); + expect(recovery).toMatchObject({ recovered: true }); + await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: "turn-2", + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-2", + }); + await expect(recovery!.session!.startTurn({ + message: { + role: "user", + text: "Do not duplicate the adopted disposition turn.", + }, + })).rejects.toThrow("session cannot start another turn"); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(0); + await recovery!.session!.close({ reason: "test complete" }); + }); + + for (const testCase of [ + { + history: "omitted", + turns: undefined, + dispositionOnlyRecoveryTurnId: null, + }, + { + history: "not an array", + turns: { unavailable: true }, + dispositionOnlyRecoveryTurnId: "turn-2", + }, + ]) { + it( + `preserves disposition recovery ownership when provider history is ${testCase.history}`, + async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + const thread: Record = { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + }; + if (testCase.turns !== undefined) thread.turns = testCase.turns; + second.readResponse = { thread }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: `run-unknown-history-${testCase.history.replaceAll(" ", "-")}`, + normalizedSessionId: + `normalized-unknown-history-${testCase.history.replaceAll(" ", "-")}`, + workingDirectory: WORKSPACE, + }); + await original.startTurn({ + message: { role: "user", text: "Complete." }, + }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ + reason: "simulate unavailable provider history", + }); + + const recovery = await driver.recoverSession?.({ + ...checkpoint, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: + testCase.dispositionOnlyRecoveryTurnId, + }); + expect(recovery).toMatchObject({ recovered: true }); + await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: + testCase.dispositionOnlyRecoveryTurnId, + }); + await expect(recovery!.session!.startTurn({ + message: { + role: "user", + text: "Do not repeat the task while provider history is unknown.", + }, + })).rejects.toThrow("session cannot start another turn"); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(0); + await recovery!.session!.close({ reason: "test complete" }); + }, + ); + } + + it("preserves disposition recovery ownership when provider history omits the checkpoint terminal", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-2", status: "inProgress", items: [] }], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-unanchored-history", + normalizedSessionId: "normalized-unanchored-history", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ + message: { role: "user", text: "Complete." }, + }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "simulate truncated provider history" }); + + const recovery = await driver.recoverSession?.({ + ...checkpoint, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: null, + }); + expect(recovery).toMatchObject({ recovered: true }); + await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: null, + }); + await expect(recovery!.session!.startTurn({ + message: { + role: "user", + text: "Do not duplicate the accepted disposition turn.", + }, + })).rejects.toThrow("session cannot start another turn"); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(0); + await recovery!.session!.close({ reason: "test complete" }); + }); + + it("releases a legacy disposition marker when provider history has no accepted turn", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }], + }, + }; + second.turnStartResponse = Promise.resolve({ + turn: { id: "turn-2", status: "inProgress", items: [] }, + }); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-missing-disposition", + normalizedSessionId: "normalized-missing-disposition", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "simulate pre-acceptance checkpoint" }); + + const recovery = await driver.recoverSession?.({ + ...checkpoint, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: null, + }); + expect(recovery).toMatchObject({ recovered: true }); + await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, + }); + await expect(recovery!.session!.startTurn({ + message: { role: "user", text: "Recover disposition only." }, + })).resolves.toMatchObject({ turnId: "turn-2" }); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(1); + await recovery!.session!.close({ reason: "test complete" }); + }); + + it("releases a missing bound disposition turn without restoring the task envelope", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "completed", items: [] }], + }, + }; + second.turnStartResponse = Promise.resolve({ + turn: { id: "turn-2", status: "inProgress", items: [] }, + }); + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-missing-bound-disposition", + normalizedSessionId: "normalized-missing-bound-disposition", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "simulate missing provider acceptance" }); + + const recovery = await driver.recoverSession?.({ + ...checkpoint, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-missing-disposition", + }); + expect(recovery).toMatchObject({ recovered: true }); + await expect(recovery!.session!.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, + }); + const recoveryMessage = "Recover disposition only; do not repeat work."; + await expect(recovery!.session!.startTurn({ + message: { role: "user", text: recoveryMessage }, + })).resolves.toMatchObject({ turnId: "turn-2" }); + const recoveryStarts = second.calls.filter( + (call) => call.method === "turn/start", + ); + expect(recoveryStarts).toHaveLength(1); + expect(recoveryStarts[0]!.params.input).toEqual([ + { type: "text", text: recoveryMessage, text_elements: [] }, + ]); + expect(JSON.stringify(recoveryStarts[0]!.params.input)).not.toContain( + "Complete.", + ); + await recovery!.session!.close({ reason: "test complete" }); + }); + + it("fails closed when provider history substitutes a different disposition turn", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { id: "turn-1", status: "completed", items: [] }, + { id: "turn-unrelated", status: "inProgress", items: [] }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-conflicting-bound-disposition", + normalizedSessionId: "normalized-conflicting-bound-disposition", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const checkpoint = await original.snapshot(); + await original.close({ reason: "simulate conflicting provider history" }); + + await expect(driver.recoverSession?.({ + ...checkpoint, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-missing-disposition", + })).resolves.toEqual({ + recovered: false, + reason: "provider changed the bound disposition recovery turn", + }); + expect( + second.calls.filter((call) => call.method === "turn/start"), + ).toHaveLength(0); + }); + + it("keeps a newer result-bearing recovery turn active beside an older result-less terminal", async () => { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: [ + { id: "turn-1", status: "completed", items: [] }, + { id: "turn-2", status: "inProgress", items: [] }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "run-result-recovery-loss", + normalizedSessionId: "normalized-result-recovery-loss", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Complete." } }); + expect(await first.invoke({ + id: "result-before-loss", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "result-before-loss", + tool: "paperclip_finish", + arguments: result, + }, + })).toMatchObject({ success: true }); + first.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(original.events()); + const snapshot = await original.snapshot(); + if (snapshot.semanticResult === null || snapshot.semanticResult === undefined) { + throw new Error("expected persisted semantic result"); + } + snapshot.activeTurnId = "turn-2"; + snapshot.semanticResult = { + ...snapshot.semanticResult, + turnId: "turn-2", + }; + snapshot.terminalTurns![0]!.fingerprint = + '{"error":null,"result":null,"terminalState":"completed"}'; + await original.close({ reason: "transport lost during recovery turn" }); + + const recovery = await driver.recoverSession?.(snapshot); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + const collecting = collectUntilTerminal(recovered!.events()); + await recovered!.reconcile?.(); + second.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-2", status: "completed", items: [] }, + }); + const events = await collecting; + expect(events.at(-1)).toMatchObject({ + eventType: "turn.completed", + turnId: "turn-2", + }); + await recovered!.close({ reason: "test complete" }); + }); + + it("fails recovery explicitly when the persisted active turn is missing or replaced", async () => { + for (const testCase of [ + { + label: "missing", + turns: [{ id: "turn-old", status: "completed", items: [] }], + message: "persisted active turn turn-1 is missing", + }, + { + label: "replaced", + turns: [{ id: "turn-2", status: "inProgress", items: [] }], + message: "active turn turn-2 instead of persisted active turn turn-1", + }, + ]) { + const first = new FakeCodexTransport(); + const second = new FakeCodexTransport(); + second.readResponse = { + thread: { + id: "thread-1", + sessionId: "provider-session-1", + cwd: WORKSPACE, + turns: testCase.turns, + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: `run-${testCase.label}-turn`, + normalizedSessionId: `normalized-${testCase.label}-turn`, + workingDirectory: WORKSPACE, + }); + await original.startTurn({ message: { role: "user", text: "Work." } }); + const snapshot = await original.snapshot(); + await original.close({ reason: "transport lost" }); + const recovery = await driver.recoverSession?.(snapshot); + expect(recovery?.session).toBeDefined(); + await expect( + recovery!.session!.reconcile!(), + ).rejects.toMatchObject({ + name: "HarnessReconciliationError", + recoverable: true, + message: expect.stringContaining(testCase.message), + }); + await recovery!.session!.close({ reason: "test complete" }); + } + }); + + it("reconciles a turn completed during transport loss without replacing either session identity", async () => { + const first = new FakeCodexTransport( + "driver-thread-loss", + "provider-session-loss", + ); + const second = new FakeCodexTransport( + "driver-thread-loss", + "provider-session-loss", + ); + second.readResponse = { + thread: { + id: "driver-thread-loss", + sessionId: "provider-session-loss", + cwd: WORKSPACE, + tokenUsage: { total: { inputTokens: 21, outputTokens: 8 } }, + turns: [ + { + id: "turn-1", + status: "completed", + items: [ + { + id: "answer-loss", + type: "agentMessage", + text: JSON.stringify(result), + }, + ], + }, + ], + }, + }; + const driver = makeDriver([first, second]); + const original = await driver.openSession({ + runId: "runtime-run-loss", + normalizedSessionId: "controller-session-loss", + workingDirectory: WORKSPACE, + }); + await original.startTurn({ + message: { role: "user", text: "Work through loss." }, + }); + const snapshot = await original.snapshot(); + await original.close({ reason: "transport lost before terminal delivery" }); + + const recovery = await driver.recoverSession?.(snapshot); + expect(recovery).toMatchObject({ recovered: true }); + const recovered = recovery?.session; + expect(recovered).toBeDefined(); + const collecting = collectUntilTerminal(recovered!.events()); + await recovered!.reconcile?.(); + const events = await collecting; + + expect(recovered!.ids()).toEqual({ + driverSessionId: "driver-thread-loss", + providerSessionId: "provider-session-loss", + displayId: "driver-thread-loss", + }); + expect(await recovered!.snapshot()).toMatchObject({ + runId: "runtime-run-loss", + normalizedSessionId: "controller-session-loss", + driverSessionId: "driver-thread-loss", + providerSessionId: "provider-session-loss", + }); + expect( + events.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.every( + (event, index) => + event.sourceSeq === snapshot.lastSourceSequence! + index + 1, + ), + ).toBe(true); + expect(await recovered!.usage?.()).toEqual({ + total: { inputTokens: 21, outputTokens: 8 }, + }); + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.runtime-requests.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.runtime-requests.test.ts new file mode 100644 index 0000000000..fb7eaae47d --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.runtime-requests.test.ts @@ -0,0 +1,757 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("validates runtime request resolutions against the kind of request they answer", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-resolution-shapes", + normalizedSessionId: "normalized-resolution-shapes", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask me things." }, + }); + + // Not `async`: an async function would flatten and await the pending + // provider response, which only settles once the request is resolved. + function open( + id: string, + method: string, + ): Promise> { + return transport.invoke({ + id, + method, + params: { + threadId: "thread-1", + turnId, + itemId: `item-${id}`, + reason: `fixture ${id}`, + }, + }); + } + const settled = () => new Promise((resolve) => setTimeout(resolve, 0)); + + const userInput = open("ask-input", "item/tool/requestUserInput"); + await settled(); + for (const resolution of [ + { action: "submit" }, + { action: "submit", answers: {} }, + { action: "submit", answers: { field: {} } }, + { action: "submit", answers: { field: { answers: [7] } } }, + { action: "submit", content: { answer: "wrong shape" } }, + { action: "accept" }, + ]) { + await expect( + session.resolveRuntimeRequest?.({ + requestId: "ask-input", + turnId, + resolution: resolution as unknown as HarnessRuntimeRequestResolution, + }), + ).rejects.toThrow(/user_input rejected its resolution/); + // A rejected resolution must leave the request answerable. + expect(session.pendingRuntimeRequests?.()).toHaveLength(1); + } + await session.resolveRuntimeRequest?.({ + requestId: "ask-input", + turnId, + resolution: { + action: "submit", + answers: { field: { answers: ["staging"] } }, + }, + }); + expect(await userInput).toEqual({ + answers: { field: { answers: ["staging"] } }, + }); + + const elicitation = open( + "ask-elicitation", + "mcpServer/elicitation/request", + ); + await settled(); + for (const resolution of [ + { action: "submit" }, + { action: "submit", content: {} }, + { action: "submit", answers: { field: { answers: ["wrong shape"] } } }, + { action: "accept_for_session" }, + ]) { + await expect( + session.resolveRuntimeRequest?.({ + requestId: "ask-elicitation", + turnId, + resolution: resolution as unknown as HarnessRuntimeRequestResolution, + }), + ).rejects.toThrow(/elicitation rejected its resolution/); + expect(session.pendingRuntimeRequests?.()).toHaveLength(1); + } + await session.resolveRuntimeRequest?.({ + requestId: "ask-elicitation", + turnId, + resolution: { action: "submit", content: { answer: "green" } }, + }); + expect(await elicitation).toEqual({ + action: "accept", + content: { answer: "green" }, + _meta: null, + }); + + const approval = open( + "ask-approval", + "item/commandExecution/requestApproval", + ); + await settled(); + await expect( + session.resolveRuntimeRequest?.({ + requestId: "ask-approval", + turnId, + resolution: { + action: "submit", + answers: { field: { answers: ["nope"] } }, + } as unknown as HarnessRuntimeRequestResolution, + }), + ).rejects.toThrow(/command_approval rejected its resolution/); + await session.resolveRuntimeRequest?.({ + requestId: "ask-approval", + turnId, + resolution: { action: "accept" }, + }); + expect(await approval).toEqual({ decision: "accept" }); + await session.close({ reason: "fixture complete" }); + }); + + it("replays the DOT-185 requestUserInput shape as one canonical three-question request", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-dot-185", + normalizedSessionId: "normalized-dot-185", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask the deployment questions." }, + }); + const resolvedEvent = (async () => { + for await (const event of session.events()) { + if (event.eventType === "runtime_request.resolved" + && event.payload.requestId === "dot-185-request") return event; + } + return null; + })(); + const nativeResponse = transport.invoke({ + id: "dot-185-request", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "dot-185-item", + questions: [ + { + id: "environment", + header: "Environment", + question: "Where should we deploy?", + options: [ + { label: "Staging", description: "Deploy to staging first." }, + { label: "Production", description: "Deploy directly to production." }, + ], + isOther: true, + }, + { + id: "regions", + header: "Regions", + question: "Which regions should receive the release?", + options: [{ label: "US" }, { label: "EU" }], + multiSelect: true, + }, + { + id: "notes", + header: "Notes", + question: "Anything else we should know?", + required: false, + }, + ], + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const [pending] = session.pendingRuntimeRequests?.() ?? []; + expect(pending?.input).toMatchObject({ + schema: "paperclip.question_set.v1", + questions: [ + { + id: "environment", + required: false, + answerMode: "single_select", + customAnswer: { enabled: true }, + options: [ + { id: "option-1", label: "Staging", description: "Deploy to staging first." }, + { id: "option-2", label: "Production", description: "Deploy directly to production." }, + ], + }, + { id: "regions", answerMode: "multi_select", required: false }, + { id: "notes", answerMode: "text", required: false }, + ], + }); + await session.resolveRuntimeRequest?.({ + requestId: "dot-185-request", + turnId, + resolution: { + action: "submit", + response: { + schema: "paperclip.question_response.v1", + answers: { + environment: { selectedOptionIds: ["option-1"] }, + regions: { selectedOptionIds: ["option-1", "option-2"] }, + notes: { text: "Ship during the maintenance window." }, + }, + }, + }, + }); + expect(await nativeResponse).toEqual({ + answers: { + environment: { answers: ["Staging"] }, + regions: { answers: ["US", "EU"] }, + notes: { answers: ["Ship during the maintenance window."] }, + }, + }); + expect((await resolvedEvent)?.payload).toMatchObject({ + action: "submit", + response: { + schema: "paperclip.question_response.v1", + answers: { + environment: { selectedOptionIds: ["option-1"] }, + regions: { selectedOptionIds: ["option-1", "option-2"] }, + notes: { text: "Ship during the maintenance window." }, + }, + }, + }); + await session.close({ reason: "fixture complete" }); + }); + + it("keeps redacted native option values through the cloned pending-request lifecycle", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-private-option", + normalizedSessionId: "normalized-private-option", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Choose the configured credential." }, + }); + const nativeResponse = transport.invoke({ + id: "private-option-request", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "private-option-item", + questions: [{ + id: "credential", + question: "Choose the configured credential.", + options: [{ id: "configured", label: "token=native-option-secret" }], + }], + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + + const [publicRequest] = structuredClone(session.pendingRuntimeRequests?.() ?? []); + expect(JSON.stringify(publicRequest)).not.toContain("native-option-secret"); + expect(publicRequest?.input?.questions[0]?.options?.[0]?.label).toContain("[REDACTED]"); + + await session.resolveRuntimeRequest?.({ + requestId: "private-option-request", + turnId, + resolution: { + action: "submit", + response: { + schema: "paperclip.question_response.v1", + answers: { credential: { selectedOptionIds: ["configured"] } }, + }, + }, + }); + expect(await nativeResponse).toEqual({ + answers: { credential: { answers: ["token=native-option-secret"] } }, + }); + await session.close({ reason: "fixture complete" }); + }); + + it("rejects an explicit malformed Codex form without falling back to v1", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-malformed-input", + normalizedSessionId: "normalized-malformed-input", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask a malformed question." }, + }); + const diagnostic = (async () => { + for await (const event of session.events()) { + if (event.eventType === "harness.diagnostic" && event.payload.code === "runtime_input_rejected") { + return event; + } + } + return null; + })(); + await expect(transport.invoke({ + id: "malformed-input", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "malformed-item", + questions: [ + { id: "duplicate", question: "First?" }, + { id: "duplicate", question: "Second?" }, + ], + }, + })).resolves.toEqual({ answers: {} }); + expect(session.pendingRuntimeRequests?.()).toEqual([]); + expect(await diagnostic).toEqual(expect.objectContaining({ + eventType: "harness.diagnostic", + payload: expect.objectContaining({ + code: "runtime_input_rejected", + adapter: "codex-app-server", + }), + })); + await session.close({ reason: "fixture complete" }); + }); + + it("emits one canonical outcome payload for every terminal runtime request fact", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-outcome", + normalizedSessionId: "normalized-outcome", + workingDirectory: WORKSPACE, + }); + const events: PrpEvent[] = []; + void (async () => { + for await (const event of session.events()) events.push(event); + })(); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Approve this." }, + }); + const resolved = transport.invoke({ + id: "outcome-resolved", + method: "item/commandExecution/requestApproval", + params: { + threadId: "thread-1", + turnId, + itemId: "item-resolved", + reason: "approve", + }, + }); + const cancelled = transport.invoke({ + id: "outcome-cancelled", + method: "item/commandExecution/requestApproval", + params: { + threadId: "thread-1", + turnId, + itemId: "item-cancelled", + reason: "approve", + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + await session.resolveRuntimeRequest?.({ + requestId: "outcome-resolved", + turnId, + resolution: { action: "accept_for_session" }, + }); + await resolved; + await session.close({ reason: "operator_closed" }); + await cancelled; + await new Promise((resolve) => setTimeout(resolve, 0)); + + const terminal = events.filter( + ({ eventType }) => + eventType === "runtime_request.resolved" || + eventType === "runtime_request.cancelled", + ); + expect( + terminal.map(({ eventType, turnId, itemId, payload }) => ({ + eventType, + turnId, + itemId, + payload, + })), + ).toEqual([ + { + eventType: "runtime_request.resolved", + turnId, + itemId: "item-resolved", + payload: { + requestId: "outcome-resolved", + requestKind: "command_approval", + turnId, + itemId: "item-resolved", + action: "accept_for_session", + }, + }, + { + eventType: "runtime_request.cancelled", + turnId, + itemId: "item-cancelled", + payload: { + requestId: "outcome-cancelled", + requestKind: "command_approval", + turnId, + itemId: "item-cancelled", + reason: "session_closed", + }, + }, + ]); + for (const event of terminal) expect(validatePrpEvent(event).ok).toBe(true); + }); + + it("expires a canonical input with its full question set when the provider transport is lost", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-provider-loss", + normalizedSessionId: "normalized-provider-loss", + workingDirectory: WORKSPACE, + }); + const events: PrpEvent[] = []; + const consume = (async () => { + for await (const event of session.events()) events.push(event); + })(); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask before continuing." }, + }); + const pending = transport.invoke({ + id: "lost-input", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "lost-input-item", + questions: [{ + id: "environment", + header: "Environment", + question: "Where should we deploy?", + options: [{ label: "Staging" }, { label: "Production" }], + }], + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + transport.queue.fail(new Error("provider exited")); + expect(await pending).toEqual({ answers: {} }); + await consume; + + const expired = events.find((event) => event.eventType === "runtime_request.expired"); + expect(expired).toMatchObject({ + turnId, + itemId: "lost-input-item", + payload: { + requestId: "lost-input", + reason: "provider_process_lost", + replayAllowed: false, + requestType: "input", + request: { + schema: "paperclip.runtime_request.v2", + requestKind: "runtime", + requestId: "lost-input", + type: "input", + input: { + schema: "paperclip.question_set.v1", + questions: [expect.objectContaining({ id: "environment" })], + }, + }, + }, + }); + expect(events.filter((event) => event.eventType === "runtime_request.cancelled")).toHaveLength(0); + expect(validatePrpEvent(expired!)).toMatchObject({ ok: true }); + }); + + it("hands a live canonical input off exactly once when the durable window expires", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-durable-handoff", + normalizedSessionId: "normalized-durable-handoff", + workingDirectory: WORKSPACE, + }); + const iterator = session.events()[Symbol.asyncIterator](); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask before continuing." }, + }); + const pending = transport.invoke({ + id: "handoff-input", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "handoff-input-item", + questions: [{ + id: "environment", + header: "Environment", + question: "Where should we deploy?", + options: [{ label: "Staging" }, { label: "Production" }], + }], + }, + }); + let created: PrpEvent | null = null; + for (let count = 0; count < 20; count += 1) { + const event = await iterator.next(); + if (event.done) break; + if (event.value.eventType === "runtime_request.created") { + created = event.value; + break; + } + } + expect(created).not.toBeNull(); + + const firstHandoff = session.handoffRuntimeRequest!({ + requestId: "handoff-input", + turnId, + reason: "durable_handoff", + signal: new AbortController().signal, + }); + expect(firstHandoff.result).toBe("handed_off"); + await expect(firstHandoff.cleanup).resolves.toBeUndefined(); + const repeatedHandoff = session.handoffRuntimeRequest!({ + requestId: "handoff-input", + turnId, + reason: "durable_handoff", + signal: new AbortController().signal, + }); + expect(repeatedHandoff.result).toBe("already_settled"); + await expect(repeatedHandoff.cleanup).resolves.toBeUndefined(); + expect(await pending).toEqual({ answers: {} }); + + let expired: PrpEvent | null = null; + for (let count = 0; count < 20; count += 1) { + const event = await iterator.next(); + if (event.done) break; + if (event.value.eventType === "runtime_request.expired") { + expired = event.value; + break; + } + } + expect(expired).toMatchObject({ + turnId, + itemId: "handoff-input-item", + payload: { + requestId: "handoff-input", + reason: "durable_handoff", + replayAllowed: false, + requestType: "input", + request: { + schema: "paperclip.runtime_request.v2", + requestId: "handoff-input", + type: "input", + }, + }, + }); + await session.close({ reason: "test complete" }); + }); + + it("does not commit a durable handoff after runtime ownership is revoked", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-aborted-handoff", + normalizedSessionId: "normalized-aborted-handoff", + workingDirectory: WORKSPACE, + }); + const iterator = session.events()[Symbol.asyncIterator](); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask before continuing." }, + }); + const pending = transport.invoke({ + id: "aborted-handoff-input", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "aborted-handoff-input-item", + questions: [{ + id: "environment", + header: "Environment", + question: "Where should we deploy?", + options: [{ label: "Staging" }, { label: "Production" }], + }], + }, + }); + for (let count = 0; count < 20; count += 1) { + const event = await iterator.next(); + if (event.done || event.value.eventType === "runtime_request.created") break; + } + + const ownership = new AbortController(); + ownership.abort(); + const abortedHandoff = session.handoffRuntimeRequest!({ + requestId: "aborted-handoff-input", + turnId, + reason: "durable_handoff", + signal: ownership.signal, + }); + expect(abortedHandoff.result).toBe("already_settled"); + await expect(abortedHandoff.cleanup).resolves.toBeUndefined(); + expect(session.pendingRuntimeRequests?.()).toHaveLength(1); + expect(transport.calls).not.toContainEqual(expect.objectContaining({ + method: "turn/interrupt", + })); + + const liveHandoff = session.handoffRuntimeRequest!({ + requestId: "aborted-handoff-input", + turnId, + reason: "durable_handoff", + signal: new AbortController().signal, + }); + expect(liveHandoff.result).toBe("handed_off"); + await expect(liveHandoff.cleanup).resolves.toBeUndefined(); + await expect(pending).resolves.toEqual({ answers: {} }); + await session.close({ reason: "test complete" }); + }); + + it("lets an answer claimed before expiry win the terminal-event race", async () => { + const transport = new FakeCodexTransport(); + let releaseResolution!: () => void; + transport.runtimeRequestResolver = () => new Promise((resolve) => { + releaseResolution = resolve; + }); + const session = await makeDriver([transport]).openSession({ + runId: "run-answer-handoff-race", + normalizedSessionId: "normalized-answer-handoff-race", + workingDirectory: WORKSPACE, + }); + const iterator = session.events()[Symbol.asyncIterator](); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Ask before continuing." }, + }); + const providerRequest = transport.invoke({ + id: "race-input", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "race-input-item", + questions: [{ + id: "environment", + header: "Environment", + question: "Where should we deploy?", + options: [{ label: "Staging" }, { label: "Production" }], + }], + }, + }); + for (let count = 0; count < 20; count += 1) { + const event = await iterator.next(); + if (event.done || event.value.eventType === "runtime_request.created") break; + } + const resolution = session.resolveRuntimeRequest!({ + requestId: "race-input", + turnId, + resolution: { + action: "submit", + response: { + schema: "paperclip.question_response.v1", + answers: { environment: { selectedOptionIds: ["option-1"] } }, + }, + }, + }); + await Promise.resolve(); + const handoff = session.handoffRuntimeRequest!({ + requestId: "race-input", + turnId, + reason: "durable_handoff", + signal: new AbortController().signal, + }); + expect(handoff.result).toBe("already_settled"); + await expect(handoff.cleanup).resolves.toBeUndefined(); + releaseResolution(); + await resolution; + await providerRequest; + + let terminalEvent: PrpEvent | null = null; + for (let count = 0; count < 20; count += 1) { + const event = await iterator.next(); + if (event.done) break; + if ( + event.value.payload.requestId === "race-input" + && ["runtime_request.resolved", "runtime_request.cancelled", "runtime_request.expired"].includes(event.value.eventType) + ) { + terminalEvent = event.value; + break; + } + } + expect(terminalEvent?.eventType).toBe("runtime_request.resolved"); + expect(session.pendingRuntimeRequests?.()).toEqual([]); + await session.close({ reason: "test complete" }); + }); + + it("redacts browser-visible request details and diagnostics from the fixture markers", async () => { + const fixture = await loadLiveConsoleConformanceFixture( + liveConsoleFixturePath, + ); + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-redaction", + normalizedSessionId: "normalized-redaction", + workingDirectory: WORKSPACE, + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Request input." }, + }); + const pending = transport.invoke({ + id: "redacted-request", + method: "item/tool/requestUserInput", + params: { + threadId: "thread-1", + turnId, + itemId: "redacted-item", + reason: fixture.redactionMarkers.join(" "), + authorization: "Bearer browser-secret", + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const serialized = JSON.stringify(session.pendingRuntimeRequests?.()); + for (const marker of fixture.redactionMarkers) + expect(serialized).not.toContain(marker); + expect(serialized).toContain("[REDACTED]"); + await session.resolveRuntimeRequest?.({ + requestId: "redacted-request", + turnId, + resolution: { action: "cancel" }, + }); + await pending; + await session.close({ reason: "fixture complete" }); + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.semantic-results.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.semantic-results.test.ts new file mode 100644 index 0000000000..2885a30675 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.semantic-results.test.ts @@ -0,0 +1,703 @@ +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CODEX_RESULT_OUTPUT_SCHEMA, + CodexAppServerDriver, + CodexRpcError, + FakeCodexTransport, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + WORKSPACE, + applyPrpEvent, + collectUntilTerminal, + createCodexTaskEnvelope, + createIsolatedCodexAppServerArgs, + createSessionSnapshotFromMetadata, + describe, + envelope, + expect, + isSkilllessCodexContext, + it, + liveConsoleFixturePath, + loadLiveConsoleConformanceFixture, + makeDriver, + replayPersistedCodexEvents, + result, + reverseObjectKeys, + runCodexCodexTracer, + traceCompletedProposal, + validateCodexResultProposal, + validatePrpEvent, + vi, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, + type HarnessRuntimeRequestResolution, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "./codex-app-server-driver.test-support.js"; + +describe("Codex app-server Codex driver", () => { + it("makes duplicate semantic completion idempotent and rejects changed payloads", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-result", + normalizedSessionId: "normalized-result", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + const request = { + id: 1, + method: "item/tool/call", + paperclipTrace: { + sourceEventId: "event_runner_000101", + sourceEventType: "semantic_tool.input", + }, + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-1", + tool: "paperclip_finish", + arguments: result, + }, + }; + expect(await transport.invoke(request)).toMatchObject({ success: true }); + const resultMapping = transport.traceInterpretations.find( + (entry) => + entry.sourceEventId === "event_runner_000101" && + entry.providerMethod === "item/tool/call", + ); + expect(resultMapping).toMatchObject({ + sourceEventType: "semantic_tool.input", + disposition: "mapped", + }); + expect(resultMapping?.emittedEventIds).toHaveLength(2); + expect(resultMapping?.emittedEventIds).toEqual( + expect.arrayContaining([ + expect.stringContaining(":run-result:"), + ]), + ); + expect( + await transport.invoke({ + ...request, + id: 2, + params: { + ...request.params, + callId: "call-2", + arguments: reverseObjectKeys(result), + }, + }), + ).toMatchObject({ success: true }); + expect((await session.snapshot()).semanticResult?.callId).toBe("call-1"); + const changed = structuredClone(result); + changed.summary = "Changed after commit."; + expect( + await transport.invoke({ + ...request, + id: 3, + params: { ...request.params, arguments: changed }, + }), + ).toMatchObject({ success: false }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + }); + + it("accepts a normalized runnerd echo of a tool-committed semantic result", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-result-normalized-echo", + normalizedSessionId: "normalized-result-normalized-echo", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + const toolShaped = structuredClone(result) as unknown as Record; + toolShaped.verification = [{ + commandOrCheck: "read hello.txt", + status: "passed", + result: "hello", + }]; + delete toolShaped.attentionRequests; + expect( + await transport.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: toolShaped, + }, + }), + ).toMatchObject({ success: true }); + transport.push("paperclip/runResult", { + threadId: "thread-1", + turnId: "turn-1", + itemId: "tool-result", + result: { + ...result, + verification: [{ + commandOrCheck: "read hello.txt", + status: "passed", + detail: "hello", + }], + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect(events.some((event) => event.eventType === "session.failed")).toBe( + false, + ); + }); + + it("advertises and dispatches run-authorized control-plane tools", async () => { + const transport = new FakeCodexTransport(); + const handler = vi.fn(async (call) => ({ + task: { id: "issue-1" }, + callId: call.callId, + })); + const session = await makeDriver([transport], { + dynamicTools: [ + { + name: "get_task_context", + description: "Read the assigned task.", + inputSchema: { type: "object", additionalProperties: false }, + }, + ], + dynamicToolHandler: handler, + }).openSession({ + runId: "run-tools", + normalizedSessionId: "normalized-tools", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ + message: { role: "user", text: "Inspect the task." }, + }); + + expect( + transport.calls.find((call) => call.method === "thread/start")?.params + .dynamicTools, + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: "get_task_context" }), + ]), + ); + const response = await transport.invoke({ + id: "rpc-tool", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-context", + tool: "get_task_context", + arguments: {}, + }, + }); + expect(response).toMatchObject({ success: true }); + expect( + JSON.parse( + String( + (response.contentItems as Array>)[0]?.text, + ), + ), + ).toEqual({ task: { id: "issue-1" }, callId: "call-context" }); + expect(handler).toHaveBeenCalledWith( + expect.objectContaining({ + tool: "get_task_context", + callId: "call-context", + arguments: {}, + }), + ); + }); + + it("fails closed when an agent message changes a tool-committed result", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-tool-then-message-conflict", + normalizedSessionId: "normalized-tool-then-message-conflict", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await transport.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + const changed = structuredClone(result); + changed.summary = "Conflicting agent-message result."; + transport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { + id: "message-result", + type: "agentMessage", + text: JSON.stringify(changed), + }, + }); + + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.find((event) => event.eventType === "session.failed")?.payload, + ).toMatchObject({ + code: "conflicting_semantic_result", + recoverable: false, + }); + expect((await session.snapshot()).semanticResult?.result).toEqual(result); + }); + + it("rejects a tool result that changes an agent-message commitment", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-message-then-tool-conflict", + normalizedSessionId: "normalized-message-then-tool-conflict", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { + id: "message-result", + type: "agentMessage", + text: JSON.stringify(result), + }, + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + const changed = structuredClone(result); + changed.summary = "Conflicting tool result."; + expect( + await transport.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: changed, + }, + }), + ).toMatchObject({ success: false }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect(events.some((event) => event.eventType === "session.failed")).toBe( + false, + ); + expect((await session.snapshot()).semanticResult).toMatchObject({ + callId: "message-result", + result, + }); + }); + + it("treats a canonically identical cross-channel result as a no-op", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-identical-cross-channel", + normalizedSessionId: "normalized-identical-cross-channel", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await transport.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + transport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { + id: "message-result", + type: "agentMessage", + text: JSON.stringify(reverseObjectKeys(result)), + }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect((await session.snapshot()).semanticResult?.callId).toBe( + "tool-result", + ); + }); + + it("fails closed when a live terminal embeds a changed semantic result", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-terminal-result-conflict", + normalizedSessionId: "normalized-terminal-result-conflict", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await transport.invoke({ + id: "tool-result", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "tool-result", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: true }); + const changed = structuredClone(result); + changed.summary = "Conflicting terminal result."; + transport.push("turn/completed", { + threadId: "thread-1", + turn: { + id: "turn-1", + status: "completed", + items: [ + { + id: "terminal-result", + type: "agentMessage", + text: JSON.stringify(changed), + }, + ], + }, + }); + + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + expect( + events.find((event) => event.eventType === "session.failed")?.payload, + ).toMatchObject({ + code: "conflicting_semantic_result", + recoverable: false, + }); + expect(events.some((event) => event.eventType === "turn.completed")).toBe( + false, + ); + expect((await session.snapshot()).semanticResult?.result).toEqual(result); + }); + + it.each([ + { threadId: "other-thread", turnId: "turn-1", label: "another thread" }, + { threadId: "thread-1", turnId: "other-turn", label: "another turn" }, + ])( + "rejects a semantic result for $label without committing it", + async ({ threadId, turnId }) => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-hostile-tool", + normalizedSessionId: "normalized-hostile-tool", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await transport.invoke({ + id: "hostile-call", + method: "item/tool/call", + params: { + threadId, + turnId, + callId: "hostile-call", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: false }); + const events = await collectUntilTerminal(session.events()); + expect( + events.some((event) => event.eventType === "run.result.proposed"), + ).toBe(false); + expect( + events.find((event) => event.eventType === "session.failed")?.payload, + ).toMatchObject({ code: "tool_binding_mismatch", recoverable: false }); + expect( + events.find((event) => event.eventType === "turn.failed")?.turnId, + ).toBe("turn-1"); + }, + ); + + it("rejects pre-turn, cross-thread, and post-terminal notifications", async () => { + const preTurnTransport = new FakeCodexTransport(); + const preTurn = await makeDriver([preTurnTransport]).openSession({ + runId: "run-pre-turn", + normalizedSessionId: "normalized-pre-turn", + workingDirectory: WORKSPACE, + }); + preTurnTransport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { + id: "answer-pre", + type: "agentMessage", + text: JSON.stringify(result), + }, + }); + const preTurnEvents: PrpEvent[] = []; + for await (const event of preTurn.events()) preTurnEvents.push(event); + expect( + preTurnEvents.some((event) => event.eventType === "run.result.proposed"), + ).toBe(false); + expect( + preTurnEvents.find((event) => event.eventType === "session.failed") + ?.payload, + ).toMatchObject({ code: "turn_binding_mismatch" }); + + const crossThreadTransport = new FakeCodexTransport(); + const crossThread = await makeDriver([crossThreadTransport]).openSession({ + runId: "run-cross-thread", + normalizedSessionId: "normalized-cross-thread", + workingDirectory: WORKSPACE, + }); + await crossThread.startTurn({ + message: { role: "user", text: "Complete." }, + }); + crossThreadTransport.push("item/completed", { + threadId: "other-thread", + turnId: "turn-1", + item: { + id: "answer-other", + type: "agentMessage", + text: JSON.stringify(result), + }, + }); + const crossThreadEvents = await collectUntilTerminal(crossThread.events()); + expect( + crossThreadEvents.some( + (event) => event.eventType === "run.result.proposed", + ), + ).toBe(false); + expect( + crossThreadEvents.find((event) => event.eventType === "session.failed") + ?.payload, + ).toMatchObject({ code: "thread_binding_mismatch" }); + + const postTerminalTransport = new FakeCodexTransport(); + const postTerminal = await makeDriver([postTerminalTransport]).openSession({ + runId: "run-post-terminal", + normalizedSessionId: "normalized-post-terminal", + workingDirectory: WORKSPACE, + }); + await postTerminal.startTurn({ + message: { role: "user", text: "Complete." }, + }); + postTerminalTransport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + await collectUntilTerminal(postTerminal.events()); + expect( + await postTerminalTransport.invoke({ + id: "late-call", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "late-call", + tool: "paperclip_finish", + arguments: result, + }, + }), + ).toMatchObject({ success: false }); + }); + + it("rejects duplicate and conflicting terminal facts", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-terminal-replay", + normalizedSessionId: "normalized-terminal-replay", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { + id: "turn-1", + status: "failed", + error: { message: "changed" }, + items: [], + }, + }); + const events: PrpEvent[] = []; + for await (const event of session.events()) events.push(event); + expect( + events.filter((event) => event.eventType === "turn.completed"), + ).toHaveLength(1); + expect( + events.filter((event) => event.eventType === "turn.failed"), + ).toHaveLength(0); + expect( + events.find((event) => event.eventType === "session.failed")?.payload, + ).toMatchObject({ code: "conflicting_turn_terminal", recoverable: false }); + }); + + it("rejects oversized semantic results without retaining provider payloads", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-large-result", + normalizedSessionId: "normalized-large-result", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ message: { role: "user", text: "Complete." } }); + expect( + await transport.invoke({ + id: "large-call", + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "large-call", + tool: "paperclip_finish", + arguments: { ...result, summary: "x".repeat(70 * 1024) }, + }, + }), + ).toEqual({ + success: false, + contentItems: [ + { + type: "inputText", + text: "Semantic result exceeded the retained payload limit.", + }, + ], + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + const events = await collectUntilTerminal(session.events()); + expect( + events.some((event) => JSON.stringify(event).includes("x".repeat(1024))), + ).toBe(false); + expect( + events.some((event) => event.eventType === "run.result.proposed"), + ).toBe(false); + }); + + it("normalizes finish and block tools into one canonical result contract", async () => { + const transport = new FakeCodexTransport(); + const session = await makeDriver([transport]).openSession({ + runId: "run-tools", + normalizedSessionId: "normalized-tools", + workingDirectory: WORKSPACE, + }); + await session.startTurn({ + message: { role: "user", text: "Complete or block." }, + }); + const blocked: PrpStructuredRunResult = { + ...structuredClone(result), + reportedWorkDisposition: "blocked", + summary: "Waiting on a fixture owner.", + completionClaim: { + ...structuredClone(result.completionClaim), + objectiveSatisfied: false, + criteria: [ + { criterionId: "file", status: "not_satisfied", evidenceRefs: [] }, + ], + remainingWork: [ + { + description: "Fixture owner must provide input.", + blocksCompletion: true, + }, + ], + }, + blocker: { + reasonCode: "fixture_input_missing", + owner: { kind: "external", name: "fixture owner" }, + unblockAction: "Provide the fixture input.", + scope: "task_wide", + }, + artifacts: [], + }; + const request = { + id: 10, + method: "item/tool/call", + params: { + threadId: "thread-1", + turnId: "turn-1", + callId: "call-block", + tool: "paperclip_block", + arguments: blocked, + }, + }; + expect( + await transport.invoke({ + ...request, + id: 9, + params: { ...request.params, tool: "paperclip_finish" }, + }), + ).toMatchObject({ success: false }); + expect(await transport.invoke(request)).toMatchObject({ success: true }); + expect(await transport.invoke({ ...request, id: 11 })).toMatchObject({ + success: true, + }); + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + const events = await collectUntilTerminal(session.events()); + expect( + events.filter((event) => event.eventType === "run.result.proposed"), + ).toHaveLength(1); + expect( + events.find((event) => event.eventType === "run.result.proposed") + ?.payload, + ).toMatchObject({ reportedWorkDisposition: "blocked" }); + }); + +}); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts new file mode 100644 index 0000000000..a916308719 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.test-support.ts @@ -0,0 +1,427 @@ +import { realpathSync } from "node:fs"; +import { fileURLToPath } from "node:url"; + +import { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_RESULT_OUTPUT_SCHEMA, + createCodexTaskEnvelope, + isSkilllessCodexContext, +} from "../../contracts/codex.js"; +import { + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + type HarnessRuntimeRequestResolution, +} from "../../contracts/harness-driver.js"; +import { + applyPrpEvent, + createSessionSnapshotFromMetadata, +} from "../../reducer/session-reducer.js"; +import { + validatePrpEvent, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, +} from "../../protocol/replay-contract.js"; +import { loadLiveConsoleConformanceFixture } from "../../protocol/live-console-fixture.js"; +import { + CodexAppServerDriver, + createIsolatedCodexAppServerArgs, +} from "./codex-app-server-driver.js"; +import { + runCodexCodexTracer, + replayPersistedCodexEvents, + validateCodexResultProposal, +} from "../../mock-core/codex-runner.js"; +import { + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CodexRpcError, + type CodexAppServerTransport, + type CodexRpcNotification, + type CodexRpcServerRequest, + type CodexServerRequestHandler, + type CodexTraceInterpretation, +} from "./app-server-transport.js"; + +export const WORKSPACE = realpathSync.native(process.cwd()); + +export class TestQueue implements AsyncIterable { + values: T[] = []; + waiters: Array<{ + resolve: (value: IteratorResult) => void; + reject: (error: Error) => void; + }> = []; + closed = false; + error: Error | null = null; + + push(value: T): void { + const waiter = this.waiters.shift(); + if (waiter) waiter.resolve({ value, done: false }); + else this.values.push(value); + } + + close(): void { + this.closed = true; + for (const waiter of this.waiters.splice(0)) + waiter.resolve({ value: undefined, done: true }); + } + + fail(error: Error): void { + this.error = error; + for (const waiter of this.waiters.splice(0)) waiter.reject(error); + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: async () => { + const value = this.values.shift(); + if (value) return { value, done: false }; + if (this.error) throw this.error; + if (this.closed) return { value: undefined, done: true }; + return new Promise((resolve, reject) => this.waiters.push({ resolve, reject })); + }, + }; + } +} + +export class FakeCodexTransport implements CodexAppServerTransport { + readonly calls: Array<{ method: string; params: Record }> = + []; + readonly sentNotifications: Array<{ + method: string; + params?: Record; + }> = []; + readonly traceInterpretations: CodexTraceInterpretation[] = []; + readonly queue = new TestQueue(); + handler: CodexServerRequestHandler = async () => ({}); + rejectMethods = new Map(); + readResponse: Record | null = null; + turnStartResponse: Promise> | null = null; + goalState: Record | null = null; + confirmCollaborationMode = true; + runtimeRequestResolver: ((input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }) => Promise) | null = null; + + constructor( + readonly threadId = "thread-1", + readonly providerSessionId = "provider-session-1", + ) {} + + async request( + method: string, + params: Record, + ): Promise> { + this.calls.push({ method, params: structuredClone(params) }); + const rejection = this.rejectMethods.get(method); + if (rejection) throw rejection; + if (method === "initialize") { + return { + userAgent: "codex-cli/0.132.0", + codexHome: "/isolated/codex", + platformFamily: "unix", + platformOs: "linux", + }; + } + if (method === "collaborationMode/list") { + return this.confirmCollaborationMode + ? { + data: [ + { + name: "Plan", + mode: "plan", + model: "gpt-test", + reasoning_effort: "high", + }, + ], + } + : { data: [{ name: "Default", mode: "default", model: "gpt-test" }] }; + } + if (method === "thread/start" || method === "thread/resume") { + const planMode = + params.permissions === "paperclip-runner-workspace-read-only"; + return { + thread: { + id: this.threadId, + sessionId: this.providerSessionId, + modelProvider: "openai", + cwd: WORKSPACE, + turns: [], + activePermissionProfile: { + id: planMode + ? "paperclip-runner-workspace-read-only" + : "paperclip-runner-workspace-only", + }, + }, + model: "gpt-test", + modelProvider: "openai", + cwd: WORKSPACE, + sandbox: { type: "workspaceWrite" }, + approvalPolicy: params.approvalPolicy, + instructionSources: [], + }; + } + if (method === "turn/start") { + return ( + this.turnStartResponse ?? { + turn: { id: "turn-1", status: "inProgress", items: [] }, + } + ); + } + if (method === "thread/goal/get") return { goal: this.goalState }; + if (method === "thread/goal/set") { + this.goalState = { + threadId: this.threadId, + objective: + typeof params.objective === "string" + ? params.objective + : String( + this.goalState?.objective ?? "Ship the Live console tracer", + ), + status: params.status ?? this.goalState?.status ?? "active", + tokenBudget: params.tokenBudget ?? this.goalState?.tokenBudget ?? null, + tokensUsed: 0, + timeUsedSeconds: 0, + createdAt: 1, + updatedAt: 2, + }; + return { goal: this.goalState }; + } + if (method === "thread/goal/clear") { + this.goalState = null; + return {}; + } + if (method === "thread/read") { + return ( + this.readResponse ?? { + thread: { + id: this.threadId, + sessionId: this.providerSessionId, + cwd: WORKSPACE, + turns: [{ id: "turn-1", status: "inProgress", items: [] }], + }, + } + ); + } + return {}; + } + + notify(method: string, params?: Record): void { + this.sentNotifications.push( + params === undefined ? { method } : { method, params }, + ); + } + + notifications(): AsyncIterable { + return this.queue; + } + + setServerRequestHandler(handler: CodexServerRequestHandler): void { + this.handler = handler; + } + + async resolveRuntimeRequest(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise { + await this.runtimeRequestResolver?.(input); + } + + recordTraceInterpretation(input: CodexTraceInterpretation): void { + this.traceInterpretations.push(structuredClone(input)); + } + + async close(): Promise { + this.queue.close(); + } + + push(method: string, params: Record): void { + this.queue.push({ method, params }); + } + + pushTraced( + method: string, + params: Record, + sourceEventId: string, + sourceEventType: string, + ): void { + this.queue.push({ + method, + params, + paperclipTrace: { sourceEventId, sourceEventType }, + }); + } + + invoke(request: CodexRpcServerRequest): Promise> { + return this.handler(request); + } +} + +export const envelope = createCodexTaskEnvelope({ + objective: "Create hello.txt with the text hello.", + criteria: [{ id: "file", requirement: "hello.txt contains hello" }], +}); + +export const liveConsoleFixturePath = fileURLToPath( + new URL( + "../../../protocol/fixtures/codex-driver/driver-conformance.json", + import.meta.url, + ), +); + +export const result: PrpStructuredRunResult = { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Created hello.txt.", + completionClaim: { + contractRevision: "codex-demo-v1", + objectiveSatisfied: true, + criteria: [ + { criterionId: "file", status: "satisfied", evidenceRefs: ["hello.txt"] }, + ], + remainingWork: [], + }, + evidence: [{ ref: "hello.txt" }], + verification: [{ commandOrCheck: "read hello.txt", status: "passed" }], + attentionRequests: [], + artifacts: [{ kind: "file", ref: "hello.txt" }], +}; + +export function makeDriver( + transports: FakeCodexTransport[], + options: Record = {}, +) { + let index = 0; + return new CodexAppServerDriver({ + taskEnvelope: envelope, + environment: { + PATH: "/bin", + HOME: "/isolated/home", + CODEX_HOME: "/isolated/codex", + LANG: "C.UTF-8", + PAPERCLIP_API_KEY: "must-not-pass", + RANDOM_SKILL_PATH: "/skills/unrelated", + }, + now: () => new Date("2026-08-08T12:00:00.000Z"), + transportFactory: () => transports[index++]!, + ...options, + }); +} + +export async function collectUntilTerminal( + events: AsyncIterable, +): Promise { + const collected: PrpEvent[] = []; + for await (const event of events) { + collected.push(event); + if ( + [ + "turn.completed", + "turn.failed", + "turn.interrupted", + "turn.cancelled", + ].includes(event.eventType) + ) + break; + } + return collected; +} + +export function reverseObjectKeys(value: unknown): unknown { + if (Array.isArray(value)) return value.map(reverseObjectKeys); + if (typeof value !== "object" || value === null) return value; + return Object.fromEntries( + Object.entries(value as Record) + .reverse() + .map(([key, entry]) => [key, reverseObjectKeys(entry)]), + ); +} + +export async function traceCompletedProposal( + proposal: PrpStructuredRunResult | null, + options: { + runId?: string; + normalizedSessionId?: string; + capabilities?: Record; + steer?: string; + interrupt?: boolean; + } = {}, +) { + const transport = new FakeCodexTransport(); + const driver = makeDriver([transport], { + capabilities: options.capabilities, + }); + const traced = runCodexCodexTracer({ + driver, + taskEnvelope: envelope, + workingDirectory: WORKSPACE, + runId: options.runId, + normalizedSessionId: options.normalizedSessionId, + steer: options.steer, + interrupt: options.interrupt, + }); + while (!transport.calls.some((call) => call.method === "turn/start")) { + await new Promise((resolve) => setTimeout(resolve, 0)); + } + transport.push("turn/started", { + threadId: "thread-1", + turn: { id: "turn-1", status: "inProgress" }, + }); + if (proposal !== null) { + transport.push("item/completed", { + threadId: "thread-1", + turnId: "turn-1", + item: { + id: "answer-1", + type: "agentMessage", + text: JSON.stringify(proposal), + }, + }); + } + transport.push("turn/completed", { + threadId: "thread-1", + turn: { id: "turn-1", status: "completed", items: [] }, + }); + return { trace: await traced, transport }; +} + + +export { describe, expect, it, vi } from "vitest"; +export { + CODEX_BLOCK_RESULT_OUTPUT_SCHEMA, + CODEX_RESULT_OUTPUT_SCHEMA, + createCodexTaskEnvelope, + isSkilllessCodexContext, + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + applyPrpEvent, + createSessionSnapshotFromMetadata, + validatePrpEvent, + loadLiveConsoleConformanceFixture, + CodexAppServerDriver, + createIsolatedCodexAppServerArgs, + runCodexCodexTracer, + replayPersistedCodexEvents, + validateCodexResultProposal, + CODEX_INVALID_REQUEST, + CODEX_METHOD_NOT_FOUND, + CodexRpcError, +}; +export type { + HarnessRuntimeRequestResolution, + PrpCapabilities, + PrpEvent, + PrpStructuredRunResult, + CodexAppServerTransport, + CodexRpcNotification, + CodexRpcServerRequest, + CodexServerRequestHandler, + CodexTraceInterpretation, +}; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts new file mode 100644 index 0000000000..3d0ec4a9c8 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.ts @@ -0,0 +1,9 @@ +export { parseCodexTurnDiff } from "./codex-turn-diff.js"; +export { + createIsolatedCodexAppServerArgs, + createSkilllessCodexThreadConfig, +} from "./codex-security-config.js"; +export { normalizeCodexQuestionSet } from "./codex-question-adapter.js"; +export { CodexAppServerDriver } from "./codex-app-server-driver-impl.js"; +export type { CodexAppServerDriverOptions } from "./codex-driver-types.js"; +export { codexSemanticToolSpecs } from "./codex-driver-values.js"; diff --git a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts index 49e95d7b85..d2ebec8dc8 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-boundaries.ts @@ -116,7 +116,7 @@ function pathContains(parent: string, candidate: string): boolean { ); } -function boundedCodexValue(value: unknown, depth = 0): unknown { +export function boundedCodexValue(value: unknown, depth = 0): unknown { if (depth > 8) return { truncated: true, reason: "maximum depth" }; if (typeof value === "string") { return value.length <= MAX_RETAINED_CODEX_STRING_CHARS diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts new file mode 100644 index 0000000000..f19b1ef4f0 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-types.ts @@ -0,0 +1,97 @@ +import type { + HarnessRuntimeRequest, + HarnessRuntimeRequestResolution, + HarnessThreadLineageEntry, + PersistedHarnessSession, +} from "../../contracts/harness-driver.js"; +import type { + CodexModelContextSnapshot, + CodexTaskEnvelope, +} from "../../contracts/codex.js"; +import type { CodexAppServerTransport } from "./app-server-transport.js"; +import type { CodexQuestionResponseContext } from "./codex-question-adapter.js"; + +export interface CodexAppServerDriverOptions { + taskEnvelope: CodexTaskEnvelope; + /** Explicit provider model selected by the persisted native execution. */ + model?: string; + approvalPolicy?: "never" | "on-request" | "untrusted"; + baseInstructions?: string; + includeSkillInstructions?: boolean; + conversationMode?: "task" | "direct"; + requestedCollaborationMode?: "default" | "plan"; + /** + * Include Codex's built-in collaboration instructions. Defaults to true so + * interactive runs receive native commentary/preambles. Deterministic evals + * may opt out explicitly without changing the production default. + */ + includeCollaborationModeInstructions?: boolean; + transportFactory?: (context?: { + providerRecoveryPolicy?: PersistedHarnessSession["providerRecoveryPolicy"]; + }) => CodexAppServerTransport; + /** Additional control-plane tools exposed to the provider for this run. */ + dynamicTools?: readonly Readonly>[]; + /** Executes an admitted additional tool call. Completion tools remain driver-owned. */ + dynamicToolHandler?: (call: { + tool: string; + callId: string; + threadId: string; + turnId: string; + arguments: unknown; + }) => Promise; + environment?: NodeJS.ProcessEnv; + now?: () => Date; + runnerInstanceId?: string; + onDiagnostic?: (message: string) => void; + onSpawn?: (meta: { + pid: number; + processGroupId: number | null; + startedAt: string; + }) => Promise; + capabilities?: Partial<{ + resume: boolean; + read: boolean; + steering: boolean; + interruption: boolean; + usage: boolean; + reconciliation: boolean; + dynamicTools: boolean; + runtimeRequestResolution: boolean; + goals: boolean; + threadLineage: boolean; + }>; + /** Provider-specific identity retained when the Codex protocol facade is backed by runnerd. */ + driverIdentity?: { + kind: string; + displayName: string; + version: string; + }; + collaborationModes?: readonly ("default" | "plan")[]; + requireProviderSessionIdentity?: boolean; +} + +export type CodexCapabilities = Required< + NonNullable +>; + +export type SemanticResultAdmission = "committed" | "identical" | "conflict"; + +export interface TerminalReplayConflict { + code: "conflicting_semantic_result" | "conflicting_turn_terminal"; + message: string; +} + +export interface OpenedCodexThread { + threadId: string; + providerSessionId: string | null; + collaborationMode: Record | null; + context: CodexModelContextSnapshot; + lineage: HarnessThreadLineageEntry; +} + +export interface PendingRuntimeRequest { + request: HarnessRuntimeRequest; + responseContext: CodexQuestionResponseContext; + settle: (response: Record) => void; + settlingResolution?: HarnessRuntimeRequestResolution; +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts new file mode 100644 index 0000000000..1aa59070ff --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-driver-values.ts @@ -0,0 +1,170 @@ +import type { NativeUserMessage } from "../../contracts/types.js"; +import { + CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + CODEX_BLOCK_TOOL_NAME, + CODEX_COMPLETION_TOOL_NAME, + CODEX_RESULT_PROVIDER_INPUT_SCHEMA, +} from "../../contracts/codex.js"; +import { + validatePrpStructuredRunResult, + type PrpStructuredRunResult, +} from "../../protocol/replay-contract.js"; +import { boundedCodexValue, isRetainableCodexPayload } from "./codex-boundaries.js"; + +export function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : {}; +} + +export function text(value: unknown, fallback = ""): string { + return typeof value === "string" ? value : fallback; +} + +export function boundedText( + value: unknown, + fallback = "unknown", + maxCharacters = 1024, +): string { + const candidate = text(value, fallback); + return candidate.length <= maxCharacters + ? candidate + : `${candidate.slice(0, maxCharacters)}...[truncated]`; +} + +export function itemFromParams( + params: Record, +): Record { + return record(params.item); +} + +export function itemText(item: Record): string { + for (const key of ["text", "aggregatedOutput", "patch", "delta"]) { + if (typeof item[key] === "string") return item[key]; + } + return ""; +} + +export function userInput(message: NativeUserMessage): Record { + return { type: "text", text: message.text, text_elements: [] }; +} + +export function terminalState( + status: string, +): "completed" | "failed" | "interrupted" | "cancelled" { + if (status === "failed") return "failed"; + if (status === "interrupted") return "interrupted"; + if (status === "cancelled") return "cancelled"; + return "completed"; +} + +export function tryParseResult(value: unknown): PrpStructuredRunResult | null { + let candidate = value; + if (typeof value === "string") { + try { + candidate = JSON.parse(value); + } catch { + return null; + } + } + const validation = validatePrpStructuredRunResult(candidate); + return validation.ok ? validation.result : null; +} + +export function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const object = record(value); + if ( + Object.keys(object).length > 0 || + (typeof value === "object" && value !== null) + ) { + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} + +export function differingJsonPaths( + left: unknown, + right: unknown, + prefix = "", + limit = 12, +): string[] { + if (canonicalJson(left) === canonicalJson(right)) return []; + if (limit <= 0) return []; + if (Array.isArray(left) && Array.isArray(right)) { + const paths: string[] = []; + for (let index = 0; index < Math.max(left.length, right.length); index += 1) { + paths.push(...differingJsonPaths( + left[index], + right[index], + `${prefix}[${index}]`, + limit - paths.length, + )); + if (paths.length >= limit) break; + } + return paths.length > 0 ? paths : [prefix || "result"]; + } + const leftRecord = record(left); + const rightRecord = record(right); + if ( + (typeof left === "object" && left !== null) && + (typeof right === "object" && right !== null) && + !Array.isArray(left) && + !Array.isArray(right) + ) { + const paths: string[] = []; + const keys = [...new Set([ + ...Object.keys(leftRecord), + ...Object.keys(rightRecord), + ])].sort(); + for (const key of keys) { + paths.push(...differingJsonPaths( + leftRecord[key], + rightRecord[key], + prefix ? `${prefix}.${key}` : key, + limit - paths.length, + )); + if (paths.length >= limit) break; + } + return paths.length > 0 ? paths : [prefix || "result"]; + } + return [prefix || "result"]; +} + +function finishToolSpec(): Record { + return { + name: CODEX_COMPLETION_TOOL_NAME, + description: "Return the one semantic completion result for this task.", + inputSchema: CODEX_RESULT_PROVIDER_INPUT_SCHEMA, + }; +} + +function blockToolSpec(): Record { + return { + name: CODEX_BLOCK_TOOL_NAME, + description: + "Return the one semantic result when the task cannot continue.", + inputSchema: CODEX_BLOCK_RESULT_PROVIDER_INPUT_SCHEMA, + }; +} + +export function codexSemanticToolSpecs(): readonly Readonly< + Record +>[] { + return [finishToolSpec(), blockToolSpec()]; +} + +export function dynamicToolResponse(value: unknown): Record { + return { + success: true, + contentItems: [ + { + type: "inputText", + text: typeof value === "string" ? value : JSON.stringify(value), + }, + ], + }; +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts new file mode 100644 index 0000000000..cebdd76bd0 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-harness-session.ts @@ -0,0 +1,656 @@ +import { randomUUID } from "node:crypto"; + +import type { + HarnessGoalOperation, + HarnessRuntimeRequest, + HarnessRuntimeRequestHandoff, + HarnessRuntimeRequestResolution, + HarnessSession, + HarnessThreadGoal, + HarnessThreadLineageEntry, + PersistedHarnessSession, +} from "../../contracts/harness-driver.js"; +import { + HarnessCapabilityUnavailableError, + HarnessOperationAlreadyTerminalError, + HarnessReconciliationError, + HarnessStaleTurnError, + harnessRuntimeInputExpiredOutcome, + harnessRuntimeRequestOutcome, + parseHarnessRuntimeRequestResolution, +} from "../../contracts/harness-driver.js"; +import type { NativeUserMessage } from "../../contracts/types.js"; +import { + CODEX_RESULT_OUTPUT_SCHEMA, + type CodexModelContextSnapshot, +} from "../../contracts/codex.js"; +import type { PrpEvent } from "../../protocol/replay-contract.js"; +import { boundedCodexPayload as boundedPayload } from "./codex-boundaries.js"; +import { CodexRpcError, redactCodexDiagnostic } from "./app-server-transport.js"; +import { runtimeRequestResponse } from "./codex-question-adapter.js"; +import { + CODEX_PLANNING_PERMISSION_PROFILE as PLANNING_PERMISSION_PROFILE, + CODEX_SKILLLESS_PERMISSION_PROFILE as SKILLLESS_PERMISSION_PROFILE, +} from "./codex-security-config.js"; +import { + parseCodexThreadGoal as parseThreadGoal, + safeCodexRequestResponse as safeRequestResponse, +} from "./codex-thread-normalization.js"; +import { + CodexSessionState, + initializeCodexSessionEvents, + type CodexSessionStateInput, +} from "./codex-session-state.js"; +import { pumpNotifications } from "./codex-session-notifications.js"; +import { handleServerRequest } from "./codex-session-server-requests.js"; +import { mapTerminalTurn, terminalReplayConflict } from "./codex-session-terminal.js"; +import { boundedText, record, text, userInput } from "./codex-driver-values.js"; + +export class CodexHarnessSession extends CodexSessionState implements HarnessSession { + constructor(input: CodexSessionStateInput) { + super(input); + this.transport.setServerRequestHandler((request) => + handleServerRequest(this, request), + ); + initializeCodexSessionEvents(this, input); + if (this.terminal) { + this.eventQueue.close(); + } else { + void pumpNotifications(this); + } + } + + ids(): ReturnType { + return { + driverSessionId: this.opened.threadId, + providerSessionId: this.opened.providerSessionId, + displayId: this.opened.threadId, + }; + } + + async attachRun(input: { runId: string }): Promise { + if ( + this.activeTurnId !== null || + this.turnStartPending || + this.pendingRuntimeRequestMap.size > 0 + ) { + throw new Error("codex_run_attach_busy"); + } + if (!input.runId) throw new Error("codex_run_attach_invalid"); + await this.transport.attachRun?.({ + runId: input.runId, + turnId: `turn_attachment_${randomUUID().replaceAll("-", "")}`, + itemId: `item_attachment_${randomUUID().replaceAll("-", "")}`, + }); + this.runId = input.runId; + this.result = null; + this.resultFingerprint = null; + this.resultCallId = null; + this.resultTurnId = null; + this.dispositionOnlyRecoveryConsumed = false; + this.dispositionOnlyRecoveryTurnId = null; + this.terminal = false; + this.terminalTurns.clear(); + this.turnStarted = false; + this.protocolFailed = false; + this.protocolFailureCode = null; + this.protocolFailureMessage = null; + this.emit("run.attached", { runId: input.runId, sameSession: true }); + } + + contextSnapshot(): CodexModelContextSnapshot { + return structuredClone(this.opened.context); + } + + events(): AsyncIterable { + return this.eventQueue; + } + + async startTurn(input: { + message: NativeUserMessage; + requestedCollaborationMode?: "default" | "plan"; + }): Promise<{ + turnId: string; + effectiveCollaborationMode: "default" | "plan"; + }> { + if ( + this.terminal || + this.protocolFailed || + this.activeTurnId !== null || + this.turnStartPending + ) { + throw this.unsupported( + "turn start", + this.protocolFailureCode === null + ? "session cannot start another turn" + : `session failed protocol validation: ${this.protocolFailureCode} (${this.protocolFailureMessage ?? "no detail"})`, + ); + } + const dispositionOnlyRecovery = this.dispositionOnlyRecoveryAvailable; + const taskText = + this.conversationMode === "direct" + ? input.message.text + : dispositionOnlyRecovery + ? input.message.text + : JSON.stringify({ + task: this.taskEnvelope, + message: input.message.text, + }); + const effectiveCollaborationMode = this.opened.context.collaborationMode; + if ( + input.requestedCollaborationMode && + input.requestedCollaborationMode !== effectiveCollaborationMode + ) { + throw new Error( + `collaboration_mode_mismatch: requested ${input.requestedCollaborationMode}, effective ${effectiveCollaborationMode}`, + ); + } + if (dispositionOnlyRecovery) { + // Prevent a second submission in this process. The resulting accepted + // turn id is checkpointed by orchestration; if the process dies before + // that checkpoint, recoverSession adopts the provider-side turn. + this.dispositionOnlyRecoveryAvailable = false; + this.dispositionOnlyRecoveryConsumed = true; + this.dispositionOnlyRecoveryTurnId = null; + } + // The submitted text is part of the canonical record so a tracer can show + // the operator's own message without keeping shadow state next to the + // reducer. + this.emit("turn.submitted", { + envelopeSchema: this.taskEnvelope.schema, + text: input.message.text, + requestedCollaborationMode: + input.requestedCollaborationMode ?? effectiveCollaborationMode, + effectiveCollaborationMode, + }); + this.turnStartPending = true; + let response: Record; + const requestedMode = this.opened.context.collaborationMode; + try { + response = await this.transport.request("turn/start", { + threadId: this.opened.threadId, + cwd: this.opened.context.workingDirectory, + permissions: + requestedMode === "plan" + ? PLANNING_PERMISSION_PROFILE + : SKILLLESS_PERMISSION_PROFILE, + runtimeWorkspaceRoots: [this.opened.context.workingDirectory], + ...(this.opened.collaborationMode === null + ? {} + : { collaborationMode: this.opened.collaborationMode }), + input: [userInput({ role: "user", text: taskText })], + ...(this.conversationMode === "direct" + ? {} + : { outputSchema: CODEX_RESULT_OUTPUT_SCHEMA }), + }); + } catch (error) { + if (dispositionOnlyRecovery) { + if (error instanceof CodexRpcError) { + // A JSON-RPC error is a definite provider rejection: no turn was + // accepted, so the one-shot recovery allowance remains available. + this.dispositionOnlyRecoveryAvailable = true; + this.dispositionOnlyRecoveryConsumed = false; + this.dispositionOnlyRecoveryTurnId = null; + } else { + // A transport failure is ambiguous. Recovery must inspect the + // provider thread before deciding whether this submission landed. + this.terminal = true; + } + } + throw error; + } finally { + this.turnStartPending = false; + } + const turn = record(response.turn); + const turnId = text(turn.id); + if (turnId.length === 0) + throw new Error("Codex turn response omitted turn.id"); + if (this.activeTurnId !== null && this.activeTurnId !== turnId) { + this.failProtocol( + "turn_start_mismatch", + "turn/start response disagreed with turn/started", + ); + throw new Error("Codex turn identity changed during start"); + } + this.activeTurnId ??= turnId; + if (dispositionOnlyRecovery) { + this.dispositionOnlyRecoveryTurnId = turnId; + } + this.emit("turn.accepted", { turnId }, { turnId }); + if (this.interruptQueued) { + this.interruptQueued = false; + await this.#sendInterrupt(turnId, "queued_before_start"); + } + return { + turnId, + effectiveCollaborationMode, + }; + } + + async steer(input: { + turnId: string; + message: NativeUserMessage; + correlationId?: string; + }): Promise { + this.requireCapability("steering"); + this.requireActiveTurn(input.turnId, "steering"); + if (input.correlationId) { + const acknowledgedTurnId = this.acknowledgedSteeringCorrelations.get( + input.correlationId, + ); + if (acknowledgedTurnId) { + if (acknowledgedTurnId !== input.turnId) + throw new HarnessOperationAlreadyTerminalError("steering"); + return; + } + } + try { + await this.transport.request("turn/steer", { + threadId: this.opened.threadId, + input: [userInput(input.message)], + expectedTurnId: input.turnId, + correlationId: input.correlationId, + }); + if (this.activeTurnId !== input.turnId) { + throw new HarnessOperationAlreadyTerminalError("steering"); + } + if (input.correlationId) { + this.acknowledgedSteeringCorrelations.set( + input.correlationId, + input.turnId, + ); + } + this.emit( + "item.completed", + { + kind: "steering_acknowledgement", + text: "Steering acknowledged for the active turn.", + status: "acknowledged", + }, + { + turnId: input.turnId, + itemId: input.correlationId + ? `${input.turnId}:steer:${input.correlationId}` + : `${input.turnId}:steer:${++this.steerSequence}`, + }, + ); + } catch (error) { + if (error instanceof HarnessOperationAlreadyTerminalError) throw error; + const detail = redactCodexDiagnostic(String(error)); + if (/unsupported|unavailable|capability|method not found/i.test(detail)) { + throw this.unsupported("steering", detail); + } + // Steering rejection is a retryable operation failure, not evidence + // that the provider lacks the capability. Preserve that distinction for + // the control plane while keeping provider diagnostics redacted. + throw new Error(detail); + } + } + + async interrupt(input: { turnId?: string; reason?: string }): Promise { + this.requireCapability("interruption"); + if (this.turnStartPending && this.activeTurnId === null) { + this.interruptQueued = true; + this.emit( + "item.completed", + { + kind: "interrupt_acknowledgement", + text: "Interrupt queued until the provider assigns the turn identity.", + status: "queued", + }, + { itemId: `interrupt:queued:${++this.interruptSequence}` }, + ); + return; + } + if (this.terminal || this.activeTurnId === null) { + throw new HarnessOperationAlreadyTerminalError("interruption"); + } + const turnId = input.turnId ?? this.activeTurnId; + this.requireActiveTurn(turnId, "interruption"); + await this.#sendInterrupt(turnId, input.reason); + } + + async #sendInterrupt(turnId: string, reason?: string): Promise { + try { + await this.transport.request("turn/interrupt", { + threadId: this.opened.threadId, + turnId, + }); + if (this.activeTurnId !== turnId) { + throw new HarnessOperationAlreadyTerminalError("interruption"); + } + this.emit( + "item.completed", + { + kind: "interrupt_acknowledgement", + text: "Interrupt accepted for the active turn.", + status: "acknowledged", + reason: boundedText(reason), + }, + { + turnId, + itemId: `${turnId}:interrupt:${++this.interruptSequence}`, + }, + ); + } catch (error) { + if (error instanceof HarnessOperationAlreadyTerminalError) throw error; + throw this.unsupported("interruption", error); + } + } + + pendingRuntimeRequests(): HarnessRuntimeRequest[] { + return [...this.pendingRuntimeRequestMap.values()].map(({ request }) => + structuredClone(request), + ); + } + + async resolveRuntimeRequest(input: { + requestId: string; + turnId: string; + resolution: HarnessRuntimeRequestResolution; + }): Promise { + this.requireCapability("runtimeRequestResolution"); + const pending = this.pendingRuntimeRequestMap.get(input.requestId); + if (pending === undefined) { + throw new HarnessCapabilityUnavailableError( + "runtime request resolution", + `request ${input.requestId} is no longer pending`, + ); + } + if ( + pending.request.turnId !== input.turnId || + this.activeTurnId !== input.turnId + ) { + throw new HarnessStaleTurnError(input.turnId); + } + const resolution = parseHarnessRuntimeRequestResolution( + pending.request.requestKind, + input.resolution, + pending.request.input, + ); + if (pending.settlingResolution !== undefined) { + throw new HarnessCapabilityUnavailableError( + "runtime request resolution", + `request ${input.requestId} is already settling`, + ); + } + pending.settlingResolution = structuredClone(resolution); + const response = runtimeRequestResponse( + pending.request, + resolution, + pending.responseContext, + ); + try { + await this.transport.resolveRuntimeRequest?.({ + requestId: input.requestId, + turnId: input.turnId, + resolution, + }); + } catch (error) { + if (this.pendingRuntimeRequestMap.get(input.requestId) === pending) { + pending.settlingResolution = undefined; + } + throw error; + } + if (!this.pendingRuntimeRequestMap.delete(input.requestId)) return; + this.emit( + "runtime_request.resolved", + harnessRuntimeRequestOutcome(pending.request, { + action: resolution.action, + ...(resolution.action === "submit" && "response" in resolution + ? { response: resolution.response } + : {}), + }), + { turnId: input.turnId, itemId: pending.request.itemId }, + ); + pending.settle(response); + } + + handoffRuntimeRequest(input: { + requestId: string; + turnId: string; + reason: "durable_handoff"; + signal: AbortSignal; + }): HarnessRuntimeRequestHandoff { + if (input.signal.aborted) { + return { result: "already_settled", cleanup: Promise.resolve() }; + } + this.requireCapability("runtimeRequestResolution"); + const pending = this.pendingRuntimeRequestMap.get(input.requestId); + if ( + pending === undefined + || pending.request.input === undefined + || pending.request.turnId !== input.turnId + || this.activeTurnId !== input.turnId + || pending.settlingResolution !== undefined + ) return { result: "already_settled", cleanup: Promise.resolve() }; + if (!this.pendingRuntimeRequestMap.delete(input.requestId)) { + return { result: "already_settled", cleanup: Promise.resolve() }; + } + this.emit( + "runtime_request.expired", + harnessRuntimeInputExpiredOutcome(pending.request, input.reason), + { turnId: input.turnId, itemId: pending.request.itemId }, + ); + pending.settle(safeRequestResponse(pending.request.method, "cancel")); + const cleanup = Promise.allSettled([ + Promise.resolve().then(() => this.transport.resolveRuntimeRequest?.({ + requestId: input.requestId, + turnId: input.turnId, + resolution: { action: "cancel" }, + })), + Promise.resolve().then(() => this.transport.request("turn/interrupt", { + threadId: this.opened.threadId, + turnId: input.turnId, + })), + ]).then(() => undefined); + return { result: "handed_off", cleanup }; + } + + async goal(input: HarnessGoalOperation): Promise { + this.requireCapability("goals"); + let method: string; + let params: Record = { threadId: this.opened.threadId }; + if (input.action === "get") { + method = "thread/goal/get"; + } else if (input.action === "clear") { + method = "thread/goal/clear"; + } else { + method = "thread/goal/set"; + if (input.action === "set") { + params = { + ...params, + objective: input.objective, + status: "active", + ...(input.tokenBudget !== undefined + ? { tokenBudget: input.tokenBudget } + : {}), + }; + } else { + params = { + ...params, + status: input.action === "pause" ? "paused" : "active", + }; + } + } + try { + const response = await this.transport.request(method, params); + const goal = + input.action === "clear" ? null : parseThreadGoal(response.goal); + if (!["get", "clear"].includes(input.action) && goal === null) { + throw new Error(`${method} omitted its goal`); + } + this.currentGoal = goal; + const kind = input.action === "clear" ? "goal_cleared" : "goal"; + this.emit( + "item.completed", + { + kind, + text: + goal === null + ? input.action === "clear" + ? "Thread goal cleared." + : "No thread goal is configured." + : `Thread goal ${goal.status}: ${goal.objective}`, + action: input.action, + goal, + }, + { itemId: `${this.opened.threadId}:goal:${this.sourceSequence + 1}` }, + ); + return goal === null ? null : structuredClone(goal); + } catch (error) { + throw this.unsupported(`goal ${input.action}`, error); + } + } + + lineage(): HarnessThreadLineageEntry[] { + return [...this.lineageByThread.values()].map((entry) => structuredClone(entry)); + } + + async read(): Promise> { + this.requireCapability("read"); + try { + return await this.transport.request("thread/read", { + threadId: this.opened.threadId, + includeTurns: true, + }); + } catch (error) { + throw this.unsupported("read", error); + } + } + + async reconcile(): Promise> { + this.requireCapability("reconciliation"); + const snapshot = await this.read(); + const thread = record(snapshot.thread); + if (text(thread.id) !== this.opened.threadId) { + throw new HarnessReconciliationError( + "thread/read returned a different driver session", + ); + } + const providerSessionId = text(thread.sessionId); + if ( + this.opened.providerSessionId !== null && + providerSessionId !== this.opened.providerSessionId + ) { + throw new HarnessReconciliationError( + "thread/read returned a different provider session", + ); + } + const turns = Array.isArray(thread.turns) ? thread.turns.map(record) : []; + const reconciledUsage = boundedPayload( + record(thread.tokenUsage ?? snapshot.tokenUsage), + ); + if (Object.keys(reconciledUsage).length > 0) this.usageSnapshot = reconciledUsage; + const activeTurns = turns.filter( + (turn) => text(turn.status) === "inProgress", + ); + const expectedTurnId = this.activeTurnId; + const unexpectedActive = activeTurns.find( + (turn) => text(turn.id) !== expectedTurnId, + ); + if (unexpectedActive !== undefined) { + throw new HarnessReconciliationError( + `thread/read exposed active turn ${text(unexpectedActive.id)} instead of persisted active turn ${expectedTurnId ?? "none"}`, + ); + } + + let reconciledTerminalTurnId: string | null = null; + if (expectedTurnId !== null) { + const expectedTurn = turns.find( + (turn) => text(turn.id) === expectedTurnId, + ); + if (expectedTurn === undefined) { + throw new HarnessReconciliationError( + `persisted active turn ${expectedTurnId} is missing from thread/read`, + ); + } + const status = text(expectedTurn.status); + if (status === "inProgress") { + this.activeTurnId = expectedTurnId; + } else if ( + ["completed", "failed", "interrupted", "cancelled"].includes(status) + ) { + reconciledTerminalTurnId = expectedTurnId; + mapTerminalTurn(this, expectedTurn, expectedTurnId, true); + } else { + throw new HarnessReconciliationError( + `persisted active turn ${expectedTurnId} has unreconcilable status ${status || "missing"}`, + ); + } + } + + for (const [turnId, fingerprint] of this.terminalTurns) { + const observed = turns.find((turn) => text(turn.id) === turnId); + if (observed === undefined) continue; + if ( + !["completed", "failed", "interrupted", "cancelled"].includes( + text(observed.status), + ) + ) { + throw new HarnessReconciliationError( + `previously terminal turn ${turnId} is no longer terminal in thread/read`, + ); + } + const conflict = terminalReplayConflict(this, observed, fingerprint); + if (conflict !== null) { + throw new HarnessReconciliationError( + `previously terminal turn ${turnId} ${conflict.message}`, + ); + } + } + this.emit("session.reconciled", { + providerSessionId: this.opened.providerSessionId, + turnCount: turns.length, + activeTurnId: this.activeTurnId, + terminalTurnId: reconciledTerminalTurnId, + }); + return snapshot; + } + + async usage(): Promise | null> { + this.requireCapability("usage"); + return this.usageSnapshot === null ? null : structuredClone(this.usageSnapshot); + } + + async snapshot(): Promise { + return { + driverKind: this.driverKind, + driverSessionId: this.opened.threadId, + providerSessionId: this.opened.providerSessionId, + runId: this.runId, + normalizedSessionId: this.normalizedSessionId, + activeTurnId: this.activeTurnId, + semanticResult: + this.result === null || + this.resultFingerprint === null || + this.resultTurnId === null + ? null + : { + result: structuredClone(this.result), + fingerprint: this.resultFingerprint, + callId: this.resultCallId, + turnId: this.resultTurnId, + }, + terminalTurns: [...this.terminalTurns].map(([turnId, fingerprint]) => ({ + turnId, + fingerprint, + })), + dispositionOnlyRecoveryConsumed: + this.dispositionOnlyRecoveryConsumed, + dispositionOnlyRecoveryTurnId: + this.dispositionOnlyRecoveryTurnId, + pendingRuntimeRequests: this.pendingRuntimeRequests(), + goal: this.currentGoal === null ? null : structuredClone(this.currentGoal), + lineage: this.lineage(), + lastSourceSequence: this.sourceSequence, + }; + } + + async close(): Promise { + this.cancelPendingRequests("session_closed"); + this.eventQueue.close(); + await this.transport.close(); + } + +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts index 0b80b835cb..7da56d171b 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-security-config.ts @@ -1,7 +1,7 @@ import { resolve } from "node:path"; -const SKILLLESS_PERMISSION_PROFILE = "paperclip-runner-workspace-only"; -const PLANNING_PERMISSION_PROFILE = "paperclip-runner-workspace-read-only"; +export const CODEX_SKILLLESS_PERMISSION_PROFILE = "paperclip-runner-workspace-only"; +export const CODEX_PLANNING_PERMISSION_PROFILE = "paperclip-runner-workspace-read-only"; const SKILLLESS_BASE_CONFIG = { "skills.include_instructions": false, @@ -14,7 +14,7 @@ const SKILLLESS_BASE_CONFIG = { "features.image_generation": false, } as const; -function commandEnvironment( +export function codexCommandEnvironment( source: NodeJS.ProcessEnv = process.env, ): Record { const environment: Record = {}; @@ -90,20 +90,20 @@ export function createIsolatedCodexAppServerArgs( ...readOnlyRoots.map((path) => `${tomlString(resolve(path))}="read"`), `":workspace_roots"={"."="read"}`, ].join(","); - const commandEnv = Object.entries(commandEnvironment(source)) + const commandEnv = Object.entries(codexCommandEnvironment(source)) .map(([key, value]) => `${key}=${tomlString(value)}`) .join(","); return [ "-c", - `default_permissions=${tomlString(SKILLLESS_PERMISSION_PROFILE)}`, + `default_permissions=${tomlString(CODEX_SKILLLESS_PERMISSION_PROFILE)}`, "-c", - `permissions.${SKILLLESS_PERMISSION_PROFILE}.filesystem={${filesystemRules}}`, + `permissions.${CODEX_SKILLLESS_PERMISSION_PROFILE}.filesystem={${filesystemRules}}`, "-c", - `permissions.${SKILLLESS_PERMISSION_PROFILE}.network.enabled=false`, + `permissions.${CODEX_SKILLLESS_PERMISSION_PROFILE}.network.enabled=false`, "-c", - `permissions.${PLANNING_PERMISSION_PROFILE}.filesystem={${planningFilesystemRules}}`, + `permissions.${CODEX_PLANNING_PERMISSION_PROFILE}.filesystem={${planningFilesystemRules}}`, "-c", - `permissions.${PLANNING_PERMISSION_PROFILE}.network.enabled=false`, + `permissions.${CODEX_PLANNING_PERMISSION_PROFILE}.network.enabled=false`, "-c", `shell_environment_policy.inherit="none"`, "-c", @@ -125,8 +125,8 @@ export function createSecuredCodexThreadParams( ): Record { const permissionProfile = mode === "plan" - ? PLANNING_PERMISSION_PROFILE - : SKILLLESS_PERMISSION_PROFILE; + ? CODEX_PLANNING_PERMISSION_PROFILE + : CODEX_SKILLLESS_PERMISSION_PROFILE; return { cwd: workingDirectory, config: collaborationThreadConfig( diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts new file mode 100644 index 0000000000..60a342e3ee --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-notifications.ts @@ -0,0 +1,509 @@ +import { paperclipWorkspaceFileReferencesFromText } from "../../live/workspace-file-reference.js"; +import { canonicalProviderEventsFromCodex } from "../../provider-events.js"; +import { harnessRuntimeRequestOutcome } from "../../contracts/harness-driver.js"; +import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; +import type { CodexRpcNotification, CodexTraceInterpretation } from "./app-server-transport.js"; +import { redactCodexDiagnostic } from "./app-server-transport.js"; +import { boundedCodexPayload as boundedPayload, boundedCodexValue, isRetainableCodexPayload } from "./codex-boundaries.js"; +import { runtimeRequestResponse } from "./codex-question-adapter.js"; +import { + isBoundCodexNotification, + isSupportedCodexNotificationMethod, + codexThreadLineage as lineageFromThread, + codexThreadStatus as threadStatus, + parseCodexThreadGoal as parseThreadGoal, + safeCodexRequestResponse as safeRequestResponse, +} from "./codex-thread-normalization.js"; +import type { CodexSessionState } from "./codex-session-state.js"; +import { + admitResult, + captureResultFromItem, + mapTerminalTurn, +} from "./codex-session-terminal.js"; +import { + recordCanonicalWorkspaceChange, + recordTurnDiff, + recordWorkspaceChanges, +} from "./codex-session-workspace.js"; +import { + boundedText, + differingJsonPaths, + itemFromParams, + itemText, + record, + text, +} from "./codex-driver-values.js"; + +export async function pumpNotifications(state: CodexSessionState): Promise { + try { + for await (const notification of state.transport.notifications()) { + mapNotification(state, notification); + } + } catch (error) { + state.emit("harness.diagnostic", { + code: "notification_transport_failed", + message: redactCodexDiagnostic(String(error)), + }); + state.expirePendingInputRequestsAfterProviderLoss(); + state.failProtocol( + "notification_transport_failed", + "Provider notification transport failed closed.", + ); + } + } + +function mapNotification(state: CodexSessionState, notification: CodexRpcNotification): void { + const sourceSequenceBefore = state.sourceSequence; + let rejected = false; + try { + mapNotificationBody(state, notification); + } catch (error) { + rejected = true; + throw error; + } finally { + const correlation = notification.paperclipTrace; + if (correlation !== undefined) { + const emittedEventIds: string[] = []; + for ( + let sourceSeq = sourceSequenceBefore + 1; + sourceSeq <= state.sourceSequence; + sourceSeq += 1 + ) { + emittedEventIds.push( + `${state.runnerInstanceId}:${state.runId}:${sourceSeq}`, + ); + } + const disposition: CodexTraceInterpretation["disposition"] = rejected + ? "rejected" + : emittedEventIds.length > 0 + ? "mapped" + : "ignored"; + try { + state.transport.recordTraceInterpretation?.({ + sourceEventId: correlation.sourceEventId, + sourceEventType: correlation.sourceEventType, + providerMethod: notification.method, + disposition, + emittedEventIds, + reason: rejected + ? "Codex driver rejected the rehydrated provider notification" + : emittedEventIds.length > 0 + ? "Codex driver normalized the rehydrated provider notification into persisted canonical PRP events" + : "Codex driver accepted the rehydrated provider notification but emitted no canonical PRP event", + }); + } catch { + // Trace delivery is deliberately outside run authority. + } + } + } + } + +function mapNotificationBody(state: CodexSessionState, notification: CodexRpcNotification): void { + if (!isSupportedCodexNotificationMethod(notification.method)) return; + if (!isBoundCodexNotification(notification, { + runId: state.runId, + threadIds: [...state.lineageByThread.keys()], + })) { + const params = notification.params; + const claimedThreadId = text( + params.threadId, + text(record(params.thread).id, text(record(params.turn).threadId)), + ); + const claimedRunId = text(params.runId, text(params.paperclipRunId)); + if (claimedThreadId.length > 0 || claimedRunId.length > 0) { + state.failProtocol( + "thread_binding_mismatch", + `Provider ${notification.method} message did not name the active run or a known thread.`, + ); + } + return; + } + const params = notification.params; + const turn = record(params.turn); + const item = itemFromParams(params); + const threadId = text(params.threadId); + const turnId = text(params.turnId, text(turn.id)); + const itemId = text(item.id, text(params.itemId)); + if (notification.method === "paperclip/workspaceChange/updated") { + if (!state.notificationNamesActiveTurn(turnId, "workspace change")) return; + if (threadId.length > 0 && threadId !== state.opened.threadId) return; + recordCanonicalWorkspaceChange(state, + turnId, + params.workspaceChange ?? params, + ); + return; + } + if ( + (threadId.length === 0 || threadId === state.opened.threadId) && + (turnId.length === 0 || + state.activeTurnId === null || + turnId === state.activeTurnId) + ) { + for (const canonical of canonicalProviderEventsFromCodex( + notification.method, + params, + )) { + state.emit(canonical.eventType, canonical.payload, { + turnId: turnId || undefined, + itemId: canonical.itemId, + }); + } + } + if ( + notification.method === "error" || + notification.method === "warning" || + notification.method === "configWarning" + ) { + state.emit("harness.diagnostic", { + code: notification.method.replaceAll("/", "_"), + message: redactCodexDiagnostic( + text(params.message, JSON.stringify(boundedCodexValue(params))), + ), + }); + return; + } + if (notification.method === "thread/started") { + if (!state.capabilities.threadLineage) return; + const thread = record(params.thread); + const lineage = lineageFromThread(thread); + if ( + lineage.threadId.length === 0 || + lineage.threadId === state.opened.threadId || + lineage.parentThreadId === null || + !state.lineageByThread.has(lineage.parentThreadId) + ) { + return; + } + state.lineageByThread.set(lineage.threadId, lineage); + state.emit( + "item.started", + { + kind: "thread_lineage", + text: `${lineage.nickname ?? lineage.role ?? "Child agent"} started.`, + lineage, + }, + { itemId: `thread:${lineage.threadId}` }, + ); + return; + } + if (notification.method === "thread/status/changed") { + const lineage = state.lineageByThread.get(threadId); + if (lineage === undefined || threadId === state.opened.threadId) return; + lineage.status = threadStatus(params.status); + state.emit( + "item.delta", + { + kind: "thread_lineage", + text: `${lineage.nickname ?? lineage.role ?? "Child agent"}: ${lineage.status}`, + lineage, + }, + { itemId: `thread:${lineage.threadId}` }, + ); + return; + } + if (notification.method === "thread/closed") { + const lineage = state.lineageByThread.get(threadId); + if (lineage === undefined || threadId === state.opened.threadId) return; + lineage.status = "closed"; + state.emit( + "item.completed", + { + kind: "thread_lineage", + text: `${lineage.nickname ?? lineage.role ?? "Child agent"} closed.`, + lineage, + }, + { itemId: `thread:${lineage.threadId}` }, + ); + return; + } + if (notification.method === "thread/goal/updated") { + if (threadId !== state.opened.threadId) return; + const goal = parseThreadGoal(params.goal); + if (goal === null) return; + state.currentGoal = goal; + state.emit( + "item.completed", + { + kind: "goal", + text: `Thread goal ${goal.status}: ${goal.objective}`, + action: "notification", + goal, + }, + { + turnId: turnId || undefined, + itemId: `${threadId}:goal:update:${state.sourceSequence + 1}`, + }, + ); + return; + } + if (notification.method === "thread/goal/cleared") { + if (threadId !== state.opened.threadId) return; + state.currentGoal = null; + state.emit( + "item.completed", + { + kind: "goal_cleared", + text: "Thread goal cleared.", + action: "notification", + goal: null, + }, + { itemId: `${threadId}:goal:clear:${state.sourceSequence + 1}` }, + ); + return; + } + if (notification.method === "serverRequest/resolved") { + const requestId = String(params.requestId ?? ""); + const pending = state.pendingRuntimeRequestMap.get(requestId); + if (pending === undefined || threadId !== state.opened.threadId) return; + state.pendingRuntimeRequestMap.delete(requestId); + const resolution = pending.settlingResolution; + state.emit( + resolution === undefined ? "runtime_request.cancelled" : "runtime_request.resolved", + harnessRuntimeRequestOutcome( + pending.request, + resolution === undefined + ? { reason: "provider_resolved" } + : { + action: resolution.action, + ...(resolution.action === "submit" && "response" in resolution + ? { response: resolution.response } + : {}), + }, + ), + { turnId: pending.request.turnId, itemId: pending.request.itemId }, + ); + pending.settle( + resolution === undefined + ? safeRequestResponse(pending.request.method, "cancel") + : runtimeRequestResponse( + pending.request, + resolution, + pending.responseContext, + ), + ); + return; + } + if ( + notification.method === "item/completed" + && text(params.kind) === "steering_acknowledgement" + && Object.keys(item).length === 0 + ) { + // runnerd persists its own command acknowledgement as a canonical PRP + // item. The request() call is already the authoritative acknowledgement + // and steer() emits the user-visible item with the active turn binding. + // Do not reinterpret this transport-level echo as an unbound Codex item. + return; + } + if (notification.method === "paperclip/runResult") { + if ( + threadId !== state.opened.threadId + || !state.notificationNamesActiveTurn(turnId, "semantic result") + ) { + if (threadId !== state.opened.threadId) { + state.failProtocol( + "thread_binding_mismatch", + "Provider semantic result did not name the opened thread.", + ); + } + return; + } + if (!isRetainableCodexPayload(params.result)) { + state.failProtocol( + "invalid_semantic_result", + "Provider semantic result exceeded the retained payload limit.", + ); + return; + } + const validation = validatePrpStructuredRunResult(params.result); + if (!validation.ok) { + state.failProtocol( + "invalid_semantic_result", + "Provider semantic result did not match the run-result contract.", + ); + return; + } + const differingFields = state.result === null + ? [] + : differingJsonPaths(state.result, validation.result); + if (admitResult(state, validation.result, itemId, turnId) === "conflict") { + state.failProtocol( + "conflicting_semantic_result", + `Provider supplied a different schema-valid semantic result after one was committed. Differing fields: ${differingFields.join(", ") || "unknown"}.`, + ); + } + return; + } + if (threadId !== state.opened.threadId) { + state.failProtocol( + "thread_binding_mismatch", + `Provider ${notification.method} message did not name the opened thread.`, + ); + return; + } + if (notification.method === "turn/started") { + if ( + turnId.length === 0 || + state.terminal || + state.protocolFailed || + state.turnStarted || + (!state.turnStartPending && state.activeTurnId === null) || + (state.activeTurnId !== null && state.activeTurnId !== turnId) + ) { + state.failProtocol( + "turn_binding_mismatch", + "Provider started an unexpected turn.", + ); + return; + } + state.activeTurnId = turnId; + state.turnStarted = true; + state.emit( + "turn.started", + { status: text(turn.status, "inProgress") }, + { turnId }, + ); + return; + } + if (notification.method === "turn/completed") { + if (state.terminalTurns.has(turnId)) { + mapTerminalTurn(state, turn, turnId); + return; + } + if (!state.notificationNamesActiveTurn(turnId, "turn terminal")) return; + mapTerminalTurn(state, turn, turnId); + return; + } + if (notification.method === "item/started") { + if (!state.notificationNamesActiveTurn(turnId, "item start")) return; + const channel = channelForStartedItem(state, item); + if (itemId) state.itemChannels.set(itemId, channel); + state.emit( + "item.started", + boundedPayload({ + kind: text(item.type, "unknown"), + channel, + providerPhase: text(item.phase) || undefined, + text: itemText(item), + item, + }), + { turnId, itemId }, + ); + return; + } + if (notification.method === "item/completed") { + if (!state.notificationNamesActiveTurn(turnId, "item completion")) return; + if (!captureResultFromItem(state, item, turnId)) return; + const channel = itemId + ? (state.itemChannels.get(itemId) ?? channelForStartedItem(state, item)) + : channelForStartedItem(state, item); + state.emit( + "item.completed", + boundedPayload({ + kind: text(item.type, "unknown"), + channel, + providerPhase: text(item.phase) || undefined, + text: itemText(item), + item, + }), + { turnId, itemId }, + ); + if (text(item.type) === "agentMessage") { + for (const reference of paperclipWorkspaceFileReferencesFromText( + state.opened.context.workingDirectory, + text(item.text), + turnId, + )) { + if (state.emittedFileReferences.has(reference.referenceId)) continue; + state.emittedFileReferences.add(reference.referenceId); + state.emit( + "workspace.file.referenced", + { ...reference }, + { turnId, itemId: reference.referenceId }, + ); + } + } + if (itemId) state.itemChannels.delete(itemId); + if (text(item.type) === "fileChange") + recordWorkspaceChanges(state, turnId, item.changes, true); + return; + } + const deltaKinds: Record = { + "item/agentMessage/delta": "agentMessage", + "item/plan/delta": "plan", + "item/reasoning/summaryTextDelta": "reasoning", + "item/reasoning/textDelta": "reasoning", + "item/commandExecution/outputDelta": "commandExecution", + "item/fileChange/outputDelta": "fileChange", + "item/fileChange/patchUpdated": "fileChange", + "turn/diff/updated": "diff", + "turn/plan/updated": "plan", + }; + const deltaKind = deltaKinds[notification.method]; + if (deltaKind !== undefined) { + if (!state.notificationNamesActiveTurn(turnId, "item update")) return; + const methodChannel = channelForDelta(state, notification.method); + const channel = + methodChannel !== "unknown" + ? methodChannel + : itemId + ? (state.itemChannels.get(itemId) ?? "unknown") + : "unknown"; + state.emit( + "item.delta", + boundedPayload({ + kind: deltaKind, + channel, + providerMethod: notification.method, + text: text(params.delta, text(params.patch, text(params.output))), + update: params, + }), + { turnId, itemId: itemId || `${turnId}:${deltaKind}` }, + ); + if (notification.method === "item/fileChange/patchUpdated") { + recordWorkspaceChanges(state, turnId, params.changes, false); + } else if (notification.method === "turn/diff/updated") { + recordTurnDiff(state, turnId, params.diff); + } + return; + } + if (notification.method === "thread/tokenUsage/updated") { + state.usageSnapshot = boundedPayload(record(params.tokenUsage)); + // Codex can replay a thread-scoped usage snapshot while a resumed thread + // is being attached, before the next turn has started. Keep the snapshot, + // but do not turn that benign replay into a fatal turn-binding violation. + if (state.activeTurnId === null || turnId !== state.activeTurnId) return; + state.emit( + "item.completed", + { kind: "usage", usage: state.usageSnapshot }, + { + turnId, + itemId: `${turnId}:usage:${state.sourceSequence + 1}`, + }, + ); + return; + } + } + +function channelForStartedItem( + state: CodexSessionState, + item: Record, + ): "progress" | "final" | "summary" | "detail" | "unknown" { + const type = text(item.type); + const phase = text(item.phase).toLowerCase(); + if (type === "agentMessage") { + if (phase === "commentary") return "progress"; + if (phase === "final_answer") return "final"; + return "unknown"; + } + if (type === "reasoning") return "summary"; + return "unknown"; + } + +function channelForDelta( + state: CodexSessionState, + method: string, + ): "progress" | "final" | "summary" | "detail" | "unknown" { + if (method === "item/reasoning/summaryTextDelta") return "summary"; + if (method === "item/reasoning/textDelta") return "detail"; + return "unknown"; + } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts new file mode 100644 index 0000000000..76d0f7be09 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-server-requests.ts @@ -0,0 +1,303 @@ +import type { HarnessRuntimeRequest, PaperclipQuestionSet } from "../../contracts/harness-driver.js"; +import { + CODEX_BLOCK_TOOL_NAME, +} from "../../contracts/codex.js"; +import { validatePrpStructuredRunResult } from "../../protocol/replay-contract.js"; +import type { CodexRpcServerRequest } from "./app-server-transport.js"; +import { + boundedCodexValue, + codexToolAcceptsDisposition as toolAcceptsDisposition, + isCodexSemanticTool as isSemanticTool, + isRetainableCodexPayload, + redactCodexValue, + rejectedCodexToolCall as rejectedToolCall, +} from "./codex-boundaries.js"; +import { + createCodexQuestionResponseContext, + hasCodexQuestionForm, + normalizeCodexQuestionSet, + runtimeRequestKind, + runtimeRequestPrompt, + runtimeRequestProtocolPayload, +} from "./codex-question-adapter.js"; +import { safeCodexRequestResponse as safeRequestResponse } from "./codex-thread-normalization.js"; +import type { CodexSessionState } from "./codex-session-state.js"; +import { admitResult } from "./codex-session-terminal.js"; +import { boundedText, dynamicToolResponse, record, text } from "./codex-driver-values.js"; + +export async function handleServerRequest( + state: CodexSessionState, + request: CodexRpcServerRequest, + ): Promise> { + const sourceSequenceBefore = state.sourceSequence; + let rejected = false; + try { + const response = await handleServerRequestBody(state, request); + rejected = response.success === false; + return response; + } catch (error) { + rejected = true; + throw error; + } finally { + const correlation = request.paperclipTrace; + if (correlation !== undefined) { + const emittedEventIds: string[] = []; + for ( + let sourceSeq = sourceSequenceBefore + 1; + sourceSeq <= state.sourceSequence; + sourceSeq += 1 + ) { + emittedEventIds.push( + `${state.runnerInstanceId}:${state.runId}:${sourceSeq}`, + ); + } + try { + state.transport.recordTraceInterpretation?.({ + sourceEventId: correlation.sourceEventId, + sourceEventType: correlation.sourceEventType, + providerMethod: request.method, + disposition: rejected + ? "rejected" + : emittedEventIds.length > 0 + ? "mapped" + : "ignored", + emittedEventIds, + reason: rejected + ? "Codex driver rejected the correlated provider server request" + : emittedEventIds.length > 0 + ? "Codex driver mapped the correlated provider server request into canonical PRP events" + : "Codex driver accepted the correlated provider server request without emitting a canonical PRP event", + }); + } catch { + // Trace delivery is deliberately outside run authority. + } + } + } + } + +async function handleServerRequestBody( + state: CodexSessionState, + request: CodexRpcServerRequest, + ): Promise> { + if (request.method === "item/tool/call") { + const tool = text(request.params.tool); + const threadId = text(request.params.threadId); + const turnId = text(request.params.turnId); + const callId = text(request.params.callId); + if ( + state.protocolFailed || + state.terminal || + threadId !== state.opened.threadId || + turnId.length === 0 || + turnId !== state.activeTurnId || + callId.length === 0 + ) { + state.failProtocol( + "tool_binding_mismatch", + "Semantic tool call did not name the active thread and turn.", + ); + return rejectedToolCall( + "Semantic tool call was outside the active thread and turn.", + ); + } + if (!isSemanticTool(tool)) { + const admitted = state.dynamicTools.some( + (candidate) => candidate.name === tool, + ); + if (admitted && state.dynamicToolHandler !== undefined) { + state.emit( + "item.started", + { + kind: "dynamicToolCall", + item: { + type: "tool_use", + id: callId, + name: tool, + input: request.params.arguments, + }, + }, + { turnId, itemId: callId }, + ); + try { + const result = await state.dynamicToolHandler({ + tool, + callId, + threadId, + turnId, + arguments: request.params.arguments, + }); + state.emit( + "item.completed", + { + kind: "dynamicToolCall", + item: { + type: "tool_result", + id: callId, + tool_use_id: callId, + result, + }, + }, + { turnId, itemId: callId }, + ); + return dynamicToolResponse(result); + } catch (error) { + const message = boundedText( + error instanceof Error ? error.message : error, + ); + state.emit( + "item.completed", + { + kind: "dynamicToolCall", + item: { + type: "tool_result", + id: callId, + tool_use_id: callId, + error: message, + is_error: true, + }, + }, + { turnId, itemId: callId }, + ); + return { + success: false, + contentItems: [{ type: "inputText", text: message }], + }; + } + } + state.diagnoseUnsupported(`dynamic tool ${tool}`); + return rejectedToolCall("Unsupported tool."); + } + if (!isRetainableCodexPayload(request.params.arguments)) { + return rejectedToolCall( + "Semantic result exceeded the retained payload limit.", + ); + } + const validation = validatePrpStructuredRunResult( + request.params.arguments, + ); + if (!validation.ok) { + return { + success: false, + contentItems: [ + { type: "inputText", text: "Invalid semantic result." }, + ], + }; + } + if ( + !toolAcceptsDisposition(tool, validation.result.reportedWorkDisposition) + ) { + return { + success: false, + contentItems: [ + { + type: "inputText", + text: + tool === CODEX_BLOCK_TOOL_NAME + ? "paperclip_block requires reportedWorkDisposition=blocked." + : "paperclip_finish accepts only done or needs_review.", + }, + ], + }; + } + const admission = admitResult(state, validation.result, callId, turnId); + if (admission === "conflict") { + return rejectedToolCall( + "A different semantic result was already committed.", + ); + } + return { + success: true, + contentItems: [ + { type: "inputText", text: "Semantic completion accepted." }, + ], + }; + } + + const requestKind = runtimeRequestKind(request.method); + if (requestKind === null) return safeRequestResponse(request.method); + const requestTurnId = text(request.params.turnId); + if ( + text(request.params.threadId) !== state.opened.threadId || + requestTurnId.length === 0 || + requestTurnId !== state.activeTurnId || + state.terminal || + state.protocolFailed + ) { + state.failProtocol( + "runtime_request_binding_mismatch", + "Runtime request did not name the active thread and turn.", + ); + return safeRequestResponse(request.method); + } + const requestId = String(request.id); + if (state.pendingRuntimeRequestMap.has(requestId)) { + state.failProtocol( + "runtime_request_duplicate", + "Provider reused a pending runtime request identity.", + ); + return safeRequestResponse(request.method); + } + let input: PaperclipQuestionSet | null = null; + const responseContext = createCodexQuestionResponseContext(); + try { + input = normalizeCodexQuestionSet( + request.method, + request.params, + responseContext, + ); + } catch { + state.emit("harness.diagnostic", { + code: "runtime_input_rejected", + adapter: "codex-app-server", + method: request.method, + reason: "The provider input request contained an invalid question form.", + }, { turnId: requestTurnId, itemId: String(request.id) }); + return safeRequestResponse(request.method); + } + if ( + (requestKind === "user_input" || requestKind === "elicitation") && + input === null && + hasCodexQuestionForm(request.method, request.params) + ) { + state.emit("harness.diagnostic", { + code: "runtime_input_rejected", + adapter: "codex-app-server", + method: request.method, + reason: "The provider input request did not contain a supported question form.", + }, { turnId: requestTurnId, itemId: String(request.id) }); + return safeRequestResponse(request.method); + } + const runtimeRequest: HarnessRuntimeRequest = { + requestId, + requestKind, + method: request.method, + turnId: requestTurnId, + itemId: text(request.params.itemId, requestId), + status: "pending", + prompt: runtimeRequestPrompt(requestKind, request.params), + details: record(redactCodexValue(boundedCodexValue(request.params))), + ...(input !== null ? { input } : {}), + origin: { + adapter: "codex-app-server", + provider: "codex", + method: request.method, + }, + }; + state.emit( + "runtime_request.created", + { + request: runtimeRequestProtocolPayload(runtimeRequest), + }, + { + turnId: requestTurnId, + itemId: runtimeRequest.itemId, + }, + ); + return new Promise>((settle) => { + state.pendingRuntimeRequestMap.set(requestId, { + request: runtimeRequest, + responseContext, + settle, + }); + }); + } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts new file mode 100644 index 0000000000..0ff9a479b2 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-state.ts @@ -0,0 +1,403 @@ +import type { + HarnessRuntimeRequest, + HarnessThreadGoal, + HarnessThreadLineageEntry, + PersistedHarnessSemanticResult, + PersistedHarnessTurnTerminal, +} from "../../contracts/harness-driver.js"; +import { + HarnessCapabilityUnavailableError, + HarnessReconciliationError, + HarnessStaleTurnError, + harnessRuntimeInputExpiredOutcome, + harnessRuntimeRequestOutcome, +} from "../../contracts/harness-driver.js"; +import type { CodexTaskEnvelope } from "../../contracts/codex.js"; +import { + validatePrpStructuredRunResult, + type PrpEvent, + type PrpStructuredRunResult, +} from "../../protocol/replay-contract.js"; +import type { CodexAppServerTransport } from "./app-server-transport.js"; +import { redactCodexDiagnostic } from "./app-server-transport.js"; +import { safeCodexRequestResponse as safeRequestResponse } from "./codex-thread-normalization.js"; +import type { + CodexAppServerDriverOptions, + CodexCapabilities, + OpenedCodexThread, + PendingRuntimeRequest, +} from "./codex-driver-types.js"; +import { canonicalJson, record } from "./codex-driver-values.js"; + +class AsyncQueue implements AsyncIterable { + #values: T[] = []; + #waiters: Array<(value: IteratorResult) => void> = []; + #closed = false; + + push(value: T): void { + if (this.#closed) return; + const waiter = this.#waiters.shift(); + if (waiter === undefined) this.#values.push(value); + else waiter({ value, done: false }); + } + + close(): void { + if (this.#closed) return; + this.#closed = true; + for (const waiter of this.#waiters.splice(0)) + waiter({ value: undefined, done: true }); + } + + [Symbol.asyncIterator](): AsyncIterator { + return { + next: async () => { + const value = this.#values.shift(); + if (value !== undefined) return { value, done: false }; + if (this.#closed) return { value: undefined, done: true }; + return new Promise((resolve) => this.#waiters.push(resolve)); + }, + }; + } +} + +export class CodexSessionState { + readonly transport: CodexAppServerTransport; + runId: string; + readonly normalizedSessionId: string; + readonly opened: OpenedCodexThread; + readonly taskEnvelope: CodexTaskEnvelope; + readonly conversationMode: "task" | "direct"; + readonly now: () => Date; + readonly runnerInstanceId: string; + readonly driverKind: string; + readonly capabilities: CodexCapabilities; + readonly dynamicTools: readonly Readonly>[]; + readonly dynamicToolHandler: CodexAppServerDriverOptions["dynamicToolHandler"]; + readonly eventQueue = new AsyncQueue(); + sourceSequence: number; + activeTurnId: string | null; + usageSnapshot: Record | null = null; + result: PrpStructuredRunResult | null = null; + resultFingerprint: string | null = null; + resultCallId: string | null = null; + resultTurnId: string | null = null; + turnStartPending = false; + protocolFailed = false; + protocolFailureCode: string | null = null; + protocolFailureMessage: string | null = null; + terminal = false; + dispositionOnlyRecoveryAvailable = false; + dispositionOnlyRecoveryConsumed = false; + dispositionOnlyRecoveryTurnId: string | null = null; + turnStarted = false; + readonly terminalTurns = new Map(); + readonly workspaceChangesByTurn = new Map>(); + readonly emittedFileReferences = new Set(); + readonly itemChannels = new Map< + string, + "progress" | "final" | "summary" | "detail" | "unknown" + >(); + readonly pendingRuntimeRequestMap = new Map(); + readonly lineageByThread = new Map(); + currentGoal: HarnessThreadGoal | null = null; + interruptQueued = false; + steerSequence = 0; + readonly acknowledgedSteeringCorrelations = new Map(); + interruptSequence = 0; + + constructor(input: { + transport: CodexAppServerTransport; + runId: string; + normalizedSessionId: string; + opened: OpenedCodexThread; + taskEnvelope: CodexTaskEnvelope; + conversationMode: "task" | "direct"; + resumed: boolean; + activeTurnId?: string | null; + semanticResult?: PersistedHarnessSemanticResult | null; + terminalTurns?: PersistedHarnessTurnTerminal[]; + dispositionOnlyRecoveryConsumed?: boolean; + dispositionOnlyRecoveryTurnId?: string | null; + stalePendingRuntimeRequests?: HarnessRuntimeRequest[]; + lineage?: HarnessThreadLineageEntry[]; + goal?: HarnessThreadGoal | null; + sourceSequence: number; + now: () => Date; + runnerInstanceId: string; + driverKind: string; + capabilities: CodexCapabilities; + dynamicTools: readonly Readonly>[]; + dynamicToolHandler?: CodexAppServerDriverOptions["dynamicToolHandler"]; + }) { + this.transport = input.transport; + this.runId = input.runId; + this.sourceSequence = 0; + this.normalizedSessionId = input.normalizedSessionId; + this.opened = input.opened; + this.taskEnvelope = input.taskEnvelope; + this.conversationMode = input.conversationMode; + this.activeTurnId = input.activeTurnId ?? null; + this.sourceSequence = input.sourceSequence; + this.now = input.now; + this.runnerInstanceId = input.runnerInstanceId; + this.driverKind = input.driverKind; + this.capabilities = input.capabilities; + this.dynamicTools = input.dynamicTools; + this.dynamicToolHandler = input.dynamicToolHandler; + this.currentGoal = input.goal === undefined ? null : structuredClone(input.goal); + for (const entry of input.lineage ?? [input.opened.lineage]) { + this.lineageByThread.set(entry.threadId, structuredClone(entry)); + } + if (!this.lineageByThread.has(input.opened.lineage.threadId)) { + this.lineageByThread.set( + input.opened.lineage.threadId, + structuredClone(input.opened.lineage), + ); + } + if (input.semanticResult) { + const validation = validatePrpStructuredRunResult( + input.semanticResult.result, + ); + if ( + !validation.ok || + canonicalJson(validation.result) !== input.semanticResult.fingerprint + ) { + throw new HarnessReconciliationError( + "persisted semantic result fingerprint is invalid", + ); + } + this.result = structuredClone(validation.result); + this.resultFingerprint = input.semanticResult.fingerprint; + this.resultCallId = input.semanticResult.callId ?? null; + this.resultTurnId = input.semanticResult.turnId; + } + for (const terminal of input.terminalTurns ?? []) { + if ( + !terminal.turnId || + !terminal.fingerprint || + this.terminalTurns.has(terminal.turnId) + ) { + throw new HarnessReconciliationError( + "persisted terminal turn fingerprints are invalid", + ); + } + this.terminalTurns.set(terminal.turnId, terminal.fingerprint); + } + if (this.activeTurnId && this.terminalTurns.has(this.activeTurnId)) { + this.activeTurnId = null; + } + const dispositionOnlyRecoveryPreviouslyConsumed = + input.dispositionOnlyRecoveryConsumed ?? false; + const dispositionOnlyRecoveryTurnId = + input.dispositionOnlyRecoveryTurnId ?? null; + const settledSemanticResult = + this.conversationMode === "task" + && this.result !== null + && this.resultTurnId !== null + && this.terminalTurns.has(this.resultTurnId); + const consumedResultlessRecovery = + input.resumed + && this.conversationMode === "task" + && this.result === null + && this.activeTurnId === null + && dispositionOnlyRecoveryPreviouslyConsumed; + // Once the one-shot disposition allowance was consumed, only affirmative + // provider history can release it or recover its active turn. Missing or + // malformed history is ambiguous, so a reconstructed session with no + // active turn must remain closed to further provider submissions. A bound + // terminal fingerprint is stronger completion evidence, but it is not + // required to preserve ownership across an ambiguous crash boundary. + this.terminal = settledSemanticResult || consumedResultlessRecovery; + this.dispositionOnlyRecoveryAvailable = + input.resumed && + this.conversationMode === "task" && + this.terminalTurns.size > 0 && + this.result === null && + !dispositionOnlyRecoveryPreviouslyConsumed; + // Recovery itself does not consume the allowance. A checkpoint can occur + // before startTurn, and consuming it here would strand the run if the + // process crashed at that boundary. startTurn consumes it in memory; a + // later recovery adopts provider evidence for an accepted, uncheckpointed + // turn before deciding whether another submission is safe. + this.dispositionOnlyRecoveryConsumed = + dispositionOnlyRecoveryPreviouslyConsumed; + this.dispositionOnlyRecoveryTurnId = dispositionOnlyRecoveryTurnId; + } + + requireActiveTurn(turnId: string, operation: string): void { + if (this.activeTurnId !== turnId) { + this.emit("harness.diagnostic", { + code: "stale_turn_rejected", + operation, + turnId, + activeTurnId: this.activeTurnId, + message: `Rejected ${operation} for a stale turn identity.`, + }); + throw new HarnessStaleTurnError(turnId); + } + } + + cancelPendingRequests(reason: string): void { + for (const pending of this.pendingRuntimeRequestMap.values()) { + this.emit( + pending.request.input === undefined ? "runtime_request.cancelled" : "runtime_request.expired", + pending.request.input === undefined + ? harnessRuntimeRequestOutcome(pending.request, { reason }) + : harnessRuntimeInputExpiredOutcome(pending.request, "provider_process_lost"), + { turnId: pending.request.turnId, itemId: pending.request.itemId }, + ); + pending.settle(safeRequestResponse(pending.request.method, "cancel")); + } + this.pendingRuntimeRequestMap.clear(); + } + + expirePendingInputRequestsAfterProviderLoss(): void { + for (const [requestId, pending] of this.pendingRuntimeRequestMap) { + if (pending.request.input === undefined) continue; + this.emit( + "runtime_request.expired", + harnessRuntimeInputExpiredOutcome(pending.request, "provider_process_lost"), + { turnId: pending.request.turnId, itemId: pending.request.itemId }, + ); + pending.settle(safeRequestResponse(pending.request.method, "cancel")); + this.pendingRuntimeRequestMap.delete(requestId); + } + } + + notificationNamesActiveTurn(turnId: string, kind: string): boolean { + if ( + this.protocolFailed || + this.terminal || + turnId.length === 0 || + this.activeTurnId === null || + turnId !== this.activeTurnId + ) { + this.failProtocol( + "turn_binding_mismatch", + `Provider ${kind} did not name the active turn.`, + ); + return false; + } + return true; + } + + requireCapability(operation: keyof CodexCapabilities): void { + if (!this.capabilities[operation]) + throw this.unsupported(operation, "capability not advertised"); + } + + unsupported( + operation: string, + detail: unknown, + ): HarnessCapabilityUnavailableError { + const error = new HarnessCapabilityUnavailableError( + operation, + redactCodexDiagnostic(String(detail)), + ); + this.diagnoseUnsupported(operation, error.message); + return error; + } + + diagnoseUnsupported( + operation: string, + detail = "operation is not available", + ): void { + this.emit("harness.diagnostic", { + code: "unsupported_operation", + operation, + message: redactCodexDiagnostic(detail), + }); + } + + failProtocol(code: string, message: string): void { + if (this.protocolFailed) return; + this.protocolFailed = true; + this.protocolFailureCode = code; + this.protocolFailureMessage = redactCodexDiagnostic(message); + this.cancelPendingRequests("protocol_failed"); + this.emit("session.failed", { + code, + message: redactCodexDiagnostic(message), + recoverable: false, + }); + if (!this.terminal && this.activeTurnId !== null) { + const turnId = this.activeTurnId; + this.emit( + "turn.failed", + { status: "failed", error: { code } }, + { turnId }, + ); + this.terminalTurns.set(turnId, canonicalJson({ protocolFailure: code })); + this.activeTurnId = null; + } + this.terminal = true; + this.eventQueue.close(); + void this.transport.close(); + } + + emit( + eventType: PrpEvent["eventType"], + payload: Record, + refs: { turnId?: string; itemId?: string } = {}, + ): void { + const sourceSeq = ++this.sourceSequence; + this.eventQueue.push({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${this.runnerInstanceId}:${this.runId}:${sourceSeq}`, + sourceSeq, + sourceInstanceId: this.runnerInstanceId, + sourceKind: "runner", + runId: this.runId, + normalizedSessionId: this.normalizedSessionId, + ...(refs.turnId ? { turnId: refs.turnId } : {}), + ...(refs.itemId ? { itemId: refs.itemId } : {}), + eventType, + schemaVersion: 1, + priority: eventType === "run.result.proposed" ? 0 : 1, + emittedAt: this.now().toISOString(), + payload, + }); + } +} + +export type CodexSessionStateInput = ConstructorParameters[0]; + +export function initializeCodexSessionEvents( + state: CodexSessionState, + input: CodexSessionStateInput, +): void { + state.emit(input.resumed ? "session.resumed" : "session.started", { + driverSessionId: input.opened.threadId, + providerSessionId: input.opened.providerSessionId, + context: input.opened.context, + }); + for (const stale of input.stalePendingRuntimeRequests ?? []) { + state.emit( + "runtime_request.cancelled", + harnessRuntimeRequestOutcome(stale, { reason: "transport_recovered" }), + { turnId: stale.turnId, itemId: stale.itemId }, + ); + } + state.emit( + "item.completed", + { + kind: "thread_lineage", + text: `Root thread ${input.opened.threadId}`, + lineage: input.opened.lineage, + }, + { itemId: `thread:${input.opened.threadId}` }, + ); + state.emit( + "item.completed", + { + kind: "model", + text: `${input.opened.context.model} (${input.opened.context.modelProvider})`, + model: { + name: input.opened.context.model, + provider: input.opened.context.modelProvider, + codexVersion: input.opened.context.codexVersion, + }, + }, + { itemId: `${input.opened.threadId}:model` }, + ); +} diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts new file mode 100644 index 0000000000..23ad91499b --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-terminal.ts @@ -0,0 +1,231 @@ +import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; +import type { PrpStructuredRunResult } from "../../protocol/replay-contract.js"; +import { boundedCodexPayload as boundedPayload, isRetainableCodexPayload } from "./codex-boundaries.js"; +import type { CodexSessionState } from "./codex-session-state.js"; +import type { SemanticResultAdmission, TerminalReplayConflict } from "./codex-driver-types.js"; +import { canonicalJson, record, terminalState, text, tryParseResult } from "./codex-driver-values.js"; + +export function captureResultFromItem( + state: CodexSessionState, + item: Record, + turnId: string, + ): boolean { + if (state.conversationMode === "direct") return true; + if ( + text(item.type) !== "agentMessage" || + !isRetainableCodexPayload(item.text) + ) + return true; + const result = tryParseResult(item.text); + if (result !== null && isRetainableCodexPayload(result)) { + const admission = admitResult(state, result, text(item.id), turnId); + if (admission === "conflict") { + state.failProtocol( + "conflicting_semantic_result", + "Provider agentMessage supplied a different schema-valid semantic result after one was committed.", + ); + return false; + } + } + return true; + } + +export function admitResult( + state: CodexSessionState, + result: PrpStructuredRunResult, + itemId: string, + turnId?: string, + ): SemanticResultAdmission { + const fingerprint = canonicalJson(result); + if (state.resultFingerprint !== null) { + return state.resultFingerprint === fingerprint ? "identical" : "conflict"; + } + state.result = structuredClone(result); + state.resultFingerprint = fingerprint; + state.resultCallId = itemId || null; + state.resultTurnId = turnId || state.activeTurnId; + result.verification.forEach((verification, index) => { + state.emit( + "item.completed", + { + kind: "verification", + text: `${verification.status}: ${verification.commandOrCheck}`, + verification, + }, + { + turnId: turnId || state.activeTurnId || undefined, + itemId: `${itemId || "semantic-result"}:verification:${index + 1}`, + }, + ); + }); + state.emit("run.result.proposed", result, { + turnId: turnId || state.activeTurnId || undefined, + itemId: itemId || undefined, + }); + return "committed"; + } + +function finalize( + state: CodexSessionState,turnStatus: string): void { + if (state.conversationMode === "direct") return; + if (state.terminal) return; + if (state.result === null) { + state.emit("harness.diagnostic", { + code: "semantic_result_missing", + message: `Turn ${turnStatus} without a schema-valid semantic proposal; recovery is required.`, + }); + } + state.terminal = true; + } + +export function mapTerminalTurn( + state: CodexSessionState, + turn: Record, + fallbackTurnId: string, + reconciling = false, + ): void { + const turnId = text(turn.id, fallbackTurnId || state.activeTurnId || ""); + const status = text(turn.status, "completed"); + const previous = state.terminalTurns.get(turnId); + if (previous !== undefined) { + const conflict = terminalReplayConflict(state, turn, previous); + if (conflict !== null) { + if (reconciling) { + throw new HarnessReconciliationError( + `previously terminal turn ${turnId} ${conflict.message}`, + ); + } + state.failProtocol( + conflict.code, + `Provider terminal for turn ${turnId} ${conflict.message}.`, + ); + } + return; + } + const candidate = resultFromTurn(state, turn); + if (candidate !== null) { + const admission = admitResult(state, + candidate, + text(resultItemFromTurn(state, turn)?.id), + turnId, + ); + if (admission === "conflict") { + const message = `terminal turn ${turnId} contains a conflicting semantic result`; + if (reconciling) throw new HarnessReconciliationError(message); + state.failProtocol( + "conflicting_semantic_result", + `Provider ${message}.`, + ); + return; + } + } + state.terminalTurns.set(turnId, terminalFingerprint(state, turn)); + const workspace = state.workspaceChangesByTurn.get(turnId); + if (workspace !== undefined) { + state.emit( + "workspace.diff.recorded", + { ...workspace, source: "runner_verified", complete: true }, + { + turnId, + itemId: `${turnId}:workspace`, + }, + ); + } + const eventType = + status === "failed" + ? "turn.failed" + : status === "interrupted" + ? "turn.interrupted" + : status === "cancelled" + ? "turn.cancelled" + : "turn.completed"; + state.cancelPendingRequests("turn_terminal"); + state.activeTurnId = null; + state.turnStarted = false; + state.emit( + eventType, + boundedPayload({ + status, + error: turn.error ?? null, + }), + { turnId }, + ); + finalize(state, status); + } + +function terminalFingerprint( + state: CodexSessionState, + turn: Record, + semanticResult: PrpStructuredRunResult | null = state.result, + ): string { + return canonicalJson({ + terminalState: terminalState(text(turn.status, "completed")), + error: turn.error ?? null, + result: semanticResult, + }); + } + +export function terminalReplayConflict( + state: CodexSessionState, + turn: Record, + expectedFingerprint: string, + ): TerminalReplayConflict | null { + const turnId = text(turn.id); + const candidate = resultFromTurn(state, turn); + if ( + candidate !== null && + state.resultFingerprint !== null && + state.resultTurnId === turnId && + canonicalJson(candidate) !== state.resultFingerprint + ) { + return { + code: "conflicting_semantic_result", + message: "contains a conflicting semantic result", + }; + } + const semanticResult = state.resultTurnId === turnId + ? state.result + : candidate; + if ( + terminalFingerprint(state, turn, semanticResult) !== expectedFingerprint + ) { + return { + code: "conflicting_turn_terminal", + message: "changed from its committed terminal fingerprint", + }; + } + if (candidate !== null && state.result === null) { + admitResult(state, + candidate, + text(resultItemFromTurn(state, turn)?.id), + turnId, + ); + } + return null; + } + +function resultItemFromTurn( + state: CodexSessionState, + turn: Record, + ): Record | null { + if (!Array.isArray(turn.items)) return null; + return ( + turn.items + .map(record) + .reverse() + .find( + (value) => + text(value.type) === "agentMessage" && + isRetainableCodexPayload(value.text) && + tryParseResult(value.text) !== null, + ) ?? null + ); + } + +function resultFromTurn( + state: CodexSessionState, + turn: Record, + ): PrpStructuredRunResult | null { + const item = resultItemFromTurn(state, turn); + return item === null ? null : tryParseResult(item.text); + } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-session-workspace.ts b/packages/paperclip-runner/src/drivers/codex/codex-session-workspace.ts new file mode 100644 index 0000000000..8036589c18 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/codex/codex-session-workspace.ts @@ -0,0 +1,202 @@ +import { parseCodexTurnDiff, type ParsedCodexTurnDiffFile } from "./codex-turn-diff.js"; +import { boundedCodexWorkspaceStat as boundedWorkspaceStat, codexWorkspaceRelativePath as workspaceRelativePath } from "./codex-thread-normalization.js"; +import type { CodexSessionState } from "./codex-session-state.js"; +import { record, text } from "./codex-driver-values.js"; + +export function recordWorkspaceChanges( + state: CodexSessionState, + turnId: string, + value: unknown, + complete: boolean, + ): void { + const changes = Array.isArray(value) ? value : []; + const files = changes + .slice(0, 2_000) + .flatMap((candidate): Record[] => { + const change = record(candidate); + const path = text(change.path).replaceAll("\\", "/"); + if (!path || path.startsWith("/") || path.split("/").includes("..")) + return []; + const kind = change.kind; + const kindRecord = record(kind); + const kindText = text( + kind, + text(kindRecord.type, Object.keys(kindRecord)[0] ?? "update"), + ); + const update = record(kindRecord.update ?? change.update); + const previousPath = + text(update.move_path, text(update.movePath)) || null; + const operation = previousPath + ? "rename" + : kindText.toLowerCase().includes("add") + ? "create" + : kindText.toLowerCase().includes("delete") + ? "delete" + : "modify"; + const diff = text(change.diff).slice(0, 262_144) || null; + const diffLines = diff?.split("\n") ?? []; + return [ + { + path, + operation, + previousPath, + additions: + diff === null + ? null + : diffLines.filter( + (line) => line.startsWith("+") && !line.startsWith("+++"), + ).length, + deletions: + diff === null + ? null + : diffLines.filter( + (line) => line.startsWith("-") && !line.startsWith("---"), + ).length, + binary: diff === null, + diff, + }, + ]; + }); + if (files.length === 0) return; + const unknown = files.some( + (file) => file.additions === null || file.deletions === null, + ); + const payload = { + schema: "paperclip.workspace.diff.v1", + changeSetId: `${turnId}:workspace`, + revision: + Number(record(state.workspaceChangesByTurn.get(turnId)).revision ?? 0) + + 1, + source: "harness_reported", + complete, + files, + totals: { + files: files.length, + additions: unknown + ? null + : files.reduce((sum, file) => sum + Number(file.additions), 0), + deletions: unknown + ? null + : files.reduce((sum, file) => sum + Number(file.deletions), 0), + }, + patchArtifactRef: null, + }; + state.workspaceChangesByTurn.set(turnId, payload); + state.emit("workspace.change.updated", payload, { + turnId, + itemId: `${turnId}:workspace`, + }); + } + +export function recordTurnDiff( + state: CodexSessionState,turnId: string, value: unknown): void { + const diff = text(value); + const files = parseCodexTurnDiff(diff); + // An empty string is an authoritative empty aggregate snapshot. A + // non-empty value that cannot be parsed is left on the bounded diagnostic + // item.delta path instead of erasing the last valid workspace snapshot. + if (files.length === 0 && diff.trim()) return; + recordWorkspaceSnapshot(state, turnId, files); + } + +export function recordCanonicalWorkspaceChange( + state: CodexSessionState,turnId: string, value: unknown): void { + const candidate = record(value); + if (candidate.schema !== "paperclip.workspace.diff.v1") return; + if (!Array.isArray(candidate.files)) return; + const files = candidate.files.slice(0, 2_000).flatMap((value) => { + const file = record(value); + const path = workspaceRelativePath(file.path); + if (path === null) return []; + const operation = text(file.operation); + if ( + operation !== "create" && + operation !== "modify" && + operation !== "delete" && + operation !== "rename" && + operation !== "mode_change" + ) return []; + const previousPath = + file.previousPath === null || file.previousPath === undefined + ? null + : workspaceRelativePath(file.previousPath); + if (operation === "rename" && previousPath === null) return []; + const binary = file.binary === true; + const diff = + binary || file.diff === null || file.diff === undefined + ? null + : typeof file.diff === "string" + ? file.diff.slice(0, 262_144) + : null; + const additions = boundedWorkspaceStat(file.additions); + const deletions = boundedWorkspaceStat(file.deletions); + return [{ + path, + operation: operation as ParsedCodexTurnDiffFile["operation"], + previousPath, + additions: binary ? null : additions, + deletions: binary ? null : deletions, + binary, + diff, + }]; + }); + // Empty is an authoritative snapshot. If the provider supplied entries + // but every one failed validation, retain the previous valid revision. + if (candidate.files.length > 0 && files.length === 0) return; + recordWorkspaceSnapshot(state, + turnId, + files, + candidate.revision, + typeof candidate.patchArtifactRef === "string" + ? candidate.patchArtifactRef.slice(0, 2_048) + : null, + ); + } + +function recordWorkspaceSnapshot( + state: CodexSessionState, + turnId: string, + files: ReturnType, + requestedRevision?: unknown, + patchArtifactRef: string | null = null, + ): void { + const previous = state.workspaceChangesByTurn.get(turnId); + if ( + previous !== undefined && + JSON.stringify(record(previous).files) === JSON.stringify(files) && + record(previous).patchArtifactRef === patchArtifactRef + ) return; + const unknown = files.some( + (file) => file.additions === null || file.deletions === null, + ); + const priorRevision = Number(record(previous).revision ?? 0); + const incomingRevision = + typeof requestedRevision === "number" && + Number.isSafeInteger(requestedRevision) && + requestedRevision > 0 + ? requestedRevision + : 1; + const payload = { + schema: "paperclip.workspace.diff.v1", + changeSetId: `${turnId}:workspace`, + revision: Math.max(priorRevision + 1, incomingRevision), + source: "harness_reported", + complete: false, + files, + totals: { + files: files.length, + additions: unknown + ? null + : files.reduce((sum, file) => sum + Number(file.additions), 0), + deletions: unknown + ? null + : files.reduce((sum, file) => sum + Number(file.deletions), 0), + }, + patchArtifactRef, + }; + state.workspaceChangesByTurn.set(turnId, payload); + state.emit("workspace.change.updated", payload, { + turnId, + itemId: `${turnId}:workspace`, + }); + } diff --git a/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts b/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts index ea596093f2..00a8b5e1b8 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.ts @@ -72,7 +72,7 @@ export function parseCodexThreadGoal(value: unknown): HarnessThreadGoal | null { }; } -function threadStatus(value: unknown): string { +export function codexThreadStatus(value: unknown): string { if (typeof value === "string") return value; return text(record(value).type, "unknown"); } @@ -102,7 +102,7 @@ export function codexThreadLineage(value: unknown): HarnessThreadLineageEntry { ) || null, role: text(thread.agentRole, text(spawn.agent_role ?? spawn.agentRole)) || null, - status: threadStatus(thread.status), + status: codexThreadStatus(thread.status), }; } @@ -116,7 +116,7 @@ export interface BindableCodexNotification { params: Record; } -function supportedCodexNotificationMethod(method: string): boolean { +export function isSupportedCodexNotificationMethod(method: string): boolean { return ( method === "turn/started" || method === "turn/completed" || @@ -155,7 +155,7 @@ export function isBoundCodexNotification( notification: BindableCodexNotification, binding: CodexNotificationBinding, ): boolean { - if (!supportedCodexNotificationMethod(notification.method)) return false; + if (!isSupportedCodexNotificationMethod(notification.method)) return false; const params = record(notification.params); const claimedRunId = text(params.runId, text(params.paperclipRunId)); if (claimedRunId.length > 0 && claimedRunId !== binding.runId) return false; diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 9ea07c4b6c..60c04b33ab 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -2591,21 +2591,24 @@ describe("executeNativeSession recovery", () => { }); }); - it("starts a continuation after the driver clears a stale active terminal turn", async () => { + it("retries disposition recovery when the driver releases an absent bound provider turn", async () => { const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: "driver-recovery", identity, providerSessionId: "provider-recovery", cursor: "1", - activeTurnId: "turn-already-terminal", - terminalTurns: [{ turnId: "turn-already-terminal", fingerprint: "terminal-fingerprint" }], + activeTurnId: null, + terminalTurns: [{ turnId: "turn-work", fingerprint: "terminal-fingerprint" }], + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-missing-disposition", pendingRuntimeRequests: [], lineage: [], }; const recoveredSnapshot: PersistedNativeSession = { ...checkpoint, - activeTurnId: null, + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, }; const terminalEvent: PrpEvent = { schema: "paperclip.prp.event.v1", @@ -2647,6 +2650,292 @@ describe("executeNativeSession recovery", () => { async recoverSession() { return { recovered: true, session }; }, }; const bySource = new Map(); + const replayedPages: PrpEvent[][] = []; + const replayEvents = vi.fn(async (replay: Parameters[0]) => { + const list = bySource.get(replay.sourceInstanceId) ?? []; + const events = structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)); + replayedPages.push(events); + return { + events, + highestContiguousSourceSeq: highestContiguous(list), + }; + }); + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + replayEvents, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" }); + expect(startTurn).toHaveBeenCalledOnce(); + expect(replayEvents).toHaveBeenCalledWith({ + runId: identity.runId, + sourceInstanceId: "runner-recovery", + afterSourceSeq: 0, + limit: 1_000, + }); + expect(replayedPages.every((events) => events.length === 0)).toBe(true); + const recoveryEnvelope = JSON.parse( + startTurn.mock.calls[0]![0].message.text, + ) as { task: { prompt: string } }; + expect(recoveryEnvelope.task.prompt).toContain( + "semantic-result recovery for a prior completed provider turn", + ); + expect(recoveryEnvelope.task.prompt).toContain( + "Do not repeat implementation, tests, research, or the final answer", + ); + expect(recoveryEnvelope.task.prompt).not.toContain(input.task.prompt); + + checkpoint.dispositionOnlyRecoveryTurnId = undefined; + recoveredSnapshot.dispositionOnlyRecoveryTurnId = undefined; + startTurn.mockClear(); + bySource.clear(); + bySource.set("runner-recovery", [{ + ...terminalEvent, + sourceEventId: "runner-recovery:stale-terminal", + turnId: "turn-stale-unbound", + }]); + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + })).resolves.toMatchObject({ + turnId: "turn-continuation", + providerSessionId: "provider-recovery", + }); + expect(startTurn).toHaveBeenCalledOnce(); + }); + + it("consumes an adopted completed disposition turn without starting another turn", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + terminalTurns: [{ turnId: "turn-work", fingerprint: "work-terminal" }], + dispositionOnlyRecoveryConsumed: false, + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveredSnapshot: PersistedNativeSession = { + ...checkpoint, + cursor: "2", + terminalTurns: [ + ...checkpoint.terminalTurns!, + { turnId: "turn-disposition", fingerprint: "disposition-terminal" }, + ], + dispositionOnlyRecoveryConsumed: true, + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "provider-recovery:2", + sourceSeq: 2, + sourceInstanceId: "provider-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-disposition", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:01.000Z", + payload: {}, + }; + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + let dispositionTerminalCommitted = false; + let prematureDispositionCheckpoint = false; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield terminalEvent; }, + startTurn, + async result() { return null; }, + async snapshot() { return structuredClone(recoveredSnapshot); }, + async close() {}, + }; + const events: PrpEvent[] = []; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession(snapshot) { + if ( + snapshot.terminalTurns?.some((turn) => turn.turnId === "turn-disposition") + && !dispositionTerminalCommitted + ) prematureDispositionCheckpoint = true; + }, + async appendEvent(event) { + events.push(structuredClone(event)); + if (event.eventType === "turn.completed" && event.turnId === "turn-disposition") { + dispositionTerminalCommitted = true; + } + return { + cursor: events.length, + highestContiguousSourceSeq: highestContiguous(events), + disposition: "committed", + }; + }, + async replayEvents(replay) { + return { + events: structuredClone(events.filter((event) => + event.sourceInstanceId === replay.sourceInstanceId + && event.sourceSeq > replay.afterSourceSeq + )), + highestContiguousSourceSeq: highestContiguous(events), + }; + }, + async completeRun() {}, + }; + + await expect(executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult: async () => result, + })).resolves.toMatchObject({ + result, + turnId: "turn-disposition", + }); + expect(startTurn).not.toHaveBeenCalled(); + expect(prematureDispositionCheckpoint).toBe(false); + expect(events.map((event) => event.eventType)).toEqual([ + "turn.completed", + "run.result.accepted", + "run.terminal", + ]); + }); + + it("resolves a proposal-less durable disposition terminal through control-plane policy", async () => { + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "1", + activeTurnId: null, + terminalTurns: [{ turnId: "turn-work", fingerprint: "work-terminal" }], + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-disposition", + pendingRuntimeRequests: [], + lineage: [], + }; + const terminalEvent: PrpEvent = { + schema: "paperclip.prp.event.v1", + sourceEventId: "runner-recovery:run-native:4", + sourceSeq: 4, + sourceInstanceId: "runner-recovery", + sourceKind: "provider", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-disposition", + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: "2026-08-09T00:00:01.000Z", + payload: {}, + }; + const resultProposalEvent: PrpEvent = { + ...terminalEvent, + sourceEventId: "runner-recovery:run-native:3", + sourceSeq: 3, + eventType: "run.result.proposed", + payload: result, + }; + const originalTaskTerminal: PrpEvent = { + ...terminalEvent, + sourceEventId: "runner-recovery:run-native:2", + sourceSeq: 2, + turnId: "turn-work", + }; + const originalTaskProposal: PrpEvent = { + ...resultProposalEvent, + sourceEventId: "runner-recovery:run-native:1", + sourceSeq: 1, + turnId: "turn-work", + }; + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + let recoveredSubmissionOwned = true; + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + async *events() { yield structuredClone(terminalEvent); }, + startTurn, + async result() { return null; }, + async snapshot() { + return { + ...structuredClone(checkpoint), + dispositionOnlyRecoveryConsumed: recoveredSubmissionOwned, + terminalTurns: recoveredSubmissionOwned + ? [ + ...structuredClone(checkpoint.terminalTurns ?? []), + { + turnId: "turn-disposition", + fingerprint: "disposition-terminal", + }, + ] + : structuredClone(checkpoint.terminalTurns), + }; + }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const bySource = new Map([ + ["runner-recovery", [ + structuredClone(originalTaskProposal), + structuredClone(originalTaskTerminal), + ]], + ]); const port: ControlPlanePort = { async openRun() {}, async loadSessionCheckpoint() { return structuredClone(checkpoint); }, @@ -2671,23 +2960,177 @@ describe("executeNativeSession recovery", () => { async completeRun() {}, }; - await expect(executeNativeSession({ + const execute = () => executeNativeSession({ input, backend, controlPlane: port, runnerInstanceId: "runner-recovery", controlPlaneInstanceId: "control-recovery", - })).resolves.toMatchObject({ turnId: "turn-continuation", providerSessionId: "provider-recovery" }); - expect(startTurn).toHaveBeenCalledOnce(); - const recoveryEnvelope = JSON.parse( - startTurn.mock.calls[0]![0].message.text, - ) as { task: { prompt: string } }; - expect(recoveryEnvelope.task.prompt).toContain( - "semantic-result recovery for a prior completed provider turn", - ); - expect(recoveryEnvelope.task.prompt).toContain( - "Do not repeat implementation, tests, research, or the final answer", + resolveMissingResult: async ({ terminalEvent: replayed }) => { + expect(replayed).toEqual(terminalEvent); + return result; + }, + }); + await expect(execute()).resolves.toMatchObject({ + result, + turnId: "turn-disposition", + }); + bySource.set("runner-recovery", [ + structuredClone(originalTaskProposal), + structuredClone(originalTaskTerminal), + structuredClone(resultProposalEvent), + structuredClone(terminalEvent), + ]); + // Provider recovery may clear a legacy pre-acceptance marker when thread + // history has no matching turn. Durable replay remains authoritative and + // must still prevent a duplicate disposition submission. + recoveredSubmissionOwned = false; + + await expect(execute()).resolves.toMatchObject({ + result, + turnId: "turn-disposition", + }); + expect(startTurn).not.toHaveBeenCalled(); + expect(bySource.get("runner-recovery")).toEqual([ + originalTaskProposal, + originalTaskTerminal, + resultProposalEvent, + terminalEvent, + ]); + expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([ + "run.result.accepted", + "run.terminal", + ]); + + }); + + it("resolves a checkpointed result-less disposition without resubmitting when its terminal event is missing", async () => { + const workProposal: PrpEvent = { + ...runnerEvent(1, "run.result.proposed", result), + turnId: "turn-work", + }; + const workTerminal: PrpEvent = { + ...runnerEvent(2, "turn.completed"), + turnId: "turn-work", + }; + const checkpoint: PersistedNativeSession = { + backendKind: "mock", + sessionId: "driver-recovery", + identity, + providerSessionId: "provider-recovery", + cursor: "3", + activeTurnId: null, + terminalTurns: [ + { turnId: "turn-work", fingerprint: "work-terminal" }, + { turnId: "turn-disposition", fingerprint: "disposition-terminal" }, + ], + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-disposition", + pendingRuntimeRequests: [], + lineage: [], + }; + const recoveredCheckpoint: PersistedNativeSession = structuredClone(checkpoint); + const startTurn = vi.fn(async () => ({ turnId: "unexpected-turn" })); + const events = vi.fn(() => (async function* () { + throw new Error("checkpoint fallback must not consume provider events"); + })()); + const session: NativeSession = { + identity: () => identity, + async capabilities() { + return { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }; + }, + events, + startTurn, + async result() { return null; }, + async snapshot() { return structuredClone(recoveredCheckpoint); }, + async close() {}, + }; + const backend: NativeSessionBackend = { + async descriptor() { + return { + kind: "mock", + name: "recovery-backend", + version: "1", + capabilities: { resume: true, typedEvents: true, steering: false, interruption: true, structuredResult: true }, + }; + }, + async openSession() { throw new Error("must recover the provider session"); }, + async recoverSession() { return { recovered: true, session }; }, + }; + const bySource = new Map([ + ["runner-recovery", [workProposal, workTerminal]], + ]); + const completeRun = vi.fn(async () => undefined); + const port: ControlPlanePort = { + async openRun() {}, + async loadSessionCheckpoint() { return structuredClone(checkpoint); }, + async checkpointSession() {}, + async appendEvent(event) { + const list = bySource.get(event.sourceInstanceId) ?? []; + list.push(structuredClone(event)); + bySource.set(event.sourceInstanceId, list); + return { + cursor: list.length, + highestContiguousSourceSeq: highestContiguous(list), + disposition: "committed", + }; + }, + async replayEvents(replay) { + const list = bySource.get(replay.sourceInstanceId) ?? []; + return { + events: structuredClone(list.filter((event) => event.sourceSeq > replay.afterSourceSeq)), + highestContiguousSourceSeq: highestContiguous(list), + }; + }, + completeRun, + }; + + const execute = () => executeNativeSession({ + input, + backend, + controlPlane: port, + runnerInstanceId: "runner-recovery", + controlPlaneInstanceId: "control-recovery", + resolveMissingResult: async ({ turnId, terminalEvent }) => { + expect(turnId).toBe("turn-disposition"); + expect(terminalEvent).toMatchObject({ + sourceInstanceId: "control-recovery", + sourceKind: "control_plane", + runId: identity.runId, + normalizedSessionId: identity.sessionId, + turnId: "turn-disposition", + eventType: "turn.completed", + payload: { + recovery: "checkpointed_resultless_disposition", + terminalFingerprint: "disposition-terminal", + }, + }); + return result; + }, + }); + await expect(execute()).resolves.toMatchObject({ + result, + turnId: "turn-disposition", + }); + expect(startTurn).not.toHaveBeenCalled(); + expect(events).not.toHaveBeenCalled(); + expect(completeRun).toHaveBeenCalledOnce(); + expect(bySource.get("runner-recovery")).toEqual([ + workProposal, + workTerminal, + ]); + expect(bySource.get("control-recovery")?.map((event) => event.eventType)).toEqual([ + "run.result.accepted", + "run.terminal", + ]); + + recoveredCheckpoint.terminalTurns![1]!.fingerprint = "conflicting-terminal"; + await expect(execute()).rejects.toThrow( + "native_disposition_recovery_checkpoint_conflict", ); + expect(startTurn).not.toHaveBeenCalled(); + expect(events).not.toHaveBeenCalled(); + expect(completeRun).toHaveBeenCalledOnce(); }); it("recovers a completed checkpoint and appends only a missing control terminal fact", async () => { diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 66e4696131..31d89f3ddf 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -617,6 +617,126 @@ async function reconcileRecoveryCursor(input: { return { ...input.checkpoint, cursor: String(persistedHighWater) }; } +async function replayCheckpointedTurnTerminal(input: { + controlPlane: ControlPlanePort; + runId: string; + sourceInstanceId: string; + priorTerminalTurnIds: readonly string[]; + expectedTurnId?: string | null; +}): Promise<{ terminal: PrpEvent; hasPriorResultProposal: boolean } | null> { + let afterSourceSeq = 0; + const terminals: PrpEvent[] = []; + const latestResultProposalByTurn = new Map(); + const priorTerminalTurnIds = new Set(input.priorTerminalTurnIds); + while (true) { + const replay = await input.controlPlane.replayEvents({ + runId: input.runId, + sourceInstanceId: input.sourceInstanceId, + afterSourceSeq, + limit: 1_000, + }); + if (replay.events.length === 0) { + // Disposition recovery owns the newest durable provider terminal. An + // older task turn may also have a valid proposal, but selecting it would + // finalize stale work and strand the actual recovery terminal. + const terminal = [...terminals] + .sort((left, right) => right.sourceSeq - left.sourceSeq)[0] ?? null; + const proposalSequence = latestResultProposalByTurn.get(terminal?.turnId ?? "") ?? 0; + return terminal === null + ? null + : { + terminal, + hasPriorResultProposal: + proposalSequence > 0 && proposalSequence < terminal.sourceSeq, + }; + } + for (const event of replay.events) { + if (event.turnId && event.eventType === "run.result.proposed") { + latestResultProposalByTurn.set( + event.turnId, + Math.max( + latestResultProposalByTurn.get(event.turnId) ?? 0, + event.sourceSeq, + ), + ); + } + if ( + event.turnId && + isTurnTerminal(event) && + ( + input.expectedTurnId !== undefined && input.expectedTurnId !== null + ? event.turnId === input.expectedTurnId + : !priorTerminalTurnIds.has(event.turnId) + ) + ) { + terminals.push(structuredClone(event)); + } + } + const pageHighWater = replay.events.reduce( + (highest, event) => Math.max(highest, event.sourceSeq), + afterSourceSeq, + ); + if (pageHighWater <= afterSourceSeq) { + throw new Error("native_recovery_replay_did_not_advance"); + } + afterSourceSeq = pageHighWater; + } +} + +function checkpointedResultlessDispositionFallback(input: { + persisted: PersistedNativeSession; + recovered: PersistedNativeSession; + controlPlaneInstanceId: string; +}): PrpEvent | null { + const turnId = input.persisted.dispositionOnlyRecoveryTurnId; + if ( + !input.persisted.dispositionOnlyRecoveryConsumed + || input.persisted.semanticResult + || input.persisted.activeTurnId + || typeof turnId !== "string" + || turnId.length === 0 + ) return null; + const persistedTerminal = input.persisted.terminalTurns?.filter( + (terminal) => terminal.turnId === turnId, + ) ?? []; + if (persistedTerminal.length !== 1 || persistedTerminal[0]!.fingerprint.length === 0) { + return null; + } + const recoveredTurnId = input.recovered.dispositionOnlyRecoveryTurnId; + const recoveredTerminal = input.recovered.terminalTurns?.filter( + (terminal) => terminal.turnId === recoveredTurnId, + ) ?? []; + if ( + !input.recovered.dispositionOnlyRecoveryConsumed + || input.recovered.semanticResult + || input.recovered.activeTurnId + || recoveredTurnId !== turnId + || recoveredTerminal.length !== 1 + || recoveredTerminal[0]!.fingerprint !== persistedTerminal[0]!.fingerprint + || canonicalJson(input.recovered.identity) !== canonicalJson(input.persisted.identity) + ) { + throw new Error("native_disposition_recovery_checkpoint_conflict"); + } + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `${input.controlPlaneInstanceId}:${input.persisted.identity.runId}:checkpointed-disposition-terminal`, + sourceSeq: 1, + sourceInstanceId: input.controlPlaneInstanceId, + sourceKind: "control_plane", + runId: input.persisted.identity.runId, + normalizedSessionId: input.persisted.identity.sessionId, + turnId, + eventType: "turn.completed", + schemaVersion: 1, + priority: 0, + emittedAt: new Date().toISOString(), + payload: { + recovery: "checkpointed_resultless_disposition", + terminalFingerprint: persistedTerminal[0]!.fingerprint, + }, + }; +} + /** * Package-owned normalized session loop. Paperclip supplies persistence and * authority through ControlPlanePort; provider/session behavior stays here. @@ -866,6 +986,16 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) await persistCheckpoint(snapshot, signal); }; const recoveredSnapshot = await session.snapshot(); + const recoveredActiveTurnId = recovered + ? recoveredSnapshot.activeTurnId ?? null + : persistedSession?.activeTurnId ?? null; + const adoptedDispositionTerminal = Boolean( + recovered + && recoveredSnapshot.dispositionOnlyRecoveryConsumed + && !recoveredActiveTurnId + && (recoveredSnapshot.terminalTurns?.length ?? 0) + > (persistedSession?.terminalTurns?.length ?? 0) + ); if (continuityBreak) { await options.onContinuityBreak?.({ ...continuityBreak, @@ -874,7 +1004,15 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) recoveredSnapshot.providerSessionId ?? null, }); } - await persistCheckpoint(recoveredSnapshot); + // recoverSession may adopt a provider terminal and enqueue its normalized + // event before returning. Do not checkpoint that terminal fingerprint + // until consumeTurn has durably appended the event: if this process dies + // first, retaining the older checkpoint lets the next recovery adopt and + // emit the same provider terminal again instead of reconstructing a closed + // session with no event to finalize. + if (!adoptedDispositionTerminal) { + await persistCheckpoint(recoveredSnapshot); + } let consumed = { event: null as PrpEvent | null, @@ -893,34 +1031,96 @@ export async function executeNativeSession(options: ExecuteNativeSessionOptions) } : null; if (!completed) { - const consumptionAbort = new AbortController(); - const consuming = consumeTurn( - session, - options.controlPlane, - options.timeoutMs ?? 900_000, - options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, - closeSession, - quarantineSession, - options.resolveGovernedWait, - consumptionAbort.signal, - ); - // Event consumption must begin before startTurn so an eager provider cannot - // outrun us. Observe its rejection immediately, though: if startTurn or - // checkpointing fails first, the outer finally closes the session and the - // abandoned consumer will reject when its stream closes. Without a handler - // that later rejection becomes process-fatal under Node's strict policy. - void consuming.catch(() => undefined); // A recovered driver is authoritative about whether a provider turn is // still active. In particular, drivers normalize the checkpoint race // where a terminal fingerprint was persisted before activeTurnId was // cleared. Falling back to the older control-plane checkpoint here // resurrects that terminal turn and waits forever for an event that was // already consumed. - const recoveredActiveTurnId = recovered - ? recoveredSnapshot.activeTurnId ?? null - : persistedSession?.activeTurnId ?? null; + const dispositionRecoveryWasSubmitted = Boolean( + recovered + && persistedSession?.dispositionOnlyRecoveryConsumed + && !recoveredSnapshot.semanticResult + && !recoveredActiveTurnId + ); + const dispositionRecoveryTurnId = + persistedSession?.dispositionOnlyRecoveryTurnId + ?? recoveredSnapshot.dispositionOnlyRecoveryTurnId + ?? null; + const recoveredDispositionTurnObserved = Boolean( + dispositionRecoveryTurnId + && recoveredSnapshot.terminalTurns?.some( + (terminal) => terminal.turnId === dispositionRecoveryTurnId, + ) + ); + const dispositionRecoveryStillOwned = Boolean( + dispositionRecoveryWasSubmitted + && recoveredSnapshot.dispositionOnlyRecoveryConsumed + && dispositionRecoveryTurnId !== null + && recoveredDispositionTurnObserved + ); + const replayedDisposition = + dispositionRecoveryWasSubmitted && dispositionRecoveryTurnId !== null + ? await replayCheckpointedTurnTerminal({ + controlPlane: options.controlPlane, + runId: input.binding.runId, + sourceInstanceId: options.runnerInstanceId, + priorTerminalTurnIds: (persistedSession?.terminalTurns ?? []) + .map((terminal) => terminal.turnId), + expectedTurnId: dispositionRecoveryTurnId, + }) + : null; + // Durable replay remains authoritative. If it has no terminal, an exact + // consumed marker bound to the same terminal fingerprint in both + // checkpoints proves provider completion without reconstructing provider + // output. Give only that non-provider fact to control-plane policy; a + // mismatch fails closed and a null policy result fails finalization. + const dispositionFallback = + dispositionRecoveryWasSubmitted && replayedDisposition === null && persistedSession + ? checkpointedResultlessDispositionFallback({ + persisted: persistedSession, + recovered: recoveredSnapshot, + controlPlaneInstanceId: options.controlPlaneInstanceId, + }) + : null; + const checkpointedDispositionTerminal = + replayedDisposition !== null || dispositionFallback !== null; + const recoveryTerminal = replayedDisposition?.terminal ?? dispositionFallback; + const consumptionAbort = new AbortController(); + const consuming = recoveryTerminal === null + ? consumeTurn( + session, + options.controlPlane, + options.timeoutMs ?? 900_000, + options.runtimeInputLiveWindowMs ?? DEFAULT_NATIVE_RUNTIME_INPUT_LIVE_WINDOW_MS, + closeSession, + quarantineSession, + options.resolveGovernedWait, + consumptionAbort.signal, + ) + : Promise.resolve({ + event: recoveryTerminal, + eventCount: 0, + highestContiguousSourceSeq: + replayedDisposition === null ? 0 : recoveryTerminal.sourceSeq, + governedResult: null, + }); + // Event consumption must begin before startTurn so an eager provider cannot + // outrun us. Observe its rejection immediately, though: if startTurn or + // checkpointing fails first, the outer finally closes the session and the + // abandoned consumer will reject when its stream closes. Without a handler + // that later rejection becomes process-fatal under Node's strict policy. + void consuming.catch(() => undefined); try { - if (!recovered || !recoveredActiveTurnId) { + if ( + !recovered + || ( + !recoveredActiveTurnId + && !adoptedDispositionTerminal + && !checkpointedDispositionTerminal + && !dispositionRecoveryStillOwned + ) + ) { const modelEnvelope = buildNativeModelEnvelope(input); const dispositionOnlyRecovery = Boolean( recovered &&