diff --git a/packages/paperclip-runner/protocol/fixtures/codex-driver/driver-conformance.json b/packages/paperclip-runner/protocol/fixtures/codex-driver/driver-conformance.json new file mode 100644 index 0000000000..4461a1fe8d --- /dev/null +++ b/packages/paperclip-runner/protocol/fixtures/codex-driver/driver-conformance.json @@ -0,0 +1,111 @@ +{ + "schema": "paperclip.runner.live-console.conformance.v1", + "codexVersion": "0.132.0", + "runtimeRequests": [ + { + "id": "command-accept-session", + "method": "item/commandExecution/requestApproval", + "requestKind": "command_approval", + "resolution": { "action": "accept_for_session" }, + "expectedResponse": { "decision": "acceptForSession" } + }, + { + "id": "file-reject", + "method": "item/fileChange/requestApproval", + "requestKind": "file_approval", + "resolution": { "action": "decline" }, + "expectedResponse": { "decision": "decline" } + }, + { + "id": "permission-accept-turn", + "method": "item/permissions/requestApproval", + "requestKind": "permission_approval", + "resolution": { "action": "accept" }, + "expectedResponse": { "permissions": {}, "scope": "turn" } + }, + { + "id": "user-input-submit", + "method": "item/tool/requestUserInput", + "requestKind": "user_input", + "resolution": { + "action": "submit", + "answers": { "preferred_name": { "answers": ["Ada"] } } + }, + "expectedResponse": { + "answers": { "preferred_name": { "answers": ["Ada"] } } + } + }, + { + "id": "elicitation-cancel", + "method": "mcpServer/elicitation/request", + "requestKind": "elicitation", + "resolution": { "action": "cancel" }, + "expectedResponse": { "action": "cancel", "content": null, "_meta": null } + } + ], + "goals": [ + { + "action": "get", + "method": "thread/goal/get", + "params": {} + }, + { + "action": "set", + "method": "thread/goal/set", + "params": { "objective": "Ship the Live console tracer", "status": "active", "tokenBudget": 4096 } + }, + { + "action": "pause", + "method": "thread/goal/set", + "params": { "status": "paused" } + }, + { + "action": "resume", + "method": "thread/goal/set", + "params": { "status": "active" } + }, + { + "action": "clear", + "method": "thread/goal/clear", + "params": {} + } + ], + "lineage": { + "rootThreadId": "thread-root", + "childThread": { + "id": "thread-child", + "sessionId": "provider-session-1", + "source": { + "subAgent": { + "thread_spawn": { + "parent_thread_id": "thread-root", + "depth": 1, + "agent_path": ["researcher"], + "agent_nickname": "Scout", + "agent_role": "researcher" + } + } + }, + "agentNickname": "Scout", + "agentRole": "researcher" + } + }, + "controls": { + "sameTurnSteer": { "turnId": "turn-1", "expected": "acknowledged" }, + "staleTurnSteer": { "turnId": "turn-stale", "expected": "stale_turn" }, + "interruptBeforeStart": { "expected": "queued" }, + "interruptAfterTerminal": { "expected": "already_terminal" } + }, + "reconnect": { + "runId": "run-liveConsole", + "normalizedSessionId": "normalized-liveConsole", + "driverSessionId": "thread-root", + "providerSessionId": "provider-session-1", + "lastSourceSequence": 17 + }, + "redactionMarkers": [ + "Bearer browser-secret", + "OPENAI_API_KEY=provider-secret", + "https://user:password@example.test/path?token=query-secret" + ] +} diff --git a/packages/paperclip-runner/protocol/manifest.json b/packages/paperclip-runner/protocol/manifest.json index cba9dcbeee..591bbd88ac 100644 --- a/packages/paperclip-runner/protocol/manifest.json +++ b/packages/paperclip-runner/protocol/manifest.json @@ -109,6 +109,12 @@ } ], "fixtures": [ + { + "path": "fixtures/codex-driver/driver-conformance.json", + "sha256": "55bbd66bb0d6641a64eedfaf02bb067c65a8b51c46d76d3052eeb99343f53176", + "expectation": "accept", + "compatibilityCase": "canonical" + }, { "path": "fixtures/conformance-expected-output.json", "sha256": "5a643639c9df0a2925ba95c5c5dfa2217605018bfa0dd3954ab4aca5fcfdc2c6", diff --git a/packages/paperclip-runner/src/mock-core/codex-runner.test.ts b/packages/paperclip-runner/src/mock-core/codex-runner.test.ts new file mode 100644 index 0000000000..c193b91219 --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/codex-runner.test.ts @@ -0,0 +1,376 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { describe, expect, it } from "vitest"; + +import { + CODEX_CODEX_PROTOCOL_VERSION, + CODEX_SKILLLESS_BASE_INSTRUCTIONS, + createCodexTaskEnvelope, + type CodexModelContextSnapshot, +} from "../contracts/codex.js"; +import type { + HarnessDriver, + HarnessSession, + OpenHarnessSessionInput, +} from "../contracts/harness-driver.js"; +import type { + PrpEvent, + PrpStructuredRunResult, +} from "../protocol/replay-contract.js"; +import { loadLiveConsoleConformanceFixture } from "../protocol/live-console-fixture.js"; +import { + runCodexCodexTracer, + validateCodexResultProposal, +} from "./codex-runner.js"; + +const envelope = createCodexTaskEnvelope({ + objective: "Create hello.txt with the text hello.", + criteria: [{ id: "file", requirement: "hello.txt contains hello" }], +}); + +function completedResult(): PrpStructuredRunResult { + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Created hello.txt.", + completionClaim: { + contractRevision: envelope.completionContract.revision, + objectiveSatisfied: true, + criteria: [{ + criterionId: "file", + status: "satisfied", + evidenceRefs: ["hello.txt"], + }], + remainingWork: [], + }, + evidence: [{ ref: "hello.txt" }], + verification: [{ commandOrCheck: "read hello.txt", status: "passed" }], + attentionRequests: [], + artifacts: [{ kind: "file", ref: "hello.txt" }], + }; +} + +class TraceConformanceDriver implements HarnessDriver { + constructor( + private readonly result: PrpStructuredRunResult = completedResult(), + private readonly terminalEvent: "turn.completed" | "turn.interrupted" = "turn.completed", + ) {} + + async descriptor() { + return { + kind: "codex-trace-fixture", + displayName: "Codex trace fixture", + version: "1.0.0", + protocolVersion: CODEX_CODEX_PROTOCOL_VERSION, + capabilities: { + resume: false, + typedEvents: true, + steering: false, + interruption: false, + structuredResult: true, + dynamicTools: false, + }, + }; + } + + async openSession(input: OpenHarnessSessionInput): Promise { + const turnId = "turn-trace"; + let releaseEvents: (() => void) | undefined; + const started = new Promise((resolve) => { + releaseEvents = resolve; + }); + const events: PrpEvent[] = []; + const result = structuredClone(this.result); + const context: CodexModelContextSnapshot = { + protocolVersion: CODEX_CODEX_PROTOCOL_VERSION, + codexVersion: "trace-fixture", + clientInfo: { + name: "paperclip-runner", + title: "Paperclip Runner", + version: "1.0.0", + }, + model: "codex-fixture", + modelProvider: "openai", + workingDirectory: input.workingDirectory, + collaborationMode: "default", + sandbox: { type: "workspaceWrite" }, + approvalPolicy: "never", + baseInstructions: CODEX_SKILLLESS_BASE_INSTRUCTIONS, + instructionSources: [], + instructionPolicy: { + skillInstructions: false, + appInstructions: false, + collaborationInstructions: true, + }, + environmentKeys: [], + dynamicToolNames: [], + modelInputKinds: ["text"], + envelope, + }; + const event = ( + sourceSeq: number, + eventType: PrpEvent["eventType"], + payload: Record, + withTurn = true, + ): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `provider:${sourceSeq}`, + sourceSeq, + sourceInstanceId: "codex-trace-provider", + sourceKind: "runner", + runId: input.runId, + normalizedSessionId: input.normalizedSessionId, + ...(withTurn ? { turnId } : {}), + eventType, + schemaVersion: 1, + priority: eventType === "run.result.proposed" ? 0 : 1, + emittedAt: new Date(Date.UTC(2026, 7, 27, 12, 0, sourceSeq)).toISOString(), + payload, + }); + + return { + ids: () => ({ + driverSessionId: "driver-trace", + providerSessionId: "provider-trace", + }), + events: async function* () { + await started; + for (const candidate of events) yield structuredClone(candidate); + }, + startTurn: async () => { + events.push( + event(1, "session.started", { context }, false), + event(2, "turn.started", {}), + event(3, "run.result.proposed", result), + event(4, this.terminalEvent, {}), + ); + releaseEvents?.(); + return { turnId }; + }, + snapshot: async () => ({ + driverKind: "codex-trace-fixture", + driverSessionId: "driver-trace", + providerSessionId: "provider-trace", + runId: input.runId, + normalizedSessionId: input.normalizedSessionId, + activeTurnId: null, + lastSourceSequence: 4, + }), + close: async () => undefined, + }; + } +} + +describe("Codex trace conformance", () => { + it("accepts only results that satisfy the exact controller envelope", () => { + expect(validateCodexResultProposal(completedResult(), envelope)).toMatchObject({ + status: "accepted", + }); + + const wrongRevision = completedResult(); + wrongRevision.completionClaim.contractRevision = "wrong-revision"; + expect(validateCodexResultProposal(wrongRevision, envelope)).toMatchObject({ + status: "rejected", + issues: [{ code: "contract_revision_mismatch" }], + }); + + const unknownCriterion = completedResult(); + unknownCriterion.completionClaim.criteria = [{ + criterionId: "not-in-envelope", + status: "satisfied", + evidenceRefs: [], + }]; + const decision = validateCodexResultProposal(unknownCriterion, envelope); + expect(decision).toMatchObject({ status: "rejected" }); + if (decision.status === "rejected") { + expect(decision.issues.map((issue) => issue.code)).toEqual( + expect.arrayContaining(["unknown_criterion", "missing_criterion"]), + ); + } + }); + + it("loads the checked-in provider conformance fixture through validation", async () => { + const fixturePath = fileURLToPath(new URL( + "../../protocol/fixtures/codex-driver/driver-conformance.json", + import.meta.url, + )); + await expect(loadLiveConsoleConformanceFixture(fixturePath)).resolves.toMatchObject({ + schema: "paperclip.runner.live-console.conformance.v1", + runtimeRequests: expect.arrayContaining([ + expect.objectContaining({ requestKind: "user_input" }), + ]), + reconnect: { lastSourceSequence: 17 }, + }); + }); + + it("rejects malformed runtime request, goal, and control declarations", async () => { + const fixturePath = fileURLToPath(new URL( + "../../protocol/fixtures/codex-driver/driver-conformance.json", + import.meta.url, + )); + const source = JSON.parse(await readFile(fixturePath, "utf8")); + const directory = await mkdtemp(join(tmpdir(), "paperclip-live-fixture-")); + const candidatePath = join(directory, "candidate.json"); + try { + source.runtimeRequests[0].requestKind = "file_approval"; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("invalid runtime request case"); + + source.runtimeRequests[0].requestKind = "command_approval"; + source.runtimeRequests[0].expectedResponse.decision = "decline"; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("invalid runtime request case"); + + source.runtimeRequests[0].expectedResponse.decision = "acceptForSession"; + source.goals[0].method = "thread/goal/wrong"; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("invalid goal operation"); + + source.goals[0].method = "thread/goal/get"; + delete source.goals[1].params.objective; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("invalid goal operation"); + + source.goals[1].params.objective = "Ship the Live console tracer"; + source.controls.sameTurnSteer.expected = 42; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("controls are invalid"); + + source.controls.sameTurnSteer.expected = "arbitrary_outcome"; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("controls are invalid"); + + source.controls.sameTurnSteer.expected = "acknowledged"; + source.controls.unsupportedControl = { expected: "queued" }; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("controls are invalid"); + + delete source.controls.unsupportedControl; + delete source.controls.sameTurnSteer.turnId; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("controls are invalid"); + + source.controls.sameTurnSteer.turnId = "turn-active"; + const spawn = source.lineage.childThread.source.subAgent.thread_spawn; + delete spawn.parent_thread_id; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("identity or lineage is incomplete"); + + spawn.parent_thread_id = source.lineage.rootThreadId; + spawn.agent_path = [""]; + await writeFile(candidatePath, JSON.stringify(source)); + await expect(loadLiveConsoleConformanceFixture(candidatePath)) + .rejects.toThrow("identity or lineage is incomplete"); + } finally { + await rm(directory, { recursive: true, force: true }); + } + }); + + it("executes live trace and persisted replay with controller-owned terminal facts", async () => { + const trace = await runCodexCodexTracer({ + driver: new TraceConformanceDriver(), + taskEnvelope: envelope, + workingDirectory: "/trace-workspace", + timeoutMs: 1_000, + }); + + expect(trace.resultDecision.status).toBe("accepted"); + expect(trace.events.filter((event) => event.eventType === "run.terminal")) + .toHaveLength(1); + expect(trace.assertions).toEqual({ + exactlyOneTerminalResult: true, + proposalAccepted: true, + liveReplayParity: true, + stableIdentity: true, + sourceSequenceContinuous: true, + stableItemIdentity: true, + contextIsSkillless: true, + unrelatedSkillsAbsent: true, + credentialsAbsent: true, + }); + expect(trace.replaySnapshot).toEqual(trace.liveSnapshot); + }); + + it("fails the run when the controller rejects a completed provider proposal", async () => { + const rejectedResult = completedResult(); + rejectedResult.completionClaim.contractRevision = "stale-contract-revision"; + const trace = await runCodexCodexTracer({ + driver: new TraceConformanceDriver(rejectedResult), + taskEnvelope: envelope, + workingDirectory: "/trace-workspace", + timeoutMs: 1_000, + }); + + expect(trace.resultDecision.status).toBe("rejected"); + expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload) + .toMatchObject({ + turnTerminalState: "completed", + runTerminalState: "failed", + reportedWorkDisposition: "yielded", + }); + expect(trace.liveSnapshot.terminal).toMatchObject({ + turnTerminalState: "completed", + runTerminalState: "failed", + }); + expect(trace.replaySnapshot).toEqual(trace.liveSnapshot); + }); + + it("preserves cancellation when an interrupted provider proposal is rejected", async () => { + const rejectedResult = completedResult(); + rejectedResult.completionClaim.contractRevision = "stale-contract-revision"; + const trace = await runCodexCodexTracer({ + driver: new TraceConformanceDriver(rejectedResult, "turn.interrupted"), + taskEnvelope: envelope, + workingDirectory: "/trace-workspace", + timeoutMs: 1_000, + }); + + expect(trace.resultDecision.status).toBe("rejected"); + expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload) + .toMatchObject({ + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }); + expect(trace.replaySnapshot).toEqual(trace.liveSnapshot); + }); + + it("does not report completed work when an accepted proposal is interrupted", async () => { + const trace = await runCodexCodexTracer({ + driver: new TraceConformanceDriver(completedResult(), "turn.interrupted"), + taskEnvelope: envelope, + workingDirectory: "/trace-workspace", + timeoutMs: 1_000, + }); + + expect(trace.result).toBeNull(); + expect(trace.resultDecision).toMatchObject({ + status: "rejected", + result: null, + issues: [{ path: "/turn/terminal" }], + }); + expect(trace.assertions.proposalAccepted).toBe(false); + expect(trace.events.some((event) => event.eventType === "run.result.accepted")).toBe(false); + expect(trace.events.find((event) => event.eventType === "run.result.rejected")?.payload) + .toMatchObject({ reasonCode: "provider_turn_did_not_complete" }); + expect(trace.events.find((event) => event.eventType === "run.terminal")?.payload) + .toMatchObject({ + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }); + expect(trace.replaySnapshot).toEqual(trace.liveSnapshot); + }); +}); diff --git a/packages/paperclip-runner/src/mock-core/codex-runner.ts b/packages/paperclip-runner/src/mock-core/codex-runner.ts new file mode 100644 index 0000000000..c690d512d7 --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/codex-runner.ts @@ -0,0 +1,574 @@ +import type { HarnessDriver, HarnessSession } from "../contracts/harness-driver.js"; +import { + isSkilllessCodexContext, + type CodexResultDecision, + type CodexResultValidationIssue, + type CodexRunMetadata, + type CodexRunTrace, + type CodexTaskEnvelope, +} from "../contracts/codex.js"; +import type { NativeUserMessage } from "../contracts/types.js"; +import { + applyPrpEvent, + createSessionSnapshotFromMetadata, +} from "../reducer/session-reducer.js"; +import { + validatePrpEvent, + validatePrpStructuredRunResult, + type PrpCapabilities, + type PrpEvent, + type PrpStructuredRunResult, + type PrpTerminalState, +} from "../protocol/replay-contract.js"; + +const MAX_PERSISTED_CODEX_BYTES = 8 * 1024 * 1024; +const MAX_PERSISTED_CODEX_LINE_BYTES = 256 * 1024; +const MAX_PERSISTED_CODEX_EVENTS = 4096; + +export interface CodexCodexTracerInput { + driver: HarnessDriver; + taskEnvelope: CodexTaskEnvelope; + workingDirectory: string; + message?: NativeUserMessage; + runId?: string; + normalizedSessionId?: string; + companyId?: string; + issueId?: string; + environmentLeaseId?: string; + runnerInstanceId?: string; + controlPlaneInstanceId?: string; + steer?: string; + interrupt?: boolean; + timeoutMs?: number; +} + +function prpCapabilities( + descriptor: Awaited>, +): PrpCapabilities { + return { + schema: "paperclip.prp.capabilities.v1", + sessionReusePolicy: "reuse_per_issue", + driver: { kind: descriptor.kind, version: descriptor.version }, + steer: descriptor.capabilities.steering, + interrupt: descriptor.capabilities.interruption, + resume: descriptor.capabilities.resume, + runtimeRequests: true, + structuredResult: descriptor.capabilities.structuredResult, + typedEvents: descriptor.capabilities.typedEvents, + ...(descriptor.capabilities.unsupported?.length + ? { unsupported: descriptor.capabilities.unsupported } + : {}), + }; +} + +function contextFrom(events: PrpEvent[]) { + const event = events.find( + (candidate) => + candidate.eventType === "session.started" || candidate.eventType === "session.resumed", + ); + const context = event?.payload.context; + if (typeof context !== "object" || context === null || Array.isArray(context)) { + throw new Error("Harness session did not emit its model-context snapshot"); + } + return context as CodexRunTrace["context"]; +} + +function dispositionIssues( + result: PrpStructuredRunResult, +): CodexResultValidationIssue[] { + const issues: CodexResultValidationIssue[] = []; + const claim = result.completionClaim; + if (result.reportedWorkDisposition === "done") { + if ( + claim.objectiveSatisfied !== true || + claim.criteria.some((criterion) => criterion.status !== "satisfied") || + claim.remainingWork.some((remaining) => remaining.blocksCompletion) || + result.blocker !== undefined + ) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "done requires a satisfied objective and criteria with no blocking work or blocker", + }); + } + } else if (result.reportedWorkDisposition === "needs_review") { + if (result.blocker !== undefined || result.attentionRequests.length === 0) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "needs_review requires an attention request and must not include a blocker", + }); + } + } else if (result.reportedWorkDisposition === "blocked") { + if ( + claim.objectiveSatisfied !== false || + result.blocker === undefined || + !claim.remainingWork.some((remaining) => remaining.blocksCompletion) + ) { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: "blocked requires an unsatisfied objective, blocker details, and blocking remaining work", + }); + } + } else { + issues.push({ + code: "invalid_disposition", + path: "/reportedWorkDisposition", + message: `${result.reportedWorkDisposition} is not a Codex terminal proposal`, + }); + } + return issues; +} + +/** Validate an advisory provider proposal against the exact controller-owned task envelope. */ +export function validateCodexResultProposal( + proposal: unknown, + envelope: CodexTaskEnvelope, +): CodexResultDecision { + const schema = validatePrpStructuredRunResult(proposal); + if (!schema.ok) { + return { + status: "rejected", + result: null, + issues: schema.issues.map((issue) => ({ + code: "schema_validation", + path: issue.path, + message: issue.message, + })), + }; + } + + const result = schema.result; + const issues: CodexResultValidationIssue[] = []; + if (result.completionClaim.contractRevision !== envelope.completionContract.revision) { + issues.push({ + code: "contract_revision_mismatch", + path: "/completionClaim/contractRevision", + message: `expected exact contract revision ${envelope.completionContract.revision}`, + }); + } + + const expectedIds = new Set(envelope.completionContract.criteria.map((criterion) => criterion.id)); + const observed = new Map(); + result.completionClaim.criteria.forEach((criterion, index) => { + observed.set(criterion.criterionId, (observed.get(criterion.criterionId) ?? 0) + 1); + if (!expectedIds.has(criterion.criterionId)) { + issues.push({ + code: "unknown_criterion", + path: `/completionClaim/criteria/${index}/criterionId`, + message: `criterion ${criterion.criterionId} is not in the task envelope`, + }); + } + }); + for (const criterionId of expectedIds) { + const count = observed.get(criterionId) ?? 0; + if (count === 0) { + issues.push({ + code: "missing_criterion", + path: "/completionClaim/criteria", + message: `criterion ${criterionId} is missing`, + }); + } else if (count > 1) { + issues.push({ + code: "duplicate_criterion", + path: "/completionClaim/criteria", + message: `criterion ${criterionId} appears more than once`, + }); + } + } + issues.push(...dispositionIssues(result)); + + return issues.length === 0 + ? { status: "accepted", result: structuredClone(result), issues: [] } + : { status: "rejected", result: null, issues }; +} + +async function consumeToTurnTerminal( + session: HarnessSession, + timeoutMs: number, + onEvent?: (event: PrpEvent) => void, +): Promise { + const consume = async () => { + const events: PrpEvent[] = []; + for await (const event of session.events()) { + events.push(event); + onEvent?.(event); + if (isTurnTerminal(event)) return events; + } + throw new Error("Harness event stream closed before a turn terminal fact"); + }; + let timer: ReturnType | undefined; + try { + return await Promise.race([ + consume(), + new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`Codex tracer timed out after ${timeoutMs}ms`)), + timeoutMs, + ); + }), + ]); + } finally { + if (timer !== undefined) clearTimeout(timer); + } +} + +function isTurnTerminal(event: PrpEvent): boolean { + return ["turn.completed", "turn.failed", "turn.interrupted", "turn.cancelled"].includes( + event.eventType, + ); +} + +function runtimeTerminal(event: PrpEvent): Pick { + if (event.eventType === "turn.completed") { + return { turnTerminalState: "completed", runTerminalState: "succeeded" }; + } + if (event.eventType === "turn.failed") { + return { turnTerminalState: "failed", runTerminalState: "failed" }; + } + if (event.eventType === "turn.interrupted") { + return { turnTerminalState: "interrupted", runTerminalState: "cancelled" }; + } + return { turnTerminalState: "cancelled", runTerminalState: "cancelled" }; +} + +function sourceSequenceContinuous(events: PrpEvent[]): boolean { + const cursors = new Map(); + for (const event of events) { + const key = `${event.sourceKind}:${event.sourceInstanceId}`; + const previous = cursors.get(key); + if (previous !== undefined && event.sourceSeq !== previous + 1) return false; + cursors.set(key, event.sourceSeq); + } + return true; +} + +/** Parse and validate the persisted boundary before replaying controller state. */ +export function replayPersistedCodexEvents( + serialized: string, + metadata: CodexRunMetadata, +) { + if (!metadata.identity.driverSessionId) { + throw new Error("Persisted Codex replay requires a driver session identity"); + } + if (Buffer.byteLength(serialized) > MAX_PERSISTED_CODEX_BYTES) { + throw new Error("Persisted Codex event stream exceeded its byte limit"); + } + const lines = serialized.endsWith("\n") + ? serialized.slice(0, -1).split("\n") + : serialized.split("\n"); + if (lines.length === 0 || lines.length > MAX_PERSISTED_CODEX_EVENTS) { + throw new Error("Persisted Codex event stream exceeded its event limit"); + } + const events: PrpEvent[] = []; + const sourceEventIds = new Set(); + const sourceSequences = new Map(); + let runTerminalCount = 0; + for (const line of lines) { + if (line.length === 0 || Buffer.byteLength(line) > MAX_PERSISTED_CODEX_LINE_BYTES) { + throw new Error("Persisted Codex event line was empty or oversized"); + } + let candidate: unknown; + try { + candidate = JSON.parse(line); + } catch { + throw new Error("Persisted Codex event line was malformed JSON"); + } + const validation = validatePrpEvent(candidate); + if (!validation.ok) throw new Error("Persisted Codex event failed schema validation"); + const event = validation.event; + if ( + event.runId !== metadata.identity.runId || + event.normalizedSessionId !== metadata.identity.normalizedSessionId + ) { + throw new Error("Persisted Codex event identity did not match the opened run"); + } + if (sourceEventIds.has(event.sourceEventId)) { + throw new Error("Persisted Codex event id was duplicated"); + } + sourceEventIds.add(event.sourceEventId); + const source = `${event.sourceKind}:${event.sourceInstanceId}`; + const previous = sourceSequences.get(source); + if (previous !== undefined && event.sourceSeq !== previous + 1) { + throw new Error("Persisted Codex source sequence was discontinuous"); + } + sourceSequences.set(source, event.sourceSeq); + if (runTerminalCount > 0) { + throw new Error("Persisted Codex event appeared after the run terminal"); + } + if (event.eventType === "run.terminal") runTerminalCount += 1; + events.push(event); + } + if (runTerminalCount !== 1) { + throw new Error("Persisted Codex event stream requires exactly one run terminal"); + } + return events.reduce( + applyPrpEvent, + createSessionSnapshotFromMetadata({ + fixtureName: metadata.fixtureName, + identity: { + ...metadata.identity, + driverSessionId: metadata.identity.driverSessionId, + }, + capabilities: metadata.capabilities, + }), + ); +} + +/** + * Small in-memory Codex mock core. The driver reports provider facts; this + * controller validates semantic proposals and alone emits the run terminal. + */ +export async function runCodexCodexTracer(input: CodexCodexTracerInput): Promise { + const runId = input.runId ?? "codex-safe-run"; + const normalizedSessionId = input.normalizedSessionId ?? "codex-normalized-session"; + const descriptor = await input.driver.descriptor(); + const capabilities = prpCapabilities(descriptor); + const metadata: CodexRunMetadata = { + schema: "paperclip.runner.codex.metadata.v1", + fixtureName: "codex-safe-codex-task", + identity: { + schema: "paperclip.prp.identity.v1", + companyId: input.companyId ?? "mock-company", + issueId: input.issueId ?? "mock-issue", + runId, + environmentLeaseId: input.environmentLeaseId ?? "mock-lease", + runnerInstanceId: input.runnerInstanceId ?? "runner-codex", + normalizedSessionId, + }, + capabilities, + }; + const session = await input.driver.openSession({ + runId, + normalizedSessionId, + workingDirectory: input.workingDirectory, + }); + const capabilityDiagnostics: string[] = []; + let signalTurnStarted: (() => void) | undefined; + const turnStarted = new Promise((resolve) => { + signalTurnStarted = resolve; + }); + const consuming = consumeToTurnTerminal(session, input.timeoutMs ?? 180_000, (event) => { + if (event.eventType === "turn.started") signalTurnStarted?.(); + }); + + try { + const turn = await session.startTurn({ + message: input.message ?? { role: "user", text: "Complete the supplied task envelope." }, + }); + if (input.steer !== undefined || input.interrupt) { + await Promise.race([ + turnStarted, + consuming.then(() => { + throw new Error("Harness turn reached terminal state before accepting the control command"); + }), + ]); + } + if (input.steer !== undefined) { + if (!descriptor.capabilities.steering || session.steer === undefined) { + capabilityDiagnostics.push("steering is unavailable by driver capability contract"); + } else { + try { + await session.steer({ + turnId: turn.turnId, + message: { role: "user", text: input.steer }, + }); + } catch (error) { + capabilityDiagnostics.push(`steering degraded: ${String(error)}`); + } + } + } + if (input.interrupt) { + if (!descriptor.capabilities.interruption || session.interrupt === undefined) { + capabilityDiagnostics.push("interruption is unavailable by driver capability contract"); + } else { + try { + await session.interrupt({ turnId: turn.turnId, reason: "Codex tutorial interrupt" }); + } catch (error) { + capabilityDiagnostics.push(`interruption degraded: ${String(error)}`); + } + } + } + + const providerEvents = await consuming; + const ids = session.ids(); + const identity = { + ...metadata.identity, + driverSessionId: ids.driverSessionId, + ...(ids.providerSessionId ? { providerSessionId: ids.providerSessionId } : {}), + }; + metadata.identity = identity; + + const proposals = providerEvents.filter( + (event) => event.eventType === "run.result.proposed", + ); + const proposedResult = proposals.length === 1 ? structuredClone(proposals[0]!.payload) : null; + const proposalDecision: CodexResultDecision = + proposals.length === 1 + ? validateCodexResultProposal(proposedResult, input.taskEnvelope) + : { + status: "rejected", + result: null, + issues: [{ + code: "schema_validation", + path: "/run.result.proposed", + message: + proposals.length === 0 + ? "the completed turn emitted no semantic proposal" + : "the completed turn emitted multiple semantic proposals", + }], + }; + + let controlSequence = 0; + const controlPlaneInstanceId = input.controlPlaneInstanceId ?? "mock-core-codex"; + const controlEvent = ( + eventType: PrpEvent["eventType"], + payload: Record, + refs: { turnId?: string; itemId?: string } = {}, + ): PrpEvent => ({ + schema: "paperclip.prp.event.v1", + sourceEventId: `${controlPlaneInstanceId}:${runId}:${++controlSequence}`, + sourceSeq: controlSequence, + sourceInstanceId: controlPlaneInstanceId, + sourceKind: "control_plane", + runId, + normalizedSessionId, + ...(refs.turnId ? { turnId: refs.turnId } : {}), + ...(refs.itemId ? { itemId: refs.itemId } : {}), + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" || eventType.startsWith("run.result.") ? 0 : 1, + emittedAt: new Date().toISOString(), + payload, + }); + const controlEvents = capabilityDiagnostics.map((message) => + controlEvent("runner.diagnostic", { + code: "capability_unavailable", + message, + }), + ); + const providerTerminal = providerEvents.findLast(isTurnTerminal); + if (providerTerminal === undefined) throw new Error("turn terminal invariant failed"); + const providerRuntime = runtimeTerminal(providerTerminal); + const resultAccepted = + proposalDecision.status === "accepted" && providerRuntime.runTerminalState === "succeeded"; + const resultDecision: CodexResultDecision = resultAccepted + ? proposalDecision + : proposalDecision.status === "rejected" + ? proposalDecision + : { + status: "rejected", + result: null, + issues: [{ + code: "schema_validation", + path: "/turn/terminal", + message: `the provider turn ended as ${providerRuntime.turnTerminalState}`, + }], + }; + controlEvents.push( + resultAccepted + ? controlEvent("run.result.accepted", { + contractRevision: input.taskEnvelope.completionContract.revision, + result: resultDecision.result, + }) + : controlEvent("run.result.rejected", { + reasonCode: + proposalDecision.status === "rejected" + ? "semantic_result_rejected" + : "provider_turn_did_not_complete", + issues: resultDecision.issues, + recovery: { required: true, recoverable: true }, + }), + ); + // A provider may complete its turn after proposing a result that the + // controller cannot accept. Preserve the provider's turn fact, but never + // promote a rejected semantic result to a successful run. + const runtime = resultDecision.status === "accepted" || providerRuntime.runTerminalState === "cancelled" + ? providerRuntime + : { ...providerRuntime, runTerminalState: "failed" as const }; + controlEvents.push(controlEvent("run.terminal", { + schema: "paperclip.prp.terminal.v1", + ...runtime, + reportedWorkDisposition: + resultDecision.status === "accepted" && runtime.runTerminalState === "succeeded" + ? resultDecision.result.reportedWorkDisposition + : "yielded", + }, providerTerminal.turnId ? { turnId: providerTerminal.turnId } : {})); + const events = [...providerEvents, ...controlEvents]; + + const liveSnapshot = events.reduce( + applyPrpEvent, + createSessionSnapshotFromMetadata({ + fixtureName: metadata.fixtureName, + identity, + capabilities, + }), + ); + const persistedEvents = `${events.map((event) => JSON.stringify(event)).join("\n")}\n`; + const replaySnapshot = replayPersistedCodexEvents(persistedEvents, metadata); + const context = contextFrom(events); + const terminalCount = events.filter((event) => event.eventType === "run.terminal").length; + const turnTerminalCount = events.filter(isTurnTerminal).length; + const decisionCount = events.filter( + (event) => + event.eventType === "run.result.accepted" || event.eventType === "run.result.rejected", + ).length; + const serialized = JSON.stringify(context).toLowerCase(); + const itemEvents = events.filter((event) => event.eventType.startsWith("item.")); + const providerSessionId = ids.providerSessionId ?? null; + return { + schema: "paperclip.runner.codex.trace.v1", + metadata, + context, + events, + proposedResult, + result: resultDecision.result, + resultDecision, + liveSnapshot, + replaySnapshot, + diagnostics: events + .filter( + (event) => + event.eventType === "runner.diagnostic" || + event.eventType === "harness.diagnostic" || + event.eventType === "run.result.rejected", + ) + .map((event) => String(event.payload.message ?? event.payload.reasonCode ?? event.payload.code ?? "diagnostic")), + assertions: { + exactlyOneTerminalResult: + terminalCount === 1 && turnTerminalCount === 1 && decisionCount === 1, + proposalAccepted: resultDecision.status === "accepted", + liveReplayParity: JSON.stringify(liveSnapshot) === JSON.stringify(replaySnapshot), + stableIdentity: + providerSessionId !== null && + runId.length > 0 && + normalizedSessionId.length > 0 && + ids.driverSessionId.length > 0 && + providerSessionId.length > 0 && + events.every( + (event) => + event.runId === runId && event.normalizedSessionId === normalizedSessionId, + ), + sourceSequenceContinuous: sourceSequenceContinuous(events), + stableItemIdentity: itemEvents.every( + (event) => typeof event.itemId === "string" && event.itemId.length > 0, + ), + contextIsSkillless: isSkilllessCodexContext(context, { + dynamicTools: descriptor.capabilities.dynamicTools === true, + }), + unrelatedSkillsAbsent: + context.instructionSources.length === 0 && + context.instructionPolicy.skillInstructions === false && + context.modelInputKinds.length === 1 && + context.modelInputKinds[0] === "text", + credentialsAbsent: + !serialized.includes("paperclip_api_key") && + !serialized.includes("authorization: bearer") && + !serialized.includes("/api/issues/"), + }, + }; + } finally { + await session.close({ reason: "Codex tracer complete" }); + } +} diff --git a/packages/paperclip-runner/src/protocol/live-console-fixture.ts b/packages/paperclip-runner/src/protocol/live-console-fixture.ts new file mode 100644 index 0000000000..b01f840286 --- /dev/null +++ b/packages/paperclip-runner/src/protocol/live-console-fixture.ts @@ -0,0 +1,291 @@ +import { readFile, stat } from "node:fs/promises"; +import { isDeepStrictEqual } from "node:util"; + +import { + parseHarnessRuntimeRequestResolution, + type HarnessRuntimeRequest, +} from "../contracts/harness-driver.js"; +import { + createCodexQuestionResponseContext, + runtimeRequestKind, + runtimeRequestResponse, +} from "../drivers/codex/codex-question-adapter.js"; + +export const LIVE_CONSOLE_CONFORMANCE_SCHEMA = + "paperclip.runner.live-console.conformance.v1" as const; + +export interface LiveConsoleRuntimeRequestFixture { + id: string; + method: string; + requestKind: string; + resolution: Record & { action: string }; + expectedResponse: Record; +} + +export interface LiveConsoleGoalFixture { + action: "get" | "set" | "pause" | "resume" | "clear"; + method: string; + params: Record; +} + +export interface LiveConsoleControlFixture { + turnId?: string; + expected: string; +} + +export interface LiveConsoleConformanceFixture { + schema: typeof LIVE_CONSOLE_CONFORMANCE_SCHEMA; + codexVersion: string; + runtimeRequests: LiveConsoleRuntimeRequestFixture[]; + goals: LiveConsoleGoalFixture[]; + lineage: { + rootThreadId: string; + childThread: Record & { + id: string; + sessionId: string; + agentNickname: string; + agentRole: string; + source: { + subAgent: { + thread_spawn: { + parent_thread_id: string; + depth: number; + agent_path: string[]; + agent_nickname: string; + agent_role: string; + }; + }; + }; + }; + }; + controls: { + sameTurnSteer: LiveConsoleControlFixture & { turnId: string }; + staleTurnSteer: LiveConsoleControlFixture & { turnId: string }; + interruptBeforeStart: LiveConsoleControlFixture; + interruptAfterTerminal: LiveConsoleControlFixture; + }; + reconnect: { + runId: string; + normalizedSessionId: string; + driverSessionId: string; + providerSessionId: string; + lastSourceSequence: number; + }; + redactionMarkers: string[]; +} + +const MAX_LIVE_CONSOLE_FIXTURE_BYTES = 1024 * 1024; +const MAX_RUNTIME_REQUEST_CASES = 64; +const MAX_REDACTION_MARKERS = 64; +const MAX_CONTROL_CASES = 64; +const GOAL_METHODS = { + get: "thread/goal/get", + set: "thread/goal/set", + pause: "thread/goal/set", + resume: "thread/goal/set", + clear: "thread/goal/clear", +} as const; +const CONTROL_EXPECTATIONS = { + sameTurnSteer: { expected: "acknowledged", requiresTurnId: true }, + staleTurnSteer: { expected: "stale_turn", requiresTurnId: true }, + interruptBeforeStart: { expected: "queued", requiresTurnId: false }, + interruptAfterTerminal: { + expected: "already_terminal", + requiresTurnId: false, + }, +} as const; + +function record(value: unknown): Record | null { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? (value as Record) + : null; +} + +function nonEmpty(value: unknown): value is string { + return typeof value === "string" && value.length > 0; +} + +/** Load the deterministic Live console wire fixture without trusting its shape. */ +export async function loadLiveConsoleConformanceFixture( + path: string, +): Promise { + const info = await stat(path); + if (!info.isFile() || info.size > MAX_LIVE_CONSOLE_FIXTURE_BYTES) { + throw new Error("Live console fixture exceeded its file-size limit"); + } + const value = JSON.parse(await readFile(path, "utf8")) as unknown; + const fixture = record(value); + if (fixture?.schema !== LIVE_CONSOLE_CONFORMANCE_SCHEMA) { + throw new Error("Live console fixture has an unsupported schema"); + } + if (!nonEmpty(fixture.codexVersion)) { + throw new Error("Live console fixture must name the observed Codex version"); + } + if ( + !Array.isArray(fixture.runtimeRequests) || + fixture.runtimeRequests.length === 0 || + fixture.runtimeRequests.length > MAX_RUNTIME_REQUEST_CASES + ) { + throw new Error("Live console fixture must include runtime request cases"); + } + if (!Array.isArray(fixture.goals) || fixture.goals.length !== 5) { + throw new Error("Live console fixture must include all five goal operations"); + } + const requestIds = new Set(); + for (const candidate of fixture.runtimeRequests) { + const request = record(candidate); + const resolution = record(request?.resolution); + const kind = nonEmpty(request?.method) + ? runtimeRequestKind(request.method) + : null; + if ( + !nonEmpty(request?.id) || + requestIds.has(request.id) || + !nonEmpty(request.method) || + kind === null || + request.requestKind !== kind || + !nonEmpty(resolution?.action) || + record(request.expectedResponse) === null + ) { + throw new Error("Live console fixture contains an invalid runtime request case"); + } + try { + const parsedResolution = parseHarnessRuntimeRequestResolution(kind, resolution); + const harnessRequest: HarnessRuntimeRequest = { + requestId: request.id, + requestKind: kind, + method: request.method, + turnId: "fixture-turn", + itemId: request.id, + status: "pending", + prompt: "fixture request", + details: {}, + }; + if ( + !isDeepStrictEqual( + runtimeRequestResponse( + harnessRequest, + parsedResolution, + createCodexQuestionResponseContext(), + ), + request.expectedResponse, + ) + ) { + throw new Error("expected response does not match the declared resolution"); + } + } catch { + throw new Error("Live console fixture contains an invalid runtime request case"); + } + requestIds.add(request.id); + } + const actions = new Set(); + for (const candidate of fixture.goals) { + const goal = record(candidate); + if (goal === null) { + throw new Error("Live console fixture contains an invalid goal operation"); + } + const action = goal?.action; + if ( + typeof action !== "string" || + !(action in GOAL_METHODS) || + goal.method !== GOAL_METHODS[action as keyof typeof GOAL_METHODS] || + !validGoalParams(action as keyof typeof GOAL_METHODS, goal.params) + ) { + throw new Error("Live console fixture contains an invalid goal operation"); + } + actions.add(action); + } + if (!["get", "set", "pause", "resume", "clear"].every((action) => actions.has(action))) { + throw new Error("Live console fixture goal operation set is incomplete"); + } + const lineage = record(fixture.lineage); + const child = record(lineage?.childThread); + const childSource = record(child?.source); + const childSubAgent = record(childSource?.subAgent); + const childSpawn = record(childSubAgent?.thread_spawn); + const childAgentPath = childSpawn?.agent_path; + const controls = record(fixture.controls); + const reconnect = record(fixture.reconnect); + if ( + !nonEmpty(lineage?.rootThreadId) || + !nonEmpty(child?.id) || + !nonEmpty(child.sessionId) || + child.id === lineage.rootThreadId || + childSpawn === null || + childSpawn.parent_thread_id !== lineage.rootThreadId || + !Number.isSafeInteger(childSpawn.depth) || + (childSpawn.depth as number) < 1 || + !Array.isArray(childAgentPath) || + childAgentPath.length === 0 || + childAgentPath.length > 64 || + !childAgentPath.every(nonEmpty) || + !nonEmpty(childSpawn.agent_nickname) || + !nonEmpty(childSpawn.agent_role) || + child.agentNickname !== childSpawn.agent_nickname || + child.agentRole !== childSpawn.agent_role || + !nonEmpty(reconnect?.runId) || + !nonEmpty(reconnect.normalizedSessionId) || + !nonEmpty(reconnect.driverSessionId) || + !nonEmpty(reconnect.providerSessionId) || + !Number.isSafeInteger(reconnect.lastSourceSequence) || + (reconnect.lastSourceSequence as number) < 0 + ) { + throw new Error("Live console fixture identity or lineage is incomplete"); + } + const controlNames = Object.keys(CONTROL_EXPECTATIONS); + if ( + controls === null || + Object.keys(controls).length > MAX_CONTROL_CASES || + Object.keys(controls).length !== controlNames.length || + controlNames.some((name) => { + const expectation = + CONTROL_EXPECTATIONS[name as keyof typeof CONTROL_EXPECTATIONS]; + const control = record(controls[name]); + if ( + control === null || + control.expected !== expectation.expected || + Object.keys(control).some( + (key) => key !== "expected" && key !== "turnId", + ) + ) { + return true; + } + return expectation.requiresTurnId + ? !nonEmpty(control.turnId) + : control.turnId !== undefined; + }) + ) { + throw new Error("Live console fixture controls are invalid"); + } + if ( + !Array.isArray(fixture.redactionMarkers) || + fixture.redactionMarkers.length > MAX_REDACTION_MARKERS || + !fixture.redactionMarkers.every(nonEmpty) + ) { + throw new Error("Live console fixture redaction markers are invalid"); + } + return value as LiveConsoleConformanceFixture; +} + +function validGoalParams( + action: keyof typeof GOAL_METHODS, + value: unknown, +): boolean { + const params = record(value); + if (params === null) return false; + if (action === "get" || action === "clear") { + return Object.keys(params).length === 0; + } + if (action === "pause" || action === "resume") { + return Object.keys(params).length === 1 && + params.status === (action === "pause" ? "paused" : "active"); + } + return nonEmpty(params.objective) && + params.status === "active" && + Object.keys(params).every((key) => + key === "objective" || key === "status" || key === "tokenBudget" + ) && + (params.tokenBudget === undefined || + params.tokenBudget === null || + (Number.isSafeInteger(params.tokenBudget) && (params.tokenBudget as number) >= 0)); +}