diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts index 689800e54e..c439227f03 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js"; -import type { PrpEvent, PrpStructuredRunResult } from "../protocol/replay-contract.js"; +import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js"; import { HarnessDriverBackend } from "./harness-driver-backend.js"; const result: PrpStructuredRunResult = { @@ -271,6 +271,105 @@ describe("HarnessDriverBackend", () => { expect(receivedSignal).toBe(controller.signal); }); + it("does not resurrect a settled turn across repeated backend recovery", async () => { + const terminal: PrpTerminalState = { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: "done", + }; + const recoveredSnapshots: PersistedHarnessSession[] = []; + + class SettledRecoveredSession extends FakeHarnessSession { + readonly #snapshot: PersistedHarnessSession; + + constructor(snapshot: PersistedHarnessSession) { + super(); + this.#snapshot = structuredClone(snapshot); + } + + override async *events() { + // Model the deterministic driver's recovery rule: only an active turn + // produces a reconstructed result/terminal sequence. + if (this.#snapshot.activeTurnId === null || this.#snapshot.activeTurnId === undefined) return; + yield prpEvent(3, "run.result.proposed", result); + yield prpEvent(4, "turn.completed", { status: "completed" }); + yield prpEvent(5, "run.terminal", terminal); + } + + override async snapshot(): Promise { + return structuredClone(this.#snapshot); + } + } + + const recoveryDriver: HarnessDriver = { + ...driver, + async recoverSession(snapshot) { + recoveredSnapshots.push(structuredClone(snapshot)); + return { recovered: true, session: new SettledRecoveredSession(snapshot) }; + }, + }; + const backend = new HarnessDriverBackend(recoveryDriver); + const settledSnapshot = { + backendKind: "runner" as const, + driverKind: "fake", + sessionId: "driver-1", + providerSessionId: "provider-1", + identity: { + runId: "run-1", + sessionId: "session-1", + companyId: "company-1", + issueId: "issue-1", + agentId: "agent-1", + }, + cursor: "2", + semanticResult: result, + terminal, + activeTurnId: null, + terminalTurns: [{ turnId: "turn-1", fingerprint: "terminal-1" }], + }; + + const recoveryOptions = { signal: new AbortController().signal }; + const firstRecovery = await backend.recoverSession( + settledSnapshot, + recoveryOptions, + ); + expect(firstRecovery.recovered).toBe(true); + const firstSession = firstRecovery.session!; + const firstRecoveredSnapshot = await firstSession.snapshot(); + expect(firstRecoveredSnapshot).toMatchObject({ + activeTurnId: null, + semanticResult: result, + terminal, + }); + await expect(firstSession.result()).resolves.toEqual({ + result, + terminal, + turnId: "turn-1", + }); + + const secondRecovery = await backend.recoverSession( + firstRecoveredSnapshot, + recoveryOptions, + ); + expect(secondRecovery.recovered).toBe(true); + const secondSession = secondRecovery.session!; + await expect(secondSession.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: result, + terminal, + }); + const replayedEvents: PrpEvent[] = []; + for await (const event of secondSession.events()) replayedEvents.push(event); + expect(replayedEvents.filter((event) => + event.eventType === "run.result.proposed" || event.eventType === "run.terminal" + )).toEqual([]); + expect(recoveredSnapshots.map((snapshot) => snapshot.activeTurnId)).toEqual([null, null]); + + await secondSession.close({ reason: "double recovery complete" }); + await firstSession.close({ reason: "first recovery complete" }); + }); + it("allows only the run id to change when a harness session is attached", async () => { const attachedRunIds: string[] = []; class AttachableHarnessSession extends FakeHarnessSession { diff --git a/packages/paperclip-runner/src/backends/harness-driver-backend.ts b/packages/paperclip-runner/src/backends/harness-driver-backend.ts index c66e69b3ae..9c39f8ccb4 100644 --- a/packages/paperclip-runner/src/backends/harness-driver-backend.ts +++ b/packages/paperclip-runner/src/backends/harness-driver-backend.ts @@ -70,6 +70,12 @@ export class HarnessDriverBackend implements NativeSessionBackend { if (this.#driver.recoverSession === undefined) { return { recovered: false, reason: "driver does not support recovery" }; } + // `null` is a durable "no active turn" checkpoint. Only an omitted legacy + // field may use the compatibility fallback; nullish coalescing here would + // otherwise turn the most recent terminal turn back into an active one. + const activeTurnId = snapshot.activeTurnId === undefined + ? snapshot.terminalTurns?.at(-1)?.turnId ?? null + : snapshot.activeTurnId; const persisted: PersistedHarnessSession = { driverKind: snapshot.driverKind ?? snapshot.backendKind, driverSessionId: snapshot.sessionId, @@ -82,7 +88,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { : { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }), runId: snapshot.identity.runId, normalizedSessionId: snapshot.identity.sessionId, - activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null, + activeTurnId, lastSourceSequence: parseCursor(snapshot.cursor), ...(snapshot.semanticResult === undefined || snapshot.semanticResult === null ? {} @@ -90,7 +96,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { semanticResult: { result: snapshot.semanticResult, fingerprint: canonicalJson(snapshot.semanticResult), - turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered", + turnId: activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered", }, }), terminalTurns: snapshot.terminalTurns ?? [], @@ -125,6 +131,7 @@ export class HarnessDriverBackend implements NativeSessionBackend { session: new HarnessNativeSession( { identity: snapshot.identity }, recovered.session, + snapshot.terminal, ), }; } @@ -154,9 +161,14 @@ class HarnessNativeSession implements NativeSession { #terminal: PrpTerminalState | null = null; #explicitlyCancelled = false; - constructor(input: OpenNativeSessionInput, session: HarnessSession) { + constructor( + input: OpenNativeSessionInput, + session: HarnessSession, + terminal?: PrpTerminalState | null, + ) { this.#input = structuredClone(input); this.#session = session; + this.#terminal = terminal === undefined ? null : structuredClone(terminal); } identity() { @@ -410,7 +422,12 @@ class HarnessNativeSession implements NativeSession { : String(snapshot.lastSourceSequence), semanticResult: snapshot.semanticResult?.result ?? null, terminal: this.#terminal, - activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null, + // Harness snapshots use an explicit null to record a settled turn. Keep + // the semantic-result fallback solely for legacy snapshots that omitted + // activeTurnId, or a later recovery can replay the terminal turn. + activeTurnId: snapshot.activeTurnId === undefined + ? snapshot.semanticResult?.turnId ?? null + : snapshot.activeTurnId, terminalTurns: snapshot.terminalTurns ?? [], pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [], lineage: snapshot.lineage ?? [], diff --git a/packages/paperclip-runner/src/conformance/control-plane-port.test.ts b/packages/paperclip-runner/src/conformance/control-plane-port.test.ts new file mode 100644 index 0000000000..ceea2e8050 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/control-plane-port.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from "vitest"; + +import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js"; +import { + CONTROL_PLANE_CONFORMANCE_EVENTS, + CONTROL_PLANE_CONFORMANCE_OPEN, + runControlPlanePortConformance, +} from "./control-plane-port.js"; + +describe("ControlPlanePort conformance", () => { + it("runs unchanged against the mock adapter", async () => { + const adapter = new MockControlPlaneAdapter(); + await expect(runControlPlanePortConformance({ + port: adapter, + start: () => adapter.start(), + stop: () => adapter.stop(), + })).resolves.toEqual({ + eventCount: 3, + highestContiguousSourceSeq: 3, + duplicateDisposition: "duplicate", + terminalReplayIdempotent: true, + openBindingRejected: true, + eventIdMutationRejected: true, + eventSequenceMutationRejected: true, + replayBindingRejected: true, + resultMutationRejected: true, + }); + }); + + it("fails closed when a source identity is replayed with different bytes", async () => { + const adapter = new MockControlPlaneAdapter(); + await adapter.start(); + await adapter.openRun(CONTROL_PLANE_CONFORMANCE_OPEN); + await adapter.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[0]); + await expect(adapter.appendEvent({ + ...CONTROL_PLANE_CONFORMANCE_EVENTS[0], + payload: { backend: "forged" }, + })).rejects.toThrow("native_event_replay_conflict"); + }); +}); diff --git a/packages/paperclip-runner/src/conformance/harness-driver.test.ts b/packages/paperclip-runner/src/conformance/harness-driver.test.ts new file mode 100644 index 0000000000..9e7a57f02c --- /dev/null +++ b/packages/paperclip-runner/src/conformance/harness-driver.test.ts @@ -0,0 +1,286 @@ +import { describe, expect, it } from "vitest"; + +import { DeterministicHarnessDriver } from "../mock-core/deterministic-harness-driver.js"; +import { + loadHarnessDriverConformanceFixture, + runHarnessDriverConformance, +} from "./harness-driver.js"; + +describe("harness-driver conformance V1", () => { + it("ships a deterministic fixture covering the complete Evals hook set", async () => { + const fixture = await loadHarnessDriverConformanceFixture(); + expect(fixture.requiredCapabilities).toContain("dynamicTools"); + expect(fixture.unsupportedFeatures).toContain("steering"); + + const report = await runHarnessDriverConformance({ + driver: new DeterministicHarnessDriver(), + fixture, + }); + expect(report).toMatchObject({ + schema: "paperclip-runner/harness-driver-conformance-report/v1", + contractVersion: 1, + eventCount: 7, + semanticToolCallCount: 1, + checks: { + capabilityDescription: true, + configValidation: true, + sessionLifecycle: true, + sessionRecovery: true, + semanticTools: true, + eventValidation: true, + interruptAndCancel: true, + usage: true, + transcriptCompleteness: true, + unsupportedFeatures: true, + }, + }); + }); + + it("settles an actively recovered turn when event consumption resumes", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_active_recovery", + normalizedSessionId: "session_active_recovery", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "[conformance:interrupt]" }, + }); + const snapshot = await session.snapshot(); + expect(snapshot).toMatchObject({ activeTurnId: turnId, lastSourceSequence: 2 }); + + const firstRecovery = await driver.recoverSession(snapshot); + expect(firstRecovery.recovered).toBe(true); + expect(firstRecovery.session).toBeDefined(); + const repeatedSnapshot = await firstRecovery.session!.snapshot(); + expect(repeatedSnapshot).toMatchObject({ + activeTurnId: turnId, + lastSourceSequence: 2, + }); + const recovery = await driver.recoverSession(repeatedSnapshot); + expect(recovery.recovered).toBe(true); + expect(recovery.session).toBeDefined(); + const recovered = recovery.session!; + const recoveredEventsPromise = (async () => { + const events = []; + for await (const event of recovered.events()) events.push(event); + return events; + })(); + + const events = await recoveredEventsPromise; + expect(events.map((event) => [event.sourceSeq, event.eventType])).toEqual([ + [3, "run.result.proposed"], + [4, "turn.interrupted"], + [5, "run.terminal"], + ]); + await expect(recovered.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: { + result: { reportedWorkDisposition: "yielded" }, + turnId, + }, + lastSourceSequence: 5, + }); + expect(events[1]?.payload).toEqual({ + reason: "deterministic_active_turn_recovered_without_live_producer", + }); + await expect(recovered.transcript?.()).resolves.toMatchObject({ + schema: "paperclip-runner/harness-transcript/v1", + complete: false, + eventCount: 3, + events: events.map((event) => expect.objectContaining({ + sourceSeq: event.sourceSeq, + eventType: event.eventType, + })), + omissionReason: "pre_recovery_events_not_reconstructed", + }); + await recovered.close({ reason: "recovered_complete" }); + await firstRecovery.session!.close({ reason: "first_recovery_complete" }); + await session.close({ reason: "original_complete", force: true }); + }); + + it("interrupts a recovered turn deterministically before event consumption", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_recovery_interrupt", + normalizedSessionId: "session_recovery_interrupt", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "[conformance:interrupt]" }, + }); + const recovery = await driver.recoverSession(await session.snapshot()); + const recovered = recovery.session!; + + await new Promise((resolve) => setImmediate(resolve)); + await expect(recovered.interrupt?.({ + turnId, + reason: "cancel_before_consumption", + })).resolves.toBeUndefined(); + const events = []; + for await (const event of recovered.events()) events.push(event); + expect(events.map((event) => [event.sourceSeq, event.eventType])).toEqual([ + [3, "run.result.proposed"], + [4, "turn.interrupted"], + [5, "run.terminal"], + ]); + await expect(recovered.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: { + result: { reportedWorkDisposition: "yielded" }, + turnId, + }, + lastSourceSequence: 5, + }); + + await recovered.close({ reason: "recovered_complete" }); + await session.close({ reason: "original_complete", force: true }); + }); + + it("acknowledges interruption after recovered event settlement starts", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_recovery_interrupt_race", + normalizedSessionId: "session_recovery_interrupt_race", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "[conformance:interrupt]" }, + }); + const recovery = await driver.recoverSession(await session.snapshot()); + const recovered = recovery.session!; + const recoveredEvents = recovered.events()[Symbol.asyncIterator](); + + const first = await recoveredEvents.next(); + expect(first.value?.eventType).toBe("run.result.proposed"); + await expect(recovered.interrupt?.({ + turnId, + reason: "cancel_after_event_settlement_started", + })).resolves.toBeUndefined(); + + const events = first.done || first.value === undefined ? [] : [first.value]; + for await (const event of { [Symbol.asyncIterator]: () => recoveredEvents }) { + events.push(event); + } + expect(events.map((event) => event.eventType)).toEqual([ + "run.result.proposed", + "turn.interrupted", + "run.terminal", + ]); + expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1); + + await recovered.close({ reason: "recovered_complete" }); + await session.close({ reason: "original_complete", force: true }); + }); + + it("preserves a completed result when its terminal checkpoint lagged", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_result_first_recovery", + normalizedSessionId: "session_result_first_recovery", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Complete deterministically." }, + }); + const completedSnapshot = await session.snapshot(); + const recovery = await driver.recoverSession({ + ...completedSnapshot, + activeTurnId: turnId, + terminalTurns: [], + }); + const recovered = recovery.session!; + const events = []; + for await (const event of recovered.events()) events.push(event); + + expect(events.map((event) => event.eventType)).toEqual([ + "run.result.proposed", + "turn.completed", + "run.terminal", + ]); + expect(events[0]?.payload).toMatchObject({ reportedWorkDisposition: "done" }); + expect(events[2]?.payload).toMatchObject({ runTerminalState: "succeeded" }); + await expect(recovered.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: completedSnapshot.semanticResult, + }); + await recovered.close({ reason: "recovered_complete" }); + await session.close({ reason: "original_complete" }); + }); + + it("does not let an interrupt overwrite a completed result-first recovery", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_result_first_interrupt", + normalizedSessionId: "session_result_first_interrupt", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "Complete deterministically." }, + }); + const completedSnapshot = await session.snapshot(); + const recovery = await driver.recoverSession({ + ...completedSnapshot, + activeTurnId: turnId, + terminalTurns: [], + }); + const recovered = recovery.session!; + + await expect(recovered.interrupt?.({ + turnId, + reason: "late cancellation", + })).resolves.toBeUndefined(); + const events = []; + for await (const event of recovered.events()) events.push(event); + expect(events.map((event) => event.eventType)).toEqual([ + "run.result.proposed", + "turn.completed", + "run.terminal", + ]); + await expect(recovered.snapshot()).resolves.toMatchObject({ + activeTurnId: null, + semanticResult: completedSnapshot.semanticResult, + }); + await recovered.close({ reason: "recovered_complete" }); + await session.close({ reason: "original_complete" }); + }); + + it("preserves an interrupted result when its terminal checkpoint lagged", async () => { + const driver = new DeterministicHarnessDriver(); + const session = await driver.openSession({ + runId: "run_yielded_result_first_recovery", + normalizedSessionId: "session_yielded_result_first_recovery", + workingDirectory: "/deterministic/conformance", + }); + const { turnId } = await session.startTurn({ + message: { role: "user", text: "[conformance:interrupt]" }, + }); + await session.interrupt?.({ turnId, reason: "checkpoint_after_result" }); + const interruptedSnapshot = await session.snapshot(); + const recovery = await driver.recoverSession({ + ...interruptedSnapshot, + activeTurnId: turnId, + terminalTurns: [], + }); + const recovered = recovery.session!; + const events = []; + for await (const event of recovered.events()) events.push(event); + + expect(events.map((event) => event.eventType)).toEqual([ + "run.result.proposed", + "turn.interrupted", + "run.terminal", + ]); + expect(events[0]?.payload).toMatchObject({ + reportedWorkDisposition: "yielded", + continuation: { kind: "same_agent" }, + }); + expect(events[2]?.payload).toMatchObject({ + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }); + await recovered.close({ reason: "recovered_complete" }); + await session.close({ reason: "original_complete" }); + }); +}); diff --git a/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts b/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts new file mode 100644 index 0000000000..4b4c88fd87 --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/deterministic-harness-driver.ts @@ -0,0 +1,483 @@ +import { + type HarnessDriver, + type HarnessDriverConfigValidation, + type HarnessDriverDescriptor, + type HarnessSession, + type HarnessSessionRecoveryResult, + type HarnessTranscriptSnapshot, + type OpenHarnessSessionInput, + type PersistedHarnessSession, +} from "../contracts/harness-driver.js"; +import { + createPrpSemanticToolInputEnvelope, + createPrpSemanticToolResultEnvelope, +} from "../protocol/semantic-tool-receipts.js"; +import type { + PrpEvent, + PrpStructuredRunResult, + PrpTerminalState, +} from "../protocol/replay-contract.js"; + +export const DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA = + "paperclip-runner/deterministic-harness-driver-config/v1" as const; + +export interface DeterministicHarnessDriverConfig { + schema: typeof DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA; + scenario: "semantic-tool-success"; +} + +const DEFAULT_CONFIG: DeterministicHarnessDriverConfig = { + schema: DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA, + scenario: "semantic-tool-success", +}; + +function parseConfig(value: unknown): HarnessDriverConfigValidation { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return { + ok: false, + config: null, + issues: [{ path: "/", code: "type", message: "config must be an object" }], + }; + } + const config = value as Record; + if (config.schema !== DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA) { + return { + ok: false, + config: null, + issues: [{ + path: "/schema", + code: "unsupported_schema", + message: `schema must be ${DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA}`, + }], + }; + } + if (config.scenario !== "semantic-tool-success") { + return { + ok: false, + config: null, + issues: [{ + path: "/scenario", + code: "unsupported_scenario", + message: "scenario must be semantic-tool-success", + }], + }; + } + return { ok: true, config: structuredClone(config), issues: [] }; +} + +/** Provider-free fake used by packed consumers and App/Evals conformance. */ +export class DeterministicHarnessDriver implements HarnessDriver { + readonly #config: DeterministicHarnessDriverConfig; + + constructor(config: DeterministicHarnessDriverConfig = DEFAULT_CONFIG) { + const validation = parseConfig(config); + if (!validation.ok) throw new Error(validation.issues[0]?.message ?? "invalid config"); + this.#config = structuredClone(config); + } + + async descriptor(): Promise { + return { + kind: "paperclip-deterministic", + displayName: "Paperclip deterministic harness driver", + version: "1.0.0", + protocolVersion: "prp.v1", + capabilities: { + resume: true, + typedEvents: true, + steering: false, + interruption: true, + structuredResult: true, + read: true, + reconciliation: true, + usage: true, + dynamicTools: true, + runtimeRequestResolution: false, + goals: false, + threadLineage: false, + unsupported: ["steering", "runtimeRequestResolution", "goals", "threadLineage"], + }, + }; + } + + async validateConfig(config: unknown): Promise { + return parseConfig(config); + } + + async openSession(input: OpenHarnessSessionInput): Promise { + return new DeterministicHarnessSession(input, this.#config); + } + + async recoverSession( + snapshot: PersistedHarnessSession, + ): Promise { + if ( + snapshot.driverKind !== "paperclip-deterministic" + || snapshot.runId === undefined + || snapshot.normalizedSessionId === undefined + ) { + return { recovered: false, reason: "snapshot does not belong to the deterministic driver" }; + } + return { + recovered: true, + session: new DeterministicHarnessSession({ + runId: snapshot.runId, + normalizedSessionId: snapshot.normalizedSessionId, + workingDirectory: "/deterministic/recovered", + }, this.#config, snapshot), + }; + } +} + +class DeterministicHarnessSession implements HarnessSession { + readonly #input: OpenHarnessSessionInput; + readonly #config: DeterministicHarnessDriverConfig; + #events: PrpEvent[] = []; + #semanticResult: PrpStructuredRunResult | null = null; + #turnId: string | null = null; + #closed = false; + #active = false; + #recoveredActiveTurn = false; + #recoveredTurnSettled = false; + #nextSourceSeq = 1; + readonly #transcriptOmissionReason: string | null; + readonly #eventWaiters = new Set<() => void>(); + + constructor( + input: OpenHarnessSessionInput, + config: DeterministicHarnessDriverConfig, + snapshot?: PersistedHarnessSession, + ) { + this.#input = structuredClone(input); + this.#config = structuredClone(config); + this.#turnId = snapshot?.activeTurnId ?? snapshot?.semanticResult?.turnId ?? null; + this.#semanticResult = snapshot?.semanticResult?.result ?? null; + this.#active = snapshot?.activeTurnId !== null && snapshot?.activeTurnId !== undefined; + this.#recoveredActiveTurn = this.#active; + this.#nextSourceSeq = (snapshot?.lastSourceSequence ?? 0) + 1; + this.#transcriptOmissionReason = snapshot === undefined + ? null + : "pre_recovery_events_not_reconstructed"; + } + + ids() { + return { + driverSessionId: `driver_${this.#input.normalizedSessionId}`, + providerSessionId: null, + displayId: this.#input.normalizedSessionId, + }; + } + + async *events(): AsyncIterable { + if (this.#recoveredActiveTurn && this.#active && this.#turnId !== null) { + // The deterministic fake has no provider process that can continue an + // in-flight turn after reconstruction. Produce one conservative terminal + // outcome when consumption resumes so the native loop cannot wait until + // its global timeout. An explicit interrupt issued before consumption + // still wins and supplies its own reason. + if (!this.#settleRecoveredSemanticResult()) { + this.#recoveredActiveTurn = false; + this.#semanticResult = interruptedResult( + this.#input.runId, + "The deterministic provider was unavailable after active-turn recovery.", + ); + this.#appendEvents( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.interrupted", { + reason: "deterministic_active_turn_recovered_without_live_producer", + }), + this.#event("run.terminal", cancelledTerminal(), { turn: false }), + ); + this.#recoveredTurnSettled = true; + } + this.#active = false; + } + let index = 0; + while (true) { + while (index < this.#events.length) { + const event = this.#events[index++]!; + yield structuredClone(event); + if (event.eventType === "run.terminal") return; + } + if (this.#closed) return; + await new Promise((resolve) => this.#eventWaiters.add(resolve)); + } + } + + async startTurn(input: { message: { role: "user"; text: string } }): Promise<{ turnId: string }> { + this.#assertOpen(); + if (this.#turnId !== null) throw new Error("deterministic session accepts one turn"); + this.#turnId = `turn_${this.#input.runId}`; + this.#active = true; + this.#appendEvents( + this.#event("session.started", {}, { turn: false }), + this.#event("turn.started", { message: input.message.text }), + ); + if (input.message.text.includes("[conformance:interrupt]")) { + return { turnId: this.#turnId }; + } + + const itemId = `item_${this.#input.runId}`; + const callId = `call_${this.#input.runId}`; + const correlation = { + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + turnId: this.#turnId, + itemId, + }; + const inputEnvelope = createPrpSemanticToolInputEnvelope({ + operationId: "finish_task", + callId, + correlation, + idempotencyKey: `finish_${this.#input.runId}`, + content: { summary: "Deterministic conformance complete." }, + }); + const resultEnvelope = createPrpSemanticToolResultEnvelope({ + operationId: "finish_task", + callId, + correlation, + idempotencyKey: `finish_${this.#input.runId}`, + content: { disposition: "applied", status: "done" }, + outcome: "succeeded", + code: "applied", + retryable: false, + authorizationBoundary: "active_task", + auditReceiptId: `audit_${this.#input.runId}`, + }); + this.#appendEvents( + this.#event("mcp_app.tool_input", { semantic_tool: inputEnvelope }, { itemId }), + this.#event("mcp_app.tool_result", { semantic_tool: resultEnvelope }, { itemId }), + ); + this.#semanticResult = completedResult(); + const terminal = completedTerminal(); + this.#appendEvents( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.completed", {}), + this.#event("run.terminal", terminal, { turn: false }), + ); + this.#active = false; + return { turnId: this.#turnId }; + } + + async interrupt(input: { turnId?: string; reason?: string }): Promise { + this.#assertOpen(); + if (this.#turnId === null) throw new Error("no active turn to interrupt"); + if (input.turnId !== undefined && input.turnId !== this.#turnId) { + throw new Error(`turn ${input.turnId} is not active`); + } + // Event consumption can conservatively settle a reconstructed turn before + // its first event is yielded. Treat a concurrent interrupt as an + // idempotent acknowledgement once that recovered terminal is committed, + // rather than making cancellation depend on iterator scheduling. + if (!this.#active) { + if (this.#recoveredTurnSettled) return; + throw new Error("no active turn to interrupt"); + } + if (this.#settleRecoveredSemanticResult()) return; + this.#semanticResult = interruptedResult( + this.#input.runId, + input.reason ?? "The deterministic turn was interrupted.", + ); + this.#appendEvents( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.interrupted", { reason: input.reason ?? "interrupted" }), + this.#event("run.terminal", cancelledTerminal(), { turn: false }), + ); + this.#active = false; + } + + #settleRecoveredSemanticResult(): boolean { + if ( + !this.#recoveredActiveTurn + || !this.#active + || this.#turnId === null + || this.#semanticResult === null + ) return false; + this.#recoveredActiveTurn = false; + // A result-first checkpoint can still carry activeTurnId when the + // terminal checkpoint lagged behind the provider work. Preserve the + // authoritative result even if cancellation races event consumption. + if (this.#semanticResult.reportedWorkDisposition === "yielded") { + this.#appendEvents( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.interrupted", { + reason: "deterministic_active_turn_recovered_with_yielded_result", + }), + this.#event("run.terminal", cancelledTerminal(), { turn: false }), + ); + } else { + this.#appendEvents( + this.#event("run.result.proposed", this.#semanticResult), + this.#event("turn.completed", {}), + this.#event( + "run.terminal", + completedTerminal(this.#semanticResult.reportedWorkDisposition), + { turn: false }, + ), + ); + } + this.#recoveredTurnSettled = true; + this.#active = false; + return true; + } + + async read(): Promise> { + return { scenario: this.#config.scenario, eventCount: this.#events.length }; + } + + async reconcile(): Promise> { + return { active: this.#active, closed: this.#closed, eventCount: this.#events.length }; + } + + async usage(): Promise> { + return { + inputTokens: this.#turnId === null ? 0 : 12, + outputTokens: this.#semanticResult === null ? 0 : 8, + totalTokens: this.#turnId === null ? 0 : this.#semanticResult === null ? 12 : 20, + requestCount: this.#turnId === null ? 0 : 1, + cost: { currency: "USD", amountMicros: 0 }, + }; + } + + async transcript(): Promise { + return { + schema: "paperclip-runner/harness-transcript/v1", + complete: this.#transcriptOmissionReason === null, + eventCount: this.#events.length, + events: structuredClone(this.#events), + omissionReason: this.#transcriptOmissionReason, + }; + } + + async snapshot(): Promise { + return { + driverKind: "paperclip-deterministic", + driverSessionId: `driver_${this.#input.normalizedSessionId}`, + providerSessionId: null, + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + activeTurnId: this.#active ? this.#turnId : null, + lastSourceSequence: this.#nextSourceSeq - 1, + semanticResult: this.#semanticResult === null || this.#turnId === null + ? null + : { + result: structuredClone(this.#semanticResult), + fingerprint: JSON.stringify(this.#semanticResult), + turnId: this.#turnId, + }, + terminalTurns: this.#events.some((event) => event.eventType === "run.terminal") && this.#turnId !== null + ? [{ turnId: this.#turnId, fingerprint: `terminal_${this.#input.runId}` }] + : [], + }; + } + + async close(input: { reason: string; force?: boolean }): Promise { + if (this.#closed) return; + if (this.#active) { + await this.interrupt({ reason: input.reason }); + } + this.#closed = true; + this.#notifyEventWaiters(); + } + + #appendEvents(...events: PrpEvent[]): void { + this.#events.push(...events); + this.#notifyEventWaiters(); + } + + #notifyEventWaiters(): void { + for (const resolve of this.#eventWaiters) resolve(); + this.#eventWaiters.clear(); + } + + #event( + eventType: PrpEvent["eventType"], + payload: Record | PrpStructuredRunResult | PrpTerminalState, + options: { itemId?: string; turn?: boolean } = {}, + ): PrpEvent { + const sourceSeq = this.#nextSourceSeq++; + return { + schema: "paperclip.prp.event.v1", + sourceEventId: `event_${this.#input.runId}_${sourceSeq}`, + sourceSeq, + sourceInstanceId: `runner_${this.#input.runId}`, + sourceKind: "runner", + runId: this.#input.runId, + normalizedSessionId: this.#input.normalizedSessionId, + ...(options.turn === false || this.#turnId === null ? {} : { turnId: this.#turnId }), + ...(options.itemId === undefined ? {} : { itemId: options.itemId }), + eventType, + schemaVersion: 1, + priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1, + emittedAt: new Date(Date.UTC(2026, 7, 11, 12, 0, sourceSeq)).toISOString(), + payload: structuredClone(payload), + } as PrpEvent; + } + + #assertOpen(): void { + if (this.#closed) throw new Error("deterministic session is closed"); + } +} + +function completedResult(): PrpStructuredRunResult { + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "done", + summary: "Deterministic harness-driver conformance completed.", + completionClaim: { + contractRevision: "harness-driver-conformance-v1", + objectiveSatisfied: true, + criteria: [], + remainingWork: [], + }, + evidence: [], + verification: [{ commandOrCheck: "deterministic harness flow", status: "passed" }], + attentionRequests: [], + artifacts: [], + }; +} + +function completedTerminal( + disposition: PrpTerminalState["reportedWorkDisposition"] = "done", +): PrpTerminalState { + return { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "completed", + runTerminalState: "succeeded", + reportedWorkDisposition: disposition, + }; +} + +function interruptedResult(runId: string, summary: string): PrpStructuredRunResult { + return { + schema: "paperclip.run_result.v1", + reportedWorkDisposition: "yielded", + summary, + completionClaim: { + contractRevision: "harness-driver-conformance-v1", + objectiveSatisfied: false, + criteria: [], + remainingWork: [{ + description: "Resume the interrupted work in a live provider session.", + blocksCompletion: true, + }], + }, + evidence: [], + verification: [], + attentionRequests: [], + artifacts: [], + continuation: { + kind: "same_agent", + summary: "Resume the interrupted deterministic turn.", + idempotencyKey: `resume_deterministic_${runId}`, + }, + }; +} + +function cancelledTerminal(): PrpTerminalState { + return { + schema: "paperclip.prp.terminal.v1", + turnTerminalState: "interrupted", + runTerminalState: "cancelled", + reportedWorkDisposition: "yielded", + }; +} diff --git a/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.test.ts b/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.test.ts new file mode 100644 index 0000000000..605ea6b2b1 --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { loadConformanceFixture } from "../protocol/conformance-fixture.js"; +import { replayConformanceFixture } from "../tracer/conformance-runner.js"; +import { MockControlPlaneAdapter } from "./mock-control-plane-adapter.js"; + +describe("MockControlPlaneAdapter", () => { + it("exercises the complete control-plane contract path", async () => { + const fixture = await loadConformanceFixture(); + const mockCore = new MockControlPlaneAdapter(); + + await mockCore.start(); + await replayConformanceFixture(mockCore, fixture); + + expect(mockCore.snapshot()).toMatchObject({ + lifecycle: "running", + openedRun: { identity: fixture.run, backendKind: "mock" }, + events: fixture.events, + result: fixture.result, + }); + await mockCore.stop(); + }); + + it("rejects events before startup", async () => { + const fixture = await loadConformanceFixture(); + const mockCore = new MockControlPlaneAdapter(); + + await expect(mockCore.appendEvent(fixture.events[0]!)).rejects.toThrow( + "mock control plane must be started first", + ); + }); +}); diff --git a/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.ts b/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.ts new file mode 100644 index 0000000000..0ca497af6b --- /dev/null +++ b/packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.ts @@ -0,0 +1,199 @@ +import type { + AppendedEventReceipt, + CompleteControlPlaneRunInput, + ControlPlanePort, + OpenControlPlaneRunInput, + ReplayControlPlaneEventsInput, + ReplayedControlPlaneEvents, +} from "../contracts/control-plane-port.js"; +import type { NativeRunEvent, NativeRunResult } from "../contracts/types.js"; +import { + validatePrpEvent, + validatePrpStructuredRunResult, + type PrpEvent, +} from "../protocol/replay-contract.js"; + +export interface MockControlPlaneSnapshot { + lifecycle: "stopped" | "running"; + openedRun: OpenControlPlaneRunInput | null; + events: Array; + result: NativeRunResult | CompleteControlPlaneRunInput | null; +} + +/** In-memory control-plane shell used by standalone tests and tutorials. */ +export class MockControlPlaneAdapter implements ControlPlanePort { + #lifecycle: "stopped" | "running" = "stopped"; + #openedRun: OpenControlPlaneRunInput | null = null; + #events: Array = []; + #result: NativeRunResult | CompleteControlPlaneRunInput | null = null; + + async start(): Promise { + if (this.#lifecycle === "running") { + throw new Error("mock control plane is already running"); + } + this.#lifecycle = "running"; + } + + async stop(): Promise { + this.#lifecycle = "stopped"; + } + + async openRun(input: OpenControlPlaneRunInput): Promise { + this.#assertRunning(); + if (this.#openedRun !== null) { + throw new Error("mock control plane accepts one Conformance run"); + } + this.#openedRun = structuredClone(input); + } + + async appendEvent(event: NativeRunEvent | PrpEvent): Promise { + const runId = this.#requireRunId(); + if (event.runId !== runId) { + throw new Error("event runId does not match the opened run"); + } + if ("sourceEventId" in event) { + const validation = validatePrpEvent(event); + if (!validation.ok) { + throw new Error(`event failed validation: ${validation.issues[0]?.message ?? "invalid event"}`); + } + const sameId = this.#events.find( + (candidate): candidate is PrpEvent => + "sourceEventId" in candidate && candidate.sourceEventId === event.sourceEventId, + ); + const sameSequence = this.#events.find( + (candidate): candidate is PrpEvent => + "sourceSeq" in candidate && + candidate.sourceInstanceId === event.sourceInstanceId && + candidate.sourceSeq === event.sourceSeq, + ); + for (const candidate of [sameId, sameSequence]) { + if (candidate !== undefined) { + if (canonicalJson(candidate) !== canonicalJson(event)) { + throw new Error("native_event_replay_conflict"); + } + return { + cursor: event.sourceSeq, + highestContiguousSourceSeq: this.#highestContiguous(event.sourceInstanceId), + disposition: "duplicate", + }; + } + } + this.#events.push(structuredClone(event)); + return { + cursor: event.sourceSeq, + highestContiguousSourceSeq: this.#highestContiguous(event.sourceInstanceId), + disposition: "committed", + }; + } + + const expectedSequence = this.#events.length + 1; + if (event.sequence !== expectedSequence) { + throw new Error(`event sequence must be ${expectedSequence}`); + } + this.#events.push(structuredClone(event)); + return { + cursor: event.sequence, + highestContiguousSourceSeq: event.sequence, + disposition: "committed", + }; + } + + async replayEvents( + input: ReplayControlPlaneEventsInput, + ): Promise { + const runId = this.#requireRunId(); + if (input.runId !== runId) { + throw new Error("replay runId does not match the opened run"); + } + const events = this.#events + .filter( + (event): event is PrpEvent => + "sourceSeq" in event && + event.sourceInstanceId === input.sourceInstanceId && + event.sourceSeq > input.afterSourceSeq, + ) + .sort((left, right) => left.sourceSeq - right.sourceSeq) + .slice(0, input.limit) + .map((event) => structuredClone(event)); + return { + events, + highestContiguousSourceSeq: this.#highestContiguous(input.sourceInstanceId), + }; + } + + async completeRun(result: NativeRunResult | CompleteControlPlaneRunInput): Promise { + const runId = this.#requireRunId(); + const resultRunId = "runId" in result ? result.runId : runId; + if (resultRunId !== runId) { + throw new Error("result runId does not match the opened run"); + } + if ("terminal" in result) { + const validation = validatePrpStructuredRunResult(result.result); + if (!validation.ok) { + throw new Error("native_result_invalid"); + } + const terminalEvent = this.#events.find( + (event) => "eventType" in event && event.eventType === "run.terminal", + ); + if (terminalEvent === undefined) { + throw new Error("run.terminal must be ingested before the result"); + } + } else if (this.#events.at(-1)?.type !== "run.completed") { + throw new Error("run.completed must be ingested before the result"); + } + if (this.#result !== null) { + if (canonicalJson(this.#result) === canonicalJson(result)) return; + throw new Error("native_result_replay_conflict"); + } + this.#result = structuredClone(result); + } + + snapshot(): MockControlPlaneSnapshot { + return { + lifecycle: this.#lifecycle, + openedRun: this.#openedRun === null ? null : structuredClone(this.#openedRun), + events: structuredClone(this.#events), + result: this.#result === null ? null : structuredClone(this.#result), + }; + } + + #assertRunning(): void { + if (this.#lifecycle !== "running") { + throw new Error("mock control plane must be started first"); + } + } + + #requireRunId(): string { + this.#assertRunning(); + if (this.#openedRun === null) { + throw new Error("a run must be opened first"); + } + return this.#openedRun.identity.runId; + } + + #highestContiguous(sourceInstanceId: string): number { + const sequences = new Set( + this.#events + .filter( + (event): event is PrpEvent => + "sourceSeq" in event && event.sourceInstanceId === sourceInstanceId, + ) + .map((event) => event.sourceSeq), + ); + let cursor = 0; + while (sequences.has(cursor + 1)) cursor += 1; + return cursor; + } +} + +function canonicalJson(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + if (typeof value === "object" && value !== null) { + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`) + .join(",")}}`; + } + return JSON.stringify(value) ?? "undefined"; +} diff --git a/packages/paperclip-runner/src/protocol/conformance-fixture.test.ts b/packages/paperclip-runner/src/protocol/conformance-fixture.test.ts new file mode 100644 index 0000000000..58e81d18de --- /dev/null +++ b/packages/paperclip-runner/src/protocol/conformance-fixture.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "vitest"; + +import { + loadConformanceFixture, + validateConformanceFixture, +} from "./conformance-fixture.js"; + +describe("Conformance fixture validation", () => { + it("accepts the package-owned fixture", async () => { + const fixture = await loadConformanceFixture(); + + expect(fixture.run.runId).toBe("run_conformance_0001"); + expect(fixture.events.map((event) => event.type)).toEqual([ + "run.started", + "run.completed", + ]); + expect(fixture.result.status).toBe("succeeded"); + }); + + it("rejects a non-contiguous event sequence", () => { + expect(() => + validateConformanceFixture({ + schemaVersion: "paperclip.runner.conformance.fixture.v1", + run: { + runId: "run_conformance_0001", + sessionId: "session_conformance_0001", + companyId: "company_conformance", + issueId: "issue_conformance", + agentId: "agent_conformance", + }, + events: [ + { + eventId: "event_conformance_0001", + runId: "run_conformance_0001", + sequence: 2, + type: "run.started", + }, + { + eventId: "event_conformance_0002", + runId: "run_conformance_0001", + sequence: 3, + type: "run.completed", + }, + ], + result: { + runId: "run_conformance_0001", + status: "succeeded", + summary: "Standalone Conformance fixture accepted.", + }, + }), + ).toThrow("events[0].sequence must be 1"); + }); +}); diff --git a/packages/paperclip-runner/src/protocol/conformance-fixture.ts b/packages/paperclip-runner/src/protocol/conformance-fixture.ts new file mode 100644 index 0000000000..4303a9d58d --- /dev/null +++ b/packages/paperclip-runner/src/protocol/conformance-fixture.ts @@ -0,0 +1,127 @@ +import { readFile } from "node:fs/promises"; + +import type { + NativeRunEvent, + NativeRunEventType, + NativeRunIdentity, + NativeRunResult, + NativeRunStatus, +} from "../contracts/types.js"; + +export const CONFORMANCE_FIXTURE_SCHEMA = "paperclip.runner.conformance.fixture.v1"; + +export interface ConformanceFixture { + schemaVersion: typeof CONFORMANCE_FIXTURE_SCHEMA; + run: NativeRunIdentity; + events: NativeRunEvent[]; + result: NativeRunResult; +} + +export const conformanceFixtureUrl = new URL( + "../../protocol/fixtures/conformance-minimal-run.json", + import.meta.url, +); + +function asRecord(value: unknown, path: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error(`${path} must be an object`); + } + return value as Record; +} + +function asIdentifier(value: unknown, path: string): string { + if (typeof value !== "string" || !/^[a-z][a-z0-9_]*$/.test(value)) { + throw new Error(`${path} must be a stable lowercase identifier`); + } + return value; +} + +function asString(value: unknown, path: string): string { + if (typeof value !== "string" || value.trim().length === 0) { + throw new Error(`${path} must be a non-empty string`); + } + return value; +} + +function parseRun(value: unknown): NativeRunIdentity { + const run = asRecord(value, "run"); + return { + runId: asIdentifier(run.runId, "run.runId"), + sessionId: asIdentifier(run.sessionId, "run.sessionId"), + companyId: asIdentifier(run.companyId, "run.companyId"), + issueId: asIdentifier(run.issueId, "run.issueId"), + agentId: asIdentifier(run.agentId, "run.agentId"), + }; +} + +function parseEvents(value: unknown, runId: string): NativeRunEvent[] { + if (!Array.isArray(value) || value.length < 2) { + throw new Error("events must contain at least run.started and run.completed"); + } + + const allowedTypes = new Set(["run.started", "run.completed"]); + const events = value.map((entry, index) => { + const event = asRecord(entry, `events[${index}]`); + const type = asString(event.type, `events[${index}].type`) as NativeRunEventType; + if (!allowedTypes.has(type)) { + throw new Error(`events[${index}].type is unsupported`); + } + if (event.runId !== runId) { + throw new Error(`events[${index}].runId must match run.runId`); + } + if (event.sequence !== index + 1) { + throw new Error(`events[${index}].sequence must be ${index + 1}`); + } + return { + eventId: asIdentifier(event.eventId, `events[${index}].eventId`), + runId, + sequence: index + 1, + type, + }; + }); + + if (events[0]?.type !== "run.started") { + throw new Error("the first event must be run.started"); + } + if (events.at(-1)?.type !== "run.completed") { + throw new Error("the final event must be run.completed"); + } + return events; +} + +function parseResult(value: unknown, runId: string): NativeRunResult { + const result = asRecord(value, "result"); + if (result.runId !== runId) { + throw new Error("result.runId must match run.runId"); + } + const status = asString(result.status, "result.status") as NativeRunStatus; + if (status !== "succeeded") { + throw new Error("the Conformance fixture result must be succeeded"); + } + return { + runId, + status, + summary: asString(result.summary, "result.summary"), + }; +} + +export function validateConformanceFixture(value: unknown): ConformanceFixture { + const fixture = asRecord(value, "fixture"); + if (fixture.schemaVersion !== CONFORMANCE_FIXTURE_SCHEMA) { + throw new Error(`schemaVersion must be ${CONFORMANCE_FIXTURE_SCHEMA}`); + } + const run = parseRun(fixture.run); + return { + schemaVersion: CONFORMANCE_FIXTURE_SCHEMA, + run, + events: parseEvents(fixture.events, run.runId), + result: parseResult(fixture.result, run.runId), + }; +} + +export async function loadConformanceFixture( + url: URL = conformanceFixtureUrl, +): Promise { + const contents = await readFile(url, "utf8"); + return validateConformanceFixture(JSON.parse(contents) as unknown); +} diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts index cfc6a4bb0b..14a11890a1 100644 --- a/packages/paperclip-runner/src/testing.ts +++ b/packages/paperclip-runner/src/testing.ts @@ -9,4 +9,8 @@ export * from "./index.js"; export * from "./conformance/control-plane-port.js"; export * from "./conformance/harness-driver.js"; export * from "./conformance/semantic-conformance.js"; +export * from "./mock-core/deterministic-harness-driver.js"; +export * from "./mock-core/mock-control-plane-adapter.js"; +export * from "./protocol/conformance-fixture.js"; export * from "./protocol/replay-loader.js"; +export * from "./tracer/conformance-runner.js"; diff --git a/packages/paperclip-runner/src/tracer/conformance-runner.test.ts b/packages/paperclip-runner/src/tracer/conformance-runner.test.ts new file mode 100644 index 0000000000..6470818d78 --- /dev/null +++ b/packages/paperclip-runner/src/tracer/conformance-runner.test.ts @@ -0,0 +1,21 @@ +import { readFile } from "node:fs/promises"; + +import { describe, expect, it } from "vitest"; + +import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js"; +import { runConformanceTracer } from "./conformance-runner.js"; + +const expectedOutputUrl = new URL( + "../../protocol/fixtures/conformance-expected-output.json", + import.meta.url, +); + +describe("Conformance tracer", () => { + it("prints a stable identity and result and stops the mock core", async () => { + const mockCore = new MockControlPlaneAdapter(); + const expectedOutput = (await readFile(expectedOutputUrl, "utf8")).trimEnd(); + + await expect(runConformanceTracer(mockCore)).resolves.toBe(expectedOutput); + expect(mockCore.snapshot().lifecycle).toBe("stopped"); + }); +}); diff --git a/packages/paperclip-runner/src/tracer/conformance-runner.ts b/packages/paperclip-runner/src/tracer/conformance-runner.ts new file mode 100644 index 0000000000..a598a8ea2d --- /dev/null +++ b/packages/paperclip-runner/src/tracer/conformance-runner.ts @@ -0,0 +1,59 @@ +import type { ControlPlanePort } from "../contracts/control-plane-port.js"; +import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js"; +import { + loadConformanceFixture, + type ConformanceFixture, +} from "../protocol/conformance-fixture.js"; + +export const CONFORMANCE_OUTPUT_SCHEMA = "paperclip.runner.conformance.output.v1"; + +export interface ConformanceTracerOutput { + schemaVersion: typeof CONFORMANCE_OUTPUT_SCHEMA; + runIdentity: { + runId: string; + sessionId: string; + }; + result: { + status: "succeeded"; + summary: string; + }; +} + +export async function replayConformanceFixture( + controlPlane: ControlPlanePort, + fixture: ConformanceFixture, +): Promise { + await controlPlane.openRun({ identity: fixture.run, backendKind: "mock" }); + for (const event of fixture.events) { + await controlPlane.appendEvent(event); + } + await controlPlane.completeRun(fixture.result); +} + +export function formatConformanceOutput(fixture: ConformanceFixture): string { + const output: ConformanceTracerOutput = { + schemaVersion: CONFORMANCE_OUTPUT_SCHEMA, + runIdentity: { + runId: fixture.run.runId, + sessionId: fixture.run.sessionId, + }, + result: { + status: "succeeded", + summary: fixture.result.summary, + }, + }; + return JSON.stringify(output); +} + +export async function runConformanceTracer( + mockCore = new MockControlPlaneAdapter(), +): Promise { + const fixture = await loadConformanceFixture(); + await mockCore.start(); + try { + await replayConformanceFixture(mockCore, fixture); + return formatConformanceOutput(fixture); + } finally { + await mockCore.stop(); + } +}