Add deterministic runner conformance core (#12354)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The runner needs repeatable tests at the provider and control-plane boundaries > - Network services and live providers cannot produce deterministic conformance results > - Small in-memory adapters can exercise the same public contracts without production side effects > - This pull request adds a deterministic harness driver, control-plane adapter, and fixture runner > - It exposes these helpers only from the testing entry point > - The benefit is stable cross-boundary verification for later runtime changes ## Linked Issues or Issue Description **Subsystem affected** packages/paperclip-runner testing and conformance support **Problem or motivation** The public runner contracts have unit tests, but later provider and server slices need a deterministic way to exercise session, event, replay, checkpoint, and result behavior together. **Proposed solution** Add in-memory implementations of the harness-driver and control-plane ports. Add a canonical conformance fixture and runner. Export these utilities only from `@paperclipai/paperclip-runner/testing`. **Alternatives considered** The tests could start a real provider or server. That would make the conformance gate slower, less portable, and dependent on credentials or network state. **Roadmap alignment** This supports the existing experimental Paperclip Runner rollout. It does not enable a production adapter or change current app execution. ## What Changed - Add a deterministic harness-driver implementation. - Add an in-memory control-plane adapter with replay and checkpoint support. - Add a canonical provider-neutral conformance fixture. - Add a reusable conformance runner and contract tests. - Export the helpers from the package testing entry point only. ## Verification - `pnpm --filter @paperclipai/paperclip-runner test:typescript` - `pnpm --filter @paperclipai/paperclip-runner typecheck:typescript` - `pnpm -r typecheck` - `pnpm build` - The branch changes 10 files relative to its declared base. ## Risks Low risk. This pull request adds test-only adapters and conformance helpers. It does not select a runtime or change production behavior. Contract tests verify event identity, replay, checkpoints, and deterministic results. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used OpenAI Codex, `gpt-5`, with agentic reasoning, tool use, and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either linked an existing public issue or described the issue in-PR - [x] I have not referenced internal or instance-local Paperclip issues or links - [x] My branch name describes the change and contains no internal Paperclip ticket id - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge
This commit is contained in:
Dotta
GitHub
parent
e18632ebcb
commit
773d357d8a
|
|
@ -1,7 +1,7 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import type { HarnessDriver, HarnessSession, PersistedHarnessSession } from "../contracts/harness-driver.js";
|
||||
import type { PrpEvent, PrpStructuredRunResult } from "../protocol/replay-contract.js";
|
||||
import type { PrpEvent, PrpStructuredRunResult, PrpTerminalState } from "../protocol/replay-contract.js";
|
||||
import { HarnessDriverBackend } from "./harness-driver-backend.js";
|
||||
|
||||
const result: PrpStructuredRunResult = {
|
||||
|
|
@ -271,6 +271,105 @@ describe("HarnessDriverBackend", () => {
|
|||
expect(receivedSignal).toBe(controller.signal);
|
||||
});
|
||||
|
||||
it("does not resurrect a settled turn across repeated backend recovery", async () => {
|
||||
const terminal: PrpTerminalState = {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: "completed",
|
||||
runTerminalState: "succeeded",
|
||||
reportedWorkDisposition: "done",
|
||||
};
|
||||
const recoveredSnapshots: PersistedHarnessSession[] = [];
|
||||
|
||||
class SettledRecoveredSession extends FakeHarnessSession {
|
||||
readonly #snapshot: PersistedHarnessSession;
|
||||
|
||||
constructor(snapshot: PersistedHarnessSession) {
|
||||
super();
|
||||
this.#snapshot = structuredClone(snapshot);
|
||||
}
|
||||
|
||||
override async *events() {
|
||||
// Model the deterministic driver's recovery rule: only an active turn
|
||||
// produces a reconstructed result/terminal sequence.
|
||||
if (this.#snapshot.activeTurnId === null || this.#snapshot.activeTurnId === undefined) return;
|
||||
yield prpEvent(3, "run.result.proposed", result);
|
||||
yield prpEvent(4, "turn.completed", { status: "completed" });
|
||||
yield prpEvent(5, "run.terminal", terminal);
|
||||
}
|
||||
|
||||
override async snapshot(): Promise<PersistedHarnessSession> {
|
||||
return structuredClone(this.#snapshot);
|
||||
}
|
||||
}
|
||||
|
||||
const recoveryDriver: HarnessDriver = {
|
||||
...driver,
|
||||
async recoverSession(snapshot) {
|
||||
recoveredSnapshots.push(structuredClone(snapshot));
|
||||
return { recovered: true, session: new SettledRecoveredSession(snapshot) };
|
||||
},
|
||||
};
|
||||
const backend = new HarnessDriverBackend(recoveryDriver);
|
||||
const settledSnapshot = {
|
||||
backendKind: "runner" as const,
|
||||
driverKind: "fake",
|
||||
sessionId: "driver-1",
|
||||
providerSessionId: "provider-1",
|
||||
identity: {
|
||||
runId: "run-1",
|
||||
sessionId: "session-1",
|
||||
companyId: "company-1",
|
||||
issueId: "issue-1",
|
||||
agentId: "agent-1",
|
||||
},
|
||||
cursor: "2",
|
||||
semanticResult: result,
|
||||
terminal,
|
||||
activeTurnId: null,
|
||||
terminalTurns: [{ turnId: "turn-1", fingerprint: "terminal-1" }],
|
||||
};
|
||||
|
||||
const recoveryOptions = { signal: new AbortController().signal };
|
||||
const firstRecovery = await backend.recoverSession(
|
||||
settledSnapshot,
|
||||
recoveryOptions,
|
||||
);
|
||||
expect(firstRecovery.recovered).toBe(true);
|
||||
const firstSession = firstRecovery.session!;
|
||||
const firstRecoveredSnapshot = await firstSession.snapshot();
|
||||
expect(firstRecoveredSnapshot).toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: result,
|
||||
terminal,
|
||||
});
|
||||
await expect(firstSession.result()).resolves.toEqual({
|
||||
result,
|
||||
terminal,
|
||||
turnId: "turn-1",
|
||||
});
|
||||
|
||||
const secondRecovery = await backend.recoverSession(
|
||||
firstRecoveredSnapshot,
|
||||
recoveryOptions,
|
||||
);
|
||||
expect(secondRecovery.recovered).toBe(true);
|
||||
const secondSession = secondRecovery.session!;
|
||||
await expect(secondSession.snapshot()).resolves.toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: result,
|
||||
terminal,
|
||||
});
|
||||
const replayedEvents: PrpEvent[] = [];
|
||||
for await (const event of secondSession.events()) replayedEvents.push(event);
|
||||
expect(replayedEvents.filter((event) =>
|
||||
event.eventType === "run.result.proposed" || event.eventType === "run.terminal"
|
||||
)).toEqual([]);
|
||||
expect(recoveredSnapshots.map((snapshot) => snapshot.activeTurnId)).toEqual([null, null]);
|
||||
|
||||
await secondSession.close({ reason: "double recovery complete" });
|
||||
await firstSession.close({ reason: "first recovery complete" });
|
||||
});
|
||||
|
||||
it("allows only the run id to change when a harness session is attached", async () => {
|
||||
const attachedRunIds: string[] = [];
|
||||
class AttachableHarnessSession extends FakeHarnessSession {
|
||||
|
|
|
|||
|
|
@ -70,6 +70,12 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
|||
if (this.#driver.recoverSession === undefined) {
|
||||
return { recovered: false, reason: "driver does not support recovery" };
|
||||
}
|
||||
// `null` is a durable "no active turn" checkpoint. Only an omitted legacy
|
||||
// field may use the compatibility fallback; nullish coalescing here would
|
||||
// otherwise turn the most recent terminal turn back into an active one.
|
||||
const activeTurnId = snapshot.activeTurnId === undefined
|
||||
? snapshot.terminalTurns?.at(-1)?.turnId ?? null
|
||||
: snapshot.activeTurnId;
|
||||
const persisted: PersistedHarnessSession = {
|
||||
driverKind: snapshot.driverKind ?? snapshot.backendKind,
|
||||
driverSessionId: snapshot.sessionId,
|
||||
|
|
@ -82,7 +88,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
|||
: { providerRecoveryPolicy: snapshot.providerRecoveryPolicy }),
|
||||
runId: snapshot.identity.runId,
|
||||
normalizedSessionId: snapshot.identity.sessionId,
|
||||
activeTurnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? null,
|
||||
activeTurnId,
|
||||
lastSourceSequence: parseCursor(snapshot.cursor),
|
||||
...(snapshot.semanticResult === undefined || snapshot.semanticResult === null
|
||||
? {}
|
||||
|
|
@ -90,7 +96,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
|||
semanticResult: {
|
||||
result: snapshot.semanticResult,
|
||||
fingerprint: canonicalJson(snapshot.semanticResult),
|
||||
turnId: snapshot.activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered",
|
||||
turnId: activeTurnId ?? snapshot.terminalTurns?.at(-1)?.turnId ?? "recovered",
|
||||
},
|
||||
}),
|
||||
terminalTurns: snapshot.terminalTurns ?? [],
|
||||
|
|
@ -125,6 +131,7 @@ export class HarnessDriverBackend implements NativeSessionBackend {
|
|||
session: new HarnessNativeSession(
|
||||
{ identity: snapshot.identity },
|
||||
recovered.session,
|
||||
snapshot.terminal,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
|
@ -154,9 +161,14 @@ class HarnessNativeSession implements NativeSession {
|
|||
#terminal: PrpTerminalState | null = null;
|
||||
#explicitlyCancelled = false;
|
||||
|
||||
constructor(input: OpenNativeSessionInput, session: HarnessSession) {
|
||||
constructor(
|
||||
input: OpenNativeSessionInput,
|
||||
session: HarnessSession,
|
||||
terminal?: PrpTerminalState | null,
|
||||
) {
|
||||
this.#input = structuredClone(input);
|
||||
this.#session = session;
|
||||
this.#terminal = terminal === undefined ? null : structuredClone(terminal);
|
||||
}
|
||||
|
||||
identity() {
|
||||
|
|
@ -410,7 +422,12 @@ class HarnessNativeSession implements NativeSession {
|
|||
: String(snapshot.lastSourceSequence),
|
||||
semanticResult: snapshot.semanticResult?.result ?? null,
|
||||
terminal: this.#terminal,
|
||||
activeTurnId: snapshot.activeTurnId ?? snapshot.semanticResult?.turnId ?? null,
|
||||
// Harness snapshots use an explicit null to record a settled turn. Keep
|
||||
// the semantic-result fallback solely for legacy snapshots that omitted
|
||||
// activeTurnId, or a later recovery can replay the terminal turn.
|
||||
activeTurnId: snapshot.activeTurnId === undefined
|
||||
? snapshot.semanticResult?.turnId ?? null
|
||||
: snapshot.activeTurnId,
|
||||
terminalTurns: snapshot.terminalTurns ?? [],
|
||||
pendingRuntimeRequests: snapshot.pendingRuntimeRequests ?? [],
|
||||
lineage: snapshot.lineage ?? [],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,40 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js";
|
||||
import {
|
||||
CONTROL_PLANE_CONFORMANCE_EVENTS,
|
||||
CONTROL_PLANE_CONFORMANCE_OPEN,
|
||||
runControlPlanePortConformance,
|
||||
} from "./control-plane-port.js";
|
||||
|
||||
describe("ControlPlanePort conformance", () => {
|
||||
it("runs unchanged against the mock adapter", async () => {
|
||||
const adapter = new MockControlPlaneAdapter();
|
||||
await expect(runControlPlanePortConformance({
|
||||
port: adapter,
|
||||
start: () => adapter.start(),
|
||||
stop: () => adapter.stop(),
|
||||
})).resolves.toEqual({
|
||||
eventCount: 3,
|
||||
highestContiguousSourceSeq: 3,
|
||||
duplicateDisposition: "duplicate",
|
||||
terminalReplayIdempotent: true,
|
||||
openBindingRejected: true,
|
||||
eventIdMutationRejected: true,
|
||||
eventSequenceMutationRejected: true,
|
||||
replayBindingRejected: true,
|
||||
resultMutationRejected: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a source identity is replayed with different bytes", async () => {
|
||||
const adapter = new MockControlPlaneAdapter();
|
||||
await adapter.start();
|
||||
await adapter.openRun(CONTROL_PLANE_CONFORMANCE_OPEN);
|
||||
await adapter.appendEvent(CONTROL_PLANE_CONFORMANCE_EVENTS[0]);
|
||||
await expect(adapter.appendEvent({
|
||||
...CONTROL_PLANE_CONFORMANCE_EVENTS[0],
|
||||
payload: { backend: "forged" },
|
||||
})).rejects.toThrow("native_event_replay_conflict");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,286 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { DeterministicHarnessDriver } from "../mock-core/deterministic-harness-driver.js";
|
||||
import {
|
||||
loadHarnessDriverConformanceFixture,
|
||||
runHarnessDriverConformance,
|
||||
} from "./harness-driver.js";
|
||||
|
||||
describe("harness-driver conformance V1", () => {
|
||||
it("ships a deterministic fixture covering the complete Evals hook set", async () => {
|
||||
const fixture = await loadHarnessDriverConformanceFixture();
|
||||
expect(fixture.requiredCapabilities).toContain("dynamicTools");
|
||||
expect(fixture.unsupportedFeatures).toContain("steering");
|
||||
|
||||
const report = await runHarnessDriverConformance({
|
||||
driver: new DeterministicHarnessDriver(),
|
||||
fixture,
|
||||
});
|
||||
expect(report).toMatchObject({
|
||||
schema: "paperclip-runner/harness-driver-conformance-report/v1",
|
||||
contractVersion: 1,
|
||||
eventCount: 7,
|
||||
semanticToolCallCount: 1,
|
||||
checks: {
|
||||
capabilityDescription: true,
|
||||
configValidation: true,
|
||||
sessionLifecycle: true,
|
||||
sessionRecovery: true,
|
||||
semanticTools: true,
|
||||
eventValidation: true,
|
||||
interruptAndCancel: true,
|
||||
usage: true,
|
||||
transcriptCompleteness: true,
|
||||
unsupportedFeatures: true,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("settles an actively recovered turn when event consumption resumes", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_active_recovery",
|
||||
normalizedSessionId: "session_active_recovery",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "[conformance:interrupt]" },
|
||||
});
|
||||
const snapshot = await session.snapshot();
|
||||
expect(snapshot).toMatchObject({ activeTurnId: turnId, lastSourceSequence: 2 });
|
||||
|
||||
const firstRecovery = await driver.recoverSession(snapshot);
|
||||
expect(firstRecovery.recovered).toBe(true);
|
||||
expect(firstRecovery.session).toBeDefined();
|
||||
const repeatedSnapshot = await firstRecovery.session!.snapshot();
|
||||
expect(repeatedSnapshot).toMatchObject({
|
||||
activeTurnId: turnId,
|
||||
lastSourceSequence: 2,
|
||||
});
|
||||
const recovery = await driver.recoverSession(repeatedSnapshot);
|
||||
expect(recovery.recovered).toBe(true);
|
||||
expect(recovery.session).toBeDefined();
|
||||
const recovered = recovery.session!;
|
||||
const recoveredEventsPromise = (async () => {
|
||||
const events = [];
|
||||
for await (const event of recovered.events()) events.push(event);
|
||||
return events;
|
||||
})();
|
||||
|
||||
const events = await recoveredEventsPromise;
|
||||
expect(events.map((event) => [event.sourceSeq, event.eventType])).toEqual([
|
||||
[3, "run.result.proposed"],
|
||||
[4, "turn.interrupted"],
|
||||
[5, "run.terminal"],
|
||||
]);
|
||||
await expect(recovered.snapshot()).resolves.toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: {
|
||||
result: { reportedWorkDisposition: "yielded" },
|
||||
turnId,
|
||||
},
|
||||
lastSourceSequence: 5,
|
||||
});
|
||||
expect(events[1]?.payload).toEqual({
|
||||
reason: "deterministic_active_turn_recovered_without_live_producer",
|
||||
});
|
||||
await expect(recovered.transcript?.()).resolves.toMatchObject({
|
||||
schema: "paperclip-runner/harness-transcript/v1",
|
||||
complete: false,
|
||||
eventCount: 3,
|
||||
events: events.map((event) => expect.objectContaining({
|
||||
sourceSeq: event.sourceSeq,
|
||||
eventType: event.eventType,
|
||||
})),
|
||||
omissionReason: "pre_recovery_events_not_reconstructed",
|
||||
});
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await firstRecovery.session!.close({ reason: "first_recovery_complete" });
|
||||
await session.close({ reason: "original_complete", force: true });
|
||||
});
|
||||
|
||||
it("interrupts a recovered turn deterministically before event consumption", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_recovery_interrupt",
|
||||
normalizedSessionId: "session_recovery_interrupt",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "[conformance:interrupt]" },
|
||||
});
|
||||
const recovery = await driver.recoverSession(await session.snapshot());
|
||||
const recovered = recovery.session!;
|
||||
|
||||
await new Promise<void>((resolve) => setImmediate(resolve));
|
||||
await expect(recovered.interrupt?.({
|
||||
turnId,
|
||||
reason: "cancel_before_consumption",
|
||||
})).resolves.toBeUndefined();
|
||||
const events = [];
|
||||
for await (const event of recovered.events()) events.push(event);
|
||||
expect(events.map((event) => [event.sourceSeq, event.eventType])).toEqual([
|
||||
[3, "run.result.proposed"],
|
||||
[4, "turn.interrupted"],
|
||||
[5, "run.terminal"],
|
||||
]);
|
||||
await expect(recovered.snapshot()).resolves.toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: {
|
||||
result: { reportedWorkDisposition: "yielded" },
|
||||
turnId,
|
||||
},
|
||||
lastSourceSequence: 5,
|
||||
});
|
||||
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await session.close({ reason: "original_complete", force: true });
|
||||
});
|
||||
|
||||
it("acknowledges interruption after recovered event settlement starts", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_recovery_interrupt_race",
|
||||
normalizedSessionId: "session_recovery_interrupt_race",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "[conformance:interrupt]" },
|
||||
});
|
||||
const recovery = await driver.recoverSession(await session.snapshot());
|
||||
const recovered = recovery.session!;
|
||||
const recoveredEvents = recovered.events()[Symbol.asyncIterator]();
|
||||
|
||||
const first = await recoveredEvents.next();
|
||||
expect(first.value?.eventType).toBe("run.result.proposed");
|
||||
await expect(recovered.interrupt?.({
|
||||
turnId,
|
||||
reason: "cancel_after_event_settlement_started",
|
||||
})).resolves.toBeUndefined();
|
||||
|
||||
const events = first.done || first.value === undefined ? [] : [first.value];
|
||||
for await (const event of { [Symbol.asyncIterator]: () => recoveredEvents }) {
|
||||
events.push(event);
|
||||
}
|
||||
expect(events.map((event) => event.eventType)).toEqual([
|
||||
"run.result.proposed",
|
||||
"turn.interrupted",
|
||||
"run.terminal",
|
||||
]);
|
||||
expect(events.filter((event) => event.eventType === "run.terminal")).toHaveLength(1);
|
||||
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await session.close({ reason: "original_complete", force: true });
|
||||
});
|
||||
|
||||
it("preserves a completed result when its terminal checkpoint lagged", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_result_first_recovery",
|
||||
normalizedSessionId: "session_result_first_recovery",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "Complete deterministically." },
|
||||
});
|
||||
const completedSnapshot = await session.snapshot();
|
||||
const recovery = await driver.recoverSession({
|
||||
...completedSnapshot,
|
||||
activeTurnId: turnId,
|
||||
terminalTurns: [],
|
||||
});
|
||||
const recovered = recovery.session!;
|
||||
const events = [];
|
||||
for await (const event of recovered.events()) events.push(event);
|
||||
|
||||
expect(events.map((event) => event.eventType)).toEqual([
|
||||
"run.result.proposed",
|
||||
"turn.completed",
|
||||
"run.terminal",
|
||||
]);
|
||||
expect(events[0]?.payload).toMatchObject({ reportedWorkDisposition: "done" });
|
||||
expect(events[2]?.payload).toMatchObject({ runTerminalState: "succeeded" });
|
||||
await expect(recovered.snapshot()).resolves.toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: completedSnapshot.semanticResult,
|
||||
});
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await session.close({ reason: "original_complete" });
|
||||
});
|
||||
|
||||
it("does not let an interrupt overwrite a completed result-first recovery", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_result_first_interrupt",
|
||||
normalizedSessionId: "session_result_first_interrupt",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "Complete deterministically." },
|
||||
});
|
||||
const completedSnapshot = await session.snapshot();
|
||||
const recovery = await driver.recoverSession({
|
||||
...completedSnapshot,
|
||||
activeTurnId: turnId,
|
||||
terminalTurns: [],
|
||||
});
|
||||
const recovered = recovery.session!;
|
||||
|
||||
await expect(recovered.interrupt?.({
|
||||
turnId,
|
||||
reason: "late cancellation",
|
||||
})).resolves.toBeUndefined();
|
||||
const events = [];
|
||||
for await (const event of recovered.events()) events.push(event);
|
||||
expect(events.map((event) => event.eventType)).toEqual([
|
||||
"run.result.proposed",
|
||||
"turn.completed",
|
||||
"run.terminal",
|
||||
]);
|
||||
await expect(recovered.snapshot()).resolves.toMatchObject({
|
||||
activeTurnId: null,
|
||||
semanticResult: completedSnapshot.semanticResult,
|
||||
});
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await session.close({ reason: "original_complete" });
|
||||
});
|
||||
|
||||
it("preserves an interrupted result when its terminal checkpoint lagged", async () => {
|
||||
const driver = new DeterministicHarnessDriver();
|
||||
const session = await driver.openSession({
|
||||
runId: "run_yielded_result_first_recovery",
|
||||
normalizedSessionId: "session_yielded_result_first_recovery",
|
||||
workingDirectory: "/deterministic/conformance",
|
||||
});
|
||||
const { turnId } = await session.startTurn({
|
||||
message: { role: "user", text: "[conformance:interrupt]" },
|
||||
});
|
||||
await session.interrupt?.({ turnId, reason: "checkpoint_after_result" });
|
||||
const interruptedSnapshot = await session.snapshot();
|
||||
const recovery = await driver.recoverSession({
|
||||
...interruptedSnapshot,
|
||||
activeTurnId: turnId,
|
||||
terminalTurns: [],
|
||||
});
|
||||
const recovered = recovery.session!;
|
||||
const events = [];
|
||||
for await (const event of recovered.events()) events.push(event);
|
||||
|
||||
expect(events.map((event) => event.eventType)).toEqual([
|
||||
"run.result.proposed",
|
||||
"turn.interrupted",
|
||||
"run.terminal",
|
||||
]);
|
||||
expect(events[0]?.payload).toMatchObject({
|
||||
reportedWorkDisposition: "yielded",
|
||||
continuation: { kind: "same_agent" },
|
||||
});
|
||||
expect(events[2]?.payload).toMatchObject({
|
||||
turnTerminalState: "interrupted",
|
||||
runTerminalState: "cancelled",
|
||||
reportedWorkDisposition: "yielded",
|
||||
});
|
||||
await recovered.close({ reason: "recovered_complete" });
|
||||
await session.close({ reason: "original_complete" });
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,483 @@
|
|||
import {
|
||||
type HarnessDriver,
|
||||
type HarnessDriverConfigValidation,
|
||||
type HarnessDriverDescriptor,
|
||||
type HarnessSession,
|
||||
type HarnessSessionRecoveryResult,
|
||||
type HarnessTranscriptSnapshot,
|
||||
type OpenHarnessSessionInput,
|
||||
type PersistedHarnessSession,
|
||||
} from "../contracts/harness-driver.js";
|
||||
import {
|
||||
createPrpSemanticToolInputEnvelope,
|
||||
createPrpSemanticToolResultEnvelope,
|
||||
} from "../protocol/semantic-tool-receipts.js";
|
||||
import type {
|
||||
PrpEvent,
|
||||
PrpStructuredRunResult,
|
||||
PrpTerminalState,
|
||||
} from "../protocol/replay-contract.js";
|
||||
|
||||
export const DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA =
|
||||
"paperclip-runner/deterministic-harness-driver-config/v1" as const;
|
||||
|
||||
export interface DeterministicHarnessDriverConfig {
|
||||
schema: typeof DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA;
|
||||
scenario: "semantic-tool-success";
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: DeterministicHarnessDriverConfig = {
|
||||
schema: DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA,
|
||||
scenario: "semantic-tool-success",
|
||||
};
|
||||
|
||||
function parseConfig(value: unknown): HarnessDriverConfigValidation {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
return {
|
||||
ok: false,
|
||||
config: null,
|
||||
issues: [{ path: "/", code: "type", message: "config must be an object" }],
|
||||
};
|
||||
}
|
||||
const config = value as Record<string, unknown>;
|
||||
if (config.schema !== DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA) {
|
||||
return {
|
||||
ok: false,
|
||||
config: null,
|
||||
issues: [{
|
||||
path: "/schema",
|
||||
code: "unsupported_schema",
|
||||
message: `schema must be ${DETERMINISTIC_HARNESS_DRIVER_CONFIG_SCHEMA}`,
|
||||
}],
|
||||
};
|
||||
}
|
||||
if (config.scenario !== "semantic-tool-success") {
|
||||
return {
|
||||
ok: false,
|
||||
config: null,
|
||||
issues: [{
|
||||
path: "/scenario",
|
||||
code: "unsupported_scenario",
|
||||
message: "scenario must be semantic-tool-success",
|
||||
}],
|
||||
};
|
||||
}
|
||||
return { ok: true, config: structuredClone(config), issues: [] };
|
||||
}
|
||||
|
||||
/** Provider-free fake used by packed consumers and App/Evals conformance. */
|
||||
export class DeterministicHarnessDriver implements HarnessDriver {
|
||||
readonly #config: DeterministicHarnessDriverConfig;
|
||||
|
||||
constructor(config: DeterministicHarnessDriverConfig = DEFAULT_CONFIG) {
|
||||
const validation = parseConfig(config);
|
||||
if (!validation.ok) throw new Error(validation.issues[0]?.message ?? "invalid config");
|
||||
this.#config = structuredClone(config);
|
||||
}
|
||||
|
||||
async descriptor(): Promise<HarnessDriverDescriptor> {
|
||||
return {
|
||||
kind: "paperclip-deterministic",
|
||||
displayName: "Paperclip deterministic harness driver",
|
||||
version: "1.0.0",
|
||||
protocolVersion: "prp.v1",
|
||||
capabilities: {
|
||||
resume: true,
|
||||
typedEvents: true,
|
||||
steering: false,
|
||||
interruption: true,
|
||||
structuredResult: true,
|
||||
read: true,
|
||||
reconciliation: true,
|
||||
usage: true,
|
||||
dynamicTools: true,
|
||||
runtimeRequestResolution: false,
|
||||
goals: false,
|
||||
threadLineage: false,
|
||||
unsupported: ["steering", "runtimeRequestResolution", "goals", "threadLineage"],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async validateConfig(config: unknown): Promise<HarnessDriverConfigValidation> {
|
||||
return parseConfig(config);
|
||||
}
|
||||
|
||||
async openSession(input: OpenHarnessSessionInput): Promise<HarnessSession> {
|
||||
return new DeterministicHarnessSession(input, this.#config);
|
||||
}
|
||||
|
||||
async recoverSession(
|
||||
snapshot: PersistedHarnessSession,
|
||||
): Promise<HarnessSessionRecoveryResult> {
|
||||
if (
|
||||
snapshot.driverKind !== "paperclip-deterministic"
|
||||
|| snapshot.runId === undefined
|
||||
|| snapshot.normalizedSessionId === undefined
|
||||
) {
|
||||
return { recovered: false, reason: "snapshot does not belong to the deterministic driver" };
|
||||
}
|
||||
return {
|
||||
recovered: true,
|
||||
session: new DeterministicHarnessSession({
|
||||
runId: snapshot.runId,
|
||||
normalizedSessionId: snapshot.normalizedSessionId,
|
||||
workingDirectory: "/deterministic/recovered",
|
||||
}, this.#config, snapshot),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
class DeterministicHarnessSession implements HarnessSession {
|
||||
readonly #input: OpenHarnessSessionInput;
|
||||
readonly #config: DeterministicHarnessDriverConfig;
|
||||
#events: PrpEvent[] = [];
|
||||
#semanticResult: PrpStructuredRunResult | null = null;
|
||||
#turnId: string | null = null;
|
||||
#closed = false;
|
||||
#active = false;
|
||||
#recoveredActiveTurn = false;
|
||||
#recoveredTurnSettled = false;
|
||||
#nextSourceSeq = 1;
|
||||
readonly #transcriptOmissionReason: string | null;
|
||||
readonly #eventWaiters = new Set<() => void>();
|
||||
|
||||
constructor(
|
||||
input: OpenHarnessSessionInput,
|
||||
config: DeterministicHarnessDriverConfig,
|
||||
snapshot?: PersistedHarnessSession,
|
||||
) {
|
||||
this.#input = structuredClone(input);
|
||||
this.#config = structuredClone(config);
|
||||
this.#turnId = snapshot?.activeTurnId ?? snapshot?.semanticResult?.turnId ?? null;
|
||||
this.#semanticResult = snapshot?.semanticResult?.result ?? null;
|
||||
this.#active = snapshot?.activeTurnId !== null && snapshot?.activeTurnId !== undefined;
|
||||
this.#recoveredActiveTurn = this.#active;
|
||||
this.#nextSourceSeq = (snapshot?.lastSourceSequence ?? 0) + 1;
|
||||
this.#transcriptOmissionReason = snapshot === undefined
|
||||
? null
|
||||
: "pre_recovery_events_not_reconstructed";
|
||||
}
|
||||
|
||||
ids() {
|
||||
return {
|
||||
driverSessionId: `driver_${this.#input.normalizedSessionId}`,
|
||||
providerSessionId: null,
|
||||
displayId: this.#input.normalizedSessionId,
|
||||
};
|
||||
}
|
||||
|
||||
async *events(): AsyncIterable<PrpEvent> {
|
||||
if (this.#recoveredActiveTurn && this.#active && this.#turnId !== null) {
|
||||
// The deterministic fake has no provider process that can continue an
|
||||
// in-flight turn after reconstruction. Produce one conservative terminal
|
||||
// outcome when consumption resumes so the native loop cannot wait until
|
||||
// its global timeout. An explicit interrupt issued before consumption
|
||||
// still wins and supplies its own reason.
|
||||
if (!this.#settleRecoveredSemanticResult()) {
|
||||
this.#recoveredActiveTurn = false;
|
||||
this.#semanticResult = interruptedResult(
|
||||
this.#input.runId,
|
||||
"The deterministic provider was unavailable after active-turn recovery.",
|
||||
);
|
||||
this.#appendEvents(
|
||||
this.#event("run.result.proposed", this.#semanticResult),
|
||||
this.#event("turn.interrupted", {
|
||||
reason: "deterministic_active_turn_recovered_without_live_producer",
|
||||
}),
|
||||
this.#event("run.terminal", cancelledTerminal(), { turn: false }),
|
||||
);
|
||||
this.#recoveredTurnSettled = true;
|
||||
}
|
||||
this.#active = false;
|
||||
}
|
||||
let index = 0;
|
||||
while (true) {
|
||||
while (index < this.#events.length) {
|
||||
const event = this.#events[index++]!;
|
||||
yield structuredClone(event);
|
||||
if (event.eventType === "run.terminal") return;
|
||||
}
|
||||
if (this.#closed) return;
|
||||
await new Promise<void>((resolve) => this.#eventWaiters.add(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
async startTurn(input: { message: { role: "user"; text: string } }): Promise<{ turnId: string }> {
|
||||
this.#assertOpen();
|
||||
if (this.#turnId !== null) throw new Error("deterministic session accepts one turn");
|
||||
this.#turnId = `turn_${this.#input.runId}`;
|
||||
this.#active = true;
|
||||
this.#appendEvents(
|
||||
this.#event("session.started", {}, { turn: false }),
|
||||
this.#event("turn.started", { message: input.message.text }),
|
||||
);
|
||||
if (input.message.text.includes("[conformance:interrupt]")) {
|
||||
return { turnId: this.#turnId };
|
||||
}
|
||||
|
||||
const itemId = `item_${this.#input.runId}`;
|
||||
const callId = `call_${this.#input.runId}`;
|
||||
const correlation = {
|
||||
runId: this.#input.runId,
|
||||
normalizedSessionId: this.#input.normalizedSessionId,
|
||||
turnId: this.#turnId,
|
||||
itemId,
|
||||
};
|
||||
const inputEnvelope = createPrpSemanticToolInputEnvelope({
|
||||
operationId: "finish_task",
|
||||
callId,
|
||||
correlation,
|
||||
idempotencyKey: `finish_${this.#input.runId}`,
|
||||
content: { summary: "Deterministic conformance complete." },
|
||||
});
|
||||
const resultEnvelope = createPrpSemanticToolResultEnvelope({
|
||||
operationId: "finish_task",
|
||||
callId,
|
||||
correlation,
|
||||
idempotencyKey: `finish_${this.#input.runId}`,
|
||||
content: { disposition: "applied", status: "done" },
|
||||
outcome: "succeeded",
|
||||
code: "applied",
|
||||
retryable: false,
|
||||
authorizationBoundary: "active_task",
|
||||
auditReceiptId: `audit_${this.#input.runId}`,
|
||||
});
|
||||
this.#appendEvents(
|
||||
this.#event("mcp_app.tool_input", { semantic_tool: inputEnvelope }, { itemId }),
|
||||
this.#event("mcp_app.tool_result", { semantic_tool: resultEnvelope }, { itemId }),
|
||||
);
|
||||
this.#semanticResult = completedResult();
|
||||
const terminal = completedTerminal();
|
||||
this.#appendEvents(
|
||||
this.#event("run.result.proposed", this.#semanticResult),
|
||||
this.#event("turn.completed", {}),
|
||||
this.#event("run.terminal", terminal, { turn: false }),
|
||||
);
|
||||
this.#active = false;
|
||||
return { turnId: this.#turnId };
|
||||
}
|
||||
|
||||
async interrupt(input: { turnId?: string; reason?: string }): Promise<void> {
|
||||
this.#assertOpen();
|
||||
if (this.#turnId === null) throw new Error("no active turn to interrupt");
|
||||
if (input.turnId !== undefined && input.turnId !== this.#turnId) {
|
||||
throw new Error(`turn ${input.turnId} is not active`);
|
||||
}
|
||||
// Event consumption can conservatively settle a reconstructed turn before
|
||||
// its first event is yielded. Treat a concurrent interrupt as an
|
||||
// idempotent acknowledgement once that recovered terminal is committed,
|
||||
// rather than making cancellation depend on iterator scheduling.
|
||||
if (!this.#active) {
|
||||
if (this.#recoveredTurnSettled) return;
|
||||
throw new Error("no active turn to interrupt");
|
||||
}
|
||||
if (this.#settleRecoveredSemanticResult()) return;
|
||||
this.#semanticResult = interruptedResult(
|
||||
this.#input.runId,
|
||||
input.reason ?? "The deterministic turn was interrupted.",
|
||||
);
|
||||
this.#appendEvents(
|
||||
this.#event("run.result.proposed", this.#semanticResult),
|
||||
this.#event("turn.interrupted", { reason: input.reason ?? "interrupted" }),
|
||||
this.#event("run.terminal", cancelledTerminal(), { turn: false }),
|
||||
);
|
||||
this.#active = false;
|
||||
}
|
||||
|
||||
#settleRecoveredSemanticResult(): boolean {
|
||||
if (
|
||||
!this.#recoveredActiveTurn
|
||||
|| !this.#active
|
||||
|| this.#turnId === null
|
||||
|| this.#semanticResult === null
|
||||
) return false;
|
||||
this.#recoveredActiveTurn = false;
|
||||
// A result-first checkpoint can still carry activeTurnId when the
|
||||
// terminal checkpoint lagged behind the provider work. Preserve the
|
||||
// authoritative result even if cancellation races event consumption.
|
||||
if (this.#semanticResult.reportedWorkDisposition === "yielded") {
|
||||
this.#appendEvents(
|
||||
this.#event("run.result.proposed", this.#semanticResult),
|
||||
this.#event("turn.interrupted", {
|
||||
reason: "deterministic_active_turn_recovered_with_yielded_result",
|
||||
}),
|
||||
this.#event("run.terminal", cancelledTerminal(), { turn: false }),
|
||||
);
|
||||
} else {
|
||||
this.#appendEvents(
|
||||
this.#event("run.result.proposed", this.#semanticResult),
|
||||
this.#event("turn.completed", {}),
|
||||
this.#event(
|
||||
"run.terminal",
|
||||
completedTerminal(this.#semanticResult.reportedWorkDisposition),
|
||||
{ turn: false },
|
||||
),
|
||||
);
|
||||
}
|
||||
this.#recoveredTurnSettled = true;
|
||||
this.#active = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
async read(): Promise<Record<string, unknown>> {
|
||||
return { scenario: this.#config.scenario, eventCount: this.#events.length };
|
||||
}
|
||||
|
||||
async reconcile(): Promise<Record<string, unknown>> {
|
||||
return { active: this.#active, closed: this.#closed, eventCount: this.#events.length };
|
||||
}
|
||||
|
||||
async usage(): Promise<Record<string, unknown>> {
|
||||
return {
|
||||
inputTokens: this.#turnId === null ? 0 : 12,
|
||||
outputTokens: this.#semanticResult === null ? 0 : 8,
|
||||
totalTokens: this.#turnId === null ? 0 : this.#semanticResult === null ? 12 : 20,
|
||||
requestCount: this.#turnId === null ? 0 : 1,
|
||||
cost: { currency: "USD", amountMicros: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
async transcript(): Promise<HarnessTranscriptSnapshot> {
|
||||
return {
|
||||
schema: "paperclip-runner/harness-transcript/v1",
|
||||
complete: this.#transcriptOmissionReason === null,
|
||||
eventCount: this.#events.length,
|
||||
events: structuredClone(this.#events),
|
||||
omissionReason: this.#transcriptOmissionReason,
|
||||
};
|
||||
}
|
||||
|
||||
async snapshot(): Promise<PersistedHarnessSession> {
|
||||
return {
|
||||
driverKind: "paperclip-deterministic",
|
||||
driverSessionId: `driver_${this.#input.normalizedSessionId}`,
|
||||
providerSessionId: null,
|
||||
runId: this.#input.runId,
|
||||
normalizedSessionId: this.#input.normalizedSessionId,
|
||||
activeTurnId: this.#active ? this.#turnId : null,
|
||||
lastSourceSequence: this.#nextSourceSeq - 1,
|
||||
semanticResult: this.#semanticResult === null || this.#turnId === null
|
||||
? null
|
||||
: {
|
||||
result: structuredClone(this.#semanticResult),
|
||||
fingerprint: JSON.stringify(this.#semanticResult),
|
||||
turnId: this.#turnId,
|
||||
},
|
||||
terminalTurns: this.#events.some((event) => event.eventType === "run.terminal") && this.#turnId !== null
|
||||
? [{ turnId: this.#turnId, fingerprint: `terminal_${this.#input.runId}` }]
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
async close(input: { reason: string; force?: boolean }): Promise<void> {
|
||||
if (this.#closed) return;
|
||||
if (this.#active) {
|
||||
await this.interrupt({ reason: input.reason });
|
||||
}
|
||||
this.#closed = true;
|
||||
this.#notifyEventWaiters();
|
||||
}
|
||||
|
||||
#appendEvents(...events: PrpEvent[]): void {
|
||||
this.#events.push(...events);
|
||||
this.#notifyEventWaiters();
|
||||
}
|
||||
|
||||
#notifyEventWaiters(): void {
|
||||
for (const resolve of this.#eventWaiters) resolve();
|
||||
this.#eventWaiters.clear();
|
||||
}
|
||||
|
||||
#event(
|
||||
eventType: PrpEvent["eventType"],
|
||||
payload: Record<string, unknown> | PrpStructuredRunResult | PrpTerminalState,
|
||||
options: { itemId?: string; turn?: boolean } = {},
|
||||
): PrpEvent {
|
||||
const sourceSeq = this.#nextSourceSeq++;
|
||||
return {
|
||||
schema: "paperclip.prp.event.v1",
|
||||
sourceEventId: `event_${this.#input.runId}_${sourceSeq}`,
|
||||
sourceSeq,
|
||||
sourceInstanceId: `runner_${this.#input.runId}`,
|
||||
sourceKind: "runner",
|
||||
runId: this.#input.runId,
|
||||
normalizedSessionId: this.#input.normalizedSessionId,
|
||||
...(options.turn === false || this.#turnId === null ? {} : { turnId: this.#turnId }),
|
||||
...(options.itemId === undefined ? {} : { itemId: options.itemId }),
|
||||
eventType,
|
||||
schemaVersion: 1,
|
||||
priority: eventType === "run.terminal" || eventType === "run.result.proposed" ? 0 : 1,
|
||||
emittedAt: new Date(Date.UTC(2026, 7, 11, 12, 0, sourceSeq)).toISOString(),
|
||||
payload: structuredClone(payload),
|
||||
} as PrpEvent;
|
||||
}
|
||||
|
||||
#assertOpen(): void {
|
||||
if (this.#closed) throw new Error("deterministic session is closed");
|
||||
}
|
||||
}
|
||||
|
||||
function completedResult(): PrpStructuredRunResult {
|
||||
return {
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "done",
|
||||
summary: "Deterministic harness-driver conformance completed.",
|
||||
completionClaim: {
|
||||
contractRevision: "harness-driver-conformance-v1",
|
||||
objectiveSatisfied: true,
|
||||
criteria: [],
|
||||
remainingWork: [],
|
||||
},
|
||||
evidence: [],
|
||||
verification: [{ commandOrCheck: "deterministic harness flow", status: "passed" }],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
};
|
||||
}
|
||||
|
||||
function completedTerminal(
|
||||
disposition: PrpTerminalState["reportedWorkDisposition"] = "done",
|
||||
): PrpTerminalState {
|
||||
return {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: "completed",
|
||||
runTerminalState: "succeeded",
|
||||
reportedWorkDisposition: disposition,
|
||||
};
|
||||
}
|
||||
|
||||
function interruptedResult(runId: string, summary: string): PrpStructuredRunResult {
|
||||
return {
|
||||
schema: "paperclip.run_result.v1",
|
||||
reportedWorkDisposition: "yielded",
|
||||
summary,
|
||||
completionClaim: {
|
||||
contractRevision: "harness-driver-conformance-v1",
|
||||
objectiveSatisfied: false,
|
||||
criteria: [],
|
||||
remainingWork: [{
|
||||
description: "Resume the interrupted work in a live provider session.",
|
||||
blocksCompletion: true,
|
||||
}],
|
||||
},
|
||||
evidence: [],
|
||||
verification: [],
|
||||
attentionRequests: [],
|
||||
artifacts: [],
|
||||
continuation: {
|
||||
kind: "same_agent",
|
||||
summary: "Resume the interrupted deterministic turn.",
|
||||
idempotencyKey: `resume_deterministic_${runId}`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function cancelledTerminal(): PrpTerminalState {
|
||||
return {
|
||||
schema: "paperclip.prp.terminal.v1",
|
||||
turnTerminalState: "interrupted",
|
||||
runTerminalState: "cancelled",
|
||||
reportedWorkDisposition: "yielded",
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { loadConformanceFixture } from "../protocol/conformance-fixture.js";
|
||||
import { replayConformanceFixture } from "../tracer/conformance-runner.js";
|
||||
import { MockControlPlaneAdapter } from "./mock-control-plane-adapter.js";
|
||||
|
||||
describe("MockControlPlaneAdapter", () => {
|
||||
it("exercises the complete control-plane contract path", async () => {
|
||||
const fixture = await loadConformanceFixture();
|
||||
const mockCore = new MockControlPlaneAdapter();
|
||||
|
||||
await mockCore.start();
|
||||
await replayConformanceFixture(mockCore, fixture);
|
||||
|
||||
expect(mockCore.snapshot()).toMatchObject({
|
||||
lifecycle: "running",
|
||||
openedRun: { identity: fixture.run, backendKind: "mock" },
|
||||
events: fixture.events,
|
||||
result: fixture.result,
|
||||
});
|
||||
await mockCore.stop();
|
||||
});
|
||||
|
||||
it("rejects events before startup", async () => {
|
||||
const fixture = await loadConformanceFixture();
|
||||
const mockCore = new MockControlPlaneAdapter();
|
||||
|
||||
await expect(mockCore.appendEvent(fixture.events[0]!)).rejects.toThrow(
|
||||
"mock control plane must be started first",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,199 @@
|
|||
import type {
|
||||
AppendedEventReceipt,
|
||||
CompleteControlPlaneRunInput,
|
||||
ControlPlanePort,
|
||||
OpenControlPlaneRunInput,
|
||||
ReplayControlPlaneEventsInput,
|
||||
ReplayedControlPlaneEvents,
|
||||
} from "../contracts/control-plane-port.js";
|
||||
import type { NativeRunEvent, NativeRunResult } from "../contracts/types.js";
|
||||
import {
|
||||
validatePrpEvent,
|
||||
validatePrpStructuredRunResult,
|
||||
type PrpEvent,
|
||||
} from "../protocol/replay-contract.js";
|
||||
|
||||
export interface MockControlPlaneSnapshot {
|
||||
lifecycle: "stopped" | "running";
|
||||
openedRun: OpenControlPlaneRunInput | null;
|
||||
events: Array<NativeRunEvent | PrpEvent>;
|
||||
result: NativeRunResult | CompleteControlPlaneRunInput | null;
|
||||
}
|
||||
|
||||
/** In-memory control-plane shell used by standalone tests and tutorials. */
|
||||
export class MockControlPlaneAdapter implements ControlPlanePort {
|
||||
#lifecycle: "stopped" | "running" = "stopped";
|
||||
#openedRun: OpenControlPlaneRunInput | null = null;
|
||||
#events: Array<NativeRunEvent | PrpEvent> = [];
|
||||
#result: NativeRunResult | CompleteControlPlaneRunInput | null = null;
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.#lifecycle === "running") {
|
||||
throw new Error("mock control plane is already running");
|
||||
}
|
||||
this.#lifecycle = "running";
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
this.#lifecycle = "stopped";
|
||||
}
|
||||
|
||||
async openRun(input: OpenControlPlaneRunInput): Promise<void> {
|
||||
this.#assertRunning();
|
||||
if (this.#openedRun !== null) {
|
||||
throw new Error("mock control plane accepts one Conformance run");
|
||||
}
|
||||
this.#openedRun = structuredClone(input);
|
||||
}
|
||||
|
||||
async appendEvent(event: NativeRunEvent | PrpEvent): Promise<AppendedEventReceipt> {
|
||||
const runId = this.#requireRunId();
|
||||
if (event.runId !== runId) {
|
||||
throw new Error("event runId does not match the opened run");
|
||||
}
|
||||
if ("sourceEventId" in event) {
|
||||
const validation = validatePrpEvent(event);
|
||||
if (!validation.ok) {
|
||||
throw new Error(`event failed validation: ${validation.issues[0]?.message ?? "invalid event"}`);
|
||||
}
|
||||
const sameId = this.#events.find(
|
||||
(candidate): candidate is PrpEvent =>
|
||||
"sourceEventId" in candidate && candidate.sourceEventId === event.sourceEventId,
|
||||
);
|
||||
const sameSequence = this.#events.find(
|
||||
(candidate): candidate is PrpEvent =>
|
||||
"sourceSeq" in candidate &&
|
||||
candidate.sourceInstanceId === event.sourceInstanceId &&
|
||||
candidate.sourceSeq === event.sourceSeq,
|
||||
);
|
||||
for (const candidate of [sameId, sameSequence]) {
|
||||
if (candidate !== undefined) {
|
||||
if (canonicalJson(candidate) !== canonicalJson(event)) {
|
||||
throw new Error("native_event_replay_conflict");
|
||||
}
|
||||
return {
|
||||
cursor: event.sourceSeq,
|
||||
highestContiguousSourceSeq: this.#highestContiguous(event.sourceInstanceId),
|
||||
disposition: "duplicate",
|
||||
};
|
||||
}
|
||||
}
|
||||
this.#events.push(structuredClone(event));
|
||||
return {
|
||||
cursor: event.sourceSeq,
|
||||
highestContiguousSourceSeq: this.#highestContiguous(event.sourceInstanceId),
|
||||
disposition: "committed",
|
||||
};
|
||||
}
|
||||
|
||||
const expectedSequence = this.#events.length + 1;
|
||||
if (event.sequence !== expectedSequence) {
|
||||
throw new Error(`event sequence must be ${expectedSequence}`);
|
||||
}
|
||||
this.#events.push(structuredClone(event));
|
||||
return {
|
||||
cursor: event.sequence,
|
||||
highestContiguousSourceSeq: event.sequence,
|
||||
disposition: "committed",
|
||||
};
|
||||
}
|
||||
|
||||
async replayEvents(
|
||||
input: ReplayControlPlaneEventsInput,
|
||||
): Promise<ReplayedControlPlaneEvents> {
|
||||
const runId = this.#requireRunId();
|
||||
if (input.runId !== runId) {
|
||||
throw new Error("replay runId does not match the opened run");
|
||||
}
|
||||
const events = this.#events
|
||||
.filter(
|
||||
(event): event is PrpEvent =>
|
||||
"sourceSeq" in event &&
|
||||
event.sourceInstanceId === input.sourceInstanceId &&
|
||||
event.sourceSeq > input.afterSourceSeq,
|
||||
)
|
||||
.sort((left, right) => left.sourceSeq - right.sourceSeq)
|
||||
.slice(0, input.limit)
|
||||
.map((event) => structuredClone(event));
|
||||
return {
|
||||
events,
|
||||
highestContiguousSourceSeq: this.#highestContiguous(input.sourceInstanceId),
|
||||
};
|
||||
}
|
||||
|
||||
async completeRun(result: NativeRunResult | CompleteControlPlaneRunInput): Promise<void> {
|
||||
const runId = this.#requireRunId();
|
||||
const resultRunId = "runId" in result ? result.runId : runId;
|
||||
if (resultRunId !== runId) {
|
||||
throw new Error("result runId does not match the opened run");
|
||||
}
|
||||
if ("terminal" in result) {
|
||||
const validation = validatePrpStructuredRunResult(result.result);
|
||||
if (!validation.ok) {
|
||||
throw new Error("native_result_invalid");
|
||||
}
|
||||
const terminalEvent = this.#events.find(
|
||||
(event) => "eventType" in event && event.eventType === "run.terminal",
|
||||
);
|
||||
if (terminalEvent === undefined) {
|
||||
throw new Error("run.terminal must be ingested before the result");
|
||||
}
|
||||
} else if (this.#events.at(-1)?.type !== "run.completed") {
|
||||
throw new Error("run.completed must be ingested before the result");
|
||||
}
|
||||
if (this.#result !== null) {
|
||||
if (canonicalJson(this.#result) === canonicalJson(result)) return;
|
||||
throw new Error("native_result_replay_conflict");
|
||||
}
|
||||
this.#result = structuredClone(result);
|
||||
}
|
||||
|
||||
snapshot(): MockControlPlaneSnapshot {
|
||||
return {
|
||||
lifecycle: this.#lifecycle,
|
||||
openedRun: this.#openedRun === null ? null : structuredClone(this.#openedRun),
|
||||
events: structuredClone(this.#events),
|
||||
result: this.#result === null ? null : structuredClone(this.#result),
|
||||
};
|
||||
}
|
||||
|
||||
#assertRunning(): void {
|
||||
if (this.#lifecycle !== "running") {
|
||||
throw new Error("mock control plane must be started first");
|
||||
}
|
||||
}
|
||||
|
||||
#requireRunId(): string {
|
||||
this.#assertRunning();
|
||||
if (this.#openedRun === null) {
|
||||
throw new Error("a run must be opened first");
|
||||
}
|
||||
return this.#openedRun.identity.runId;
|
||||
}
|
||||
|
||||
#highestContiguous(sourceInstanceId: string): number {
|
||||
const sequences = new Set(
|
||||
this.#events
|
||||
.filter(
|
||||
(event): event is PrpEvent =>
|
||||
"sourceSeq" in event && event.sourceInstanceId === sourceInstanceId,
|
||||
)
|
||||
.map((event) => event.sourceSeq),
|
||||
);
|
||||
let cursor = 0;
|
||||
while (sequences.has(cursor + 1)) cursor += 1;
|
||||
return cursor;
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalJson(value: unknown): string {
|
||||
if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`;
|
||||
if (typeof value === "object" && value !== null) {
|
||||
const record = value as Record<string, unknown>;
|
||||
return `{${Object.keys(record)
|
||||
.sort()
|
||||
.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`)
|
||||
.join(",")}}`;
|
||||
}
|
||||
return JSON.stringify(value) ?? "undefined";
|
||||
}
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
loadConformanceFixture,
|
||||
validateConformanceFixture,
|
||||
} from "./conformance-fixture.js";
|
||||
|
||||
describe("Conformance fixture validation", () => {
|
||||
it("accepts the package-owned fixture", async () => {
|
||||
const fixture = await loadConformanceFixture();
|
||||
|
||||
expect(fixture.run.runId).toBe("run_conformance_0001");
|
||||
expect(fixture.events.map((event) => event.type)).toEqual([
|
||||
"run.started",
|
||||
"run.completed",
|
||||
]);
|
||||
expect(fixture.result.status).toBe("succeeded");
|
||||
});
|
||||
|
||||
it("rejects a non-contiguous event sequence", () => {
|
||||
expect(() =>
|
||||
validateConformanceFixture({
|
||||
schemaVersion: "paperclip.runner.conformance.fixture.v1",
|
||||
run: {
|
||||
runId: "run_conformance_0001",
|
||||
sessionId: "session_conformance_0001",
|
||||
companyId: "company_conformance",
|
||||
issueId: "issue_conformance",
|
||||
agentId: "agent_conformance",
|
||||
},
|
||||
events: [
|
||||
{
|
||||
eventId: "event_conformance_0001",
|
||||
runId: "run_conformance_0001",
|
||||
sequence: 2,
|
||||
type: "run.started",
|
||||
},
|
||||
{
|
||||
eventId: "event_conformance_0002",
|
||||
runId: "run_conformance_0001",
|
||||
sequence: 3,
|
||||
type: "run.completed",
|
||||
},
|
||||
],
|
||||
result: {
|
||||
runId: "run_conformance_0001",
|
||||
status: "succeeded",
|
||||
summary: "Standalone Conformance fixture accepted.",
|
||||
},
|
||||
}),
|
||||
).toThrow("events[0].sequence must be 1");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,127 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import type {
|
||||
NativeRunEvent,
|
||||
NativeRunEventType,
|
||||
NativeRunIdentity,
|
||||
NativeRunResult,
|
||||
NativeRunStatus,
|
||||
} from "../contracts/types.js";
|
||||
|
||||
export const CONFORMANCE_FIXTURE_SCHEMA = "paperclip.runner.conformance.fixture.v1";
|
||||
|
||||
export interface ConformanceFixture {
|
||||
schemaVersion: typeof CONFORMANCE_FIXTURE_SCHEMA;
|
||||
run: NativeRunIdentity;
|
||||
events: NativeRunEvent[];
|
||||
result: NativeRunResult;
|
||||
}
|
||||
|
||||
export const conformanceFixtureUrl = new URL(
|
||||
"../../protocol/fixtures/conformance-minimal-run.json",
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
function asRecord(value: unknown, path: string): Record<string, unknown> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) {
|
||||
throw new Error(`${path} must be an object`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function asIdentifier(value: unknown, path: string): string {
|
||||
if (typeof value !== "string" || !/^[a-z][a-z0-9_]*$/.test(value)) {
|
||||
throw new Error(`${path} must be a stable lowercase identifier`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function asString(value: unknown, path: string): string {
|
||||
if (typeof value !== "string" || value.trim().length === 0) {
|
||||
throw new Error(`${path} must be a non-empty string`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseRun(value: unknown): NativeRunIdentity {
|
||||
const run = asRecord(value, "run");
|
||||
return {
|
||||
runId: asIdentifier(run.runId, "run.runId"),
|
||||
sessionId: asIdentifier(run.sessionId, "run.sessionId"),
|
||||
companyId: asIdentifier(run.companyId, "run.companyId"),
|
||||
issueId: asIdentifier(run.issueId, "run.issueId"),
|
||||
agentId: asIdentifier(run.agentId, "run.agentId"),
|
||||
};
|
||||
}
|
||||
|
||||
function parseEvents(value: unknown, runId: string): NativeRunEvent[] {
|
||||
if (!Array.isArray(value) || value.length < 2) {
|
||||
throw new Error("events must contain at least run.started and run.completed");
|
||||
}
|
||||
|
||||
const allowedTypes = new Set<NativeRunEventType>(["run.started", "run.completed"]);
|
||||
const events = value.map((entry, index) => {
|
||||
const event = asRecord(entry, `events[${index}]`);
|
||||
const type = asString(event.type, `events[${index}].type`) as NativeRunEventType;
|
||||
if (!allowedTypes.has(type)) {
|
||||
throw new Error(`events[${index}].type is unsupported`);
|
||||
}
|
||||
if (event.runId !== runId) {
|
||||
throw new Error(`events[${index}].runId must match run.runId`);
|
||||
}
|
||||
if (event.sequence !== index + 1) {
|
||||
throw new Error(`events[${index}].sequence must be ${index + 1}`);
|
||||
}
|
||||
return {
|
||||
eventId: asIdentifier(event.eventId, `events[${index}].eventId`),
|
||||
runId,
|
||||
sequence: index + 1,
|
||||
type,
|
||||
};
|
||||
});
|
||||
|
||||
if (events[0]?.type !== "run.started") {
|
||||
throw new Error("the first event must be run.started");
|
||||
}
|
||||
if (events.at(-1)?.type !== "run.completed") {
|
||||
throw new Error("the final event must be run.completed");
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
function parseResult(value: unknown, runId: string): NativeRunResult {
|
||||
const result = asRecord(value, "result");
|
||||
if (result.runId !== runId) {
|
||||
throw new Error("result.runId must match run.runId");
|
||||
}
|
||||
const status = asString(result.status, "result.status") as NativeRunStatus;
|
||||
if (status !== "succeeded") {
|
||||
throw new Error("the Conformance fixture result must be succeeded");
|
||||
}
|
||||
return {
|
||||
runId,
|
||||
status,
|
||||
summary: asString(result.summary, "result.summary"),
|
||||
};
|
||||
}
|
||||
|
||||
export function validateConformanceFixture(value: unknown): ConformanceFixture {
|
||||
const fixture = asRecord(value, "fixture");
|
||||
if (fixture.schemaVersion !== CONFORMANCE_FIXTURE_SCHEMA) {
|
||||
throw new Error(`schemaVersion must be ${CONFORMANCE_FIXTURE_SCHEMA}`);
|
||||
}
|
||||
const run = parseRun(fixture.run);
|
||||
return {
|
||||
schemaVersion: CONFORMANCE_FIXTURE_SCHEMA,
|
||||
run,
|
||||
events: parseEvents(fixture.events, run.runId),
|
||||
result: parseResult(fixture.result, run.runId),
|
||||
};
|
||||
}
|
||||
|
||||
export async function loadConformanceFixture(
|
||||
url: URL = conformanceFixtureUrl,
|
||||
): Promise<ConformanceFixture> {
|
||||
const contents = await readFile(url, "utf8");
|
||||
return validateConformanceFixture(JSON.parse(contents) as unknown);
|
||||
}
|
||||
|
|
@ -9,4 +9,8 @@ export * from "./index.js";
|
|||
export * from "./conformance/control-plane-port.js";
|
||||
export * from "./conformance/harness-driver.js";
|
||||
export * from "./conformance/semantic-conformance.js";
|
||||
export * from "./mock-core/deterministic-harness-driver.js";
|
||||
export * from "./mock-core/mock-control-plane-adapter.js";
|
||||
export * from "./protocol/conformance-fixture.js";
|
||||
export * from "./protocol/replay-loader.js";
|
||||
export * from "./tracer/conformance-runner.js";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js";
|
||||
import { runConformanceTracer } from "./conformance-runner.js";
|
||||
|
||||
const expectedOutputUrl = new URL(
|
||||
"../../protocol/fixtures/conformance-expected-output.json",
|
||||
import.meta.url,
|
||||
);
|
||||
|
||||
describe("Conformance tracer", () => {
|
||||
it("prints a stable identity and result and stops the mock core", async () => {
|
||||
const mockCore = new MockControlPlaneAdapter();
|
||||
const expectedOutput = (await readFile(expectedOutputUrl, "utf8")).trimEnd();
|
||||
|
||||
await expect(runConformanceTracer(mockCore)).resolves.toBe(expectedOutput);
|
||||
expect(mockCore.snapshot().lifecycle).toBe("stopped");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import type { ControlPlanePort } from "../contracts/control-plane-port.js";
|
||||
import { MockControlPlaneAdapter } from "../mock-core/mock-control-plane-adapter.js";
|
||||
import {
|
||||
loadConformanceFixture,
|
||||
type ConformanceFixture,
|
||||
} from "../protocol/conformance-fixture.js";
|
||||
|
||||
export const CONFORMANCE_OUTPUT_SCHEMA = "paperclip.runner.conformance.output.v1";
|
||||
|
||||
export interface ConformanceTracerOutput {
|
||||
schemaVersion: typeof CONFORMANCE_OUTPUT_SCHEMA;
|
||||
runIdentity: {
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
};
|
||||
result: {
|
||||
status: "succeeded";
|
||||
summary: string;
|
||||
};
|
||||
}
|
||||
|
||||
export async function replayConformanceFixture(
|
||||
controlPlane: ControlPlanePort,
|
||||
fixture: ConformanceFixture,
|
||||
): Promise<void> {
|
||||
await controlPlane.openRun({ identity: fixture.run, backendKind: "mock" });
|
||||
for (const event of fixture.events) {
|
||||
await controlPlane.appendEvent(event);
|
||||
}
|
||||
await controlPlane.completeRun(fixture.result);
|
||||
}
|
||||
|
||||
export function formatConformanceOutput(fixture: ConformanceFixture): string {
|
||||
const output: ConformanceTracerOutput = {
|
||||
schemaVersion: CONFORMANCE_OUTPUT_SCHEMA,
|
||||
runIdentity: {
|
||||
runId: fixture.run.runId,
|
||||
sessionId: fixture.run.sessionId,
|
||||
},
|
||||
result: {
|
||||
status: "succeeded",
|
||||
summary: fixture.result.summary,
|
||||
},
|
||||
};
|
||||
return JSON.stringify(output);
|
||||
}
|
||||
|
||||
export async function runConformanceTracer(
|
||||
mockCore = new MockControlPlaneAdapter(),
|
||||
): Promise<string> {
|
||||
const fixture = await loadConformanceFixture();
|
||||
await mockCore.start();
|
||||
try {
|
||||
await replayConformanceFixture(mockCore, fixture);
|
||||
return formatConformanceOutput(fixture);
|
||||
} finally {
|
||||
await mockCore.stop();
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue