diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b1f4138959..e356bb8a09 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -280,6 +280,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Verify Paperclip Runner + run: pnpm --filter @paperclipai/paperclip-runner check:all + - name: Build run: pnpm build diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index df146e7faa..55d457a24d 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -182,5 +182,8 @@ jobs: - name: Install dependencies run: pnpm install --no-frozen-lockfile + - name: Verify Paperclip Runner + run: pnpm --filter @paperclipai/paperclip-runner check:all + - name: Build run: pnpm build diff --git a/packages/paperclip-runner/README.md b/packages/paperclip-runner/README.md index b66e45a521..9d3a5d83a5 100644 --- a/packages/paperclip-runner/README.md +++ b/packages/paperclip-runner/README.md @@ -22,8 +22,16 @@ undiscoverable because no production application binding or server authority has landed. Catalog membership alone does not grant authority. See [`SEMANTIC_ACTIONS.md`](SEMANTIC_ACTIONS.md) for the catalog boundary. -The root export is intentionally narrow. The `./testing` entry point and package -release boundary will arrive with the later package-boundary change. +The package has two initial public surfaces: + +- `@paperclipai/paperclip-runner` contains runtime contracts, validation, + replay/reducer logic, the semantic catalog, and the authorization dispatcher. +- `@paperclipai/paperclip-runner/testing` adds Node-only fixture loading and a + provider-neutral semantic conformance kit for deterministic test adapters. + +No SDK, browser, React, eval, live-console, lab, or provider-experiment entry +point is exported. The package remains private in this wave, and no production +adapter starts it yet. Run the complete contract gate with: @@ -37,9 +45,11 @@ Run the Rust runner gate with: pnpm --filter @paperclipai/paperclip-runner check:runner ``` -This command checks Rust formatting, builds and tests the minimal workspace, -verifies bounded process cleanup, exercises the fake local runner, and compares -the Rust conformance and replay summaries with the shared fixtures. +This command checks Rust formatting, builds and tests the minimal workspace in +release mode, verifies bounded process cleanup, launches the real +`paperclip-runnerd` binary through the fake harness, and compares the Rust +conformance and replay summaries with the shared fixtures. The checked-in Cargo +lock and pinned Rust toolchain keep this verification reproducible. Durability and failure semantics are documented in [`runner/DURABLE_TRANSPORT.md`](runner/DURABLE_TRANSPORT.md). The fault suite diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 0d497231f3..626d410d8e 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -11,8 +11,13 @@ ".": { "types": "./dist/index.d.ts", "import": "./dist/index.js" + }, + "./testing": { + "types": "./dist/testing.d.ts", + "import": "./dist/testing.js" } }, + "sideEffects": false, "files": [ "dist", "protocol", @@ -27,7 +32,7 @@ "typecheck:rust": "cargo fmt --manifest-path runner/Cargo.toml --all -- --check && cargo check --manifest-path runner/Cargo.toml --locked --workspace", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "node --test test/protocol-contract.test.mjs && vitest run", - "test:rust": "cargo test --manifest-path runner/Cargo.toml --locked --workspace", + "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", "generate:protocol-manifest": "node scripts/generate-protocol-manifest.mjs", diff --git a/packages/paperclip-runner/rust-toolchain.toml b/packages/paperclip-runner/rust-toolchain.toml new file mode 100644 index 0000000000..cf01515b3a --- /dev/null +++ b/packages/paperclip-runner/rust-toolchain.toml @@ -0,0 +1,4 @@ +[toolchain] +channel = "1.97.1" +profile = "minimal" +components = ["rustfmt"] diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts new file mode 100644 index 0000000000..d44c42184f --- /dev/null +++ b/packages/paperclip-runner/src/conformance/semantic-conformance.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, it } from "vitest"; + +import { + SemanticConformanceMismatchError, + runSemanticConformanceKit, + type SemanticConformanceAdapter, + type SemanticConformanceObservation, +} from "./semantic-conformance.js"; + +const allowed: SemanticConformanceObservation = { + authorization: { outcome: "allowed" }, + state: { task: { status: "done" } }, + effects: [{ kind: "issue_status", status: "done" }], + audit: [{ action: "finish_task" }], +}; + +function adapter( + id: string, + observation: SemanticConformanceObservation, +): SemanticConformanceAdapter { + return { id, execute: async () => structuredClone(observation) }; +} + +describe("semantic conformance kit", () => { + it("accepts equivalent observations independent of object key order", async () => { + const report = await runSemanticConformanceKit({ + vectors: [ + { + id: "finish", + operationId: "finish_task", + input: { summary: "done" }, + }, + ], + adapters: [ + adapter("mock", allowed), + adapter("real", { + audit: [{ action: "finish_task" }], + effects: [{ status: "done", kind: "issue_status" }], + state: { task: { status: "done" } }, + authorization: { outcome: "allowed" }, + }), + ], + }); + + expect(report.schema).toBe("paperclip.semantic-conformance-report.v1"); + expect(report.rows).toHaveLength(1); + expect(report.rows[0]?.adapterIds).toEqual(["mock", "real"]); + }); + + it("fails explicitly when adapters diverge", async () => { + await expect( + runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: {} }], + adapters: [ + adapter("mock", allowed), + adapter("real", { + ...allowed, + authorization: { outcome: "denied", code: "forbidden" }, + }), + ], + }), + ).rejects.toBeInstanceOf(SemanticConformanceMismatchError); + }); + + it("requires at least two adapters", async () => { + await expect( + runSemanticConformanceKit({ + vectors: [], + adapters: [adapter("only", allowed)], + }), + ).rejects.toThrow("semantic_conformance_requires_two_adapters"); + }); + + it("requires unique adapter identities", async () => { + await expect( + runSemanticConformanceKit({ + vectors: [], + adapters: [ + adapter("duplicate", allowed), + adapter("duplicate", allowed), + ], + }), + ).rejects.toThrow("semantic_conformance_adapter_ids_must_be_unique"); + }); + + it("fails closed for non-JSON normalized observations", async () => { + await expect( + runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: {} }], + adapters: [ + adapter("mock", allowed), + { + id: "invalid", + execute: async () => + ({ + ...allowed, + state: new Date(), + }) as unknown as SemanticConformanceObservation, + }, + ], + }), + ).rejects.toThrow("semantic_conformance_non_json_observation"); + }); + + it("fails closed for sparse normalized arrays", async () => { + const sparseEffects = Array(1); + await expect( + runSemanticConformanceKit({ + vectors: [{ id: "finish", operationId: "finish_task", input: {} }], + adapters: [ + adapter("mock", allowed), + { + id: "invalid", + execute: async () => + ({ + ...allowed, + effects: sparseEffects, + }) as unknown as SemanticConformanceObservation, + }, + ], + }), + ).rejects.toThrow("semantic_conformance_non_json_observation"); + }); +}); diff --git a/packages/paperclip-runner/src/conformance/semantic-conformance.ts b/packages/paperclip-runner/src/conformance/semantic-conformance.ts new file mode 100644 index 0000000000..38408d6d98 --- /dev/null +++ b/packages/paperclip-runner/src/conformance/semantic-conformance.ts @@ -0,0 +1,167 @@ +export type SemanticConformanceAuthorization = + | { readonly outcome: "allowed" } + | { readonly outcome: "denied"; readonly code: string }; + +export type SemanticConformanceJsonValue = + | null + | boolean + | number + | string + | readonly SemanticConformanceJsonValue[] + | { readonly [key: string]: SemanticConformanceJsonValue }; + +export interface SemanticConformanceVector { + readonly id: string; + readonly operationId: string; + readonly input: SemanticConformanceJsonValue; +} + +export interface SemanticConformanceObservation { + readonly authorization: SemanticConformanceAuthorization; + readonly state: SemanticConformanceJsonValue; + readonly effects: readonly SemanticConformanceJsonValue[]; + readonly audit: readonly SemanticConformanceJsonValue[]; +} + +export interface SemanticConformanceAdapter { + readonly id: string; + execute( + vector: SemanticConformanceVector, + ): Promise; +} + +export interface SemanticConformanceReportRow { + readonly vectorId: string; + readonly operationId: string; + readonly adapterIds: readonly string[]; + readonly observation: SemanticConformanceObservation; +} + +export interface SemanticConformanceReport { + readonly schema: "paperclip.semantic-conformance-report.v1"; + readonly rows: readonly SemanticConformanceReportRow[]; +} + +export class SemanticConformanceMismatchError extends Error { + readonly code = "semantic_conformance_mismatch" as const; + + constructor( + readonly vectorId: string, + readonly baselineAdapterId: string, + readonly mismatchedAdapterId: string, + ) { + super( + `Semantic conformance mismatch for ${vectorId}: ${mismatchedAdapterId} differs from ${baselineAdapterId}`, + ); + this.name = "SemanticConformanceMismatchError"; + } +} + +/** + * Compare normalized authorization, state, effects, and audit output from two + * or more adapters. Adapters own setup and normalization; the kit owns a + * deterministic, provider-neutral comparison. + */ +export async function runSemanticConformanceKit(input: { + readonly vectors: readonly SemanticConformanceVector[]; + readonly adapters: readonly SemanticConformanceAdapter[]; +}): Promise { + if (input.adapters.length < 2) { + throw new Error("semantic_conformance_requires_two_adapters"); + } + if ( + new Set(input.adapters.map((adapter) => adapter.id)).size !== + input.adapters.length + ) { + throw new Error("semantic_conformance_adapter_ids_must_be_unique"); + } + + const rows: SemanticConformanceReportRow[] = []; + for (const vector of input.vectors) { + const observations = await Promise.all( + input.adapters.map(async (adapter) => ({ + adapter, + observation: await adapter.execute(vector), + })), + ); + const baseline = observations[0]!; + const baselineJson = canonicalJson(baseline.observation); + for (const candidate of observations.slice(1)) { + if (canonicalJson(candidate.observation) !== baselineJson) { + throw new SemanticConformanceMismatchError( + vector.id, + baseline.adapter.id, + candidate.adapter.id, + ); + } + } + rows.push( + Object.freeze({ + vectorId: vector.id, + operationId: vector.operationId, + adapterIds: Object.freeze(input.adapters.map((adapter) => adapter.id)), + observation: structuredClone(baseline.observation), + }), + ); + } + + return Object.freeze({ + schema: "paperclip.semantic-conformance-report.v1", + rows: Object.freeze(rows), + }); +} + +function canonicalJson( + value: unknown, + ancestors = new WeakSet(), +): string { + if ( + value === null || + typeof value === "boolean" || + typeof value === "string" + ) { + return JSON.stringify(value); + } + if (typeof value === "number") { + if (!Number.isFinite(value)) throw invalidObservation(); + return JSON.stringify(value); + } + if (typeof value !== "object") throw invalidObservation(); + if (ancestors.has(value)) { + throw new Error("semantic_conformance_cyclic_observation"); + } + + const prototype = Object.getPrototypeOf(value); + if ( + !Array.isArray(value) && + prototype !== Object.prototype && + prototype !== null + ) { + throw invalidObservation(); + } + ancestors.add(value); + try { + if (Array.isArray(value)) { + const entries: string[] = []; + for (let index = 0; index < value.length; index += 1) { + if (!Object.hasOwn(value, index)) throw invalidObservation(); + entries.push(canonicalJson(value[index], ancestors)); + } + return `[${entries.join(",")}]`; + } + const record = value as Record; + return `{${Object.keys(record) + .sort() + .map( + (key) => + `${JSON.stringify(key)}:${canonicalJson(record[key], ancestors)}`, + ) + .join(",")}}`; + } finally { + ancestors.delete(value); + } +} + +function invalidObservation(): Error { + return new Error("semantic_conformance_non_json_observation"); +} diff --git a/packages/paperclip-runner/src/index.ts b/packages/paperclip-runner/src/index.ts index 27d6eb7b86..c43eec2060 100644 --- a/packages/paperclip-runner/src/index.ts +++ b/packages/paperclip-runner/src/index.ts @@ -2,7 +2,6 @@ export * from "./catalog/index.js"; export * from "./contracts/completion-result.js"; export * from "./contracts/question-set.js"; export * from "./protocol/replay-contract.js"; -export * from "./protocol/replay-loader.js"; export * from "./protocol/result-normalization.js"; export * from "./reducer/session-reducer.js"; export * from "./semantic-tools/index.js"; diff --git a/packages/paperclip-runner/src/testing.ts b/packages/paperclip-runner/src/testing.ts new file mode 100644 index 0000000000..68ab11bfac --- /dev/null +++ b/packages/paperclip-runner/src/testing.ts @@ -0,0 +1,10 @@ +/** + * Public test-only surface for deterministic fixtures and conformance kits. + * + * Production consumers import the package root. Tests import this explicit + * subpath so Node-only fixture loading and comparison helpers cannot become an + * accidental production dependency. + */ +export * from "./index.js"; +export * from "./conformance/semantic-conformance.js"; +export * from "./protocol/replay-loader.js"; diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index d32f08ebcc..7ddc23bf08 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -104,6 +104,7 @@ test("release verify workflow covers the same split test surface as stable PR ve assert.match(verifyWorkflow, /node \.\/scripts\/release-package-map\.mjs check/); assert.match(verifyWorkflow, /pnpm -r typecheck/); assert.match(verifyWorkflow, /pnpm build/); + assert.match(verifyWorkflow, /pnpm --filter @paperclipai\/paperclip-runner check:all/); for (const group of ["general-server", "general-workspaces-a", "general-workspaces-b"]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); diff --git a/server/package.json b/server/package.json index e7d9b3f610..e4cc6bf6d0 100644 --- a/server/package.json +++ b/server/package.json @@ -36,7 +36,7 @@ "dev:watch": "cross-env PAPERCLIP_MIGRATION_PROMPT=never PAPERCLIP_MIGRATION_AUTO_APPLY=true tsx ./scripts/dev-watch.ts", "prepare:ui-dist": "bash ../scripts/prepare-server-ui-dist.sh", "build": "tsc && mkdir -p dist/onboarding-assets dist/built-ins dist/services/scripts && cp -R src/onboarding-assets/. dist/onboarding-assets/ && cp -R src/built-ins/. dist/built-ins/ && cp -R src/services/scripts/. dist/services/scripts/ && node scripts/write-build-stamp.mjs", - "prepack": "pnpm run prepare:ui-dist", + "prepack": "pnpm run prepare:ui-dist && pnpm run build", "postpack": "rm -rf ui-dist", "clean": "rm -rf dist", "start": "node dist/index.js", diff --git a/server/src/__tests__/server-package-build-script.test.ts b/server/src/__tests__/server-package-build-script.test.ts index 9c4d16a916..92b410ce70 100644 --- a/server/src/__tests__/server-package-build-script.test.ts +++ b/server/src/__tests__/server-package-build-script.test.ts @@ -5,6 +5,14 @@ import { describe, expect, it } from "vitest"; const packageJsonPath = fileURLToPath(new URL("../../package.json", import.meta.url)); describe("server package build script", () => { + it("builds the compiled package entry during prepack", () => { + const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { + scripts?: Record; + }; + + expect(packageJson.scripts?.prepack).toBe("pnpm run prepare:ui-dist && pnpm run build"); + }); + it("copies static runtime asset directories into dist", () => { const packageJson = JSON.parse(readFileSync(packageJsonPath, "utf8")) as { scripts?: Record;